From ced455ef7bdf7d33ae0ce8c01ee0d8b40b996388 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 9 Oct 2018 09:15:47 +0200 Subject: [PATCH 1/8] Cache redesign - Parsed and Meta cache separation * seperate Parsed and Meta cache, so that they may be locked independently * extend the RunAndParse() function to allow the caller to specify a callback which will update the Meta cache * change all methods interacting with the Meta cache to use the callback for RunAndParse() to update the Meta cache --- bird/bird.go | 162 ++++++++++++++++++++++++++++-------------- endpoints/endpoint.go | 2 +- 2 files changed, 111 insertions(+), 53 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index cf18015..f672d85 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -20,10 +20,13 @@ var RateLimitConf struct { Conf RateLimitConfig } -var Cache = struct { +type Cache struct { sync.RWMutex m map[string]Parsed -}{m: make(map[string]Parsed)} +} + +var ParsedCache = Cache{m: make(map[string]Parsed)} +var MetaCache = Cache{m: make(map[string]Parsed)} var NilParse Parsed = (Parsed)(nil) var BirdError Parsed = Parsed{"error": "bird unreachable"} @@ -32,10 +35,27 @@ func IsSpecial(ret Parsed) bool { return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError) } -func fromCache(key string) (Parsed, bool) { - Cache.RLock() - val, ok := Cache.m[key] - Cache.RUnlock() +func (c *Cache) Store(key string, val Parsed) { + var ttl int = 5 + if ClientConf.CacheTtl > 0 { + ttl = ClientConf.CacheTtl + } + cachedAt := time.Now().UTC() + cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute) + + c.Lock() + // This is not a really ... clean way of doing this. + val["ttl"] = cacheTtl + val["cached_at"] = cachedAt + + c.m[key] = val + c.Unlock() +} + +func (c *Cache) Get(key string) (Parsed, bool) { + c.RLock() + val, ok := c.m[key] + c.RUnlock() if !ok { return NilParse, false } @@ -48,23 +68,6 @@ func fromCache(key string) (Parsed, bool) { return val, ok } -func toCache(key string, val Parsed) { - var ttl int = 5 - if ClientConf.CacheTtl > 0 { - ttl = ClientConf.CacheTtl - } - cachedAt := time.Now().UTC() - cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute) - - // This is not a really ... clean way of doing this. - val["ttl"] = cacheTtl - val["cached_at"] = cachedAt - - Cache.Lock() - Cache.m[key] = val - Cache.Unlock() -} - func Run(args string) (io.Reader, error) { args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument argsList := strings.Split(args, " ") @@ -111,8 +114,8 @@ func checkRateLimit() bool { return true } -func RunAndParse(cmd string, parser func(io.Reader) Parsed) (Parsed, bool) { - if val, ok := fromCache(cmd); ok { +func RunAndParse(cmd string, parser func(io.Reader) Parsed, updateMetaCache func(Parsed)) (Parsed, bool) { + if val, ok := ParsedCache.Get(cmd); ok { return val, true } @@ -127,12 +130,18 @@ func RunAndParse(cmd string, parser func(io.Reader) Parsed) (Parsed, bool) { } parsed := parser(out) - toCache(cmd, parsed) + + ParsedCache.Store(cmd, parsed) + + if updateMetaCache != nil { + updateMetaCache(parsed) + } + return parsed, false } func Status() (Parsed, bool) { - birdStatus, from_cache := RunAndParse("status", parseStatus) + birdStatus, from_cache := RunAndParse("status", parseStatus, nil) if IsSpecial(birdStatus) { return birdStatus, from_cache } @@ -169,11 +178,32 @@ func Status() (Parsed, bool) { birdStatus["status"] = status + ParsedCache.Store("status", birdStatus) + return birdStatus, from_cache } func Protocols() (Parsed, bool) { - return RunAndParse("protocols all", parseProtocols) + initializeMetaCache := func(p Parsed) { + metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}} + + for key, _ := range p["protocols"].(Parsed) { + parsed := p["protocols"].(Parsed)[key].(Parsed) + protocol := parsed["protocol"].(string) + + birdProtocol := parsed["bird_protocol"].(string) + // Check if the structure for the current birdProtocol already exists inside the metaProtocol cache, if not create it (BGP|Pipe|etc) + if _, ok := metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol]; !ok { + metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol] = Parsed{} + } + metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed + } + + MetaCache.Store("protocol", metaProtocol) + } + + res, from_cache := RunAndParse("protocols all", parseProtocols, initializeMetaCache) + return res, from_cache } func ProtocolsBgp() (Parsed, bool) { @@ -183,11 +213,11 @@ func ProtocolsBgp() (Parsed, bool) { } bgpProtocols := Parsed{} + protocolsMeta, _ := MetaCache.Get("protocol") + metaProtocol, _ := protocolsMeta["protocols"].(Parsed) - for key, protocol := range protocols["protocols"].(Parsed) { - if protocol.(Parsed)["bird_protocol"] == "BGP" { - bgpProtocols[key] = protocol - } + for key, protocol := range metaProtocol["bird_protocol"].(Parsed)["BGP"].(Parsed) { + bgpProtocols[key] = *(protocol.(*Parsed)) } return Parsed{"protocols": bgpProtocols, @@ -196,45 +226,56 @@ func ProtocolsBgp() (Parsed, bool) { } func Symbols() (Parsed, bool) { - return RunAndParse("symbols", parseSymbols) + return RunAndParse("symbols", parseSymbols, nil) } func RoutesPrefixed(prefix string) (Parsed, bool) { cmd := routeQueryForChannel("route " + prefix + " all") - return RunAndParse(cmd, parseRoutes) + return RunAndParse(cmd, parseRoutes, nil) } func RoutesProto(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all protocol " + protocol) - return RunAndParse(cmd, parseRoutes) + return RunAndParse(cmd, parseRoutes, nil) } func RoutesProtoCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route protocol "+protocol) + " count" - return RunAndParse(cmd, parseRoutesCount) + return RunAndParse(cmd, parseRoutesCount, nil) } func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route primary protocol "+protocol) + " count" - return RunAndParse(cmd, parseRoutesCount) + return RunAndParse(cmd, parseRoutesCount, nil) } func RoutesFiltered(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all filtered protocol " + protocol) - return RunAndParse(cmd, parseRoutes) + return RunAndParse(cmd, parseRoutes, nil) } func RoutesExport(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all export " + protocol) - return RunAndParse(cmd, parseRoutes) + return RunAndParse(cmd, parseRoutes, nil) } func RoutesNoExport(protocol string) (Parsed, bool) { - // In case we have a multi table setup, we have to query // the pipe protocol. if ParserConf.PerPeerTables && strings.HasPrefix(protocol, ParserConf.PeerProtocolPrefix) { + metaProtocol, _ := MetaCache.Get("protocol") + if metaProtocol == nil { + // Warm up cache if neccessary + protocolsRes, from_cache := ProtocolsBgp() + if IsSpecial(protocolsRes) { + return protocolsRes, from_cache + } + metaProtocol, _ = MetaCache.Get("protocol") + } + if _, ok := metaProtocol["protocol"].(Parsed)[protocol]; !ok { + return NilParse, false + } // Replace prefix protocol = ParserConf.PipeProtocolPrefix + @@ -242,36 +283,37 @@ func RoutesNoExport(protocol string) (Parsed, bool) { } cmd := routeQueryForChannel("route all noexport " + protocol) - return RunAndParse(cmd, parseRoutes) + return RunAndParse(cmd, parseRoutes, nil) } func RoutesExportCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route export "+protocol) + " count" - return RunAndParse(cmd, parseRoutesCount) + return RunAndParse(cmd, parseRoutesCount, nil) } func RoutesTable(table string) (Parsed, bool) { - return RunAndParse("route table "+table+" all", parseRoutes) + return RunAndParse("route table "+table+" all", parseRoutes, nil) } func RoutesTableCount(table string) (Parsed, bool) { - return RunAndParse("route table "+table+" count", parseRoutesCount) + return RunAndParse("route table "+table+" count", parseRoutesCount, nil) } func RoutesLookupTable(net string, table string) (Parsed, bool) { - return RunAndParse("route for "+net+" table "+table+" all", parseRoutes) + return RunAndParse("route for "+net+" table "+table+" all", parseRoutes, nil) } func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) { - return RunAndParse("route for "+net+" protocol "+protocol+" all", parseRoutes) + return RunAndParse("route for "+net+" protocol "+protocol+" all", parseRoutes, nil) } func RoutesPeer(peer string) (Parsed, bool) { cmd := routeQueryForChannel("route export " + peer) - return RunAndParse(cmd, parseRoutes) + return RunAndParse(cmd, parseRoutes, nil) } func RoutesDump() (Parsed, bool) { + // TODO insert hook to update the cache with the route count information if ParserConf.PerPeerTables { return RoutesDumpPerPeerTable() } @@ -280,8 +322,14 @@ func RoutesDump() (Parsed, bool) { } func RoutesDumpSingleTable() (Parsed, bool) { - importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes) - filteredRes, _ := RunAndParse(routeQueryForChannel("route all filtered"), parseRoutes) + importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes, nil) + if IsSpecial(importedRes) { + return importedRes, cached + } + filteredRes, cached := RunAndParse(routeQueryForChannel("route all filtered"), parseRoutes, nil) + if IsSpecial(filteredRes) { + return filteredRes, cached + } imported := importedRes["routes"] filtered := filteredRes["routes"] @@ -295,12 +343,18 @@ func RoutesDumpSingleTable() (Parsed, bool) { } func RoutesDumpPerPeerTable() (Parsed, bool) { - importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes) + importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes, nil) + if IsSpecial(importedRes) { + return importedRes, cached + } imported := importedRes["routes"] filtered := []Parsed{} // Get protocols with filtered routes - protocolsRes, _ := ProtocolsBgp() + protocolsRes, cached := ProtocolsBgp() + if IsSpecial(protocolsRes) { + return protocolsRes, cached + } protocols := protocolsRes["protocols"].(Parsed) for protocol, details := range protocols { @@ -316,7 +370,7 @@ func RoutesDumpPerPeerTable() (Parsed, bool) { } // Lookup filtered routes pfilteredRes, from_cache := RoutesFiltered(protocol) - if reflect.DeepEqual(pfilteredRes, BirdError) { + if IsSpecial(pfilteredRes) { return pfilteredRes, from_cache } @@ -338,6 +392,10 @@ func RoutesDumpPerPeerTable() (Parsed, bool) { func routeQueryForChannel(cmd string) string { status, _ := Status() + if IsSpecial(status) { + return cmd + } + birdStatus, ok := status["status"].(Parsed) if !ok { return cmd diff --git a/endpoints/endpoint.go b/endpoints/endpoint.go index e967744..709e995 100644 --- a/endpoints/endpoint.go +++ b/endpoints/endpoint.go @@ -54,8 +54,8 @@ func Endpoint(wrapped endpoint) httprouter.Handle { } res := make(map[string]interface{}) - ret, from_cache := wrapped(r, ps) + if reflect.DeepEqual(ret, bird.NilParse) { w.WriteHeader(http.StatusTooManyRequests) return From 3e63268af3fe7d8dbf50199c83880696cd176267 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 9 Oct 2018 09:17:10 +0200 Subject: [PATCH 2/8] Implement a RunQeue for birdc commands This prevents running the same birdc command multiple times in parallel. --- bird/bird.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bird/bird.go b/bird/bird.go index f672d85..f621143 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -31,6 +31,8 @@ var MetaCache = Cache{m: make(map[string]Parsed)} var NilParse Parsed = (Parsed)(nil) var BirdError Parsed = Parsed{"error": "bird unreachable"} +var RunQueue sync.Map + func IsSpecial(ret Parsed) bool { return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError) } @@ -119,13 +121,30 @@ func RunAndParse(cmd string, parser func(io.Reader) Parsed, updateMetaCache func return val, true } + var wg sync.WaitGroup + wg.Add(1) + if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded { + (*queueGroup.(*sync.WaitGroup)).Wait() + + if val, ok := ParsedCache.Get(cmd); ok { + return val, true + } else { + // TODO BirdError should also be signaled somehow + return NilParse, false + } + } + if !checkRateLimit() { + wg.Done() + RunQueue.Delete(cmd) return NilParse, false } out, err := Run(cmd) if err != nil { // ignore errors for now + wg.Done() + RunQueue.Delete(cmd) return BirdError, false } @@ -137,6 +156,10 @@ func RunAndParse(cmd string, parser func(io.Reader) Parsed, updateMetaCache func updateMetaCache(parsed) } + wg.Done() + + RunQueue.Delete(cmd) + return parsed, false } From c3d0d16b48162aa11d36f624b202d1b5f3650d3d Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 9 Oct 2018 12:30:38 +0200 Subject: [PATCH 3/8] Fixed a bug in RoutesDumpPerPeerTable() The bug causes an early exit if the filtered routes for a protocol can not be retrieved. --- bird/bird.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index f621143..9d88693 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -392,11 +392,7 @@ func RoutesDumpPerPeerTable() (Parsed, bool) { continue // nothing to do here. } // Lookup filtered routes - pfilteredRes, from_cache := RoutesFiltered(protocol) - if IsSpecial(pfilteredRes) { - return pfilteredRes, from_cache - } - + pfilteredRes, _ := RoutesFiltered(protocol) pfiltered, ok := pfilteredRes["routes"].([]Parsed) if !ok { continue // something went wrong... From d931094ce7bb0f6a144097be5cf971abf625cd46 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Wed, 17 Oct 2018 09:51:55 +0200 Subject: [PATCH 4/8] Introduce a new birdc query RoutesFilteredCount() The query can be used to determine the exact number of filtered (not exported) prefixes from a table towards a pipe protocol for a certain neighbor (if multiple neighbors are connected to the given table). --- bird/bird.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bird/bird.go b/bird/bird.go index 9d88693..070abb8 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -272,6 +272,11 @@ func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) { return RunAndParse(cmd, parseRoutesCount, nil) } +func RoutesFilteredCount(table string, protocol string, neighborAddress string) (Parsed, bool) { + cmd := "route table " + table + " noexport " + protocol + " where from=" + neighborAddress + " count" + return RunAndParse(cmd, parseRoutesCount, nil) +} + func RoutesFiltered(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all filtered protocol " + protocol) return RunAndParse(cmd, parseRoutes, nil) From 50a79e7f4f123b042a208ec3d7b5990e2666d688 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Wed, 17 Oct 2018 09:52:27 +0200 Subject: [PATCH 5/8] Add the GetCacheKey() function This allows to determine the key in the cache, where the result of specific functions are stored, without requiring to know what command was executed by that function. Also updated all functions to use GetCacheKey() to store Parsed records. --- bird/bird.go | 58 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 070abb8..ff73e8d 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -37,6 +37,21 @@ func IsSpecial(ret Parsed) bool { return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError) } +// Determines the key in the cache, where the result of specific functions are stored. +// Eliminates the need to know what command was executed by that function. +func GetCacheKey(fname string, fargs ...interface{}) string { + key := strings.ToLower(fname) + + for _, arg := range fargs { + switch arg.(type) { + case string: + key += "_" + strings.ToLower(arg.(string)) + } + } + + return key +} + func (c *Cache) Store(key string, val Parsed) { var ttl int = 5 if ClientConf.CacheTtl > 0 { @@ -116,7 +131,7 @@ func checkRateLimit() bool { return true } -func RunAndParse(cmd string, parser func(io.Reader) Parsed, updateMetaCache func(Parsed)) (Parsed, bool) { +func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateMetaCache func(Parsed)) (Parsed, bool) { if val, ok := ParsedCache.Get(cmd); ok { return val, true } @@ -164,7 +179,7 @@ func RunAndParse(cmd string, parser func(io.Reader) Parsed, updateMetaCache func } func Status() (Parsed, bool) { - birdStatus, from_cache := RunAndParse("status", parseStatus, nil) + birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, nil) if IsSpecial(birdStatus) { return birdStatus, from_cache } @@ -225,7 +240,8 @@ func Protocols() (Parsed, bool) { MetaCache.Store("protocol", metaProtocol) } - res, from_cache := RunAndParse("protocols all", parseProtocols, initializeMetaCache) + res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, initializeMetaCache) + return res, from_cache } @@ -249,42 +265,42 @@ func ProtocolsBgp() (Parsed, bool) { } func Symbols() (Parsed, bool) { - return RunAndParse("symbols", parseSymbols, nil) + return RunAndParse(GetCacheKey("Symbols"), "symbols", parseSymbols, nil) } func RoutesPrefixed(prefix string) (Parsed, bool) { cmd := routeQueryForChannel("route " + prefix + " all") - return RunAndParse(cmd, parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil) } func RoutesProto(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all protocol " + protocol) - return RunAndParse(cmd, parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil) } func RoutesProtoCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route protocol "+protocol) + " count" - return RunAndParse(cmd, parseRoutesCount, nil) + return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil) } func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route primary protocol "+protocol) + " count" - return RunAndParse(cmd, parseRoutesCount, nil) + return RunAndParse(GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil) } func RoutesFilteredCount(table string, protocol string, neighborAddress string) (Parsed, bool) { cmd := "route table " + table + " noexport " + protocol + " where from=" + neighborAddress + " count" - return RunAndParse(cmd, parseRoutesCount, nil) + return RunAndParse(GetCacheKey("RoutesFilteredCount", table, protocol, neighborAddress), cmd, parseRoutesCount, nil) } func RoutesFiltered(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all filtered protocol " + protocol) - return RunAndParse(cmd, parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil) } func RoutesExport(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route all export " + protocol) - return RunAndParse(cmd, parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil) } func RoutesNoExport(protocol string) (Parsed, bool) { @@ -311,33 +327,33 @@ func RoutesNoExport(protocol string) (Parsed, bool) { } cmd := routeQueryForChannel("route all noexport " + protocol) - return RunAndParse(cmd, parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil) } func RoutesExportCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route export "+protocol) + " count" - return RunAndParse(cmd, parseRoutesCount, nil) + return RunAndParse(GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil) } func RoutesTable(table string) (Parsed, bool) { - return RunAndParse("route table "+table+" all", parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil) } func RoutesTableCount(table string) (Parsed, bool) { - return RunAndParse("route table "+table+" count", parseRoutesCount, nil) + return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil) } func RoutesLookupTable(net string, table string) (Parsed, bool) { - return RunAndParse("route for "+net+" table "+table+" all", parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil) } func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) { - return RunAndParse("route for "+net+" protocol "+protocol+" all", parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil) } func RoutesPeer(peer string) (Parsed, bool) { cmd := routeQueryForChannel("route export " + peer) - return RunAndParse(cmd, parseRoutes, nil) + return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) } func RoutesDump() (Parsed, bool) { @@ -350,11 +366,11 @@ func RoutesDump() (Parsed, bool) { } func RoutesDumpSingleTable() (Parsed, bool) { - importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes, nil) + importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil) if IsSpecial(importedRes) { return importedRes, cached } - filteredRes, cached := RunAndParse(routeQueryForChannel("route all filtered"), parseRoutes, nil) + filteredRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "filtered"), routeQueryForChannel("route all filtered"), parseRoutes, nil) if IsSpecial(filteredRes) { return filteredRes, cached } @@ -371,7 +387,7 @@ func RoutesDumpSingleTable() (Parsed, bool) { } func RoutesDumpPerPeerTable() (Parsed, bool) { - importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes, nil) + importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpPerPeerTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil) if IsSpecial(importedRes) { return importedRes, cached } From 01eb78117bb642c955da347a7247aebe74304fb0 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Wed, 17 Oct 2018 09:53:17 +0200 Subject: [PATCH 6/8] Refactor use of the Meta Cache * Update all functions that use the meta cache to only create their own records and also use GetCacheKey() to request the meta cache from other functions. * eliminates the need for Clear() -> removed * Add two new birdc commands: * PipeRoutesFiltered() * PipeRoutesFilteredCount() --- bird/bird.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index ff73e8d..40df36a 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -237,7 +237,7 @@ func Protocols() (Parsed, bool) { metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed } - MetaCache.Store("protocol", metaProtocol) + MetaCache.Store(GetCacheKey("Protocols"), metaProtocol) } res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, initializeMetaCache) @@ -251,9 +251,10 @@ func ProtocolsBgp() (Parsed, bool) { return protocols, from_cache } + protocolsMeta, _ := MetaCache.Get(GetCacheKey("Protocols")) + metaProtocol := protocolsMeta["protocols"].(Parsed) + bgpProtocols := Parsed{} - protocolsMeta, _ := MetaCache.Get("protocol") - metaProtocol, _ := protocolsMeta["protocols"].(Parsed) for key, protocol := range metaProtocol["bird_protocol"].(Parsed)["BGP"].(Parsed) { bgpProtocols[key] = *(protocol.(*Parsed)) @@ -288,9 +289,14 @@ func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) { return RunAndParse(GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil) } -func RoutesFilteredCount(table string, protocol string, neighborAddress string) (Parsed, bool) { - cmd := "route table " + table + " noexport " + protocol + " where from=" + neighborAddress + " count" - return RunAndParse(GetCacheKey("RoutesFilteredCount", table, protocol, neighborAddress), cmd, parseRoutesCount, nil) +func PipeRoutesFilteredCount(pipe string, table string, neighborAddress string) (Parsed, bool) { + cmd := "route table " + table + " noexport " + pipe + " where from=" + neighborAddress + " count" + return RunAndParse(GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil) +} + +func PipeRoutesFiltered(pipe string, table string) (Parsed, bool) { + cmd := routeQueryForChannel("route table '" + table + "' noexport '" + pipe + "' all") + return RunAndParse(GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil) } func RoutesFiltered(protocol string) (Parsed, bool) { @@ -308,16 +314,12 @@ func RoutesNoExport(protocol string) (Parsed, bool) { // the pipe protocol. if ParserConf.PerPeerTables && strings.HasPrefix(protocol, ParserConf.PeerProtocolPrefix) { - metaProtocol, _ := MetaCache.Get("protocol") - if metaProtocol == nil { - // Warm up cache if neccessary - protocolsRes, from_cache := ProtocolsBgp() - if IsSpecial(protocolsRes) { - return protocolsRes, from_cache - } - metaProtocol, _ = MetaCache.Get("protocol") + + protocolsRes, from_cache := ProtocolsBgp() + if IsSpecial(protocolsRes) { + return protocolsRes, from_cache } - if _, ok := metaProtocol["protocol"].(Parsed)[protocol]; !ok { + if _, ok := protocolsRes["protocols"].(Parsed)[protocol]; !ok { return NilParse, false } From 83628d49952ed2fae835f715529ade2687f47339 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Wed, 17 Oct 2018 09:54:09 +0200 Subject: [PATCH 7/8] Cache refactoring, allow update before store * Change the callback type to update the cache for RunAndParse() to be a pointer to the Parsed record, which allows updating the Parsed record before it will be stored inside the cache --- bird/bird.go | 84 +++++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 40df36a..29f1de8 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -131,7 +131,7 @@ func checkRateLimit() bool { return true } -func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateMetaCache func(Parsed)) (Parsed, bool) { +func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) { if val, ok := ParsedCache.Get(cmd); ok { return val, true } @@ -165,12 +165,12 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateMe parsed := parser(out) - ParsedCache.Store(cmd, parsed) - - if updateMetaCache != nil { - updateMetaCache(parsed) + if updateCache != nil { + updateCache(&parsed) } + ParsedCache.Store(cmd, parsed) + wg.Done() RunQueue.Delete(cmd) @@ -179,54 +179,45 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateMe } func Status() (Parsed, bool) { - birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, nil) - if IsSpecial(birdStatus) { - return birdStatus, from_cache + updateParsedCache := func(p *Parsed) { + status := (*p)["status"].(Parsed) + + // Last Reconfig Timestamp source: + var lastReconfig string + switch StatusConf.ReconfigTimestampSource { + case "bird": + lastReconfig = status["last_reconfig"].(string) + break + case "config_modified": + lastReconfig = lastReconfigTimestampFromFileStat( + ClientConf.ConfigFilename, + ) + case "config_regex": + lastReconfig = lastReconfigTimestampFromFileContent( + ClientConf.ConfigFilename, + StatusConf.ReconfigTimestampMatch, + ) + } + + status["last_reconfig"] = lastReconfig + + // Filter fields + for _, field := range StatusConf.FilterFields { + status[field] = nil + } } - if from_cache { - return birdStatus, from_cache - } - - status := birdStatus["status"].(Parsed) - - // Last Reconfig Timestamp source: - var lastReconfig string - switch StatusConf.ReconfigTimestampSource { - case "bird": - lastReconfig = status["last_reconfig"].(string) - break - case "config_modified": - lastReconfig = lastReconfigTimestampFromFileStat( - ClientConf.ConfigFilename, - ) - case "config_regex": - lastReconfig = lastReconfigTimestampFromFileContent( - ClientConf.ConfigFilename, - StatusConf.ReconfigTimestampMatch, - ) - } - - status["last_reconfig"] = lastReconfig - - // Filter fields - for _, field := range StatusConf.FilterFields { - status[field] = nil - } - - birdStatus["status"] = status - - ParsedCache.Store("status", birdStatus) - + birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, updateParsedCache) return birdStatus, from_cache } func Protocols() (Parsed, bool) { - initializeMetaCache := func(p Parsed) { + createMetaCache := func(p *Parsed) { metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}} - for key, _ := range p["protocols"].(Parsed) { - parsed := p["protocols"].(Parsed)[key].(Parsed) + for key, _ := range (*p)["protocols"].(Parsed) { + parsed := (*p)["protocols"].(Parsed)[key].(Parsed) + protocol := parsed["protocol"].(string) birdProtocol := parsed["bird_protocol"].(string) @@ -240,8 +231,7 @@ func Protocols() (Parsed, bool) { MetaCache.Store(GetCacheKey("Protocols"), metaProtocol) } - res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, initializeMetaCache) - + res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache) return res, from_cache } From 3eca6db9983ed623df6cb808bc5a6221fdee8aa3 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Fri, 19 Oct 2018 12:26:54 +0200 Subject: [PATCH 8/8] Add "Last Error" to protocols bgp test case. --- test/protocols_bgp_pipe.sample | 1 + 1 file changed, 1 insertion(+) diff --git a/test/protocols_bgp_pipe.sample b/test/protocols_bgp_pipe.sample index 4718922..d6c2e5d 100644 --- a/test/protocols_bgp_pipe.sample +++ b/test/protocols_bgp_pipe.sample @@ -39,4 +39,5 @@ R194_42 BGP T65001_nada_co_ripe up 2018-05-31 15:38:40 Established Route limit: 710/200000 Hold timer: 151/180 Keepalive timer: 43/60 + Last error: Socket: Connection closed