From c4dfeb253d3e7a95ccd2ade04cb73b6700dc053a Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Fri, 18 Jan 2019 18:10:06 +0100 Subject: [PATCH 1/8] API redesign - make birdwatcher more generic Removed all high level functionality e.g. endpoints with multiple invocations of birdc. Add new endpoints which are required to duplicate the removed functionality within Alice-LG. --- bird/bird.go | 112 ++++--------------------------- bird/config.go | 5 +- birdwatcher.go | 22 ++++-- endpoints/routes.go | 35 ++++++---- etc/birdwatcher/birdwatcher.conf | 15 ++++- 5 files changed, 64 insertions(+), 125 deletions(-) mode change 100644 => 100755 etc/birdwatcher/birdwatcher.conf diff --git a/bird/bird.go b/bird/bird.go index 29f1de8..ca7d005 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -217,7 +217,6 @@ func Protocols() (Parsed, bool) { for key, _ := range (*p)["protocols"].(Parsed) { parsed := (*p)["protocols"].(Parsed)[key].(Parsed) - protocol := parsed["protocol"].(string) birdProtocol := parsed["bird_protocol"].(string) @@ -269,6 +268,16 @@ func RoutesProto(protocol string) (Parsed, bool) { return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil) } +func RoutesPeer(peer string) (Parsed, bool) { + cmd := routeQueryForChannel("route all where from=" + peer) + return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) +} + +func RoutesTableAndPeer(table string, peer string) (Parsed, bool) { + cmd := routeQueryForChannel("route table " + table + " all where from=" + peer) + return RunAndParse(GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil) +} + func RoutesProtoCount(protocol string) (Parsed, bool) { cmd := routeQueryForChannel("route protocol "+protocol) + " count" return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil) @@ -300,24 +309,6 @@ func RoutesExport(protocol string) (Parsed, bool) { } func RoutesNoExport(protocol string) (Parsed, bool) { - // In case we have a multi table setup, we have to query - // the pipe protocol. - if ParserConf.PerPeerTables && - strings.HasPrefix(protocol, ParserConf.PeerProtocolPrefix) { - - protocolsRes, from_cache := ProtocolsBgp() - if IsSpecial(protocolsRes) { - return protocolsRes, from_cache - } - if _, ok := protocolsRes["protocols"].(Parsed)[protocol]; !ok { - return NilParse, false - } - - // Replace prefix - protocol = ParserConf.PipeProtocolPrefix + - protocol[len(ParserConf.PeerProtocolPrefix):] - } - cmd := routeQueryForChannel("route all noexport " + protocol) return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil) } @@ -331,6 +322,10 @@ func RoutesTable(table string) (Parsed, bool) { return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil) } +func RoutesTableFiltered(table string) (Parsed, bool) { + return RunAndParse(GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil) +} + func RoutesTableCount(table string) (Parsed, bool) { return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil) } @@ -343,85 +338,6 @@ func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) { return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil) } -func RoutesPeer(peer string) (Parsed, bool) { - cmd := routeQueryForChannel("route export " + peer) - return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) -} - -func RoutesDump() (Parsed, bool) { - // TODO insert hook to update the cache with the route count information - if ParserConf.PerPeerTables { - return RoutesDumpPerPeerTable() - } - - return RoutesDumpSingleTable() -} - -func RoutesDumpSingleTable() (Parsed, bool) { - importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil) - if IsSpecial(importedRes) { - return importedRes, cached - } - filteredRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "filtered"), routeQueryForChannel("route all filtered"), parseRoutes, nil) - if IsSpecial(filteredRes) { - return filteredRes, cached - } - - imported := importedRes["routes"] - filtered := filteredRes["routes"] - - result := Parsed{ - "imported": imported, - "filtered": filtered, - } - - return result, cached -} - -func RoutesDumpPerPeerTable() (Parsed, bool) { - importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpPerPeerTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil) - if IsSpecial(importedRes) { - return importedRes, cached - } - imported := importedRes["routes"] - filtered := []Parsed{} - - // Get protocols with filtered routes - protocolsRes, cached := ProtocolsBgp() - if IsSpecial(protocolsRes) { - return protocolsRes, cached - } - protocols := protocolsRes["protocols"].(Parsed) - - for protocol, details := range protocols { - details := details.(Parsed) - - counters, ok := details["routes"].(Parsed) - if !ok { - continue - } - filterCount := counters["filtered"] - if filterCount == 0 { - continue // nothing to do here. - } - // Lookup filtered routes - pfilteredRes, _ := RoutesFiltered(protocol) - pfiltered, ok := pfilteredRes["routes"].([]Parsed) - if !ok { - continue // something went wrong... - } - - filtered = append(filtered, pfiltered...) - } - - result := Parsed{ - "imported": imported, - "filtered": filtered, - } - - return result, cached -} - func routeQueryForChannel(cmd string) string { status, _ := Status() if IsSpecial(status) { diff --git a/bird/config.go b/bird/config.go index 37a94c0..20c81ff 100644 --- a/bird/config.go +++ b/bird/config.go @@ -17,10 +17,7 @@ type BirdConfig struct { } type ParserConfig struct { - FilterFields []string `toml:"filter_fields"` - PerPeerTables bool `toml:"per_peer_tables"` - PeerProtocolPrefix string `toml:"peer_protocol_prefix"` - PipeProtocolPrefix string `toml:"pipe_protocol_prefix"` + FilterFields []string `toml:"filter_fields"` } type RateLimitConfig struct { diff --git a/birdwatcher.go b/birdwatcher.go index 9082660..53f152b 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -16,7 +16,7 @@ import ( ) //go:generate versionize -var VERSION = "1.11.0" +var VERSION = "1.13.0" func isModuleEnabled(module string, modulesEnabled []string) bool { for _, enabled := range modulesEnabled { @@ -54,9 +54,18 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router { if isModuleEnabled("routes_protocol", whitelist) { r.GET("/routes/protocol/:protocol", endpoints.Endpoint(endpoints.ProtoRoutes)) } + if isModuleEnabled("routes_peer", whitelist) { + r.GET("/routes/peer/:peer", endpoints.Endpoint(endpoints.PeerRoutes)) + } if isModuleEnabled("routes_table", whitelist) { r.GET("/routes/table/:table", endpoints.Endpoint(endpoints.TableRoutes)) } + if isModuleEnabled("routes_table_filtered", whitelist) { + r.GET("/routes/table/:table/filtered", endpoints.Endpoint(endpoints.TableRoutesFiltered)) + } + if isModuleEnabled("routes_table_peer", whitelist) { + r.GET("/routes/table/:table/peer/:peer", endpoints.Endpoint(endpoints.TableAndPeerRoutes)) + } if isModuleEnabled("routes_count_protocol", whitelist) { r.GET("/routes/count/protocol/:protocol", endpoints.Endpoint(endpoints.ProtoCount)) } @@ -79,12 +88,13 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router { r.GET("/route/net/:net", endpoints.Endpoint(endpoints.RouteNet)) r.GET("/route/net/:net/table/:table", endpoints.Endpoint(endpoints.RouteNetTable)) } - if isModuleEnabled("routes_peer", whitelist) { - r.GET("/routes/peer", endpoints.Endpoint(endpoints.RoutesPeer)) + if isModuleEnabled("routes_pipe_filtered_count", whitelist) { + r.GET("/routes/pipe/filtered/count", endpoints.Endpoint(endpoints.PipeRoutesFilteredCount)) } - if isModuleEnabled("routes_dump", whitelist) { - r.GET("/routes/dump", endpoints.Endpoint(endpoints.RoutesDump)) + if isModuleEnabled("routes_pipe_filtered", whitelist) { + r.GET("/routes/pipe/filtered", endpoints.Endpoint(endpoints.PipeRoutesFiltered)) } + return r } @@ -108,8 +118,6 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) { for _, m := range conf.Server.ModulesEnabled { log.Println(" -", m) } - - log.Println(" Per Peer Tables:", conf.Parser.PerPeerTables) } // MyLogger is our own log.Logger wrapper so we can customize it diff --git a/endpoints/routes.go b/endpoints/routes.go index 1dee4b3..f47f4b3 100644 --- a/endpoints/routes.go +++ b/endpoints/routes.go @@ -50,6 +50,14 @@ func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { return bird.RoutesTable(ps.ByName("table")) } +func TableRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { + return bird.RoutesTableFiltered(ps.ByName("table")) +} + +func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { + return bird.RoutesTableAndPeer(ps.ByName("table"), ps.ByName("peer")) +} + func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { @@ -78,20 +86,21 @@ func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { return bird.RoutesLookupTable(ps.ByName("net"), ps.ByName("table")) } -func RoutesPeer(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { qs := r.URL.Query() - peerl := qs["peer"] - if len(peerl) != 1 { - return bird.Parsed{"error": "need a peer as single query parameter"}, false - } - - peer, err := ValidateProtocolParam(peerl[0]) - if err != nil { - return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false - } - return bird.RoutesPeer(peer) + table := qs["table"][0] + pipe := qs["pipe"][0] + return bird.PipeRoutesFiltered(pipe, table) } -func RoutesDump(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesDump() +func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { + qs := r.URL.Query() + table := qs["table"][0] + pipe := qs["pipe"][0] + address := qs["address"][0] + return bird.PipeRoutesFilteredCount(pipe, table, address) +} + +func PeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { + return bird.RoutesPeer(ps.ByName("peer")) } diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf old mode 100644 new mode 100755 index a1cc871..863b115 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -15,7 +15,10 @@ allow_from = [] # protocols # protocols_bgp # routes_protocol +# routes_peer # routes_table +# routes_table_filtered +# routes_table_peer # routes_count_protocol # routes_count_table # routes_count_primary @@ -23,8 +26,8 @@ allow_from = [] # routes_prefixed # routes_noexport # route_net -## high-level modules (aggregated data from multiple birdc invocations) -# routes_dump +# routes_pipe_filtered_count +# routes_pipe_filtered # routes_peer @@ -33,8 +36,14 @@ modules_enabled = ["status", "protocols_bgp", "routes_protocol", "routes_peer", + "routes_table", + "routes_table_filtered", + "routes_table_peer", + "routes_filtered", "routes_prefixed", - "routes_dump" + "routes_noexport", + "routes_pipe_filtered_count", + "routes_pipe_filtered" ] [status] From 1d3118b86439260dc58d8909a9e5c8c5c6f9f6ac Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Mon, 4 Feb 2019 15:48:24 +0100 Subject: [PATCH 2/8] Added missing default value (imported). --- bird/parser.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bird/parser.go b/bird/parser.go index 717922a..92d3f6a 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -542,6 +542,7 @@ func parseProtocol(lines string) Parsed { routes := Parsed{} routes["accepted"] = int64(0) routes["filtered"] = int64(0) + routes["imported"] = int64(0) routes["exported"] = int64(0) routes["preferred"] = int64(0) From e5248b21c705aacb847be4b3e799f139f33a160e Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 5 Feb 2019 17:37:53 +0100 Subject: [PATCH 3/8] Improved parameter validation. --- endpoints/routes.go | 111 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 12 deletions(-) diff --git a/endpoints/routes.go b/endpoints/routes.go index f47f4b3..394ed69 100644 --- a/endpoints/routes.go +++ b/endpoints/routes.go @@ -13,6 +13,7 @@ func ProtoRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } + return bird.RoutesProto(protocol) } @@ -21,6 +22,7 @@ func RoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } + return bird.RoutesFiltered(protocol) } @@ -29,6 +31,7 @@ func RoutesNoExport(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } + return bird.RoutesNoExport(protocol) } @@ -43,19 +46,40 @@ func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } + return bird.RoutesPrefixed(prefix) } func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesTable(ps.ByName("table")) + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTable(table) } func TableRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesTableFiltered(ps.ByName("table")) + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTableFiltered(table) } func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesTableAndPeer(ps.ByName("table"), ps.ByName("peer")) + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + peer, err := ValidatePrefixParam(ps.ByName("peer")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTableAndPeer(table, peer) } func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { @@ -63,6 +87,7 @@ func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } + return bird.RoutesProtoCount(protocol) } @@ -75,32 +100,94 @@ func ProtoPrimaryCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool } func TableCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesTableCount(ps.ByName("table")) + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTableCount(table) } func RouteNet(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesLookupTable(ps.ByName("net"), "master") + net, err := ValidatePrefixParam(ps.ByName("net")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesLookupTable(net, "master") } func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesLookupTable(ps.ByName("net"), ps.ByName("table")) + net, err := ValidatePrefixParam(ps.ByName("net")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesLookupTable(net, table) } func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { qs := r.URL.Query() - table := qs["table"][0] - pipe := qs["pipe"][0] + + if len(qs["table"]) != 1 { + return bird.Parsed{"error": "need a table as single query parameter"}, false + } + table, err := ValidateProtocolParam(qs["table"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + if len(qs["pipe"]) != 1 { + return bird.Parsed{"error": "need a pipe as single query parameter"}, false + } + pipe, err := ValidateProtocolParam(qs["pipe"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + return bird.PipeRoutesFiltered(pipe, table) } func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { qs := r.URL.Query() - table := qs["table"][0] - pipe := qs["pipe"][0] - address := qs["address"][0] + + if len(qs["table"]) != 1 { + return bird.Parsed{"error": "need a table as single query parameter"}, false + } + table, err := ValidateProtocolParam(qs["table"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + if len(qs["pipe"]) != 1 { + return bird.Parsed{"error": "need a pipe as single query parameter"}, false + } + pipe, err := ValidateProtocolParam(qs["pipe"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + if len(qs["address"]) != 1 { + return bird.Parsed{"error": "need a address as single query parameter"}, false + } + address, err := ValidatePrefixParam(qs["address"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + return bird.PipeRoutesFilteredCount(pipe, table, address) } func PeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesPeer(ps.ByName("peer")) + peer, err := ValidatePrefixParam(ps.ByName("peer")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesPeer(peer) } From 14a4ebeaaa5fb44d22bcde042b1f489185852a06 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 5 Feb 2019 17:40:59 +0100 Subject: [PATCH 4/8] Using json.Encoder to write the output JSON instead of using a buffer. --- endpoints/endpoint.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/endpoints/endpoint.go b/endpoints/endpoint.go index 709e995..8138f83 100644 --- a/endpoints/endpoint.go +++ b/endpoints/endpoint.go @@ -73,8 +73,6 @@ func Endpoint(wrapped endpoint) httprouter.Handle { res[k] = v } - js, _ := json.Marshal(res) - w.Header().Set("Content-Type", "application/json") // Check if compression is supported @@ -83,9 +81,11 @@ func Endpoint(wrapped endpoint) httprouter.Handle { w.Header().Set("Content-Encoding", "gzip") gz := gzip.NewWriter(w) defer gz.Close() - gz.Write(js) + json := json.NewEncoder(gz) + json.Encode(res) } else { - w.Write(js) // Fall back to uncompressed response + json := json.NewEncoder(w) + json.Encode(res) // Fall back to uncompressed response } } } From 119b9f6360b217038456cd388dea79cf1d03ce7a Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Thu, 28 Feb 2019 11:32:40 +0100 Subject: [PATCH 5/8] Add feature cache backends Add support for various cache backends in anticipation of the merge with master that has an additional redis backend. The current memory based cache backend is refactored to implement the new interface. --- bird/bird.go | 108 +++++++++++++++++++++++-------------------- bird/memory_cache.go | 61 ++++++++++++++++++++++++ birdwatcher.go | 8 ++++ 3 files changed, 127 insertions(+), 50 deletions(-) create mode 100644 bird/memory_cache.go diff --git a/bird/bird.go b/bird/bird.go index 181fbc3..0d1d32e 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -3,6 +3,7 @@ package bird import ( "bytes" "io" + "log" "reflect" "strconv" "strings" @@ -12,31 +13,71 @@ import ( "os/exec" ) +type Cache interface { + Set(key string, val Parsed, ttl int) error + Get(key string) (Parsed, error) +} + var ClientConf BirdConfig var StatusConf StatusConfig var IPVersion = "4" +var cache Cache // stores parsed birdc output var RateLimitConf struct { sync.RWMutex Conf RateLimitConfig } +var RunQueue sync.Map // queue birdc commands before execution -type Cache struct { - sync.RWMutex - m map[string]Parsed -} - -var ParsedCache = Cache{m: make(map[string]Parsed)} -var MetaCache = Cache{m: make(map[string]Parsed)} - -var NilParse Parsed = (Parsed)(nil) +var NilParse Parsed = (Parsed)(nil) // special Parsed values var BirdError Parsed = Parsed{"error": "bird unreachable"} -var RunQueue sync.Map - -func IsSpecial(ret Parsed) bool { +func IsSpecial(ret Parsed) bool { // test for special Parsed values return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError) } +// intitialize the Cache once during setup with either a MemoryCache or +// RedisCache implementation. +// TODO implement singleton pattern +func InitializeCache(c Cache) { + cache = c +} + +/* Convenience method to make new entries in the cache. + * Abstracts over the specific caching implementation and the ability to set + * individual TTL values for entries. Always use the default TTL value from the + * config. + */ +func toCache(key string, val Parsed) bool { + var ttl int + if ClientConf.CacheTtl > 0 { + ttl = ClientConf.CacheTtl + } else { + ttl = 5 // five minutes + } + + if err := cache.Set(key, val, ttl); err == nil { + return true + } else { + log.Println(err) + return false + } +} + +/* Convenience method to retrieve entries from the cache. + * Abstracts over the specific caching implementations. + */ +func fromCache(key string) (Parsed, bool) { + val, err := cache.Get(key) + if err != nil { + log.Println(err) + return val, false + } else if IsSpecial(val) { // cache may return NilParse e.g. if ttl is expired + return val, false + } else { + return val, true + } +} + // Determines the key in the cache, where the result of specific functions are stored. // Eliminates the need to know what command was executed by that function. func GetCacheKey(fname string, fargs ...interface{}) string { @@ -52,39 +93,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string { return key } -func (c *Cache) Store(key string, val Parsed) { - var ttl int = 5 - if ClientConf.CacheTtl > 0 { - ttl = ClientConf.CacheTtl - } - cachedAt := time.Now().UTC() - cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute) - - c.Lock() - // This is not a really ... clean way of doing this. - val["ttl"] = cacheTtl - val["cached_at"] = cachedAt - - c.m[key] = val - c.Unlock() -} - -func (c *Cache) Get(key string) (Parsed, bool) { - c.RLock() - val, ok := c.m[key] - c.RUnlock() - if !ok { - return NilParse, false - } - - ttl, correct := val["ttl"].(time.Time) - if !correct || ttl.Before(time.Now()) { - return NilParse, false - } - - return val, ok -} - func Run(args string) (io.Reader, error) { args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument argsList := strings.Split(args, " ") @@ -132,7 +140,7 @@ func checkRateLimit() bool { } func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) { - if val, ok := ParsedCache.Get(cmd); ok { + if val, ok := fromCache(cmd); ok { return val, true } @@ -141,7 +149,7 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded { (*queueGroup.(*sync.WaitGroup)).Wait() - if val, ok := ParsedCache.Get(cmd); ok { + if val, ok := fromCache(cmd); ok { return val, true } else { // TODO BirdError should also be signaled somehow @@ -169,7 +177,7 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa updateCache(&parsed) } - ParsedCache.Store(cmd, parsed) + toCache(cmd, parsed) wg.Done() @@ -228,7 +236,7 @@ func Protocols() (Parsed, bool) { metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed } - MetaCache.Store(GetCacheKey("Protocols"), metaProtocol) + toCache(GetCacheKey("Protocols"), metaProtocol) } res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache) @@ -241,7 +249,7 @@ func ProtocolsBgp() (Parsed, bool) { return protocols, from_cache } - protocolsMeta, _ := MetaCache.Get(GetCacheKey("Protocols")) + protocolsMeta, _ := fromCache(GetCacheKey("Protocols")) //TODO geht das einfach so? metaProtocol := protocolsMeta["protocols"].(Parsed) bgpProtocols := Parsed{} diff --git a/bird/memory_cache.go b/bird/memory_cache.go new file mode 100644 index 0000000..8213336 --- /dev/null +++ b/bird/memory_cache.go @@ -0,0 +1,61 @@ +package bird + +import ( + "errors" + "sync" + "time" +) + +// Implementation of the MemoryCache backend. + +type MemoryCache struct { + sync.RWMutex + m map[string]Parsed +} + +func NewMemoryCache() (*MemoryCache, error) { + var cache *MemoryCache + cache = &MemoryCache{m: make(map[string]Parsed)} + return cache, nil +} + +func (c *MemoryCache) Get(key string) (Parsed, error) { + c.RLock() + val, ok := c.m[key] + c.RUnlock() + if !ok { + return NilParse, errors.New("Could not retrive key" + key + "from MemoryCache.") + } + + ttl, correct := val["ttl"].(time.Time) + if !correct { + return NilParse, errors.New("Invalid TTL value for key" + key) + } + + if ttl.Before(time.Now()) { + return NilParse, nil // TTL expired + } else { + return val, nil // cache hit + } +} + +func (c *MemoryCache) Set(key string, val Parsed, ttl int) error { + switch { + case ttl == 0: + return nil // do not cache + case ttl > 0: + cachedAt := time.Now().UTC() + cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute) + + c.Lock() + // This is not a really ... clean way of doing this. + val["ttl"] = cacheTtl + val["cached_at"] = cachedAt + + c.m[key] = val + c.Unlock() + return nil + default: // ttl negative - invalid + return errors.New("Negative TTL value for key" + key) + } +} diff --git a/birdwatcher.go b/birdwatcher.go index 9082660..13d1072 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -163,6 +163,14 @@ func main() { bird.RateLimitConf.Conf = conf.Ratelimit bird.RateLimitConf.Unlock() bird.ParserConf = conf.Parser + + var cache bird.Cache + cache, err = bird.NewMemoryCache() // initialze the MemoryCache + if err != nil { + log.Fatal("Could not initialize MemoryCache:", err) + } + bird.InitializeCache(cache) + endpoints.Conf = conf.Server // Make server From 7e02cb23da7e7b1a1a53e488ebcc43b23d8abe6c Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Thu, 28 Feb 2019 13:02:11 +0100 Subject: [PATCH 6/8] Add test for memory cache backend Improve error handling in case value can not be retrieved. Either return the value and nil, or a value and an error. --- bird/bird.go | 11 +++--- bird/memory_cache.go | 8 ++--- bird/memory_cache_test.go | 74 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 bird/memory_cache_test.go diff --git a/bird/bird.go b/bird/bird.go index 0d1d32e..953db74 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -68,14 +68,13 @@ func toCache(key string, val Parsed) bool { */ func fromCache(key string) (Parsed, bool) { val, err := cache.Get(key) - if err != nil { - log.Println(err) - return val, false - } else if IsSpecial(val) { // cache may return NilParse e.g. if ttl is expired - return val, false - } else { + if err == nil { return val, true + } else { + return val, false } + //DEBUG log.Println(err) + } // Determines the key in the cache, where the result of specific functions are stored. diff --git a/bird/memory_cache.go b/bird/memory_cache.go index 8213336..89384b4 100644 --- a/bird/memory_cache.go +++ b/bird/memory_cache.go @@ -23,17 +23,17 @@ func (c *MemoryCache) Get(key string) (Parsed, error) { c.RLock() val, ok := c.m[key] c.RUnlock() - if !ok { - return NilParse, errors.New("Could not retrive key" + key + "from MemoryCache.") + if !ok { // cache miss + return NilParse, errors.New("Failed to retrive key '" + key + "' from MemoryCache.") } ttl, correct := val["ttl"].(time.Time) if !correct { - return NilParse, errors.New("Invalid TTL value for key" + key) + return NilParse, errors.New("Invalid TTL value for key '" + key + "'") } if ttl.Before(time.Now()) { - return NilParse, nil // TTL expired + return val, errors.New("TTL expired for key '" + key + "'") // TTL expired } else { return val, nil // cache hit } diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go new file mode 100644 index 0000000..b4a6a78 --- /dev/null +++ b/bird/memory_cache_test.go @@ -0,0 +1,74 @@ +package bird + +import ( + "testing" +) + +func Test_MemoryCacheAccess(t *testing.T) { + + cache, err := NewMemoryCache() + + parsed := Parsed{ + "foo": 23, + "bar": 42, + "baz": true, + } + + t.Log("Setting memory cache...") + err = cache.Set("testkey", parsed, 5) + if err != nil { + t.Error(err) + } + + t.Log("Fetching from memory cache...") + parsed, err = cache.Get("testkey") + if err != nil { + t.Error(err) + } + + t.Log(parsed) +} + +func Test_MemoryCacheAccessKeyMissing(t *testing.T) { + + cache, err := NewMemoryCache() + + parsed, err := cache.Get("test_missing_key") + if !IsSpecial(parsed) { + t.Error(err) + } + t.Log("Cache error:", err) + t.Log(parsed) +} + +func Test_MemoryCacheRoutes(t *testing.T) { + f, err := openFile("routes_bird1_ipv4.sample") + if err != nil { + t.Error(err) + } + defer f.Close() + + parsed := parseRoutes(f) + _, ok := parsed["routes"].([]Parsed) + if !ok { + t.Fatal("Error getting routes") + } + + cache, err := NewMemoryCache() + + err = cache.Set("routes_protocol_test", parsed, 5) + if err != nil { + t.Error(err) + } + + parsed, err = cache.Get("routes_protocol_test") + if err != nil { + t.Error(err) + return + } + routes, ok := parsed["routes"].([]Parsed) + if !ok { + t.Error("Error getting routes") + } + t.Log("Retrieved routes:", len(routes)) +} From 14c8875ae0b92f977363766311d11716ca958daf Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Thu, 28 Feb 2019 13:25:47 +0100 Subject: [PATCH 7/8] Avoid overwriting existing cache entry also fix RoutesNoExport. --- bird/bird.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 953db74..3fe2478 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -235,10 +235,10 @@ func Protocols() (Parsed, bool) { metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed } - toCache(GetCacheKey("Protocols"), metaProtocol) + toCache(GetCacheKey("metaProtocol"), metaProtocol) } - res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache) + res, from_cache := RunAndParse(GetCacheKey("metaProtocol"), "protocols all", parseProtocols, createMetaCache) return res, from_cache } @@ -248,7 +248,7 @@ func ProtocolsBgp() (Parsed, bool) { return protocols, from_cache } - protocolsMeta, _ := fromCache(GetCacheKey("Protocols")) //TODO geht das einfach so? + protocolsMeta, _ := fromCache(GetCacheKey("metaProtocol")) metaProtocol := protocolsMeta["protocols"].(Parsed) bgpProtocols := Parsed{} From 56c378109fe9d47c02ede332b36d710be8f13e04 Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Thu, 28 Feb 2019 16:14:13 +0100 Subject: [PATCH 8/8] Integrate redis backend with the Cache interface --- bird/bird.go | 4 ++++ bird/redis_cache.go | 45 ++++++++++++++++++++++++++++++---------- bird/redis_cache_test.go | 4 ++-- birdwatcher.go | 2 +- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 342678f..02403b9 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -67,6 +67,10 @@ func toCache(key string, val Parsed) bool { /* Convenience method to retrieve entries from the cache. * Abstracts over the specific caching implementations. + * If err returned by cache.Get(key) is set, the value from the cache is not + * used. There is either a fault e.g. missing entry or the ttl is expired. + * Handling of specific error conditions e.g. ttl expired but entry present is + * possible but currently not implemented. */ func fromCache(key string) (Parsed, bool) { val, err := cache.Get(key) diff --git a/bird/redis_cache.go b/bird/redis_cache.go index 6335b8b..f20f7e8 100644 --- a/bird/redis_cache.go +++ b/bird/redis_cache.go @@ -2,12 +2,15 @@ package bird import ( "encoding/json" - "github.com/go-redis/redis" + "errors" "time" + + "github.com/go-redis/redis" ) type RedisCache struct { - client *redis.Client + client *redis.Client + keyPrefix string } func NewRedisCache(config CacheConfig) (*RedisCache, error) { @@ -31,6 +34,7 @@ func NewRedisCache(config CacheConfig) (*RedisCache, error) { } func (self *RedisCache) Get(key string) (Parsed, error) { + key = self.keyPrefix + key //"B" + IPVersion + "_" + key data, err := self.client.Get(key).Result() if err != nil { return NilParse, err @@ -39,15 +43,34 @@ func (self *RedisCache) Get(key string) (Parsed, error) { parsed := Parsed{} err = json.Unmarshal([]byte(data), &parsed) - return parsed, err -} - -func (self *RedisCache) Set(key string, parsed Parsed) error { - payload, err := json.Marshal(parsed) - if err != nil { - return err + ttl, correct := parsed["ttl"].(time.Time) + if !correct { + return NilParse, errors.New("Invalid TTL value for key" + key) } - _, err = self.client.Set(key, payload, time.Minute*5).Result() - return err + if ttl.Before(time.Now()) { + return NilParse, err // TTL expired + } else { + return parsed, err // cache hit + } +} + +func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error { + switch { + case ttl == 0: + return nil // do not cache + + case ttl > 0: + key = self.keyPrefix + key //TODO "B" + IPVersion + "_" + key + payload, err := json.Marshal(parsed) + if err != nil { + return err + } + + _, err = self.client.Set(key, payload, time.Duration(ttl)*time.Minute).Result() + return err + + default: // ttl negative - invalid + return errors.New("Negative TTL value for key" + key) + } } diff --git a/bird/redis_cache_test.go b/bird/redis_cache_test.go index a8b6103..d84492c 100644 --- a/bird/redis_cache_test.go +++ b/bird/redis_cache_test.go @@ -23,7 +23,7 @@ func Test_RedisCacheAccess(t *testing.T) { } t.Log("Setting redis cache...") - err = cache.Set("testkey", parsed) + err = cache.Set("testkey", parsed, 5) if err != nil { t.Error(err) } @@ -80,7 +80,7 @@ func Test_RedisCacheRoutes(t *testing.T) { return } - err = cache.Set("routes_protocol_test", parsed) + err = cache.Set("routes_protocol_test", parsed, 5) if err != nil { t.Error(err) } diff --git a/birdwatcher.go b/birdwatcher.go index 7c6bac9..d9325f1 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -173,7 +173,7 @@ func main() { var cache bird.Cache if conf.Cache.UseRedis { - bird.CacheRedis, err = bird.NewRedisCache(conf.Cache) + cache, err = bird.NewRedisCache(conf.Cache) if err != nil { log.Fatal("Could not initialize redis cache, falling back to memory cache:", err) }