From cfa0af57ccd7c3ec555ce73ad32ce260547ea5fc Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Mon, 18 Feb 2019 17:46:32 +0100 Subject: [PATCH 1/7] Expire cache entries to save memory * Add a method to expire cache entries, based on the ttl value * Add a housekeeping method that will periodically expire cache entries and also maybe configured to force a GC/SCVG run. --- bird/bird.go | 20 ++++++++++++++ birdwatcher.go | 2 ++ config.go | 13 ++++----- etc/birdwatcher/birdwatcher.conf | 9 +++---- housekeeping.go | 45 ++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 housekeeping.go diff --git a/bird/bird.go b/bird/bird.go index 0fb8195..5b584fa 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -98,6 +98,26 @@ func GetCacheKey(fname string, fargs ...interface{}) string { return key } +func (c *MemoryCache) Expire() int { + c.Lock() + + expiredKeys := []string{} + for key, _ := range c.m { + ttl, correct := c.m[key]["ttl"].(time.Time) + if !correct || ttl.Before(time.Now()) { + expiredKeys = append(expiredKeys, key) + } + } + + for _, key := range expiredKeys { + delete(c.m, key) + } + + c.Unlock() + + return len(expiredKeys) +} + func Run(args string) (io.Reader, error) { args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument argsList := strings.Split(args, " ") diff --git a/birdwatcher.go b/birdwatcher.go index 5a3fee1..a1ebaf5 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -205,6 +205,8 @@ func main() { myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime)) mylogger := &MyLogger{myquerylog} + go Housekeeping(conf.Housekeeping) + if conf.Server.EnableTLS { if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 { log.Fatalln("You have enabled TLS support but not specified both a .crt and a .key file in the config.") diff --git a/config.go b/config.go index a722a35..12e2325 100644 --- a/config.go +++ b/config.go @@ -17,12 +17,13 @@ import ( type Config struct { Server endpoints.ServerConfig - Ratelimit bird.RateLimitConfig - Status bird.StatusConfig - Bird bird.BirdConfig - Bird6 bird.BirdConfig - Parser bird.ParserConfig - Cache bird.CacheConfig + Ratelimit bird.RateLimitConfig + Status bird.StatusConfig + Bird bird.BirdConfig + Bird6 bird.BirdConfig + Parser bird.ParserConfig + Cache bird.CacheConfig + Housekeeping HousekeepingConfig } // Try to load configfiles as specified in the files diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index 1b7356a..d817fab 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -78,12 +78,11 @@ ttl = 5 # time to live (in minutes) for caching of cli output # Remove fields e.g. interface filter_fields = [] -# Enable support for multitable configurations -per_peer_tables = true -peer_protocol_prefix = 'ID' -pipe_protocol_prefix = 'P' - [cache] use_redis = false redis_server = "myredis:6379" redis_db = 0 + +[housekeeping] +# Try to release memory via a forced GC/SCVG run on every housekeeping run +force_release_memory = true diff --git a/housekeeping.go b/housekeeping.go new file mode 100644 index 0000000..d4f3a03 --- /dev/null +++ b/housekeeping.go @@ -0,0 +1,45 @@ +package main + +import ( + "time" + "log" + "runtime/debug" + + "github.com/alice-lg/birdwatcher/bird" +) + + +type HousekeepingConfig struct { + ForceReleaseMemory bool `toml:"force_release_memory"` +} + +// This is used to run regular housekeeping tasks, currently expiring old +// Cache entries to release memory +func Housekeeping(config HousekeepingConfig) { + for { + if bird.ClientConf.CacheTtl > 0 { + time.Sleep(time.Duration(bird.ClientConf.CacheTtl) * time.Minute) + } else { + time.Sleep(5 * time.Minute) + } + + log.Println("Housekeeping started") + + if bird.ClientConf.CacheTtl > 0 { + // Expire the caches + log.Println("Expiring caches") + + count := bird.ParsedCache.Expire() + log.Println("Expired", count, "entries (ParsedCache)") + + count = bird.MetaCache.Expire() + log.Println("Expired", count, "entries (MetaCache)") + } + + if config.ForceReleaseMemory { + // Trigger a GC and SCVG run + log.Println("Freeing memory") + debug.FreeOSMemory() + } + } +} From 210d84445e274d9061c2707360f83c3fffdabf5f Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 19 Feb 2019 14:06:43 +0100 Subject: [PATCH 2/7] Made the housekeeping routine interval configurable. --- etc/birdwatcher/birdwatcher.conf | 2 ++ housekeeping.go | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index d817fab..cea27d8 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -84,5 +84,7 @@ redis_server = "myredis:6379" redis_db = 0 [housekeeping] +# Interval for the housekeeping routine in minutes +interval = 5 # Try to release memory via a forced GC/SCVG run on every housekeeping run force_release_memory = true diff --git a/housekeeping.go b/housekeeping.go index d4f3a03..b194b39 100644 --- a/housekeeping.go +++ b/housekeeping.go @@ -10,6 +10,7 @@ import ( type HousekeepingConfig struct { + Interval int `toml:"interval"` ForceReleaseMemory bool `toml:"force_release_memory"` } @@ -17,8 +18,8 @@ type HousekeepingConfig struct { // Cache entries to release memory func Housekeeping(config HousekeepingConfig) { for { - if bird.ClientConf.CacheTtl > 0 { - time.Sleep(time.Duration(bird.ClientConf.CacheTtl) * time.Minute) + if config.Interval > 0 { + time.Sleep(time.Duration(config.Interval) * time.Minute) } else { time.Sleep(5 * time.Minute) } From e6ed0cb9012f00b3eb43da63aa4628b7293c4fbe Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Wed, 20 Feb 2019 11:17:13 +0100 Subject: [PATCH 3/7] Refactor housekeeping and memory cache * run Expire() only on MemoryCaches * make initialization of the cache look pretty --- bird/bird.go | 43 ++++++++++++++------------------ bird/memory_cache.go | 20 +++++++++++++++ bird/memory_cache_test.go | 1 + bird/redis_cache.go | 6 +++++ birdwatcher.go | 19 +++----------- etc/birdwatcher/birdwatcher.conf | 3 ++- housekeeping.go | 16 +++++------- 7 files changed, 57 insertions(+), 51 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 5b584fa..c606cff 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -16,20 +16,20 @@ import ( type Cache interface { Set(key string, val Parsed, ttl int) error Get(key string) (Parsed, error) + Expire() int } var ClientConf BirdConfig var StatusConf StatusConfig var IPVersion = "4" var cache Cache // stores parsed birdc output +var CacheConf CacheConfig var RateLimitConf struct { sync.RWMutex Conf RateLimitConfig } var RunQueue sync.Map // queue birdc commands before execution -var CacheRedis *RedisCache - var NilParse Parsed = (Parsed)(nil) // special Parsed values var BirdError Parsed = Parsed{"error": "bird unreachable"} @@ -40,8 +40,23 @@ func IsSpecial(ret Parsed) bool { // test for special Parsed values // intitialize the Cache once during setup with either a MemoryCache or // RedisCache implementation. // TODO implement singleton pattern -func InitializeCache(c Cache) { - cache = c +func InitializeCache() { + var err error + if CacheConf.UseRedis { + cache, err = NewRedisCache(CacheConf) + if err != nil { + log.Println("Could not initialize redis cache, falling back to memory cache:", err) + } + } else { // initialize the MemoryCache + cache, err = NewMemoryCache() + if err != nil { + log.Fatal("Could not initialize MemoryCache:", err) + } + } +} + +func ExpireCache() int { + return cache.Expire() } /* Convenience method to make new entries in the cache. @@ -98,26 +113,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string { return key } -func (c *MemoryCache) Expire() int { - c.Lock() - - expiredKeys := []string{} - for key, _ := range c.m { - ttl, correct := c.m[key]["ttl"].(time.Time) - if !correct || ttl.Before(time.Now()) { - expiredKeys = append(expiredKeys, key) - } - } - - for _, key := range expiredKeys { - delete(c.m, key) - } - - c.Unlock() - - return len(expiredKeys) -} - func Run(args string) (io.Reader, error) { args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument argsList := strings.Split(args, " ") diff --git a/bird/memory_cache.go b/bird/memory_cache.go index 89384b4..7475e06 100644 --- a/bird/memory_cache.go +++ b/bird/memory_cache.go @@ -59,3 +59,23 @@ func (c *MemoryCache) Set(key string, val Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (c *MemoryCache) Expire() int { + c.Lock() + + expiredKeys := []string{} + for key, _ := range c.m { + ttl, correct := c.m[key]["ttl"].(time.Time) + if !correct || ttl.Before(time.Now()) { + expiredKeys = append(expiredKeys, key) + } + } + + for _, key := range expiredKeys { + delete(c.m, key) + } + + c.Unlock() + + return len(expiredKeys) +} diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go index b4a6a78..4026fae 100644 --- a/bird/memory_cache_test.go +++ b/bird/memory_cache_test.go @@ -26,6 +26,7 @@ func Test_MemoryCacheAccess(t *testing.T) { t.Error(err) } + cache.Expire() t.Log(parsed) } diff --git a/bird/redis_cache.go b/bird/redis_cache.go index f20f7e8..faf672b 100644 --- a/bird/redis_cache.go +++ b/bird/redis_cache.go @@ -3,6 +3,7 @@ package bird import ( "encoding/json" "errors" + "log" "time" "github.com/go-redis/redis" @@ -74,3 +75,8 @@ func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (self *RedisCache) Expire() int { + log.Printf("Cannot expire entries in RedisCache backend, redis does this automatically") + return 0 +} diff --git a/birdwatcher.go b/birdwatcher.go index a1ebaf5..4e60212 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -178,21 +178,8 @@ func main() { bird.RateLimitConf.Conf = conf.Ratelimit bird.RateLimitConf.Unlock() bird.ParserConf = conf.Parser - - var cache bird.Cache - if conf.Cache.UseRedis { - cache, err = bird.NewRedisCache(conf.Cache) - if err != nil { - log.Fatal("Could not initialize redis cache, falling back to memory cache:", err) - } - } else { // initialize the MemoryCache - cache, err = bird.NewMemoryCache() - if err != nil { - log.Fatal("Could not initialize MemoryCache:", err) - } else { - bird.InitializeCache(cache) - } - } + bird.CacheConf = conf.Cache + bird.InitializeCache() endpoints.Conf = conf.Server @@ -205,7 +192,7 @@ func main() { myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime)) mylogger := &MyLogger{myquerylog} - go Housekeeping(conf.Housekeeping) + go Housekeeping(conf.Housekeeping, !(bird.CacheConf.UseRedis)) // expire caches only for MemoryCache if conf.Server.EnableTLS { if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 { diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index cea27d8..ac83885 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -79,10 +79,11 @@ ttl = 5 # time to live (in minutes) for caching of cli output filter_fields = [] [cache] -use_redis = false +use_redis = false # if not using redis cache, activate housekeeping to save memory! redis_server = "myredis:6379" redis_db = 0 +# Housekeeping expires old cache entries (memory cache backend) and performs a GC/SCVG run if configured. [housekeeping] # Interval for the housekeeping routine in minutes interval = 5 diff --git a/housekeeping.go b/housekeeping.go index b194b39..3c38db3 100644 --- a/housekeeping.go +++ b/housekeeping.go @@ -1,14 +1,13 @@ package main import ( - "time" "log" "runtime/debug" + "time" "github.com/alice-lg/birdwatcher/bird" ) - type HousekeepingConfig struct { Interval int `toml:"interval"` ForceReleaseMemory bool `toml:"force_release_memory"` @@ -16,7 +15,7 @@ type HousekeepingConfig struct { // This is used to run regular housekeeping tasks, currently expiring old // Cache entries to release memory -func Housekeeping(config HousekeepingConfig) { +func Housekeeping(config HousekeepingConfig, expireCaches bool) { for { if config.Interval > 0 { time.Sleep(time.Duration(config.Interval) * time.Minute) @@ -26,15 +25,12 @@ func Housekeeping(config HousekeepingConfig) { log.Println("Housekeeping started") - if bird.ClientConf.CacheTtl > 0 { + if (bird.ClientConf.CacheTtl > 0) && expireCaches { // Expire the caches - log.Println("Expiring caches") + log.Println("Expiring MemoryCache") - count := bird.ParsedCache.Expire() - log.Println("Expired", count, "entries (ParsedCache)") - - count = bird.MetaCache.Expire() - log.Println("Expired", count, "entries (MetaCache)") + count := bird.ExpireCache() + log.Println("Expired", count, "entries (MemoryCache)") } if config.ForceReleaseMemory { From 8585003413e76d621b9473181399612023ed3ac1 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Tue, 19 Feb 2019 14:07:14 +0100 Subject: [PATCH 4/7] Added a parser and endpoint for the show protocols command. --- bird/bird.go | 5 +++++ bird/parser.go | 31 +++++++++++++++++++++++++++++++ bird/parser_test.go | 20 ++++++++++++++++++++ birdwatcher.go | 3 +++ endpoints/protocols.go | 4 ++++ etc/birdwatcher/birdwatcher.conf | 2 ++ test/protocols_short.sample | 10 ++++++++++ 7 files changed, 75 insertions(+) create mode 100644 test/protocols_short.sample diff --git a/bird/bird.go b/bird/bird.go index c606cff..103d9bf 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -239,6 +239,11 @@ func Status() (Parsed, bool) { return birdStatus, from_cache } +func ProtocolsShort() (Parsed, bool) { + res, from_cache := RunAndParse(GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil) + return res, from_cache +} + func Protocols() (Parsed, bool) { createMetaCache := func(p *Parsed) { metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}} diff --git a/bird/parser.go b/bird/parser.go index 92d3f6a..a22f9aa 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -29,6 +29,7 @@ var ( routes *regexp.Regexp stringValue *regexp.Regexp routeChanges *regexp.Regexp + short *regexp.Regexp } symbols struct { keyRx *regexp.Regexp @@ -72,6 +73,7 @@ func init() { regex.protocol.routeChanges = regexp.MustCompile(`(Import|Export) (updates|withdraws):\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s*$`) regex.routes.startDefinition = regexp.MustCompile(`^([0-9a-f\.\:\/]+)\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*`) + regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+)\s+(.*?)\s*?$`) regex.routes.second = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*$`) regex.routes.routeType = regexp.MustCompile(`^\s+Type:\s+(.*)\s*$`) regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`) @@ -132,6 +134,35 @@ func parseStatus(reader io.Reader) Parsed { return Parsed{"status": res} } +func parseProtocolsShort(reader io.Reader) Parsed { + res := Parsed{} + + lines := newLineIterator(reader, false) + for lines.next() { + line := lines.string() + + if specialLine(line) { + continue + } + + if regex.protocol.short.MatchString(line) { + // The header is skipped, because the regular expression does not + // match if the "since" field does not contain digits + matches := regex.protocol.short.FindStringSubmatch(line) + + res[matches[1]] = Parsed{ + "proto": matches[2], + "table": matches[3], + "state": matches[4], + "since": matches[5], + "info": matches[6], + } + } + } + + return Parsed{"protocols": res} +} + func parseProtocols(reader io.Reader) Parsed { res := Parsed{} diff --git a/bird/parser_test.go b/bird/parser_test.go index 54d93cd..cb026dd 100644 --- a/bird/parser_test.go +++ b/bird/parser_test.go @@ -63,6 +63,26 @@ func TestParseProtocolBgp(t *testing.T) { fmt.Println(protocols) } +func TestParseProtocolShort(t *testing.T) { + f, err := openFile("protocols_short.sample") + if err != nil { + t.Error(err) + } + defer f.Close() + + p := parseProtocolsShort(f) + log.Printf("%# v", pretty.Formatter(p)) + protocols := p["protocols"].(Parsed) + + if len(protocols) != 7 { + //log.Printf("%# v", pretty.Formatter(protocols)) + t.Fatalf("Expected 7 protocols, found: %v", len(protocols)) + } + + fmt.Println(protocols) +} + + func TestParseRoutesAllIpv4Bird1(t *testing.T) { runTestForIpv4WithFile("routes_bird1_ipv4.sample", t) } diff --git a/birdwatcher.go b/birdwatcher.go index 4e60212..e5d8d54 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -42,6 +42,9 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router { if isModuleEnabled("protocols_bgp", whitelist) { r.GET("/protocols/bgp", endpoints.Endpoint(endpoints.Bgp)) } + if isModuleEnabled("protocols_short", whitelist) { + r.GET("/protocols/short", endpoints.Endpoint(endpoints.ProtocolsShort)) + } if isModuleEnabled("symbols", whitelist) { r.GET("/symbols", endpoints.Endpoint(endpoints.Symbols)) } diff --git a/endpoints/protocols.go b/endpoints/protocols.go index da04438..eb23534 100644 --- a/endpoints/protocols.go +++ b/endpoints/protocols.go @@ -14,3 +14,7 @@ func Protocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { func Bgp(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { return bird.ProtocolsBgp() } + +func ProtocolsShort(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { + return bird.ProtocolsShort() +} diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index ac83885..3a5e17a 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -14,6 +14,7 @@ allow_from = [] # symbols_protocols # protocols # protocols_bgp +# protocols_short # routes_protocol # routes_peer # routes_table @@ -34,6 +35,7 @@ allow_from = [] modules_enabled = ["status", "protocols", "protocols_bgp", + "protocols_short", "routes_protocol", "routes_peer", "routes_table", diff --git a/test/protocols_short.sample b/test/protocols_short.sample new file mode 100644 index 0000000..c55c3d4 --- /dev/null +++ b/test/protocols_short.sample @@ -0,0 +1,10 @@ +BIRD 1.6.5 ready. +Access restricted +name proto table state since info +device1 Device master up 2019-02-15 +pp_0097_as3856 Pipe master up 2019-02-15 => t_0097_as3856 +pb_0097_as3856 BGP t_0097_as3856 up 2019-02-15 Established +pp_0175_as15169 Pipe master up 2019-02-15 => t_0175_as15169 +pb_0175_as15169 BGP t_0175_as15169 up 2019-02-15 Established +pp_0026_as20940 Pipe master up 2019-02-15 => t_0026_as20940 +pb_0026_as20940 BGP t_0026_as20940 up 2019-02-15 Established From c6e717bc61e367b15bbbfadfa5b6eba190219d15 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Wed, 20 Feb 2019 12:13:18 +0100 Subject: [PATCH 5/7] Fixed a bug regarding timestamps in the protocols parser. --- bird/parser.go | 2 +- bird/parser_test.go | 4 ++-- test/protocols_short.sample | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bird/parser.go b/bird/parser.go index a22f9aa..074e808 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -73,7 +73,7 @@ func init() { regex.protocol.routeChanges = regexp.MustCompile(`(Import|Export) (updates|withdraws):\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s*$`) regex.routes.startDefinition = regexp.MustCompile(`^([0-9a-f\.\:\/]+)\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*`) - regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+)\s+(.*?)\s*?$`) + regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+\s+[0-9\:]+?|[0-9\-]+)\s+(.*?)\s*?$`) regex.routes.second = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*$`) regex.routes.routeType = regexp.MustCompile(`^\s+Type:\s+(.*)\s*$`) regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`) diff --git a/bird/parser_test.go b/bird/parser_test.go index cb026dd..a4a81f8 100644 --- a/bird/parser_test.go +++ b/bird/parser_test.go @@ -74,9 +74,9 @@ func TestParseProtocolShort(t *testing.T) { log.Printf("%# v", pretty.Formatter(p)) protocols := p["protocols"].(Parsed) - if len(protocols) != 7 { + if len(protocols) != 27 { //log.Printf("%# v", pretty.Formatter(protocols)) - t.Fatalf("Expected 7 protocols, found: %v", len(protocols)) + t.Fatalf("Expected 27 protocols, found: %v", len(protocols)) } fmt.Println(protocols) diff --git a/test/protocols_short.sample b/test/protocols_short.sample index c55c3d4..5fabb6a 100644 --- a/test/protocols_short.sample +++ b/test/protocols_short.sample @@ -8,3 +8,24 @@ pp_0175_as15169 Pipe master up 2019-02-15 => t_0175_as15169 pb_0175_as15169 BGP t_0175_as15169 up 2019-02-15 Established pp_0026_as20940 Pipe master up 2019-02-15 => t_0026_as20940 pb_0026_as20940 BGP t_0026_as20940 up 2019-02-15 Established +direct1 Direct master down 2019-02-19 16:17:59 +kernel1 Kernel master down 2019-02-19 16:17:59 +M42_pch_radb Pipe master up 2019-02-19 16:17:59 => T42_pch_radb +C42_pch_radb Pipe Collector up 2019-02-19 16:17:59 => T42_pch_radb +R194_42 BGP T42_pch_radb up 2019-02-19 16:29:00 Established +M3856_pch_radb Pipe master up 2019-02-19 16:17:59 => T3856_pch_radb +C3856_pch_radb Pipe Collector up 2019-02-19 16:17:59 => T3856_pch_radb +R195_42 BGP T3856_pch_radb up 2019-02-19 16:18:33 Established +M112_112_ripe Pipe master up 2019-02-19 16:17:59 => T112_112_ripe +C112_112_ripe Pipe Collector up 2019-02-19 16:17:59 => T112_112_ripe +R195_77 BGP T112_112_ripe up 2019-02-19 16:24:31 Established +M286_kpn_ripe Pipe master up 2019-02-19 16:17:59 => T286_kpn_ripe +C286_kpn_ripe Pipe Collector up 2019-02-19 16:17:59 => T286_kpn_ripe +R192_22 BGP T286_kpn_ripe up 2019-02-19 16:26:10 Established +M553_belwue_ripe Pipe master up 2019-02-19 16:17:59 => T553_belwue_ripe +C553_belwue_ripe Pipe Collector up 2019-02-19 16:17:59 => T553_belwue_ripe +R192_175 BGP T553_belwue_ripe up 2019-02-19 16:18:34 Established +R194_106 BGP T553_belwue_ripe up 2019-02-19 16:18:09 Established +R194_205 BGP T52866_iveloz_radb start 2019-02-20 12:06:01 Idle BGP Error: Bad peer AS +R_janus1 BGP T6695_bh_20 start 2019-02-19 16:17:59 Idle + From 116f03fed43faa8577fe5c4eb212bdde3cc61782 Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Thu, 21 Feb 2019 13:36:28 +0100 Subject: [PATCH 6/7] Add support for uncached queries The cache is still updated with new information on every request. --- bird/bird.go | 126 ++++++++++++++++--------------- endpoints/config.go | 1 + endpoints/endpoint.go | 17 ++++- endpoints/protocols.go | 12 +-- endpoints/routes.go | 60 +++++++-------- endpoints/status.go | 4 +- endpoints/symbols.go | 12 +-- etc/birdwatcher/birdwatcher.conf | 2 + 8 files changed, 126 insertions(+), 108 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 103d9bf..8099794 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -159,12 +159,15 @@ func checkRateLimit() bool { return true } -func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) { - if val, ok := fromCache(cmd); ok { - return val, true +func RunAndParse(useCache bool, key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) { + var wg sync.WaitGroup + + if useCache { + if val, ok := fromCache(cmd); ok { + return val, true + } } - var wg sync.WaitGroup wg.Add(1) if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded { (*queueGroup.(*sync.WaitGroup)).Wait() @@ -200,13 +203,12 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa toCache(cmd, parsed) wg.Done() - RunQueue.Delete(cmd) return parsed, false } -func Status() (Parsed, bool) { +func Status(useCache bool) (Parsed, bool) { updateParsedCache := func(p *Parsed) { status := (*p)["status"].(Parsed) @@ -235,16 +237,16 @@ func Status() (Parsed, bool) { } } - birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, updateParsedCache) + birdStatus, from_cache := RunAndParse(useCache, GetCacheKey("Status"), "status", parseStatus, updateParsedCache) return birdStatus, from_cache } -func ProtocolsShort() (Parsed, bool) { - res, from_cache := RunAndParse(GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil) +func ProtocolsShort(useCache bool) (Parsed, bool) { + res, from_cache := RunAndParse(useCache, GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil) return res, from_cache } -func Protocols() (Parsed, bool) { +func Protocols(useCache bool) (Parsed, bool) { createMetaCache := func(p *Parsed) { metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}} @@ -263,12 +265,12 @@ func Protocols() (Parsed, bool) { toCache(GetCacheKey("metaProtocol"), metaProtocol) } - res, from_cache := RunAndParse(GetCacheKey("metaProtocol"), "protocols all", parseProtocols, createMetaCache) + res, from_cache := RunAndParse(useCache, GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache) return res, from_cache } -func ProtocolsBgp() (Parsed, bool) { - protocols, from_cache := Protocols() +func ProtocolsBgp(useCache bool) (Parsed, bool) { + protocols, from_cache := Protocols(useCache) if IsSpecial(protocols) { return protocols, from_cache } @@ -287,92 +289,92 @@ func ProtocolsBgp() (Parsed, bool) { "cached_at": protocols["cached_at"]}, from_cache } -func Symbols() (Parsed, bool) { - return RunAndParse(GetCacheKey("Symbols"), "symbols", parseSymbols, nil) +func Symbols(useCache bool) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("Symbols"), "symbols", parseSymbols, nil) } -func RoutesPrefixed(prefix string) (Parsed, bool) { - cmd := routeQueryForChannel("route " + prefix + " all") - return RunAndParse(GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil) +func RoutesPrefixed(useCache bool, prefix string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route "+prefix+" all") + return RunAndParse(useCache, GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil) } -func RoutesProto(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all protocol " + protocol) - return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil) +func RoutesProto(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all protocol "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil) } -func RoutesPeer(peer string) (Parsed, bool) { - cmd := routeQueryForChannel("route all where from=" + peer) - return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) +func RoutesPeer(useCache bool, peer string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all where from="+peer) + return RunAndParse(useCache, GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) } -func RoutesTableAndPeer(table string, peer string) (Parsed, bool) { - cmd := routeQueryForChannel("route table " + table + " all where from=" + peer) - return RunAndParse(GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil) +func RoutesTableAndPeer(useCache bool, table string, peer string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route table "+table+" all where from="+peer) + return RunAndParse(useCache, GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil) } -func RoutesProtoCount(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route protocol "+protocol) + " count" - return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil) +func RoutesProtoCount(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route protocol "+protocol) + " count" + return RunAndParse(useCache, GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil) } -func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route primary protocol "+protocol) + " count" - return RunAndParse(GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil) +func RoutesProtoPrimaryCount(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route primary protocol "+protocol) + " count" + return RunAndParse(useCache, GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil) } -func PipeRoutesFilteredCount(pipe string, table string, neighborAddress string) (Parsed, bool) { +func PipeRoutesFilteredCount(useCache bool, pipe string, table string, neighborAddress string) (Parsed, bool) { cmd := "route table " + table + " noexport " + pipe + " where from=" + neighborAddress + " count" - return RunAndParse(GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil) + return RunAndParse(useCache, GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil) } -func PipeRoutesFiltered(pipe string, table string) (Parsed, bool) { - cmd := routeQueryForChannel("route table '" + table + "' noexport '" + pipe + "' all") - return RunAndParse(GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil) +func PipeRoutesFiltered(useCache bool, pipe string, table string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route table '"+table+"' noexport '"+pipe+"' all") + return RunAndParse(useCache, GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil) } -func RoutesFiltered(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all filtered protocol " + protocol) - return RunAndParse(GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil) +func RoutesFiltered(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all filtered protocol "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil) } -func RoutesExport(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all export " + protocol) - return RunAndParse(GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil) +func RoutesExport(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all export "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil) } -func RoutesNoExport(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all noexport " + protocol) - return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil) +func RoutesNoExport(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all noexport "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil) } -func RoutesExportCount(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route export "+protocol) + " count" - return RunAndParse(GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil) +func RoutesExportCount(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route export "+protocol) + " count" + return RunAndParse(useCache, GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil) } -func RoutesTable(table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil) +func RoutesTable(useCache bool, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil) } -func RoutesTableFiltered(table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil) +func RoutesTableFiltered(useCache bool, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil) } -func RoutesTableCount(table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil) +func RoutesTableCount(useCache bool, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil) } -func RoutesLookupTable(net string, table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil) +func RoutesLookupTable(useCache bool, net string, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil) } -func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil) +func RoutesLookupProtocol(useCache bool, net string, protocol string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil) } -func routeQueryForChannel(cmd string) string { - status, _ := Status() +func routeQueryForChannel(useCache bool, cmd string) string { + status, _ := Status(useCache) if IsSpecial(status) { return cmd } diff --git a/endpoints/config.go b/endpoints/config.go index f49599d..4e2e370 100644 --- a/endpoints/config.go +++ b/endpoints/config.go @@ -4,6 +4,7 @@ package endpoints type ServerConfig struct { AllowFrom []string `toml:"allow_from"` ModulesEnabled []string `toml:"modules_enabled"` + AllowUncached bool `toml:"allow_uncached"` EnableTLS bool `toml:"enable_tls"` Crt string `toml:"crt"` diff --git a/endpoints/endpoint.go b/endpoints/endpoint.go index 8138f83..3dda116 100644 --- a/endpoints/endpoint.go +++ b/endpoints/endpoint.go @@ -14,7 +14,7 @@ import ( "github.com/julienschmidt/httprouter" ) -type endpoint func(*http.Request, httprouter.Params) (bird.Parsed, bool) +type endpoint func(*http.Request, httprouter.Params, bool) (bird.Parsed, bool) var Conf ServerConfig @@ -42,6 +42,17 @@ func CheckAccess(req *http.Request) error { return fmt.Errorf("%s is not allowed to access this service.", ip) } +func CheckUseCache(req *http.Request) bool { + qs := req.URL.Query() + + if Conf.AllowUncached && + len(qs["uncached"]) == 1 && qs["uncached"][0] == "true" { + return false + } + + return true +} + func Endpoint(wrapped endpoint) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, @@ -54,7 +65,9 @@ func Endpoint(wrapped endpoint) httprouter.Handle { } res := make(map[string]interface{}) - ret, from_cache := wrapped(r, ps) + + useCache := CheckUseCache(r) + ret, from_cache := wrapped(r, ps, useCache) if reflect.DeepEqual(ret, bird.NilParse) { w.WriteHeader(http.StatusTooManyRequests) diff --git a/endpoints/protocols.go b/endpoints/protocols.go index eb23534..f090cbf 100644 --- a/endpoints/protocols.go +++ b/endpoints/protocols.go @@ -7,14 +7,14 @@ import ( "github.com/julienschmidt/httprouter" ) -func Protocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.Protocols() +func Protocols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.Protocols(useCache) } -func Bgp(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.ProtocolsBgp() +func Bgp(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.ProtocolsBgp(useCache) } -func ProtocolsShort(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.ProtocolsShort() +func ProtocolsShort(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.ProtocolsShort(useCache) } diff --git a/endpoints/routes.go b/endpoints/routes.go index 394ed69..d0359b9 100644 --- a/endpoints/routes.go +++ b/endpoints/routes.go @@ -8,34 +8,34 @@ import ( "github.com/julienschmidt/httprouter" ) -func ProtoRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func ProtoRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesProto(protocol) + return bird.RoutesProto(useCache, protocol) } -func RoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesFiltered(protocol) + return bird.RoutesFiltered(useCache, protocol) } -func RoutesNoExport(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RoutesNoExport(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesNoExport(protocol) + return bird.RoutesNoExport(useCache, protocol) } -func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RoutesPrefixed(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { qs := r.URL.Query() prefixl := qs["prefix"] if len(prefixl) != 1 { @@ -47,28 +47,28 @@ func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesPrefixed(prefix) + return bird.RoutesPrefixed(useCache, prefix) } -func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func TableRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { table, err := ValidateProtocolParam(ps.ByName("table")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesTable(table) + return bird.RoutesTable(useCache, table) } -func TableRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func TableRoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { table, err := ValidateProtocolParam(ps.ByName("table")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesTableFiltered(table) + return bird.RoutesTableFiltered(useCache, table) } -func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func TableAndPeerRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { table, err := ValidateProtocolParam(ps.ByName("table")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false @@ -79,45 +79,45 @@ func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, boo return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesTableAndPeer(table, peer) + return bird.RoutesTableAndPeer(useCache, table, peer) } -func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func ProtoCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesProtoCount(protocol) + return bird.RoutesProtoCount(useCache, protocol) } -func ProtoPrimaryCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func ProtoPrimaryCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesProtoPrimaryCount(protocol) + return bird.RoutesProtoPrimaryCount(useCache, protocol) } -func TableCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func TableCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { table, err := ValidateProtocolParam(ps.ByName("table")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesTableCount(table) + return bird.RoutesTableCount(useCache, table) } -func RouteNet(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RouteNet(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { net, err := ValidatePrefixParam(ps.ByName("net")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesLookupTable(net, "master") + return bird.RoutesLookupTable(useCache, net, "master") } -func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RouteNetTable(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { net, err := ValidatePrefixParam(ps.ByName("net")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false @@ -128,10 +128,10 @@ func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesLookupTable(net, table) + return bird.RoutesLookupTable(useCache, net, table) } -func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func PipeRoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { qs := r.URL.Query() if len(qs["table"]) != 1 { @@ -150,10 +150,10 @@ func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, boo return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.PipeRoutesFiltered(pipe, table) + return bird.PipeRoutesFiltered(useCache, pipe, table) } -func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { qs := r.URL.Query() if len(qs["table"]) != 1 { @@ -180,14 +180,14 @@ func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.PipeRoutesFilteredCount(pipe, table, address) + return bird.PipeRoutesFilteredCount(useCache, pipe, table, address) } -func PeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func PeerRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { peer, err := ValidatePrefixParam(ps.ByName("peer")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesPeer(peer) + return bird.RoutesPeer(useCache, peer) } diff --git a/endpoints/status.go b/endpoints/status.go index fd18eb4..36670e2 100644 --- a/endpoints/status.go +++ b/endpoints/status.go @@ -7,6 +7,6 @@ import ( "github.com/julienschmidt/httprouter" ) -func Status(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.Status() +func Status(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.Status(useCache) } diff --git a/endpoints/symbols.go b/endpoints/symbols.go index df9245b..63fbdab 100644 --- a/endpoints/symbols.go +++ b/endpoints/symbols.go @@ -7,20 +7,20 @@ import ( "github.com/julienschmidt/httprouter" ) -func Symbols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.Symbols() +func Symbols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.Symbols(useCache) } -func SymbolTables(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - val, from_cache := bird.Symbols() +func SymbolTables(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + val, from_cache := bird.Symbols(useCache) if bird.IsSpecial(val) { return val, from_cache } return bird.Parsed{"symbols": val["symbols"].(bird.Parsed)["routing table"]}, from_cache } -func SymbolProtocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - val, from_cache := bird.Symbols() +func SymbolProtocols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + val, from_cache := bird.Symbols(useCache) if bird.IsSpecial(val) { return val, from_cache } diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index 3a5e17a..b15f4dd 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -5,6 +5,8 @@ [server] # Restrict access to certain IPs. Leave empty to allow from all. allow_from = [] +# Allow queries that bypass the cache +allow_uncached = false # Available modules: ## low-level modules (translation from birdc output to JSON objects) From e0031c61a0f6e6d5a90be5b4cb69fa994e85ac8e Mon Sep 17 00:00:00 2001 From: Patrick Seeburger Date: Fri, 22 Feb 2019 10:44:13 +0100 Subject: [PATCH 7/7] Change extended communities fomat Change format for extended BGP communities from (string,int,int) to (string,string,string) in order to support communities like (generic, 0x43000000, 0x1) --- bird/parser.go | 4 ++-- bird/parser_test.go | 32 +++++++++++++++++--------------- test/routes_bird1_ipv4.sample | 4 ++-- test/routes_bird2_ipv4.sample | 4 ++-- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/bird/parser.go b/bird/parser.go index 074e808..92ed1d9 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -79,7 +79,7 @@ func init() { regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`) regex.routes.community = regexp.MustCompile(`^\((\d+),\s*(\d+)\)`) regex.routes.largeCommunity = regexp.MustCompile(`^\((\d+),\s*(\d+),\s*(\d+)\)`) - regex.routes.extendedCommunity = regexp.MustCompile(`^\(([^,]+),\s*(\d+),\s*(\d+)\)`) + regex.routes.extendedCommunity = regexp.MustCompile(`^\(([^,]+),\s*([^,]+),\s*([^,]+)\)`) regex.routes.origin = regexp.MustCompile(`\([^\(]*\)\s*`) regex.routes.prefixBird2 = regexp.MustCompile(`^([0-9a-f\.\:\/]+)?\s+unicast\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+))?\]\s+(?:(\*)\s+)?\((\d+)(?:\/\d+)?(?:\/[^\)]*)?\).*$`) regex.routes.gatewayBird2 = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s*$`) @@ -503,7 +503,7 @@ func parseRoutesExtendedCommunities(groups []string, res Parsed) { for _, community := range regex.routes.origin.FindAllString(groups[2], -1) { if regex.routes.extendedCommunity.MatchString(community) { communityGroups := regex.routes.extendedCommunity.FindStringSubmatch(community) - communities = append(communities, []interface{}{communityGroups[1], parseInt(communityGroups[2]), parseInt(communityGroups[3])}) + communities = append(communities, []interface{}{communityGroups[1], communityGroups[2], communityGroups[3]}) } } diff --git a/bird/parser_test.go b/bird/parser_test.go index a4a81f8..82da052 100644 --- a/bird/parser_test.go +++ b/bird/parser_test.go @@ -169,7 +169,8 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"rt", int64(48858), int64(50)}, + []interface{}{"rt", "42", "1234"}, + []interface{}{"generic", "0x43000000", "0x1"}, }, metric: 100, localPref: "100", @@ -190,9 +191,9 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, metric: 100, localPref: "100", @@ -213,9 +214,9 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, metric: 100, localPref: "100", @@ -236,7 +237,8 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"rt", int64(48858), int64(50)}, + []interface{}{"rt", "42", "1234"}, + []interface{}{"generic", "0x43000000", "0x1"}, }, metric: 100, localPref: "100", @@ -332,9 +334,9 @@ func runTestForIpv6WithFile(file string, t *testing.T) { {48821, 0, 2100}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, metric: 100, localPref: "500", @@ -355,9 +357,9 @@ func runTestForIpv6WithFile(file string, t *testing.T) { {48821, 0, 3100}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, localPref: "100", metric: 100, @@ -378,7 +380,7 @@ func runTestForIpv6WithFile(file string, t *testing.T) { {48821, 0, 2100}, }, extendedCommunities: []interface{}{ - []interface{}{"unknown 0x4300", int64(0), int64(1)}, + []interface{}{"unknown 0x4300", "0", "1"}, }, metric: 100, localPref: "5000", diff --git a/test/routes_bird1_ipv4.sample b/test/routes_bird1_ipv4.sample index e56318c..814ea82 100644 --- a/test/routes_bird1_ipv4.sample +++ b/test/routes_bird1_ipv4.sample @@ -8,7 +8,7 @@ BIRD 1.6.3 ready. BGP.community: (0,5464) (0,8339) (0,8741) (0,8823) (0,12387) (0,13101) (0,16097) (0,16316) (0,20546) (0,20686) (0,20723) (0,21083) (0,21385) (0,24940) (0,25504) (0,28876) (0,29545) (0,30058) (0,31103) (0,31400) (0,39090) (0,39392) (0,39912) (0,42473) (0,43957) (0,44453) (0,47297) (0,47692) (0,48200) (0,50629) (0,51191) (0,51839) (0,51852) (0,54113) (0,56719) (0,57957) (0,60517) (0,60574) (0,61303) (0,62297) (0,62336) (0,62359) (33891,33892) (33891,50673) (48793,48793) (50673,500) (65101,11077) (65102,11000) (65103,724) (65104,150) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 200.0.0.0/24 via 1.2.3.15 on eno7 [ID8497_AS1339 2017-06-21 08:17:31] * (100) [AS1339i] Type: BGP unicast univ BGP.origin: IGP @@ -35,4 +35,4 @@ BIRD 1.6.3 ready. BGP.local_pref: 100 BGP.community: (65011,3) (9033,3251) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) diff --git a/test/routes_bird2_ipv4.sample b/test/routes_bird2_ipv4.sample index a566130..4bd1f5c 100644 --- a/test/routes_bird2_ipv4.sample +++ b/test/routes_bird2_ipv4.sample @@ -9,7 +9,7 @@ BIRD 1.6.3 ready. BGP.community: (0,5464) (0,8339) (0,8741) (0,8823) (0,12387) (0,13101) (0,16097) (0,16316) (0,20546) (0,20686) (0,20723) (0,21083) (0,21385) (0,24940) (0,25504) (0,28876) (0,29545) (0,30058) (0,31103) (0,31400) (0,39090) (0,39392) (0,39912) (0,42473) (0,43957) (0,44453) (0,47297) (0,47692) (0,48200) (0,50629) (0,51191) (0,51839) (0,51852) (0,54113) (0,56719) (0,57957) (0,60517) (0,60574) (0,61303) (0,62297) (0,62336) (0,62359) (33891,33892) (33891,50673) (48793,48793) (50673,500) (65101,11077) (65102,11000) (65103,724) (65104,150) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 200.0.0.0/24 unicast [ID8497_AS1339 2017-06-21 08:17:31] * (100) [AS1339i] via 1.2.3.15 on eno7 Type: BGP univ @@ -39,4 +39,4 @@ BIRD 1.6.3 ready. BGP.local_pref: 100 BGP.community: (65011,3) (9033,3251) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1)