diff --git a/CHANGELOG b/CHANGELOG index f091405..d43fc62 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,21 @@ +2.0.0 - + +BREAKING CHANGES AHEAD: + +In order to ease the load on the routeservers and even further +reduce the memory footprint, we decided to move the per peer table +configuration out of the birdwatcher and into alice. + +Please be aware, that you need the newest version of alice. + +Everything else: + +* Improved cach setup for inmemory and redis +* Improved housekeeping and memory footprint reduction +* Performance improvements and other good stuff. + + 1.12.4 * Add the ability to switch between redis and the classic diff --git a/README.md b/README.md index a3b9100..c1dfeff 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ of the config. ## Installation You will need to have go installed to build the package. +Please make sure your go version is `>= 1.9`. + Running `go get github.com/alice-lg/birdwatcher` will give you a binary. You might need to cross-compile it for your bird-running servive (`GOARCH` and `GOOS` are your friends). @@ -29,6 +31,13 @@ Running `make linux` will create a Linux executable (by default for `amd64`, but that is configurable by providing the `ARCH` argument to the Makefile). + +#### 2.0 Breaking Change + +The per peer table configuration is no longer done in the birdwatcher, +but directly in alice. + + ### BIRD configuration Birdwatcher parses the output of birdc and expects (for now) @@ -153,3 +162,6 @@ Initially developed by Daniel and MC from [Netnod](https://www.netnod.se/) in two days at the RIPE 73 IXP Tools Hackathon in Madrid, Spain. Running bird and parsing the results was added by [Veit Heller](https://github.com/hellerve/) on behalf of [ecix](http://ecix.net/). + +With major contributions from: Patrick Seeburger and Benedikt Rudolph on behalf of [DE-CIX](https://de-cix.net). + diff --git a/VERSION b/VERSION index 89c881b..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.4 +2.0.0 diff --git a/bird/bird.go b/bird/bird.go index 9d9797b..8099794 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -16,19 +16,20 @@ import ( type Cache interface { Set(key string, val Parsed, ttl int) error Get(key string) (Parsed, error) + Expire() int } var ClientConf BirdConfig var StatusConf StatusConfig var IPVersion = "4" var cache Cache // stores parsed birdc output +var CacheConf CacheConfig var RateLimitConf struct { sync.RWMutex Conf RateLimitConfig } var RunQueue sync.Map // queue birdc commands before execution -var CacheRedis *RedisCache var NilParse Parsed = (Parsed)(nil) // special Parsed values var BirdError Parsed = Parsed{"error": "bird unreachable"} @@ -39,8 +40,23 @@ func IsSpecial(ret Parsed) bool { // test for special Parsed values // intitialize the Cache once during setup with either a MemoryCache or // RedisCache implementation. // TODO implement singleton pattern -func InitializeCache(c Cache) { - cache = c +func InitializeCache() { + var err error + if CacheConf.UseRedis { + cache, err = NewRedisCache(CacheConf) + if err != nil { + log.Println("Could not initialize redis cache, falling back to memory cache:", err) + } + } else { // initialize the MemoryCache + cache, err = NewMemoryCache() + if err != nil { + log.Fatal("Could not initialize MemoryCache:", err) + } + } +} + +func ExpireCache() int { + return cache.Expire() } /* Convenience method to make new entries in the cache. @@ -143,12 +159,15 @@ func checkRateLimit() bool { return true } -func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) { - if val, ok := fromCache(cmd); ok { - return val, true +func RunAndParse(useCache bool, key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) { + var wg sync.WaitGroup + + if useCache { + if val, ok := fromCache(cmd); ok { + return val, true + } } - var wg sync.WaitGroup wg.Add(1) if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded { (*queueGroup.(*sync.WaitGroup)).Wait() @@ -184,13 +203,12 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa toCache(cmd, parsed) wg.Done() - RunQueue.Delete(cmd) return parsed, false } -func Status() (Parsed, bool) { +func Status(useCache bool) (Parsed, bool) { updateParsedCache := func(p *Parsed) { status := (*p)["status"].(Parsed) @@ -219,17 +237,21 @@ func Status() (Parsed, bool) { } } - birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, updateParsedCache) + birdStatus, from_cache := RunAndParse(useCache, GetCacheKey("Status"), "status", parseStatus, updateParsedCache) return birdStatus, from_cache } -func Protocols() (Parsed, bool) { +func ProtocolsShort(useCache bool) (Parsed, bool) { + res, from_cache := RunAndParse(useCache, GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil) + return res, from_cache +} + +func Protocols(useCache bool) (Parsed, bool) { createMetaCache := func(p *Parsed) { metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}} for key, _ := range (*p)["protocols"].(Parsed) { parsed := (*p)["protocols"].(Parsed)[key].(Parsed) - protocol := parsed["protocol"].(string) birdProtocol := parsed["bird_protocol"].(string) @@ -243,12 +265,12 @@ func Protocols() (Parsed, bool) { toCache(GetCacheKey("metaProtocol"), metaProtocol) } - res, from_cache := RunAndParse(GetCacheKey("metaProtocol"), "protocols all", parseProtocols, createMetaCache) + res, from_cache := RunAndParse(useCache, GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache) return res, from_cache } -func ProtocolsBgp() (Parsed, bool) { - protocols, from_cache := Protocols() +func ProtocolsBgp(useCache bool) (Parsed, bool) { + protocols, from_cache := Protocols(useCache) if IsSpecial(protocols) { return protocols, from_cache } @@ -267,167 +289,92 @@ func ProtocolsBgp() (Parsed, bool) { "cached_at": protocols["cached_at"]}, from_cache } -func Symbols() (Parsed, bool) { - return RunAndParse(GetCacheKey("Symbols"), "symbols", parseSymbols, nil) +func Symbols(useCache bool) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("Symbols"), "symbols", parseSymbols, nil) } -func RoutesPrefixed(prefix string) (Parsed, bool) { - cmd := routeQueryForChannel("route " + prefix + " all") - return RunAndParse(GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil) +func RoutesPrefixed(useCache bool, prefix string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route "+prefix+" all") + return RunAndParse(useCache, GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil) } -func RoutesProto(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all protocol " + protocol) - return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil) +func RoutesProto(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all protocol "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil) } -func RoutesProtoCount(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route protocol "+protocol) + " count" - return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil) +func RoutesPeer(useCache bool, peer string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all where from="+peer) + return RunAndParse(useCache, GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) } -func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route primary protocol "+protocol) + " count" - return RunAndParse(GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil) +func RoutesTableAndPeer(useCache bool, table string, peer string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route table "+table+" all where from="+peer) + return RunAndParse(useCache, GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil) } -func PipeRoutesFilteredCount(pipe string, table string, neighborAddress string) (Parsed, bool) { +func RoutesProtoCount(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route protocol "+protocol) + " count" + return RunAndParse(useCache, GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil) +} + +func RoutesProtoPrimaryCount(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route primary protocol "+protocol) + " count" + return RunAndParse(useCache, GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil) +} + +func PipeRoutesFilteredCount(useCache bool, pipe string, table string, neighborAddress string) (Parsed, bool) { cmd := "route table " + table + " noexport " + pipe + " where from=" + neighborAddress + " count" - return RunAndParse(GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil) + return RunAndParse(useCache, GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil) } -func PipeRoutesFiltered(pipe string, table string) (Parsed, bool) { - cmd := routeQueryForChannel("route table '" + table + "' noexport '" + pipe + "' all") - return RunAndParse(GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil) +func PipeRoutesFiltered(useCache bool, pipe string, table string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route table '"+table+"' noexport '"+pipe+"' all") + return RunAndParse(useCache, GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil) } -func RoutesFiltered(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all filtered protocol " + protocol) - return RunAndParse(GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil) +func RoutesFiltered(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all filtered protocol "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil) } -func RoutesExport(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all export " + protocol) - return RunAndParse(GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil) +func RoutesExport(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all export "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil) } -func RoutesNoExport(protocol string) (Parsed, bool) { - // In case we have a multi table setup, we have to query - // the pipe protocol. - if ParserConf.PerPeerTables && - strings.HasPrefix(protocol, ParserConf.PeerProtocolPrefix) { - - // Replace prefix - protocol = ParserConf.PipeProtocolPrefix + - protocol[len(ParserConf.PeerProtocolPrefix):] - } - - cmd := routeQueryForChannel("route all noexport " + protocol) - return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil) +func RoutesNoExport(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route all noexport "+protocol) + return RunAndParse(useCache, GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil) } -func RoutesExportCount(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route export "+protocol) + " count" - return RunAndParse(GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil) +func RoutesExportCount(useCache bool, protocol string) (Parsed, bool) { + cmd := routeQueryForChannel(useCache, "route export "+protocol) + " count" + return RunAndParse(useCache, GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil) } -func RoutesTable(table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil) +func RoutesTable(useCache bool, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil) } -func RoutesTableCount(table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil) +func RoutesTableFiltered(useCache bool, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil) } -func RoutesLookupTable(net string, table string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil) +func RoutesTableCount(useCache bool, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil) } -func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) { - return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil) +func RoutesLookupTable(useCache bool, net string, table string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil) } -func RoutesPeer(peer string) (Parsed, bool) { - cmd := routeQueryForChannel("route export " + peer) - return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil) +func RoutesLookupProtocol(useCache bool, net string, protocol string) (Parsed, bool) { + return RunAndParse(useCache, GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil) } -func RoutesDump() (Parsed, bool) { - // TODO insert hook to update the cache with the route count information - if ParserConf.PerPeerTables { - return RoutesDumpPerPeerTable() - } - - return RoutesDumpSingleTable() -} - -func RoutesDumpSingleTable() (Parsed, bool) { - importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil) - if IsSpecial(importedRes) { - return importedRes, cached - } - filteredRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "filtered"), routeQueryForChannel("route all filtered"), parseRoutes, nil) - if IsSpecial(filteredRes) { - return filteredRes, cached - } - - imported := importedRes["routes"] - filtered := filteredRes["routes"] - - result := Parsed{ - "imported": imported, - "filtered": filtered, - } - - return result, cached -} - -func RoutesDumpPerPeerTable() (Parsed, bool) { - importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpPerPeerTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil) - if IsSpecial(importedRes) { - return importedRes, cached - } - imported := importedRes["routes"] - filtered := []Parsed{} - - // Get protocols with filtered routes - protocolsRes, cached := ProtocolsBgp() - if IsSpecial(protocolsRes) { - return protocolsRes, cached - } - protocols := protocolsRes["protocols"].(Parsed) - - for protocol, details := range protocols { - details := details.(Parsed) - - counters, ok := details["routes"].(Parsed) - if !ok { - continue - } - filterCount := counters["filtered"] - if filterCount == 0 { - continue // nothing to do here. - } - // Lookup filtered routes - pfilteredRes, _ := RoutesFiltered(protocol) - pfiltered, ok := pfilteredRes["routes"].([]Parsed) - if !ok { - continue // something went wrong... - } - - filtered = append(filtered, pfiltered...) - } - - result := Parsed{ - "imported": imported, - "filtered": filtered, - } - - return result, cached -} - -func routeQueryForChannel(cmd string) string { - status, _ := Status() +func routeQueryForChannel(useCache bool, cmd string) string { + status, _ := Status(useCache) if IsSpecial(status) { return cmd } diff --git a/bird/config.go b/bird/config.go index e0adc18..a18f55a 100644 --- a/bird/config.go +++ b/bird/config.go @@ -17,10 +17,7 @@ type BirdConfig struct { } type ParserConfig struct { - FilterFields []string `toml:"filter_fields"` - PerPeerTables bool `toml:"per_peer_tables"` - PeerProtocolPrefix string `toml:"peer_protocol_prefix"` - PipeProtocolPrefix string `toml:"pipe_protocol_prefix"` + FilterFields []string `toml:"filter_fields"` } type RateLimitConfig struct { diff --git a/bird/memory_cache.go b/bird/memory_cache.go index 89384b4..7475e06 100644 --- a/bird/memory_cache.go +++ b/bird/memory_cache.go @@ -59,3 +59,23 @@ func (c *MemoryCache) Set(key string, val Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (c *MemoryCache) Expire() int { + c.Lock() + + expiredKeys := []string{} + for key, _ := range c.m { + ttl, correct := c.m[key]["ttl"].(time.Time) + if !correct || ttl.Before(time.Now()) { + expiredKeys = append(expiredKeys, key) + } + } + + for _, key := range expiredKeys { + delete(c.m, key) + } + + c.Unlock() + + return len(expiredKeys) +} diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go index b4a6a78..4026fae 100644 --- a/bird/memory_cache_test.go +++ b/bird/memory_cache_test.go @@ -26,6 +26,7 @@ func Test_MemoryCacheAccess(t *testing.T) { t.Error(err) } + cache.Expire() t.Log(parsed) } diff --git a/bird/parser.go b/bird/parser.go index 717922a..92ed1d9 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -29,6 +29,7 @@ var ( routes *regexp.Regexp stringValue *regexp.Regexp routeChanges *regexp.Regexp + short *regexp.Regexp } symbols struct { keyRx *regexp.Regexp @@ -72,12 +73,13 @@ func init() { regex.protocol.routeChanges = regexp.MustCompile(`(Import|Export) (updates|withdraws):\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s*$`) regex.routes.startDefinition = regexp.MustCompile(`^([0-9a-f\.\:\/]+)\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*`) + regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+\s+[0-9\:]+?|[0-9\-]+)\s+(.*?)\s*?$`) regex.routes.second = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*$`) regex.routes.routeType = regexp.MustCompile(`^\s+Type:\s+(.*)\s*$`) regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`) regex.routes.community = regexp.MustCompile(`^\((\d+),\s*(\d+)\)`) regex.routes.largeCommunity = regexp.MustCompile(`^\((\d+),\s*(\d+),\s*(\d+)\)`) - regex.routes.extendedCommunity = regexp.MustCompile(`^\(([^,]+),\s*(\d+),\s*(\d+)\)`) + regex.routes.extendedCommunity = regexp.MustCompile(`^\(([^,]+),\s*([^,]+),\s*([^,]+)\)`) regex.routes.origin = regexp.MustCompile(`\([^\(]*\)\s*`) regex.routes.prefixBird2 = regexp.MustCompile(`^([0-9a-f\.\:\/]+)?\s+unicast\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+))?\]\s+(?:(\*)\s+)?\((\d+)(?:\/\d+)?(?:\/[^\)]*)?\).*$`) regex.routes.gatewayBird2 = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s*$`) @@ -132,6 +134,35 @@ func parseStatus(reader io.Reader) Parsed { return Parsed{"status": res} } +func parseProtocolsShort(reader io.Reader) Parsed { + res := Parsed{} + + lines := newLineIterator(reader, false) + for lines.next() { + line := lines.string() + + if specialLine(line) { + continue + } + + if regex.protocol.short.MatchString(line) { + // The header is skipped, because the regular expression does not + // match if the "since" field does not contain digits + matches := regex.protocol.short.FindStringSubmatch(line) + + res[matches[1]] = Parsed{ + "proto": matches[2], + "table": matches[3], + "state": matches[4], + "since": matches[5], + "info": matches[6], + } + } + } + + return Parsed{"protocols": res} +} + func parseProtocols(reader io.Reader) Parsed { res := Parsed{} @@ -472,7 +503,7 @@ func parseRoutesExtendedCommunities(groups []string, res Parsed) { for _, community := range regex.routes.origin.FindAllString(groups[2], -1) { if regex.routes.extendedCommunity.MatchString(community) { communityGroups := regex.routes.extendedCommunity.FindStringSubmatch(community) - communities = append(communities, []interface{}{communityGroups[1], parseInt(communityGroups[2]), parseInt(communityGroups[3])}) + communities = append(communities, []interface{}{communityGroups[1], communityGroups[2], communityGroups[3]}) } } @@ -542,6 +573,7 @@ func parseProtocol(lines string) Parsed { routes := Parsed{} routes["accepted"] = int64(0) routes["filtered"] = int64(0) + routes["imported"] = int64(0) routes["exported"] = int64(0) routes["preferred"] = int64(0) diff --git a/bird/parser_test.go b/bird/parser_test.go index 54d93cd..82da052 100644 --- a/bird/parser_test.go +++ b/bird/parser_test.go @@ -63,6 +63,26 @@ func TestParseProtocolBgp(t *testing.T) { fmt.Println(protocols) } +func TestParseProtocolShort(t *testing.T) { + f, err := openFile("protocols_short.sample") + if err != nil { + t.Error(err) + } + defer f.Close() + + p := parseProtocolsShort(f) + log.Printf("%# v", pretty.Formatter(p)) + protocols := p["protocols"].(Parsed) + + if len(protocols) != 27 { + //log.Printf("%# v", pretty.Formatter(protocols)) + t.Fatalf("Expected 27 protocols, found: %v", len(protocols)) + } + + fmt.Println(protocols) +} + + func TestParseRoutesAllIpv4Bird1(t *testing.T) { runTestForIpv4WithFile("routes_bird1_ipv4.sample", t) } @@ -149,7 +169,8 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"rt", int64(48858), int64(50)}, + []interface{}{"rt", "42", "1234"}, + []interface{}{"generic", "0x43000000", "0x1"}, }, metric: 100, localPref: "100", @@ -170,9 +191,9 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, metric: 100, localPref: "100", @@ -193,9 +214,9 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, metric: 100, localPref: "100", @@ -216,7 +237,8 @@ func runTestForIpv4WithFile(file string, t *testing.T) { {9033, 65666, 9}, }, extendedCommunities: []interface{}{ - []interface{}{"rt", int64(48858), int64(50)}, + []interface{}{"rt", "42", "1234"}, + []interface{}{"generic", "0x43000000", "0x1"}, }, metric: 100, localPref: "100", @@ -312,9 +334,9 @@ func runTestForIpv6WithFile(file string, t *testing.T) { {48821, 0, 2100}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, metric: 100, localPref: "500", @@ -335,9 +357,9 @@ func runTestForIpv6WithFile(file string, t *testing.T) { {48821, 0, 3100}, }, extendedCommunities: []interface{}{ - []interface{}{"ro", int64(21414), int64(52001)}, - []interface{}{"ro", int64(21414), int64(52004)}, - []interface{}{"ro", int64(21414), int64(64515)}, + []interface{}{"ro", "21414", "52001"}, + []interface{}{"ro", "21414", "52004"}, + []interface{}{"ro", "21414", "64515"}, }, localPref: "100", metric: 100, @@ -358,7 +380,7 @@ func runTestForIpv6WithFile(file string, t *testing.T) { {48821, 0, 2100}, }, extendedCommunities: []interface{}{ - []interface{}{"unknown 0x4300", int64(0), int64(1)}, + []interface{}{"unknown 0x4300", "0", "1"}, }, metric: 100, localPref: "5000", diff --git a/bird/redis_cache.go b/bird/redis_cache.go index f20f7e8..faf672b 100644 --- a/bird/redis_cache.go +++ b/bird/redis_cache.go @@ -3,6 +3,7 @@ package bird import ( "encoding/json" "errors" + "log" "time" "github.com/go-redis/redis" @@ -74,3 +75,8 @@ func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (self *RedisCache) Expire() int { + log.Printf("Cannot expire entries in RedisCache backend, redis does this automatically") + return 0 +} diff --git a/birdwatcher.go b/birdwatcher.go index a186f52..3314906 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -16,7 +16,7 @@ import ( ) //go:generate versionize -var VERSION = "1.12.3" +var VERSION = "2.0.0" func isModuleEnabled(module string, modulesEnabled []string) bool { for _, enabled := range modulesEnabled { @@ -42,6 +42,9 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router { if isModuleEnabled("protocols_bgp", whitelist) { r.GET("/protocols/bgp", endpoints.Endpoint(endpoints.Bgp)) } + if isModuleEnabled("protocols_short", whitelist) { + r.GET("/protocols/short", endpoints.Endpoint(endpoints.ProtocolsShort)) + } if isModuleEnabled("symbols", whitelist) { r.GET("/symbols", endpoints.Endpoint(endpoints.Symbols)) } @@ -54,9 +57,18 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router { if isModuleEnabled("routes_protocol", whitelist) { r.GET("/routes/protocol/:protocol", endpoints.Endpoint(endpoints.ProtoRoutes)) } + if isModuleEnabled("routes_peer", whitelist) { + r.GET("/routes/peer/:peer", endpoints.Endpoint(endpoints.PeerRoutes)) + } if isModuleEnabled("routes_table", whitelist) { r.GET("/routes/table/:table", endpoints.Endpoint(endpoints.TableRoutes)) } + if isModuleEnabled("routes_table_filtered", whitelist) { + r.GET("/routes/table/:table/filtered", endpoints.Endpoint(endpoints.TableRoutesFiltered)) + } + if isModuleEnabled("routes_table_peer", whitelist) { + r.GET("/routes/table/:table/peer/:peer", endpoints.Endpoint(endpoints.TableAndPeerRoutes)) + } if isModuleEnabled("routes_count_protocol", whitelist) { r.GET("/routes/count/protocol/:protocol", endpoints.Endpoint(endpoints.ProtoCount)) } @@ -79,12 +91,13 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router { r.GET("/route/net/:net", endpoints.Endpoint(endpoints.RouteNet)) r.GET("/route/net/:net/table/:table", endpoints.Endpoint(endpoints.RouteNetTable)) } - if isModuleEnabled("routes_peer", whitelist) { - r.GET("/routes/peer", endpoints.Endpoint(endpoints.RoutesPeer)) + if isModuleEnabled("routes_pipe_filtered_count", whitelist) { + r.GET("/routes/pipe/filtered/count", endpoints.Endpoint(endpoints.PipeRoutesFilteredCount)) } - if isModuleEnabled("routes_dump", whitelist) { - r.GET("/routes/dump", endpoints.Endpoint(endpoints.RoutesDump)) + if isModuleEnabled("routes_pipe_filtered", whitelist) { + r.GET("/routes/pipe/filtered", endpoints.Endpoint(endpoints.PipeRoutesFiltered)) } + return r } @@ -115,8 +128,6 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) { for _, m := range conf.Server.ModulesEnabled { log.Println(" -", m) } - - log.Println(" Per Peer Tables:", conf.Parser.PerPeerTables) } // MyLogger is our own log.Logger wrapper so we can customize it @@ -170,21 +181,8 @@ func main() { bird.RateLimitConf.Conf = conf.Ratelimit bird.RateLimitConf.Unlock() bird.ParserConf = conf.Parser - - var cache bird.Cache - if conf.Cache.UseRedis { - cache, err = bird.NewRedisCache(conf.Cache) - if err != nil { - log.Fatal("Could not initialize redis cache, falling back to MemoryCache:", err) - } - } else { // initialze the MemoryCache - cache, err = bird.NewMemoryCache() - if err != nil { - log.Fatal("Could not initialize MemoryCache:", err) - } else { - bird.InitializeCache(cache) - } - } + bird.CacheConf = conf.Cache + bird.InitializeCache() endpoints.Conf = conf.Server @@ -197,6 +195,8 @@ func main() { myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime)) mylogger := &MyLogger{myquerylog} + go Housekeeping(conf.Housekeeping, !(bird.CacheConf.UseRedis)) // expire caches only for MemoryCache + if conf.Server.EnableTLS { if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 { log.Fatalln("You have enabled TLS support but not specified both a .crt and a .key file in the config.") diff --git a/config.go b/config.go index a722a35..12e2325 100644 --- a/config.go +++ b/config.go @@ -17,12 +17,13 @@ import ( type Config struct { Server endpoints.ServerConfig - Ratelimit bird.RateLimitConfig - Status bird.StatusConfig - Bird bird.BirdConfig - Bird6 bird.BirdConfig - Parser bird.ParserConfig - Cache bird.CacheConfig + Ratelimit bird.RateLimitConfig + Status bird.StatusConfig + Bird bird.BirdConfig + Bird6 bird.BirdConfig + Parser bird.ParserConfig + Cache bird.CacheConfig + Housekeeping HousekeepingConfig } // Try to load configfiles as specified in the files diff --git a/endpoints/config.go b/endpoints/config.go index f49599d..4e2e370 100644 --- a/endpoints/config.go +++ b/endpoints/config.go @@ -4,6 +4,7 @@ package endpoints type ServerConfig struct { AllowFrom []string `toml:"allow_from"` ModulesEnabled []string `toml:"modules_enabled"` + AllowUncached bool `toml:"allow_uncached"` EnableTLS bool `toml:"enable_tls"` Crt string `toml:"crt"` diff --git a/endpoints/endpoint.go b/endpoints/endpoint.go index 709e995..3dda116 100644 --- a/endpoints/endpoint.go +++ b/endpoints/endpoint.go @@ -14,7 +14,7 @@ import ( "github.com/julienschmidt/httprouter" ) -type endpoint func(*http.Request, httprouter.Params) (bird.Parsed, bool) +type endpoint func(*http.Request, httprouter.Params, bool) (bird.Parsed, bool) var Conf ServerConfig @@ -42,6 +42,17 @@ func CheckAccess(req *http.Request) error { return fmt.Errorf("%s is not allowed to access this service.", ip) } +func CheckUseCache(req *http.Request) bool { + qs := req.URL.Query() + + if Conf.AllowUncached && + len(qs["uncached"]) == 1 && qs["uncached"][0] == "true" { + return false + } + + return true +} + func Endpoint(wrapped endpoint) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, @@ -54,7 +65,9 @@ func Endpoint(wrapped endpoint) httprouter.Handle { } res := make(map[string]interface{}) - ret, from_cache := wrapped(r, ps) + + useCache := CheckUseCache(r) + ret, from_cache := wrapped(r, ps, useCache) if reflect.DeepEqual(ret, bird.NilParse) { w.WriteHeader(http.StatusTooManyRequests) @@ -73,8 +86,6 @@ func Endpoint(wrapped endpoint) httprouter.Handle { res[k] = v } - js, _ := json.Marshal(res) - w.Header().Set("Content-Type", "application/json") // Check if compression is supported @@ -83,9 +94,11 @@ func Endpoint(wrapped endpoint) httprouter.Handle { w.Header().Set("Content-Encoding", "gzip") gz := gzip.NewWriter(w) defer gz.Close() - gz.Write(js) + json := json.NewEncoder(gz) + json.Encode(res) } else { - w.Write(js) // Fall back to uncompressed response + json := json.NewEncoder(w) + json.Encode(res) // Fall back to uncompressed response } } } diff --git a/endpoints/protocols.go b/endpoints/protocols.go index da04438..f090cbf 100644 --- a/endpoints/protocols.go +++ b/endpoints/protocols.go @@ -7,10 +7,14 @@ import ( "github.com/julienschmidt/httprouter" ) -func Protocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.Protocols() +func Protocols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.Protocols(useCache) } -func Bgp(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.ProtocolsBgp() +func Bgp(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.ProtocolsBgp(useCache) +} + +func ProtocolsShort(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.ProtocolsShort(useCache) } diff --git a/endpoints/routes.go b/endpoints/routes.go index 1dee4b3..d0359b9 100644 --- a/endpoints/routes.go +++ b/endpoints/routes.go @@ -8,31 +8,34 @@ import ( "github.com/julienschmidt/httprouter" ) -func ProtoRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func ProtoRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesProto(protocol) + + return bird.RoutesProto(useCache, protocol) } -func RoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesFiltered(protocol) + + return bird.RoutesFiltered(useCache, protocol) } -func RoutesNoExport(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RoutesNoExport(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesNoExport(protocol) + + return bird.RoutesNoExport(useCache, protocol) } -func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func RoutesPrefixed(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { qs := r.URL.Query() prefixl := qs["prefix"] if len(prefixl) != 1 { @@ -43,55 +46,148 @@ func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesPrefixed(prefix) + + return bird.RoutesPrefixed(useCache, prefix) } -func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesTable(ps.ByName("table")) +func TableRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTable(useCache, table) } -func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func TableRoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTableFiltered(useCache, table) +} + +func TableAndPeerRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + peer, err := ValidatePrefixParam(ps.ByName("peer")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTableAndPeer(useCache, table, peer) +} + +func ProtoCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesProtoCount(protocol) + + return bird.RoutesProtoCount(useCache, protocol) } -func ProtoPrimaryCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func ProtoPrimaryCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { protocol, err := ValidateProtocolParam(ps.ByName("protocol")) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesProtoPrimaryCount(protocol) + return bird.RoutesProtoPrimaryCount(useCache, protocol) } -func TableCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesTableCount(ps.ByName("table")) +func TableCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesTableCount(useCache, table) } -func RouteNet(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesLookupTable(ps.ByName("net"), "master") +func RouteNet(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + net, err := ValidatePrefixParam(ps.ByName("net")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesLookupTable(useCache, net, "master") } -func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesLookupTable(ps.ByName("net"), ps.ByName("table")) +func RouteNetTable(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + net, err := ValidatePrefixParam(ps.ByName("net")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + table, err := ValidateProtocolParam(ps.ByName("table")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesLookupTable(useCache, net, table) } -func RoutesPeer(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { +func PipeRoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { qs := r.URL.Query() - peerl := qs["peer"] - if len(peerl) != 1 { - return bird.Parsed{"error": "need a peer as single query parameter"}, false - } - peer, err := ValidateProtocolParam(peerl[0]) + if len(qs["table"]) != 1 { + return bird.Parsed{"error": "need a table as single query parameter"}, false + } + table, err := ValidateProtocolParam(qs["table"][0]) if err != nil { return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false } - return bird.RoutesPeer(peer) + + if len(qs["pipe"]) != 1 { + return bird.Parsed{"error": "need a pipe as single query parameter"}, false + } + pipe, err := ValidateProtocolParam(qs["pipe"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.PipeRoutesFiltered(useCache, pipe, table) } -func RoutesDump(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.RoutesDump() +func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + qs := r.URL.Query() + + if len(qs["table"]) != 1 { + return bird.Parsed{"error": "need a table as single query parameter"}, false + } + table, err := ValidateProtocolParam(qs["table"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + if len(qs["pipe"]) != 1 { + return bird.Parsed{"error": "need a pipe as single query parameter"}, false + } + pipe, err := ValidateProtocolParam(qs["pipe"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + if len(qs["address"]) != 1 { + return bird.Parsed{"error": "need a address as single query parameter"}, false + } + address, err := ValidatePrefixParam(qs["address"][0]) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.PipeRoutesFilteredCount(useCache, pipe, table, address) +} + +func PeerRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + peer, err := ValidatePrefixParam(ps.ByName("peer")) + if err != nil { + return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false + } + + return bird.RoutesPeer(useCache, peer) } diff --git a/endpoints/status.go b/endpoints/status.go index fd18eb4..36670e2 100644 --- a/endpoints/status.go +++ b/endpoints/status.go @@ -7,6 +7,6 @@ import ( "github.com/julienschmidt/httprouter" ) -func Status(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.Status() +func Status(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.Status(useCache) } diff --git a/endpoints/symbols.go b/endpoints/symbols.go index df9245b..63fbdab 100644 --- a/endpoints/symbols.go +++ b/endpoints/symbols.go @@ -7,20 +7,20 @@ import ( "github.com/julienschmidt/httprouter" ) -func Symbols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - return bird.Symbols() +func Symbols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + return bird.Symbols(useCache) } -func SymbolTables(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - val, from_cache := bird.Symbols() +func SymbolTables(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + val, from_cache := bird.Symbols(useCache) if bird.IsSpecial(val) { return val, from_cache } return bird.Parsed{"symbols": val["symbols"].(bird.Parsed)["routing table"]}, from_cache } -func SymbolProtocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) { - val, from_cache := bird.Symbols() +func SymbolProtocols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) { + val, from_cache := bird.Symbols(useCache) if bird.IsSpecial(val) { return val, from_cache } diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf old mode 100644 new mode 100755 index e496fec..b15f4dd --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -5,6 +5,8 @@ [server] # Restrict access to certain IPs. Leave empty to allow from all. allow_from = [] +# Allow queries that bypass the cache +allow_uncached = false # Available modules: ## low-level modules (translation from birdc output to JSON objects) @@ -14,8 +16,12 @@ allow_from = [] # symbols_protocols # protocols # protocols_bgp +# protocols_short # routes_protocol +# routes_peer # routes_table +# routes_table_filtered +# routes_table_peer # routes_count_protocol # routes_count_table # routes_count_primary @@ -23,18 +29,25 @@ allow_from = [] # routes_prefixed # routes_noexport # route_net -## high-level modules (aggregated data from multiple birdc invocations) -# routes_dump +# routes_pipe_filtered_count +# routes_pipe_filtered # routes_peer modules_enabled = ["status", "protocols", "protocols_bgp", + "protocols_short", "routes_protocol", "routes_peer", + "routes_table", + "routes_table_filtered", + "routes_table_peer", + "routes_filtered", "routes_prefixed", - "routes_dump" + "routes_noexport", + "routes_pipe_filtered_count", + "routes_pipe_filtered" ] [status] @@ -69,12 +82,14 @@ ttl = 5 # time to live (in minutes) for caching of cli output # Remove fields e.g. interface filter_fields = [] -# Enable support for multitable configurations -per_peer_tables = true -peer_protocol_prefix = 'ID' -pipe_protocol_prefix = 'P' - [cache] -use_redis = false +use_redis = false # if not using redis cache, activate housekeeping to save memory! redis_server = "myredis:6379" redis_db = 0 + +# Housekeeping expires old cache entries (memory cache backend) and performs a GC/SCVG run if configured. +[housekeeping] +# Interval for the housekeeping routine in minutes +interval = 5 +# Try to release memory via a forced GC/SCVG run on every housekeeping run +force_release_memory = true diff --git a/housekeeping.go b/housekeeping.go new file mode 100644 index 0000000..3c38db3 --- /dev/null +++ b/housekeeping.go @@ -0,0 +1,42 @@ +package main + +import ( + "log" + "runtime/debug" + "time" + + "github.com/alice-lg/birdwatcher/bird" +) + +type HousekeepingConfig struct { + Interval int `toml:"interval"` + ForceReleaseMemory bool `toml:"force_release_memory"` +} + +// This is used to run regular housekeeping tasks, currently expiring old +// Cache entries to release memory +func Housekeeping(config HousekeepingConfig, expireCaches bool) { + for { + if config.Interval > 0 { + time.Sleep(time.Duration(config.Interval) * time.Minute) + } else { + time.Sleep(5 * time.Minute) + } + + log.Println("Housekeeping started") + + if (bird.ClientConf.CacheTtl > 0) && expireCaches { + // Expire the caches + log.Println("Expiring MemoryCache") + + count := bird.ExpireCache() + log.Println("Expired", count, "entries (MemoryCache)") + } + + if config.ForceReleaseMemory { + // Trigger a GC and SCVG run + log.Println("Freeing memory") + debug.FreeOSMemory() + } + } +} diff --git a/test/protocols_short.sample b/test/protocols_short.sample new file mode 100644 index 0000000..5fabb6a --- /dev/null +++ b/test/protocols_short.sample @@ -0,0 +1,31 @@ +BIRD 1.6.5 ready. +Access restricted +name proto table state since info +device1 Device master up 2019-02-15 +pp_0097_as3856 Pipe master up 2019-02-15 => t_0097_as3856 +pb_0097_as3856 BGP t_0097_as3856 up 2019-02-15 Established +pp_0175_as15169 Pipe master up 2019-02-15 => t_0175_as15169 +pb_0175_as15169 BGP t_0175_as15169 up 2019-02-15 Established +pp_0026_as20940 Pipe master up 2019-02-15 => t_0026_as20940 +pb_0026_as20940 BGP t_0026_as20940 up 2019-02-15 Established +direct1 Direct master down 2019-02-19 16:17:59 +kernel1 Kernel master down 2019-02-19 16:17:59 +M42_pch_radb Pipe master up 2019-02-19 16:17:59 => T42_pch_radb +C42_pch_radb Pipe Collector up 2019-02-19 16:17:59 => T42_pch_radb +R194_42 BGP T42_pch_radb up 2019-02-19 16:29:00 Established +M3856_pch_radb Pipe master up 2019-02-19 16:17:59 => T3856_pch_radb +C3856_pch_radb Pipe Collector up 2019-02-19 16:17:59 => T3856_pch_radb +R195_42 BGP T3856_pch_radb up 2019-02-19 16:18:33 Established +M112_112_ripe Pipe master up 2019-02-19 16:17:59 => T112_112_ripe +C112_112_ripe Pipe Collector up 2019-02-19 16:17:59 => T112_112_ripe +R195_77 BGP T112_112_ripe up 2019-02-19 16:24:31 Established +M286_kpn_ripe Pipe master up 2019-02-19 16:17:59 => T286_kpn_ripe +C286_kpn_ripe Pipe Collector up 2019-02-19 16:17:59 => T286_kpn_ripe +R192_22 BGP T286_kpn_ripe up 2019-02-19 16:26:10 Established +M553_belwue_ripe Pipe master up 2019-02-19 16:17:59 => T553_belwue_ripe +C553_belwue_ripe Pipe Collector up 2019-02-19 16:17:59 => T553_belwue_ripe +R192_175 BGP T553_belwue_ripe up 2019-02-19 16:18:34 Established +R194_106 BGP T553_belwue_ripe up 2019-02-19 16:18:09 Established +R194_205 BGP T52866_iveloz_radb start 2019-02-20 12:06:01 Idle BGP Error: Bad peer AS +R_janus1 BGP T6695_bh_20 start 2019-02-19 16:17:59 Idle + diff --git a/test/routes_bird1_ipv4.sample b/test/routes_bird1_ipv4.sample index e56318c..814ea82 100644 --- a/test/routes_bird1_ipv4.sample +++ b/test/routes_bird1_ipv4.sample @@ -8,7 +8,7 @@ BIRD 1.6.3 ready. BGP.community: (0,5464) (0,8339) (0,8741) (0,8823) (0,12387) (0,13101) (0,16097) (0,16316) (0,20546) (0,20686) (0,20723) (0,21083) (0,21385) (0,24940) (0,25504) (0,28876) (0,29545) (0,30058) (0,31103) (0,31400) (0,39090) (0,39392) (0,39912) (0,42473) (0,43957) (0,44453) (0,47297) (0,47692) (0,48200) (0,50629) (0,51191) (0,51839) (0,51852) (0,54113) (0,56719) (0,57957) (0,60517) (0,60574) (0,61303) (0,62297) (0,62336) (0,62359) (33891,33892) (33891,50673) (48793,48793) (50673,500) (65101,11077) (65102,11000) (65103,724) (65104,150) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 200.0.0.0/24 via 1.2.3.15 on eno7 [ID8497_AS1339 2017-06-21 08:17:31] * (100) [AS1339i] Type: BGP unicast univ BGP.origin: IGP @@ -35,4 +35,4 @@ BIRD 1.6.3 ready. BGP.local_pref: 100 BGP.community: (65011,3) (9033,3251) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) diff --git a/test/routes_bird2_ipv4.sample b/test/routes_bird2_ipv4.sample index a566130..4bd1f5c 100644 --- a/test/routes_bird2_ipv4.sample +++ b/test/routes_bird2_ipv4.sample @@ -9,7 +9,7 @@ BIRD 1.6.3 ready. BGP.community: (0,5464) (0,8339) (0,8741) (0,8823) (0,12387) (0,13101) (0,16097) (0,16316) (0,20546) (0,20686) (0,20723) (0,21083) (0,21385) (0,24940) (0,25504) (0,28876) (0,29545) (0,30058) (0,31103) (0,31400) (0,39090) (0,39392) (0,39912) (0,42473) (0,43957) (0,44453) (0,47297) (0,47692) (0,48200) (0,50629) (0,51191) (0,51839) (0,51852) (0,54113) (0,56719) (0,57957) (0,60517) (0,60574) (0,61303) (0,62297) (0,62336) (0,62359) (33891,33892) (33891,50673) (48793,48793) (50673,500) (65101,11077) (65102,11000) (65103,724) (65104,150) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 200.0.0.0/24 unicast [ID8497_AS1339 2017-06-21 08:17:31] * (100) [AS1339i] via 1.2.3.15 on eno7 Type: BGP univ @@ -39,4 +39,4 @@ BIRD 1.6.3 ready. BGP.local_pref: 100 BGP.community: (65011,3) (9033,3251) BGP.large_community: (9033, 65666, 12) (9033, 65666, 9) - BGP.ext_community: (rt, 48858, 50) + BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1)