From 37d5eeaa136c83e5d8a76f232c4bee1edf86c793 Mon Sep 17 00:00:00 2001 From: Daniel Czerwonk Date: Thu, 18 Jan 2018 18:34:08 +0100 Subject: [PATCH 1/7] fxed version check --- bird/bird.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bird/bird.go b/bird/bird.go index bb004b0..2ba5956 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "reflect" + "strconv" "strings" "sync" "time" @@ -326,7 +327,8 @@ func routeQueryForChannel(cmd string) string { return cmd } - if len(version) == 0 || int(version[0]) < 2 { + v, err := strconv.Atoi(string(version[0])) + if err != nil || v <= 2 { return cmd } From d6fc5998eeb9e68072538c824ad15e2657d279a7 Mon Sep 17 00:00:00 2001 From: Daniel Czerwonk Date: Sun, 28 Jan 2018 20:41:43 +0100 Subject: [PATCH 2/7] implemented workers to improve full table parsing time --- bird/parser.go | 109 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/bird/parser.go b/bird/parser.go index db0956e..4a80b27 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -6,13 +6,15 @@ import ( "regexp" "strconv" "strings" + "sync" ) +const workerPoolSize = 8 + var ( ParserConf ParserConfig regex struct { - lineSeperator *regexp.Regexp - status struct { + status struct { startLine *regexp.Regexp routerID *regexp.Regexp currentServer *regexp.Regexp @@ -177,15 +179,107 @@ func parseSymbols(reader io.Reader) Parsed { return Parsed{"symbols": res} } -func parseRoutes(reader io.Reader) Parsed { - res := Parsed{} - routes := []Parsed{} - route := Parsed{} +type blockJob struct { + lines []string + position int +} +type blockParsed struct { + items []Parsed + position int +} + +func parseRoutes(reader io.Reader) Parsed { + jobs := make(chan blockJob) + out := startRouteWorkers(jobs) + + res := startRouteConsumer(out) + defer close(res) + + pos := 0 + block := []string{} lines := newLineIterator(reader, true) + for lines.next() { line := lines.string() + if line[0] != 32 && line[0] != 9 && len(block) > 0 { + jobs <- blockJob{block, pos} + pos++ + block = []string{} + } + + block = append(block, line) + } + + if len(block) > 0 { + jobs <- blockJob{block, pos} + } + + close(jobs) + + return <-res +} + +func startRouteWorkers(jobs chan blockJob) chan blockParsed { + out := make(chan blockParsed) + + wg := &sync.WaitGroup{} + wg.Add(workerPoolSize) + go func() { + for i := 0; i < workerPoolSize; i++ { + go workerForRouteBlockParsing(jobs, out, wg) + } + wg.Wait() + close(out) + }() + + return out +} + +func startRouteConsumer(out <-chan blockParsed) chan Parsed { + res := make(chan Parsed) + + go func() { + byBlock := map[int][]Parsed{} + count := 0 + for r := range out { + count++ + byBlock[r.position] = r.items + } + res <- Parsed{"routes": sortedSliceForRouteBlocks(byBlock, count)} + }() + + return res +} + +func sortedSliceForRouteBlocks(byBlock map[int][]Parsed, numBlocks int) []Parsed { + res := []Parsed{} + + for i := 0; i < numBlocks; i++ { + routes, ok := byBlock[i] + if !ok { + continue + } + + res = append(res, routes...) + } + + return res +} + +func workerForRouteBlockParsing(jobs <-chan blockJob, out chan<- blockParsed, wg *sync.WaitGroup) { + for j := range jobs { + parseRouteLines(j.lines, j.position, out) + } + wg.Done() +} + +func parseRouteLines(lines []string, position int, ch chan<- blockParsed) { + route := Parsed{} + routes := []Parsed{} + + for _, line := range lines { if specialLine(line) { continue } @@ -233,8 +327,7 @@ func parseRoutes(reader io.Reader) Parsed { routes = append(routes, route) } - res["routes"] = routes - return res + ch <- blockParsed{routes, position} } func parseMainRouteDetail(groups []string, route Parsed) { From 5bffef1e07f0cceec2c116aa8cc1d5bcf579362b Mon Sep 17 00:00:00 2001 From: Daniel Czerwonk Date: Sun, 28 Jan 2018 20:52:53 +0100 Subject: [PATCH 3/7] added flag to set max workers --- bird/parser.go | 7 ++++--- birdwatcher.go | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bird/parser.go b/bird/parser.go index 4a80b27..0b58e8d 100644 --- a/bird/parser.go +++ b/bird/parser.go @@ -9,7 +9,8 @@ import ( "sync" ) -const workerPoolSize = 8 +// WorkerPoolSize is the number of go routines used to parse routing tables concurrently +var WorkerPoolSize = 8 var ( ParserConf ParserConfig @@ -225,9 +226,9 @@ func startRouteWorkers(jobs chan blockJob) chan blockParsed { out := make(chan blockParsed) wg := &sync.WaitGroup{} - wg.Add(workerPoolSize) + wg.Add(WorkerPoolSize) go func() { - for i := 0; i < workerPoolSize; i++ { + for i := 0; i < WorkerPoolSize; i++ { go workerForRouteBlockParsing(jobs, out, wg) } wg.Wait() diff --git a/birdwatcher.go b/birdwatcher.go index 71eb24b..d210d14 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -107,9 +107,12 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) { func main() { bird6 := flag.Bool("6", false, "Use bird6 instead of bird") + workerPoolSize := flag.Int("worker-pool-size", 8, "Number of go routines used to parse routing tables concurrently") configfile := flag.String("config", "./etc/ecix/birdwatcher.conf", "Configuration file location") flag.Parse() + bird.WorkerPoolSize = *workerPoolSize + endpoints.VERSION = VERSION bird.InstallRateLimitReset() // Load configurations From d5de4b8f6522fcd98447d25f630efa9e0dcdc22f Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Thu, 7 Jun 2018 11:51:43 +0200 Subject: [PATCH 4/7] Fix birdc query for RoutesFiltered --- bird/bird.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bird/bird.go b/bird/bird.go index 2ba5956..68c88ad 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -200,7 +200,7 @@ func RoutesProtoCount(protocol string) (Parsed, bool) { } func RoutesFiltered(protocol string) (Parsed, bool) { - cmd := routeQueryForChannel("route all filtered " + protocol) + cmd := routeQueryForChannel("route all filtered protocol " + protocol) return RunAndParse(cmd, parseRoutes) } From e4ec6591b9b57a4f2a2f3661098d9084337e65cd Mon Sep 17 00:00:00 2001 From: Johannes Moos Date: Wed, 11 Apr 2018 18:11:09 +0200 Subject: [PATCH 5/7] Increase default cache time to 10 minutes --- bird/bird.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bird/bird.go b/bird/bird.go index 68c88ad..285f9e8 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -49,7 +49,7 @@ func fromCache(key string) (Parsed, bool) { } func toCache(key string, val Parsed) { - val["ttl"] = time.Now().Add(5 * time.Minute) + val["ttl"] = time.Now().Add(10 * time.Minute) Cache.Lock() Cache.m[key] = val Cache.Unlock() From 46356aa9eb4eb778c49afbf26f52ac16a714dcc8 Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Thu, 12 Apr 2018 11:37:02 +0200 Subject: [PATCH 6/7] Make cache ttl for bird responses a config option For bird and bird6 you may add ttl to the respective config file sections. See example config in this commit. This allows to have individual ttls for the cache of bird cli output per daemon. --- bird/bird.go | 6 +++++- bird/config.go | 1 + birdwatcher.go | 1 + etc/ecix/birdwatcher.conf | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 285f9e8..2c923ac 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -49,7 +49,11 @@ func fromCache(key string) (Parsed, bool) { } func toCache(key string, val Parsed) { - val["ttl"] = time.Now().Add(10 * time.Minute) + var ttl int = 5 + if ClientConf.CacheTtl > 0 { + ttl = ClientConf.CacheTtl + } + val["ttl"] = time.Now().Add(time.Duration(ttl) * time.Minute) Cache.Lock() Cache.m[key] = val Cache.Unlock() diff --git a/bird/config.go b/bird/config.go index bfcdaed..37a94c0 100644 --- a/bird/config.go +++ b/bird/config.go @@ -13,6 +13,7 @@ type BirdConfig struct { Listen string ConfigFilename string `toml:"config"` BirdCmd string `toml:"birdc"` + CacheTtl int `toml:"ttl"` } type ParserConfig struct { diff --git a/birdwatcher.go b/birdwatcher.go index d210d14..c41a9fd 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -89,6 +89,7 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) { log.Println("Starting Birdwatcher") log.Println(" Using:", birdConf.BirdCmd) log.Println(" Listen:", birdConf.Listen) + log.Println(" Cache TTL:", birdConf.CacheTtl) // Endpoint Info if len(conf.Server.AllowFrom) == 0 { diff --git a/etc/ecix/birdwatcher.conf b/etc/ecix/birdwatcher.conf index ad5aab7..e48aec3 100644 --- a/etc/ecix/birdwatcher.conf +++ b/etc/ecix/birdwatcher.conf @@ -45,12 +45,13 @@ requests_per_minute = 10 listen = "0.0.0.0:29188" config = "/etc/bird.conf" birdc = "/sbin/birdc" - +ttl = 5 # time to live (in minutes) for caching of cli output [bird6] listen = "0.0.0.0:29189" config = "/etc/bird6.conf" birdc = "/sbin/birdc6" +ttl = 5 # time to live (in minutes) for caching of cli output [parser] # Remove fields e.g. interface @@ -60,4 +61,3 @@ filter_fields = [] per_peer_tables = true peer_protocol_prefix = 'ID' pipe_protocol_prefix = 'P' - From a196f8ada04660328af64cc0b793034540f54f02 Mon Sep 17 00:00:00 2001 From: Johannes Moos Date: Thu, 12 Apr 2018 11:24:22 +0200 Subject: [PATCH 7/7] Change default config location to /etc/birdwatcher --- birdwatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdwatcher.go b/birdwatcher.go index c41a9fd..1f1ebe2 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -109,7 +109,7 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) { func main() { bird6 := flag.Bool("6", false, "Use bird6 instead of bird") workerPoolSize := flag.Int("worker-pool-size", 8, "Number of go routines used to parse routing tables concurrently") - configfile := flag.String("config", "./etc/ecix/birdwatcher.conf", "Configuration file location") + configfile := flag.String("config", "etc/birdwatcher/birdwatcher.conf", "Configuration file location") flag.Parse() bird.WorkerPoolSize = *workerPoolSize