From c4dfeb253d3e7a95ccd2ade04cb73b6700dc053a Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Fri, 18 Jan 2019 18:10:06 +0100
Subject: [PATCH 01/20] API redesign - make birdwatcher more generic

Removed all high level functionality e.g. endpoints with multiple
invocations of birdc.
Add new endpoints which are required to duplicate the removed
functionality within Alice-LG.
---
 bird/bird.go                     | 112 ++++---------------------------
 bird/config.go                   |   5 +-
 birdwatcher.go                   |  22 ++++--
 endpoints/routes.go              |  35 ++++++----
 etc/birdwatcher/birdwatcher.conf |  15 ++++-
 5 files changed, 64 insertions(+), 125 deletions(-)
 mode change 100644 => 100755 etc/birdwatcher/birdwatcher.conf

diff --git a/bird/bird.go b/bird/bird.go
index 29f1de8..ca7d005 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -217,7 +217,6 @@ func Protocols() (Parsed, bool) {
 
 		for key, _ := range (*p)["protocols"].(Parsed) {
 			parsed := (*p)["protocols"].(Parsed)[key].(Parsed)
-
 			protocol := parsed["protocol"].(string)
 
 			birdProtocol := parsed["bird_protocol"].(string)
@@ -269,6 +268,16 @@ func RoutesProto(protocol string) (Parsed, bool) {
 	return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil)
 }
 
+func RoutesPeer(peer string) (Parsed, bool) {
+	cmd := routeQueryForChannel("route all where from=" + peer)
+	return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil)
+}
+
+func RoutesTableAndPeer(table string, peer string) (Parsed, bool) {
+	cmd := routeQueryForChannel("route table " + table + " all where from=" + peer)
+	return RunAndParse(GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil)
+}
+
 func RoutesProtoCount(protocol string) (Parsed, bool) {
 	cmd := routeQueryForChannel("route protocol "+protocol) + " count"
 	return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil)
@@ -300,24 +309,6 @@ func RoutesExport(protocol string) (Parsed, bool) {
 }
 
 func RoutesNoExport(protocol string) (Parsed, bool) {
-	// In case we have a multi table setup, we have to query
-	// the pipe protocol.
-	if ParserConf.PerPeerTables &&
-		strings.HasPrefix(protocol, ParserConf.PeerProtocolPrefix) {
-
-		protocolsRes, from_cache := ProtocolsBgp()
-		if IsSpecial(protocolsRes) {
-			return protocolsRes, from_cache
-		}
-		if _, ok := protocolsRes["protocols"].(Parsed)[protocol]; !ok {
-			return NilParse, false
-		}
-
-		// Replace prefix
-		protocol = ParserConf.PipeProtocolPrefix +
-			protocol[len(ParserConf.PeerProtocolPrefix):]
-	}
-
 	cmd := routeQueryForChannel("route all noexport " + protocol)
 	return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil)
 }
@@ -331,6 +322,10 @@ func RoutesTable(table string) (Parsed, bool) {
 	return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil)
 }
 
+func RoutesTableFiltered(table string) (Parsed, bool) {
+	return RunAndParse(GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil)
+}
+
 func RoutesTableCount(table string) (Parsed, bool) {
 	return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil)
 }
@@ -343,85 +338,6 @@ func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) {
 	return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil)
 }
 
-func RoutesPeer(peer string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route export " + peer)
-	return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil)
-}
-
-func RoutesDump() (Parsed, bool) {
-	// TODO insert hook to update the cache with the route count information
-	if ParserConf.PerPeerTables {
-		return RoutesDumpPerPeerTable()
-	}
-
-	return RoutesDumpSingleTable()
-}
-
-func RoutesDumpSingleTable() (Parsed, bool) {
-	importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil)
-	if IsSpecial(importedRes) {
-		return importedRes, cached
-	}
-	filteredRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "filtered"), routeQueryForChannel("route all filtered"), parseRoutes, nil)
-	if IsSpecial(filteredRes) {
-		return filteredRes, cached
-	}
-
-	imported := importedRes["routes"]
-	filtered := filteredRes["routes"]
-
-	result := Parsed{
-		"imported": imported,
-		"filtered": filtered,
-	}
-
-	return result, cached
-}
-
-func RoutesDumpPerPeerTable() (Parsed, bool) {
-	importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpPerPeerTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil)
-	if IsSpecial(importedRes) {
-		return importedRes, cached
-	}
-	imported := importedRes["routes"]
-	filtered := []Parsed{}
-
-	// Get protocols with filtered routes
-	protocolsRes, cached := ProtocolsBgp()
-	if IsSpecial(protocolsRes) {
-		return protocolsRes, cached
-	}
-	protocols := protocolsRes["protocols"].(Parsed)
-
-	for protocol, details := range protocols {
-		details := details.(Parsed)
-
-		counters, ok := details["routes"].(Parsed)
-		if !ok {
-			continue
-		}
-		filterCount := counters["filtered"]
-		if filterCount == 0 {
-			continue // nothing to do here.
-		}
-		// Lookup filtered routes
-		pfilteredRes, _ := RoutesFiltered(protocol)
-		pfiltered, ok := pfilteredRes["routes"].([]Parsed)
-		if !ok {
-			continue // something went wrong...
-		}
-
-		filtered = append(filtered, pfiltered...)
-	}
-
-	result := Parsed{
-		"imported": imported,
-		"filtered": filtered,
-	}
-
-	return result, cached
-}
-
 func routeQueryForChannel(cmd string) string {
 	status, _ := Status()
 	if IsSpecial(status) {
diff --git a/bird/config.go b/bird/config.go
index 37a94c0..20c81ff 100644
--- a/bird/config.go
+++ b/bird/config.go
@@ -17,10 +17,7 @@ type BirdConfig struct {
 }
 
 type ParserConfig struct {
-	FilterFields       []string `toml:"filter_fields"`
-	PerPeerTables      bool     `toml:"per_peer_tables"`
-	PeerProtocolPrefix string   `toml:"peer_protocol_prefix"`
-	PipeProtocolPrefix string   `toml:"pipe_protocol_prefix"`
+	FilterFields []string `toml:"filter_fields"`
 }
 
 type RateLimitConfig struct {
diff --git a/birdwatcher.go b/birdwatcher.go
index 9082660..53f152b 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -16,7 +16,7 @@ import (
 )
 
 //go:generate versionize
-var VERSION = "1.11.0"
+var VERSION = "1.13.0"
 
 func isModuleEnabled(module string, modulesEnabled []string) bool {
 	for _, enabled := range modulesEnabled {
@@ -54,9 +54,18 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router {
 	if isModuleEnabled("routes_protocol", whitelist) {
 		r.GET("/routes/protocol/:protocol", endpoints.Endpoint(endpoints.ProtoRoutes))
 	}
+	if isModuleEnabled("routes_peer", whitelist) {
+		r.GET("/routes/peer/:peer", endpoints.Endpoint(endpoints.PeerRoutes))
+	}
 	if isModuleEnabled("routes_table", whitelist) {
 		r.GET("/routes/table/:table", endpoints.Endpoint(endpoints.TableRoutes))
 	}
+	if isModuleEnabled("routes_table_filtered", whitelist) {
+		r.GET("/routes/table/:table/filtered", endpoints.Endpoint(endpoints.TableRoutesFiltered))
+	}
+	if isModuleEnabled("routes_table_peer", whitelist) {
+		r.GET("/routes/table/:table/peer/:peer", endpoints.Endpoint(endpoints.TableAndPeerRoutes))
+	}
 	if isModuleEnabled("routes_count_protocol", whitelist) {
 		r.GET("/routes/count/protocol/:protocol", endpoints.Endpoint(endpoints.ProtoCount))
 	}
@@ -79,12 +88,13 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router {
 		r.GET("/route/net/:net", endpoints.Endpoint(endpoints.RouteNet))
 		r.GET("/route/net/:net/table/:table", endpoints.Endpoint(endpoints.RouteNetTable))
 	}
-	if isModuleEnabled("routes_peer", whitelist) {
-		r.GET("/routes/peer", endpoints.Endpoint(endpoints.RoutesPeer))
+	if isModuleEnabled("routes_pipe_filtered_count", whitelist) {
+		r.GET("/routes/pipe/filtered/count", endpoints.Endpoint(endpoints.PipeRoutesFilteredCount))
 	}
-	if isModuleEnabled("routes_dump", whitelist) {
-		r.GET("/routes/dump", endpoints.Endpoint(endpoints.RoutesDump))
+	if isModuleEnabled("routes_pipe_filtered", whitelist) {
+		r.GET("/routes/pipe/filtered", endpoints.Endpoint(endpoints.PipeRoutesFiltered))
 	}
+
 	return r
 }
 
@@ -108,8 +118,6 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) {
 	for _, m := range conf.Server.ModulesEnabled {
 		log.Println("       -", m)
 	}
-
-	log.Println("   Per Peer Tables:", conf.Parser.PerPeerTables)
 }
 
 // MyLogger is our own log.Logger wrapper so we can customize it
diff --git a/endpoints/routes.go b/endpoints/routes.go
index 1dee4b3..f47f4b3 100644
--- a/endpoints/routes.go
+++ b/endpoints/routes.go
@@ -50,6 +50,14 @@ func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	return bird.RoutesTable(ps.ByName("table"))
 }
 
+func TableRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+	return bird.RoutesTableFiltered(ps.ByName("table"))
+}
+
+func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+	return bird.RoutesTableAndPeer(ps.ByName("table"), ps.ByName("peer"))
+}
+
 func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	protocol, err := ValidateProtocolParam(ps.ByName("protocol"))
 	if err != nil {
@@ -78,20 +86,21 @@ func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	return bird.RoutesLookupTable(ps.ByName("net"), ps.ByName("table"))
 }
 
-func RoutesPeer(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	qs := r.URL.Query()
-	peerl := qs["peer"]
-	if len(peerl) != 1 {
-		return bird.Parsed{"error": "need a peer as single query parameter"}, false
-	}
-
-	peer, err := ValidateProtocolParam(peerl[0])
-	if err != nil {
-		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
-	}
-	return bird.RoutesPeer(peer)
+	table := qs["table"][0]
+	pipe := qs["pipe"][0]
+	return bird.PipeRoutesFiltered(pipe, table)
 }
 
-func RoutesDump(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesDump()
+func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+	qs := r.URL.Query()
+	table := qs["table"][0]
+	pipe := qs["pipe"][0]
+	address := qs["address"][0]
+	return bird.PipeRoutesFilteredCount(pipe, table, address)
+}
+
+func PeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+	return bird.RoutesPeer(ps.ByName("peer"))
 }
diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf
old mode 100644
new mode 100755
index a1cc871..863b115
--- a/etc/birdwatcher/birdwatcher.conf
+++ b/etc/birdwatcher/birdwatcher.conf
@@ -15,7 +15,10 @@ allow_from = []
 #   protocols
 #   protocols_bgp
 #   routes_protocol
+#   routes_peer
 #   routes_table
+#   routes_table_filtered
+#   routes_table_peer
 #   routes_count_protocol
 #   routes_count_table
 #   routes_count_primary
@@ -23,8 +26,8 @@ allow_from = []
 #   routes_prefixed
 #   routes_noexport
 #   route_net
-## high-level modules (aggregated data from multiple birdc invocations)
-#   routes_dump
+#   routes_pipe_filtered_count
+#   routes_pipe_filtered
 #   routes_peer
 
 
@@ -33,8 +36,14 @@ modules_enabled = ["status",
                    "protocols_bgp",
                    "routes_protocol",
                    "routes_peer",
+                   "routes_table",
+                   "routes_table_filtered",
+                   "routes_table_peer",
+                   "routes_filtered",
                    "routes_prefixed",
-                   "routes_dump"
+                   "routes_noexport",
+                   "routes_pipe_filtered_count",
+                   "routes_pipe_filtered"
                   ]
 
 [status]

From 1d3118b86439260dc58d8909a9e5c8c5c6f9f6ac Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Mon, 4 Feb 2019 15:48:24 +0100
Subject: [PATCH 02/20] Added missing default value (imported).

---
 bird/parser.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bird/parser.go b/bird/parser.go
index 717922a..92d3f6a 100644
--- a/bird/parser.go
+++ b/bird/parser.go
@@ -542,6 +542,7 @@ func parseProtocol(lines string) Parsed {
 		routes := Parsed{}
 		routes["accepted"] = int64(0)
 		routes["filtered"] = int64(0)
+		routes["imported"] = int64(0)
 		routes["exported"] = int64(0)
 		routes["preferred"] = int64(0)
 

From e5248b21c705aacb847be4b3e799f139f33a160e Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Tue, 5 Feb 2019 17:37:53 +0100
Subject: [PATCH 03/20] Improved parameter validation.

---
 endpoints/routes.go | 111 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 99 insertions(+), 12 deletions(-)

diff --git a/endpoints/routes.go b/endpoints/routes.go
index f47f4b3..394ed69 100644
--- a/endpoints/routes.go
+++ b/endpoints/routes.go
@@ -13,6 +13,7 @@ func ProtoRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
+
 	return bird.RoutesProto(protocol)
 }
 
@@ -21,6 +22,7 @@ func RoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
+
 	return bird.RoutesFiltered(protocol)
 }
 
@@ -29,6 +31,7 @@ func RoutesNoExport(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
+
 	return bird.RoutesNoExport(protocol)
 }
 
@@ -43,19 +46,40 @@ func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
+
 	return bird.RoutesPrefixed(prefix)
 }
 
 func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesTable(ps.ByName("table"))
+	table, err := ValidateProtocolParam(ps.ByName("table"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesTable(table)
 }
 
 func TableRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesTableFiltered(ps.ByName("table"))
+	table, err := ValidateProtocolParam(ps.ByName("table"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesTableFiltered(table)
 }
 
 func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesTableAndPeer(ps.ByName("table"), ps.ByName("peer"))
+	table, err := ValidateProtocolParam(ps.ByName("table"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	peer, err := ValidatePrefixParam(ps.ByName("peer"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesTableAndPeer(table, peer)
 }
 
 func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
@@ -63,6 +87,7 @@ func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
+
 	return bird.RoutesProtoCount(protocol)
 }
 
@@ -75,32 +100,94 @@ func ProtoPrimaryCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool
 }
 
 func TableCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesTableCount(ps.ByName("table"))
+	table, err := ValidateProtocolParam(ps.ByName("table"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesTableCount(table)
 }
 
 func RouteNet(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesLookupTable(ps.ByName("net"), "master")
+	net, err := ValidatePrefixParam(ps.ByName("net"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesLookupTable(net, "master")
 }
 
 func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesLookupTable(ps.ByName("net"), ps.ByName("table"))
+	net, err := ValidatePrefixParam(ps.ByName("net"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	table, err := ValidateProtocolParam(ps.ByName("table"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesLookupTable(net, table)
 }
 
 func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	qs := r.URL.Query()
-	table := qs["table"][0]
-	pipe := qs["pipe"][0]
+
+	if len(qs["table"]) != 1 {
+		return bird.Parsed{"error": "need a table as single query parameter"}, false
+	}
+	table, err := ValidateProtocolParam(qs["table"][0])
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	if len(qs["pipe"]) != 1 {
+		return bird.Parsed{"error": "need a pipe as single query parameter"}, false
+	}
+	pipe, err := ValidateProtocolParam(qs["pipe"][0])
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
 	return bird.PipeRoutesFiltered(pipe, table)
 }
 
 func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	qs := r.URL.Query()
-	table := qs["table"][0]
-	pipe := qs["pipe"][0]
-	address := qs["address"][0]
+
+	if len(qs["table"]) != 1 {
+		return bird.Parsed{"error": "need a table as single query parameter"}, false
+	}
+	table, err := ValidateProtocolParam(qs["table"][0])
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	if len(qs["pipe"]) != 1 {
+		return bird.Parsed{"error": "need a pipe as single query parameter"}, false
+	}
+	pipe, err := ValidateProtocolParam(qs["pipe"][0])
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	if len(qs["address"]) != 1 {
+		return bird.Parsed{"error": "need a address as single query parameter"}, false
+	}
+	address, err := ValidatePrefixParam(qs["address"][0])
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
 	return bird.PipeRoutesFilteredCount(pipe, table, address)
 }
 
 func PeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.RoutesPeer(ps.ByName("peer"))
+	peer, err := ValidatePrefixParam(ps.ByName("peer"))
+	if err != nil {
+		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
+	}
+
+	return bird.RoutesPeer(peer)
 }

From 14a4ebeaaa5fb44d22bcde042b1f489185852a06 Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Tue, 5 Feb 2019 17:40:59 +0100
Subject: [PATCH 04/20] Using json.Encoder to write the output JSON instead of
 using a buffer.

---
 endpoints/endpoint.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/endpoints/endpoint.go b/endpoints/endpoint.go
index 709e995..8138f83 100644
--- a/endpoints/endpoint.go
+++ b/endpoints/endpoint.go
@@ -73,8 +73,6 @@ func Endpoint(wrapped endpoint) httprouter.Handle {
 			res[k] = v
 		}
 
-		js, _ := json.Marshal(res)
-
 		w.Header().Set("Content-Type", "application/json")
 
 		// Check if compression is supported
@@ -83,9 +81,11 @@ func Endpoint(wrapped endpoint) httprouter.Handle {
 			w.Header().Set("Content-Encoding", "gzip")
 			gz := gzip.NewWriter(w)
 			defer gz.Close()
-			gz.Write(js)
+			json := json.NewEncoder(gz)
+			json.Encode(res)
 		} else {
-			w.Write(js) // Fall back to uncompressed response
+			json := json.NewEncoder(w)
+			json.Encode(res) // Fall back to uncompressed response
 		}
 	}
 }

From 119b9f6360b217038456cd388dea79cf1d03ce7a Mon Sep 17 00:00:00 2001
From: Benedikt Rudolph <benedikt.rudolph@de-cix.net>
Date: Thu, 28 Feb 2019 11:32:40 +0100
Subject: [PATCH 05/20] Add feature cache backends

Add support for various cache backends in anticipation of the
merge with master that has an additional redis backend.
The current memory based cache backend is refactored to implement
the new interface.
---
 bird/bird.go         | 108 +++++++++++++++++++++++--------------------
 bird/memory_cache.go |  61 ++++++++++++++++++++++++
 birdwatcher.go       |   8 ++++
 3 files changed, 127 insertions(+), 50 deletions(-)
 create mode 100644 bird/memory_cache.go

diff --git a/bird/bird.go b/bird/bird.go
index 181fbc3..0d1d32e 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -3,6 +3,7 @@ package bird
 import (
 	"bytes"
 	"io"
+	"log"
 	"reflect"
 	"strconv"
 	"strings"
@@ -12,31 +13,71 @@ import (
 	"os/exec"
 )
 
+type Cache interface {
+	Set(key string, val Parsed, ttl int) error
+	Get(key string) (Parsed, error)
+}
+
 var ClientConf BirdConfig
 var StatusConf StatusConfig
 var IPVersion = "4"
+var cache Cache // stores parsed birdc output
 var RateLimitConf struct {
 	sync.RWMutex
 	Conf RateLimitConfig
 }
+var RunQueue sync.Map // queue birdc commands before execution
 
-type Cache struct {
-	sync.RWMutex
-	m map[string]Parsed
-}
-
-var ParsedCache = Cache{m: make(map[string]Parsed)}
-var MetaCache = Cache{m: make(map[string]Parsed)}
-
-var NilParse Parsed = (Parsed)(nil)
+var NilParse Parsed = (Parsed)(nil) // special Parsed values
 var BirdError Parsed = Parsed{"error": "bird unreachable"}
 
-var RunQueue sync.Map
-
-func IsSpecial(ret Parsed) bool {
+func IsSpecial(ret Parsed) bool { // test for special Parsed values
 	return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError)
 }
 
+// intitialize the Cache once during setup with either a MemoryCache or
+// RedisCache implementation.
+// TODO implement singleton pattern
+func InitializeCache(c Cache) {
+	cache = c
+}
+
+/* Convenience method to make new entries in the cache.
+ * Abstracts over the specific caching implementation and the ability to set
+ * individual TTL values for entries. Always use the default TTL value from the
+ * config.
+ */
+func toCache(key string, val Parsed) bool {
+	var ttl int
+	if ClientConf.CacheTtl > 0 {
+		ttl = ClientConf.CacheTtl
+	} else {
+		ttl = 5 // five minutes
+	}
+
+	if err := cache.Set(key, val, ttl); err == nil {
+		return true
+	} else {
+		log.Println(err)
+		return false
+	}
+}
+
+/* Convenience method to retrieve entries from the cache.
+ * Abstracts over the specific caching implementations.
+ */
+func fromCache(key string) (Parsed, bool) {
+	val, err := cache.Get(key)
+	if err != nil {
+		log.Println(err)
+		return val, false
+	} else if IsSpecial(val) { // cache may return NilParse e.g. if ttl is expired
+		return val, false
+	} else {
+		return val, true
+	}
+}
+
 // Determines the key in the cache, where the result of specific functions are stored.
 // Eliminates the need to know what command was executed by that function.
 func GetCacheKey(fname string, fargs ...interface{}) string {
@@ -52,39 +93,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string {
 	return key
 }
 
-func (c *Cache) Store(key string, val Parsed) {
-	var ttl int = 5
-	if ClientConf.CacheTtl > 0 {
-		ttl = ClientConf.CacheTtl
-	}
-	cachedAt := time.Now().UTC()
-	cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute)
-
-	c.Lock()
-	// This is not a really ... clean way of doing this.
-	val["ttl"] = cacheTtl
-	val["cached_at"] = cachedAt
-
-	c.m[key] = val
-	c.Unlock()
-}
-
-func (c *Cache) Get(key string) (Parsed, bool) {
-	c.RLock()
-	val, ok := c.m[key]
-	c.RUnlock()
-	if !ok {
-		return NilParse, false
-	}
-
-	ttl, correct := val["ttl"].(time.Time)
-	if !correct || ttl.Before(time.Now()) {
-		return NilParse, false
-	}
-
-	return val, ok
-}
-
 func Run(args string) (io.Reader, error) {
 	args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument
 	argsList := strings.Split(args, " ")
@@ -132,7 +140,7 @@ func checkRateLimit() bool {
 }
 
 func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) {
-	if val, ok := ParsedCache.Get(cmd); ok {
+	if val, ok := fromCache(cmd); ok {
 		return val, true
 	}
 
@@ -141,7 +149,7 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa
 	if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded {
 		(*queueGroup.(*sync.WaitGroup)).Wait()
 
-		if val, ok := ParsedCache.Get(cmd); ok {
+		if val, ok := fromCache(cmd); ok {
 			return val, true
 		} else {
 			// TODO BirdError should also be signaled somehow
@@ -169,7 +177,7 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa
 		updateCache(&parsed)
 	}
 
-	ParsedCache.Store(cmd, parsed)
+	toCache(cmd, parsed)
 
 	wg.Done()
 
@@ -228,7 +236,7 @@ func Protocols() (Parsed, bool) {
 			metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed
 		}
 
-		MetaCache.Store(GetCacheKey("Protocols"), metaProtocol)
+		toCache(GetCacheKey("Protocols"), metaProtocol)
 	}
 
 	res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache)
@@ -241,7 +249,7 @@ func ProtocolsBgp() (Parsed, bool) {
 		return protocols, from_cache
 	}
 
-	protocolsMeta, _ := MetaCache.Get(GetCacheKey("Protocols"))
+	protocolsMeta, _ := fromCache(GetCacheKey("Protocols")) //TODO geht das einfach so?
 	metaProtocol := protocolsMeta["protocols"].(Parsed)
 
 	bgpProtocols := Parsed{}
diff --git a/bird/memory_cache.go b/bird/memory_cache.go
new file mode 100644
index 0000000..8213336
--- /dev/null
+++ b/bird/memory_cache.go
@@ -0,0 +1,61 @@
+package bird
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+// Implementation of the MemoryCache backend.
+
+type MemoryCache struct {
+	sync.RWMutex
+	m map[string]Parsed
+}
+
+func NewMemoryCache() (*MemoryCache, error) {
+	var cache *MemoryCache
+	cache = &MemoryCache{m: make(map[string]Parsed)}
+	return cache, nil
+}
+
+func (c *MemoryCache) Get(key string) (Parsed, error) {
+	c.RLock()
+	val, ok := c.m[key]
+	c.RUnlock()
+	if !ok {
+		return NilParse, errors.New("Could not retrive key" + key + "from MemoryCache.")
+	}
+
+	ttl, correct := val["ttl"].(time.Time)
+	if !correct {
+		return NilParse, errors.New("Invalid TTL value for key" + key)
+	}
+
+	if ttl.Before(time.Now()) {
+		return NilParse, nil // TTL expired
+	} else {
+		return val, nil // cache hit
+	}
+}
+
+func (c *MemoryCache) Set(key string, val Parsed, ttl int) error {
+	switch {
+	case ttl == 0:
+		return nil // do not cache
+	case ttl > 0:
+		cachedAt := time.Now().UTC()
+		cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute)
+
+		c.Lock()
+		// This is not a really ... clean way of doing this.
+		val["ttl"] = cacheTtl
+		val["cached_at"] = cachedAt
+
+		c.m[key] = val
+		c.Unlock()
+		return nil
+	default: // ttl negative - invalid
+		return errors.New("Negative TTL value for key" + key)
+	}
+}
diff --git a/birdwatcher.go b/birdwatcher.go
index 9082660..13d1072 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -163,6 +163,14 @@ func main() {
 	bird.RateLimitConf.Conf = conf.Ratelimit
 	bird.RateLimitConf.Unlock()
 	bird.ParserConf = conf.Parser
+
+	var cache bird.Cache
+	cache, err = bird.NewMemoryCache() // initialze the MemoryCache
+	if err != nil {
+		log.Fatal("Could not initialize MemoryCache:", err)
+	}
+	bird.InitializeCache(cache)
+
 	endpoints.Conf = conf.Server
 
 	// Make server

From 7e02cb23da7e7b1a1a53e488ebcc43b23d8abe6c Mon Sep 17 00:00:00 2001
From: Benedikt Rudolph <benedikt.rudolph@de-cix.net>
Date: Thu, 28 Feb 2019 13:02:11 +0100
Subject: [PATCH 06/20] Add test for memory cache backend

Improve error handling in case value can not be retrieved.
Either return the value and nil, or a value and an error.
---
 bird/bird.go              | 11 +++---
 bird/memory_cache.go      |  8 ++---
 bird/memory_cache_test.go | 74 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 10 deletions(-)
 create mode 100644 bird/memory_cache_test.go

diff --git a/bird/bird.go b/bird/bird.go
index 0d1d32e..953db74 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -68,14 +68,13 @@ func toCache(key string, val Parsed) bool {
  */
 func fromCache(key string) (Parsed, bool) {
 	val, err := cache.Get(key)
-	if err != nil {
-		log.Println(err)
-		return val, false
-	} else if IsSpecial(val) { // cache may return NilParse e.g. if ttl is expired
-		return val, false
-	} else {
+	if err == nil {
 		return val, true
+	} else {
+		return val, false
 	}
+	//DEBUG log.Println(err)
+
 }
 
 // Determines the key in the cache, where the result of specific functions are stored.
diff --git a/bird/memory_cache.go b/bird/memory_cache.go
index 8213336..89384b4 100644
--- a/bird/memory_cache.go
+++ b/bird/memory_cache.go
@@ -23,17 +23,17 @@ func (c *MemoryCache) Get(key string) (Parsed, error) {
 	c.RLock()
 	val, ok := c.m[key]
 	c.RUnlock()
-	if !ok {
-		return NilParse, errors.New("Could not retrive key" + key + "from MemoryCache.")
+	if !ok { // cache miss
+		return NilParse, errors.New("Failed to retrive key '" + key + "' from MemoryCache.")
 	}
 
 	ttl, correct := val["ttl"].(time.Time)
 	if !correct {
-		return NilParse, errors.New("Invalid TTL value for key" + key)
+		return NilParse, errors.New("Invalid TTL value for key '" + key + "'")
 	}
 
 	if ttl.Before(time.Now()) {
-		return NilParse, nil // TTL expired
+		return val, errors.New("TTL expired for key '" + key + "'") // TTL expired
 	} else {
 		return val, nil // cache hit
 	}
diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go
new file mode 100644
index 0000000..b4a6a78
--- /dev/null
+++ b/bird/memory_cache_test.go
@@ -0,0 +1,74 @@
+package bird
+
+import (
+	"testing"
+)
+
+func Test_MemoryCacheAccess(t *testing.T) {
+
+	cache, err := NewMemoryCache()
+
+	parsed := Parsed{
+		"foo": 23,
+		"bar": 42,
+		"baz": true,
+	}
+
+	t.Log("Setting memory cache...")
+	err = cache.Set("testkey", parsed, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log("Fetching from memory cache...")
+	parsed, err = cache.Get("testkey")
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log(parsed)
+}
+
+func Test_MemoryCacheAccessKeyMissing(t *testing.T) {
+
+	cache, err := NewMemoryCache()
+
+	parsed, err := cache.Get("test_missing_key")
+	if !IsSpecial(parsed) {
+		t.Error(err)
+	}
+	t.Log("Cache error:", err)
+	t.Log(parsed)
+}
+
+func Test_MemoryCacheRoutes(t *testing.T) {
+	f, err := openFile("routes_bird1_ipv4.sample")
+	if err != nil {
+		t.Error(err)
+	}
+	defer f.Close()
+
+	parsed := parseRoutes(f)
+	_, ok := parsed["routes"].([]Parsed)
+	if !ok {
+		t.Fatal("Error getting routes")
+	}
+
+	cache, err := NewMemoryCache()
+
+	err = cache.Set("routes_protocol_test", parsed, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	parsed, err = cache.Get("routes_protocol_test")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	routes, ok := parsed["routes"].([]Parsed)
+	if !ok {
+		t.Error("Error getting routes")
+	}
+	t.Log("Retrieved routes:", len(routes))
+}

From 14c8875ae0b92f977363766311d11716ca958daf Mon Sep 17 00:00:00 2001
From: Benedikt Rudolph <benedikt.rudolph@de-cix.net>
Date: Thu, 28 Feb 2019 13:25:47 +0100
Subject: [PATCH 07/20] Avoid overwriting existing cache entry

also fix RoutesNoExport.
---
 bird/bird.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bird/bird.go b/bird/bird.go
index 953db74..3fe2478 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -235,10 +235,10 @@ func Protocols() (Parsed, bool) {
 			metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed
 		}
 
-		toCache(GetCacheKey("Protocols"), metaProtocol)
+		toCache(GetCacheKey("metaProtocol"), metaProtocol)
 	}
 
-	res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache)
+	res, from_cache := RunAndParse(GetCacheKey("metaProtocol"), "protocols all", parseProtocols, createMetaCache)
 	return res, from_cache
 }
 
@@ -248,7 +248,7 @@ func ProtocolsBgp() (Parsed, bool) {
 		return protocols, from_cache
 	}
 
-	protocolsMeta, _ := fromCache(GetCacheKey("Protocols")) //TODO geht das einfach so?
+	protocolsMeta, _ := fromCache(GetCacheKey("metaProtocol"))
 	metaProtocol := protocolsMeta["protocols"].(Parsed)
 
 	bgpProtocols := Parsed{}

From 56c378109fe9d47c02ede332b36d710be8f13e04 Mon Sep 17 00:00:00 2001
From: Benedikt Rudolph <benedikt.rudolph@de-cix.net>
Date: Thu, 28 Feb 2019 16:14:13 +0100
Subject: [PATCH 08/20] Integrate redis backend with the Cache interface

---
 bird/bird.go             |  4 ++++
 bird/redis_cache.go      | 45 ++++++++++++++++++++++++++++++----------
 bird/redis_cache_test.go |  4 ++--
 birdwatcher.go           |  2 +-
 4 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/bird/bird.go b/bird/bird.go
index 342678f..02403b9 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -67,6 +67,10 @@ func toCache(key string, val Parsed) bool {
 
 /* Convenience method to retrieve entries from the cache.
  * Abstracts over the specific caching implementations.
+ * If err returned by cache.Get(key) is set, the value from the cache is not
+ * used. There is either a fault e.g. missing entry or the ttl is expired.
+ * Handling of specific error conditions e.g. ttl expired but entry present is
+ * possible but currently not implemented.
  */
 func fromCache(key string) (Parsed, bool) {
 	val, err := cache.Get(key)
diff --git a/bird/redis_cache.go b/bird/redis_cache.go
index 6335b8b..f20f7e8 100644
--- a/bird/redis_cache.go
+++ b/bird/redis_cache.go
@@ -2,12 +2,15 @@ package bird
 
 import (
 	"encoding/json"
-	"github.com/go-redis/redis"
+	"errors"
 	"time"
+
+	"github.com/go-redis/redis"
 )
 
 type RedisCache struct {
-	client *redis.Client
+	client    *redis.Client
+	keyPrefix string
 }
 
 func NewRedisCache(config CacheConfig) (*RedisCache, error) {
@@ -31,6 +34,7 @@ func NewRedisCache(config CacheConfig) (*RedisCache, error) {
 }
 
 func (self *RedisCache) Get(key string) (Parsed, error) {
+	key = self.keyPrefix + key //"B" + IPVersion + "_" + key
 	data, err := self.client.Get(key).Result()
 	if err != nil {
 		return NilParse, err
@@ -39,15 +43,34 @@ func (self *RedisCache) Get(key string) (Parsed, error) {
 	parsed := Parsed{}
 	err = json.Unmarshal([]byte(data), &parsed)
 
-	return parsed, err
-}
-
-func (self *RedisCache) Set(key string, parsed Parsed) error {
-	payload, err := json.Marshal(parsed)
-	if err != nil {
-		return err
+	ttl, correct := parsed["ttl"].(time.Time)
+	if !correct {
+		return NilParse, errors.New("Invalid TTL value for key" + key)
 	}
 
-	_, err = self.client.Set(key, payload, time.Minute*5).Result()
-	return err
+	if ttl.Before(time.Now()) {
+		return NilParse, err // TTL expired
+	} else {
+		return parsed, err // cache hit
+	}
+}
+
+func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error {
+	switch {
+	case ttl == 0:
+		return nil // do not cache
+
+	case ttl > 0:
+		key = self.keyPrefix + key //TODO "B" + IPVersion + "_" + key
+		payload, err := json.Marshal(parsed)
+		if err != nil {
+			return err
+		}
+
+		_, err = self.client.Set(key, payload, time.Duration(ttl)*time.Minute).Result()
+		return err
+
+	default: // ttl negative - invalid
+		return errors.New("Negative TTL value for key" + key)
+	}
 }
diff --git a/bird/redis_cache_test.go b/bird/redis_cache_test.go
index a8b6103..d84492c 100644
--- a/bird/redis_cache_test.go
+++ b/bird/redis_cache_test.go
@@ -23,7 +23,7 @@ func Test_RedisCacheAccess(t *testing.T) {
 	}
 
 	t.Log("Setting redis cache...")
-	err = cache.Set("testkey", parsed)
+	err = cache.Set("testkey", parsed, 5)
 	if err != nil {
 		t.Error(err)
 	}
@@ -80,7 +80,7 @@ func Test_RedisCacheRoutes(t *testing.T) {
 		return
 	}
 
-	err = cache.Set("routes_protocol_test", parsed)
+	err = cache.Set("routes_protocol_test", parsed, 5)
 	if err != nil {
 		t.Error(err)
 	}
diff --git a/birdwatcher.go b/birdwatcher.go
index 7c6bac9..d9325f1 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -173,7 +173,7 @@ func main() {
 
 	var cache bird.Cache
 	if conf.Cache.UseRedis {
-		bird.CacheRedis, err = bird.NewRedisCache(conf.Cache)
+		cache, err = bird.NewRedisCache(conf.Cache)
 		if err != nil {
 			log.Fatal("Could not initialize redis cache, falling back to memory cache:", err)
 		}

From cfa0af57ccd7c3ec555ce73ad32ce260547ea5fc Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Mon, 18 Feb 2019 17:46:32 +0100
Subject: [PATCH 09/20] Expire cache entries to save memory

* Add a method to expire cache entries, based on the ttl value
* Add a housekeeping method that will periodically expire cache
  entries and also maybe configured to force a GC/SCVG run.
---
 bird/bird.go                     | 20 ++++++++++++++
 birdwatcher.go                   |  2 ++
 config.go                        | 13 ++++-----
 etc/birdwatcher/birdwatcher.conf |  9 +++----
 housekeeping.go                  | 45 ++++++++++++++++++++++++++++++++
 5 files changed, 78 insertions(+), 11 deletions(-)
 create mode 100644 housekeeping.go

diff --git a/bird/bird.go b/bird/bird.go
index 0fb8195..5b584fa 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -98,6 +98,26 @@ func GetCacheKey(fname string, fargs ...interface{}) string {
 	return key
 }
 
+func (c *MemoryCache) Expire() int {
+	c.Lock()
+
+	expiredKeys := []string{}
+	for key, _ := range c.m {
+		ttl, correct := c.m[key]["ttl"].(time.Time)
+		if !correct || ttl.Before(time.Now()) {
+			expiredKeys = append(expiredKeys, key)
+		}
+	}
+
+	for _, key := range expiredKeys {
+		delete(c.m, key)
+	}
+
+	c.Unlock()
+
+	return len(expiredKeys)
+}
+
 func Run(args string) (io.Reader, error) {
 	args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument
 	argsList := strings.Split(args, " ")
diff --git a/birdwatcher.go b/birdwatcher.go
index 5a3fee1..a1ebaf5 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -205,6 +205,8 @@ func main() {
 	myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime))
 	mylogger := &MyLogger{myquerylog}
 
+	go Housekeeping(conf.Housekeeping)
+
 	if conf.Server.EnableTLS {
 		if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 {
 			log.Fatalln("You have enabled TLS support but not specified both a .crt and a .key file in the config.")
diff --git a/config.go b/config.go
index a722a35..12e2325 100644
--- a/config.go
+++ b/config.go
@@ -17,12 +17,13 @@ import (
 type Config struct {
 	Server endpoints.ServerConfig
 
-	Ratelimit bird.RateLimitConfig
-	Status    bird.StatusConfig
-	Bird      bird.BirdConfig
-	Bird6     bird.BirdConfig
-	Parser    bird.ParserConfig
-	Cache     bird.CacheConfig
+	Ratelimit    bird.RateLimitConfig
+	Status       bird.StatusConfig
+	Bird         bird.BirdConfig
+	Bird6        bird.BirdConfig
+	Parser       bird.ParserConfig
+	Cache        bird.CacheConfig
+	Housekeeping HousekeepingConfig
 }
 
 // Try to load configfiles as specified in the files
diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf
index 1b7356a..d817fab 100755
--- a/etc/birdwatcher/birdwatcher.conf
+++ b/etc/birdwatcher/birdwatcher.conf
@@ -78,12 +78,11 @@ ttl = 5 # time to live (in minutes) for caching of cli output
 # Remove fields e.g. interface
 filter_fields = []
 
-# Enable support for multitable configurations
-per_peer_tables = true
-peer_protocol_prefix = 'ID'
-pipe_protocol_prefix = 'P'
-
 [cache]
 use_redis = false 
 redis_server = "myredis:6379"
 redis_db = 0
+
+[housekeeping]
+# Try to release memory via a forced GC/SCVG run on every housekeeping run
+force_release_memory = true
diff --git a/housekeeping.go b/housekeeping.go
new file mode 100644
index 0000000..d4f3a03
--- /dev/null
+++ b/housekeeping.go
@@ -0,0 +1,45 @@
+package main
+
+import (
+	"time"
+	"log"
+	"runtime/debug"
+
+	"github.com/alice-lg/birdwatcher/bird"
+)
+
+
+type HousekeepingConfig struct {
+	ForceReleaseMemory bool `toml:"force_release_memory"`
+}
+
+// This is used to run regular housekeeping tasks, currently expiring old
+// Cache entries to release memory
+func Housekeeping(config HousekeepingConfig) {
+	for {
+		if bird.ClientConf.CacheTtl > 0 {
+			time.Sleep(time.Duration(bird.ClientConf.CacheTtl) * time.Minute)
+		} else {
+			time.Sleep(5 * time.Minute)
+		}
+
+		log.Println("Housekeeping started")
+
+		if bird.ClientConf.CacheTtl > 0 {
+			// Expire the caches
+			log.Println("Expiring caches")
+
+			count := bird.ParsedCache.Expire()
+			log.Println("Expired", count, "entries (ParsedCache)")
+
+			count = bird.MetaCache.Expire()
+			log.Println("Expired", count, "entries (MetaCache)")
+		}
+
+		if config.ForceReleaseMemory {
+			// Trigger a GC and SCVG run
+			log.Println("Freeing memory")
+			debug.FreeOSMemory()
+		}
+	}
+}

From 210d84445e274d9061c2707360f83c3fffdabf5f Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Tue, 19 Feb 2019 14:06:43 +0100
Subject: [PATCH 10/20] Made the housekeeping routine interval configurable.

---
 etc/birdwatcher/birdwatcher.conf | 2 ++
 housekeeping.go                  | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf
index d817fab..cea27d8 100755
--- a/etc/birdwatcher/birdwatcher.conf
+++ b/etc/birdwatcher/birdwatcher.conf
@@ -84,5 +84,7 @@ redis_server = "myredis:6379"
 redis_db = 0
 
 [housekeeping]
+# Interval for the housekeeping routine in minutes
+interval = 5
 # Try to release memory via a forced GC/SCVG run on every housekeeping run
 force_release_memory = true
diff --git a/housekeeping.go b/housekeeping.go
index d4f3a03..b194b39 100644
--- a/housekeeping.go
+++ b/housekeeping.go
@@ -10,6 +10,7 @@ import (
 
 
 type HousekeepingConfig struct {
+	Interval           int  `toml:"interval"`
 	ForceReleaseMemory bool `toml:"force_release_memory"`
 }
 
@@ -17,8 +18,8 @@ type HousekeepingConfig struct {
 // Cache entries to release memory
 func Housekeeping(config HousekeepingConfig) {
 	for {
-		if bird.ClientConf.CacheTtl > 0 {
-			time.Sleep(time.Duration(bird.ClientConf.CacheTtl) * time.Minute)
+		if config.Interval > 0 {
+			time.Sleep(time.Duration(config.Interval) * time.Minute)
 		} else {
 			time.Sleep(5 * time.Minute)
 		}

From e6ed0cb9012f00b3eb43da63aa4628b7293c4fbe Mon Sep 17 00:00:00 2001
From: Benedikt Rudolph <benedikt.rudolph@de-cix.net>
Date: Wed, 20 Feb 2019 11:17:13 +0100
Subject: [PATCH 11/20] Refactor housekeeping and memory cache

* run Expire() only on MemoryCaches
* make initialization of the cache look pretty
---
 bird/bird.go                     | 43 ++++++++++++++------------------
 bird/memory_cache.go             | 20 +++++++++++++++
 bird/memory_cache_test.go        |  1 +
 bird/redis_cache.go              |  6 +++++
 birdwatcher.go                   | 19 +++-----------
 etc/birdwatcher/birdwatcher.conf |  3 ++-
 housekeeping.go                  | 16 +++++-------
 7 files changed, 57 insertions(+), 51 deletions(-)

diff --git a/bird/bird.go b/bird/bird.go
index 5b584fa..c606cff 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -16,20 +16,20 @@ import (
 type Cache interface {
 	Set(key string, val Parsed, ttl int) error
 	Get(key string) (Parsed, error)
+	Expire() int
 }
 
 var ClientConf BirdConfig
 var StatusConf StatusConfig
 var IPVersion = "4"
 var cache Cache // stores parsed birdc output
+var CacheConf CacheConfig
 var RateLimitConf struct {
 	sync.RWMutex
 	Conf RateLimitConfig
 }
 var RunQueue sync.Map // queue birdc commands before execution
 
-var CacheRedis *RedisCache
-
 var NilParse Parsed = (Parsed)(nil) // special Parsed values
 var BirdError Parsed = Parsed{"error": "bird unreachable"}
 
@@ -40,8 +40,23 @@ func IsSpecial(ret Parsed) bool { // test for special Parsed values
 // intitialize the Cache once during setup with either a MemoryCache or
 // RedisCache implementation.
 // TODO implement singleton pattern
-func InitializeCache(c Cache) {
-	cache = c
+func InitializeCache() {
+	var err error
+	if CacheConf.UseRedis {
+		cache, err = NewRedisCache(CacheConf)
+		if err != nil {
+			log.Println("Could not initialize redis cache, falling back to memory cache:", err)
+		}
+	} else { // initialize the MemoryCache
+		cache, err = NewMemoryCache()
+		if err != nil {
+			log.Fatal("Could not initialize MemoryCache:", err)
+		}
+	}
+}
+
+func ExpireCache() int {
+	return cache.Expire()
 }
 
 /* Convenience method to make new entries in the cache.
@@ -98,26 +113,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string {
 	return key
 }
 
-func (c *MemoryCache) Expire() int {
-	c.Lock()
-
-	expiredKeys := []string{}
-	for key, _ := range c.m {
-		ttl, correct := c.m[key]["ttl"].(time.Time)
-		if !correct || ttl.Before(time.Now()) {
-			expiredKeys = append(expiredKeys, key)
-		}
-	}
-
-	for _, key := range expiredKeys {
-		delete(c.m, key)
-	}
-
-	c.Unlock()
-
-	return len(expiredKeys)
-}
-
 func Run(args string) (io.Reader, error) {
 	args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument
 	argsList := strings.Split(args, " ")
diff --git a/bird/memory_cache.go b/bird/memory_cache.go
index 89384b4..7475e06 100644
--- a/bird/memory_cache.go
+++ b/bird/memory_cache.go
@@ -59,3 +59,23 @@ func (c *MemoryCache) Set(key string, val Parsed, ttl int) error {
 		return errors.New("Negative TTL value for key" + key)
 	}
 }
+
+func (c *MemoryCache) Expire() int {
+	c.Lock()
+
+	expiredKeys := []string{}
+	for key, _ := range c.m {
+		ttl, correct := c.m[key]["ttl"].(time.Time)
+		if !correct || ttl.Before(time.Now()) {
+			expiredKeys = append(expiredKeys, key)
+		}
+	}
+
+	for _, key := range expiredKeys {
+		delete(c.m, key)
+	}
+
+	c.Unlock()
+
+	return len(expiredKeys)
+}
diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go
index b4a6a78..4026fae 100644
--- a/bird/memory_cache_test.go
+++ b/bird/memory_cache_test.go
@@ -26,6 +26,7 @@ func Test_MemoryCacheAccess(t *testing.T) {
 		t.Error(err)
 	}
 
+	cache.Expire()
 	t.Log(parsed)
 }
 
diff --git a/bird/redis_cache.go b/bird/redis_cache.go
index f20f7e8..faf672b 100644
--- a/bird/redis_cache.go
+++ b/bird/redis_cache.go
@@ -3,6 +3,7 @@ package bird
 import (
 	"encoding/json"
 	"errors"
+	"log"
 	"time"
 
 	"github.com/go-redis/redis"
@@ -74,3 +75,8 @@ func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error {
 		return errors.New("Negative TTL value for key" + key)
 	}
 }
+
+func (self *RedisCache) Expire() int {
+	log.Printf("Cannot expire entries in RedisCache backend, redis does this automatically")
+	return 0
+}
diff --git a/birdwatcher.go b/birdwatcher.go
index a1ebaf5..4e60212 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -178,21 +178,8 @@ func main() {
 	bird.RateLimitConf.Conf = conf.Ratelimit
 	bird.RateLimitConf.Unlock()
 	bird.ParserConf = conf.Parser
-
-	var cache bird.Cache
-	if conf.Cache.UseRedis {
-		cache, err = bird.NewRedisCache(conf.Cache)
-		if err != nil {
-			log.Fatal("Could not initialize redis cache, falling back to memory cache:", err)
-		}
-	} else { // initialize the MemoryCache
-		cache, err = bird.NewMemoryCache()
-		if err != nil {
-			log.Fatal("Could not initialize MemoryCache:", err)
-		} else {
-			bird.InitializeCache(cache)
-		}
-	}
+	bird.CacheConf = conf.Cache
+	bird.InitializeCache()
 
 	endpoints.Conf = conf.Server
 
@@ -205,7 +192,7 @@ func main() {
 	myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime))
 	mylogger := &MyLogger{myquerylog}
 
-	go Housekeeping(conf.Housekeeping)
+	go Housekeeping(conf.Housekeeping, !(bird.CacheConf.UseRedis)) // expire caches only for MemoryCache
 
 	if conf.Server.EnableTLS {
 		if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 {
diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf
index cea27d8..ac83885 100755
--- a/etc/birdwatcher/birdwatcher.conf
+++ b/etc/birdwatcher/birdwatcher.conf
@@ -79,10 +79,11 @@ ttl = 5 # time to live (in minutes) for caching of cli output
 filter_fields = []
 
 [cache]
-use_redis = false 
+use_redis = false # if not using redis cache, activate housekeeping to save memory! 
 redis_server = "myredis:6379"
 redis_db = 0
 
+# Housekeeping expires old cache entries (memory cache backend) and performs a GC/SCVG run if configured.
 [housekeeping]
 # Interval for the housekeeping routine in minutes
 interval = 5
diff --git a/housekeeping.go b/housekeeping.go
index b194b39..3c38db3 100644
--- a/housekeeping.go
+++ b/housekeeping.go
@@ -1,14 +1,13 @@
 package main
 
 import (
-	"time"
 	"log"
 	"runtime/debug"
+	"time"
 
 	"github.com/alice-lg/birdwatcher/bird"
 )
 
-
 type HousekeepingConfig struct {
 	Interval           int  `toml:"interval"`
 	ForceReleaseMemory bool `toml:"force_release_memory"`
@@ -16,7 +15,7 @@ type HousekeepingConfig struct {
 
 // This is used to run regular housekeeping tasks, currently expiring old
 // Cache entries to release memory
-func Housekeeping(config HousekeepingConfig) {
+func Housekeeping(config HousekeepingConfig, expireCaches bool) {
 	for {
 		if config.Interval > 0 {
 			time.Sleep(time.Duration(config.Interval) * time.Minute)
@@ -26,15 +25,12 @@ func Housekeeping(config HousekeepingConfig) {
 
 		log.Println("Housekeeping started")
 
-		if bird.ClientConf.CacheTtl > 0 {
+		if (bird.ClientConf.CacheTtl > 0) && expireCaches {
 			// Expire the caches
-			log.Println("Expiring caches")
+			log.Println("Expiring MemoryCache")
 
-			count := bird.ParsedCache.Expire()
-			log.Println("Expired", count, "entries (ParsedCache)")
-
-			count = bird.MetaCache.Expire()
-			log.Println("Expired", count, "entries (MetaCache)")
+			count := bird.ExpireCache()
+			log.Println("Expired", count, "entries (MemoryCache)")
 		}
 
 		if config.ForceReleaseMemory {

From 8585003413e76d621b9473181399612023ed3ac1 Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Tue, 19 Feb 2019 14:07:14 +0100
Subject: [PATCH 12/20] Added a parser and endpoint for the show protocols
 command.

---
 bird/bird.go                     |  5 +++++
 bird/parser.go                   | 31 +++++++++++++++++++++++++++++++
 bird/parser_test.go              | 20 ++++++++++++++++++++
 birdwatcher.go                   |  3 +++
 endpoints/protocols.go           |  4 ++++
 etc/birdwatcher/birdwatcher.conf |  2 ++
 test/protocols_short.sample      | 10 ++++++++++
 7 files changed, 75 insertions(+)
 create mode 100644 test/protocols_short.sample

diff --git a/bird/bird.go b/bird/bird.go
index c606cff..103d9bf 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -239,6 +239,11 @@ func Status() (Parsed, bool) {
 	return birdStatus, from_cache
 }
 
+func ProtocolsShort() (Parsed, bool) {
+	res, from_cache := RunAndParse(GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil)
+	return res, from_cache
+}
+
 func Protocols() (Parsed, bool) {
 	createMetaCache := func(p *Parsed) {
 		metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}}
diff --git a/bird/parser.go b/bird/parser.go
index 92d3f6a..a22f9aa 100644
--- a/bird/parser.go
+++ b/bird/parser.go
@@ -29,6 +29,7 @@ var (
 			routes       *regexp.Regexp
 			stringValue  *regexp.Regexp
 			routeChanges *regexp.Regexp
+			short        *regexp.Regexp
 		}
 		symbols struct {
 			keyRx *regexp.Regexp
@@ -72,6 +73,7 @@ func init() {
 	regex.protocol.routeChanges = regexp.MustCompile(`(Import|Export) (updates|withdraws):\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s*$`)
 
 	regex.routes.startDefinition = regexp.MustCompile(`^([0-9a-f\.\:\/]+)\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*`)
+	regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+)\s+(.*?)\s*?$`)
 	regex.routes.second = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*$`)
 	regex.routes.routeType = regexp.MustCompile(`^\s+Type:\s+(.*)\s*$`)
 	regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`)
@@ -132,6 +134,35 @@ func parseStatus(reader io.Reader) Parsed {
 	return Parsed{"status": res}
 }
 
+func parseProtocolsShort(reader io.Reader) Parsed {
+	res := Parsed{}
+
+	lines := newLineIterator(reader, false)
+	for lines.next() {
+		line := lines.string()
+
+		if specialLine(line) {
+			continue
+		}
+
+		if regex.protocol.short.MatchString(line) {
+			// The header is skipped, because the regular expression does not
+			// match if the "since" field does not contain digits
+			matches := regex.protocol.short.FindStringSubmatch(line)
+
+			res[matches[1]] = Parsed{
+				"proto": matches[2],
+				"table": matches[3],
+				"state": matches[4],
+				"since": matches[5],
+				"info": matches[6],
+			}
+		}
+	}
+
+	return Parsed{"protocols": res}
+}
+
 func parseProtocols(reader io.Reader) Parsed {
 	res := Parsed{}
 
diff --git a/bird/parser_test.go b/bird/parser_test.go
index 54d93cd..cb026dd 100644
--- a/bird/parser_test.go
+++ b/bird/parser_test.go
@@ -63,6 +63,26 @@ func TestParseProtocolBgp(t *testing.T) {
 	fmt.Println(protocols)
 }
 
+func TestParseProtocolShort(t *testing.T) {
+	f, err := openFile("protocols_short.sample")
+	if err != nil {
+		t.Error(err)
+	}
+	defer f.Close()
+
+	p := parseProtocolsShort(f)
+	log.Printf("%# v", pretty.Formatter(p))
+	protocols := p["protocols"].(Parsed)
+
+	if len(protocols) != 7 {
+		//log.Printf("%# v", pretty.Formatter(protocols))
+		t.Fatalf("Expected 7 protocols, found: %v", len(protocols))
+	}
+
+	fmt.Println(protocols)
+}
+
+
 func TestParseRoutesAllIpv4Bird1(t *testing.T) {
 	runTestForIpv4WithFile("routes_bird1_ipv4.sample", t)
 }
diff --git a/birdwatcher.go b/birdwatcher.go
index 4e60212..e5d8d54 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -42,6 +42,9 @@ func makeRouter(config endpoints.ServerConfig) *httprouter.Router {
 	if isModuleEnabled("protocols_bgp", whitelist) {
 		r.GET("/protocols/bgp", endpoints.Endpoint(endpoints.Bgp))
 	}
+	if isModuleEnabled("protocols_short", whitelist) {
+		r.GET("/protocols/short", endpoints.Endpoint(endpoints.ProtocolsShort))
+	}
 	if isModuleEnabled("symbols", whitelist) {
 		r.GET("/symbols", endpoints.Endpoint(endpoints.Symbols))
 	}
diff --git a/endpoints/protocols.go b/endpoints/protocols.go
index da04438..eb23534 100644
--- a/endpoints/protocols.go
+++ b/endpoints/protocols.go
@@ -14,3 +14,7 @@ func Protocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 func Bgp(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 	return bird.ProtocolsBgp()
 }
+
+func ProtocolsShort(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+	return bird.ProtocolsShort()
+}
diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf
index ac83885..3a5e17a 100755
--- a/etc/birdwatcher/birdwatcher.conf
+++ b/etc/birdwatcher/birdwatcher.conf
@@ -14,6 +14,7 @@ allow_from = []
 #   symbols_protocols
 #   protocols
 #   protocols_bgp
+#   protocols_short
 #   routes_protocol
 #   routes_peer
 #   routes_table
@@ -34,6 +35,7 @@ allow_from = []
 modules_enabled = ["status",
                    "protocols",
                    "protocols_bgp",
+                   "protocols_short",
                    "routes_protocol",
                    "routes_peer",
                    "routes_table",
diff --git a/test/protocols_short.sample b/test/protocols_short.sample
new file mode 100644
index 0000000..c55c3d4
--- /dev/null
+++ b/test/protocols_short.sample
@@ -0,0 +1,10 @@
+BIRD 1.6.5 ready.
+Access restricted
+name     proto    table    state  since       info
+device1  Device   master   up     2019-02-15  
+pp_0097_as3856 Pipe     master   up     2019-02-15  => t_0097_as3856
+pb_0097_as3856 BGP      t_0097_as3856 up     2019-02-15  Established   
+pp_0175_as15169 Pipe     master   up     2019-02-15  => t_0175_as15169
+pb_0175_as15169 BGP      t_0175_as15169 up     2019-02-15  Established   
+pp_0026_as20940 Pipe     master   up     2019-02-15  => t_0026_as20940
+pb_0026_as20940 BGP      t_0026_as20940 up     2019-02-15  Established   

From c6e717bc61e367b15bbbfadfa5b6eba190219d15 Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Wed, 20 Feb 2019 12:13:18 +0100
Subject: [PATCH 13/20] Fixed a bug regarding timestamps in the protocols
 parser.

---
 bird/parser.go              |  2 +-
 bird/parser_test.go         |  4 ++--
 test/protocols_short.sample | 21 +++++++++++++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/bird/parser.go b/bird/parser.go
index a22f9aa..074e808 100644
--- a/bird/parser.go
+++ b/bird/parser.go
@@ -73,7 +73,7 @@ func init() {
 	regex.protocol.routeChanges = regexp.MustCompile(`(Import|Export) (updates|withdraws):\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s+(\d+|---)\s*$`)
 
 	regex.routes.startDefinition = regexp.MustCompile(`^([0-9a-f\.\:\/]+)\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*`)
-	regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+)\s+(.*?)\s*?$`)
+	regex.protocol.short = regexp.MustCompile(`^(?:1002\-)?([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([0-9\-]+\s+[0-9\:]+?|[0-9\-]+)\s+(.*?)\s*?$`)
 	regex.routes.second = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+)){0,1}\]\s+(?:(\*)\s+){0,1}\((\d+)(?:\/\d+){0,1}\).*$`)
 	regex.routes.routeType = regexp.MustCompile(`^\s+Type:\s+(.*)\s*$`)
 	regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`)
diff --git a/bird/parser_test.go b/bird/parser_test.go
index cb026dd..a4a81f8 100644
--- a/bird/parser_test.go
+++ b/bird/parser_test.go
@@ -74,9 +74,9 @@ func TestParseProtocolShort(t *testing.T) {
 	log.Printf("%# v", pretty.Formatter(p))
 	protocols := p["protocols"].(Parsed)
 
-	if len(protocols) != 7 {
+	if len(protocols) != 27 {
 		//log.Printf("%# v", pretty.Formatter(protocols))
-		t.Fatalf("Expected 7 protocols, found: %v", len(protocols))
+		t.Fatalf("Expected 27 protocols, found: %v", len(protocols))
 	}
 
 	fmt.Println(protocols)
diff --git a/test/protocols_short.sample b/test/protocols_short.sample
index c55c3d4..5fabb6a 100644
--- a/test/protocols_short.sample
+++ b/test/protocols_short.sample
@@ -8,3 +8,24 @@ pp_0175_as15169 Pipe     master   up     2019-02-15  => t_0175_as15169
 pb_0175_as15169 BGP      t_0175_as15169 up     2019-02-15  Established   
 pp_0026_as20940 Pipe     master   up     2019-02-15  => t_0026_as20940
 pb_0026_as20940 BGP      t_0026_as20940 up     2019-02-15  Established   
+direct1  Direct   master   down   2019-02-19 16:17:59  
+kernel1  Kernel   master   down   2019-02-19 16:17:59  
+M42_pch_radb Pipe     master   up     2019-02-19 16:17:59  => T42_pch_radb
+C42_pch_radb Pipe     Collector up     2019-02-19 16:17:59  => T42_pch_radb
+R194_42  BGP      T42_pch_radb up     2019-02-19 16:29:00  Established   
+M3856_pch_radb Pipe     master   up     2019-02-19 16:17:59  => T3856_pch_radb
+C3856_pch_radb Pipe     Collector up     2019-02-19 16:17:59  => T3856_pch_radb
+R195_42  BGP      T3856_pch_radb up     2019-02-19 16:18:33  Established   
+M112_112_ripe Pipe     master   up     2019-02-19 16:17:59  => T112_112_ripe
+C112_112_ripe Pipe     Collector up     2019-02-19 16:17:59  => T112_112_ripe
+R195_77  BGP      T112_112_ripe up     2019-02-19 16:24:31  Established   
+M286_kpn_ripe Pipe     master   up     2019-02-19 16:17:59  => T286_kpn_ripe
+C286_kpn_ripe Pipe     Collector up     2019-02-19 16:17:59  => T286_kpn_ripe
+R192_22  BGP      T286_kpn_ripe up     2019-02-19 16:26:10  Established   
+M553_belwue_ripe Pipe     master   up     2019-02-19 16:17:59  => T553_belwue_ripe
+C553_belwue_ripe Pipe     Collector up     2019-02-19 16:17:59  => T553_belwue_ripe
+R192_175 BGP      T553_belwue_ripe up     2019-02-19 16:18:34  Established   
+R194_106 BGP      T553_belwue_ripe up     2019-02-19 16:18:09  Established   
+R194_205 BGP      T52866_iveloz_radb start  2019-02-20 12:06:01  Idle          BGP Error: Bad peer AS
+R_janus1 BGP      T6695_bh_20 start  2019-02-19 16:17:59  Idle          
+

From 116f03fed43faa8577fe5c4eb212bdde3cc61782 Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Thu, 21 Feb 2019 13:36:28 +0100
Subject: [PATCH 14/20] Add support for uncached queries

The cache is still updated with new information on every request.
---
 bird/bird.go                     | 126 ++++++++++++++++---------------
 endpoints/config.go              |   1 +
 endpoints/endpoint.go            |  17 ++++-
 endpoints/protocols.go           |  12 +--
 endpoints/routes.go              |  60 +++++++--------
 endpoints/status.go              |   4 +-
 endpoints/symbols.go             |  12 +--
 etc/birdwatcher/birdwatcher.conf |   2 +
 8 files changed, 126 insertions(+), 108 deletions(-)

diff --git a/bird/bird.go b/bird/bird.go
index 103d9bf..8099794 100644
--- a/bird/bird.go
+++ b/bird/bird.go
@@ -159,12 +159,15 @@ func checkRateLimit() bool {
 	return true
 }
 
-func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) {
-	if val, ok := fromCache(cmd); ok {
-		return val, true
+func RunAndParse(useCache bool, key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) {
+	var wg sync.WaitGroup
+
+	if useCache {
+		if val, ok := fromCache(cmd); ok {
+			return val, true
+		}
 	}
 
-	var wg sync.WaitGroup
 	wg.Add(1)
 	if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded {
 		(*queueGroup.(*sync.WaitGroup)).Wait()
@@ -200,13 +203,12 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa
 	toCache(cmd, parsed)
 
 	wg.Done()
-
 	RunQueue.Delete(cmd)
 
 	return parsed, false
 }
 
-func Status() (Parsed, bool) {
+func Status(useCache bool) (Parsed, bool) {
 	updateParsedCache := func(p *Parsed) {
 		status := (*p)["status"].(Parsed)
 
@@ -235,16 +237,16 @@ func Status() (Parsed, bool) {
 		}
 	}
 
-	birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, updateParsedCache)
+	birdStatus, from_cache := RunAndParse(useCache, GetCacheKey("Status"), "status", parseStatus, updateParsedCache)
 	return birdStatus, from_cache
 }
 
-func ProtocolsShort() (Parsed, bool) {
-	res, from_cache := RunAndParse(GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil)
+func ProtocolsShort(useCache bool) (Parsed, bool) {
+	res, from_cache := RunAndParse(useCache, GetCacheKey("ProtocolsShort"), "protocols", parseProtocolsShort, nil)
 	return res, from_cache
 }
 
-func Protocols() (Parsed, bool) {
+func Protocols(useCache bool) (Parsed, bool) {
 	createMetaCache := func(p *Parsed) {
 		metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}}
 
@@ -263,12 +265,12 @@ func Protocols() (Parsed, bool) {
 		toCache(GetCacheKey("metaProtocol"), metaProtocol)
 	}
 
-	res, from_cache := RunAndParse(GetCacheKey("metaProtocol"), "protocols all", parseProtocols, createMetaCache)
+	res, from_cache := RunAndParse(useCache, GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache)
 	return res, from_cache
 }
 
-func ProtocolsBgp() (Parsed, bool) {
-	protocols, from_cache := Protocols()
+func ProtocolsBgp(useCache bool) (Parsed, bool) {
+	protocols, from_cache := Protocols(useCache)
 	if IsSpecial(protocols) {
 		return protocols, from_cache
 	}
@@ -287,92 +289,92 @@ func ProtocolsBgp() (Parsed, bool) {
 		"cached_at": protocols["cached_at"]}, from_cache
 }
 
-func Symbols() (Parsed, bool) {
-	return RunAndParse(GetCacheKey("Symbols"), "symbols", parseSymbols, nil)
+func Symbols(useCache bool) (Parsed, bool) {
+	return RunAndParse(useCache, GetCacheKey("Symbols"), "symbols", parseSymbols, nil)
 }
 
-func RoutesPrefixed(prefix string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route " + prefix + " all")
-	return RunAndParse(GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil)
+func RoutesPrefixed(useCache bool, prefix string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route "+prefix+" all")
+	return RunAndParse(useCache, GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil)
 }
 
-func RoutesProto(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route all protocol " + protocol)
-	return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil)
+func RoutesProto(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route all protocol "+protocol)
+	return RunAndParse(useCache, GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil)
 }
 
-func RoutesPeer(peer string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route all where from=" + peer)
-	return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil)
+func RoutesPeer(useCache bool, peer string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route all where from="+peer)
+	return RunAndParse(useCache, GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil)
 }
 
-func RoutesTableAndPeer(table string, peer string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route table " + table + " all where from=" + peer)
-	return RunAndParse(GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil)
+func RoutesTableAndPeer(useCache bool, table string, peer string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route table "+table+" all where from="+peer)
+	return RunAndParse(useCache, GetCacheKey("RoutesTableAndPeer", table, peer), cmd, parseRoutes, nil)
 }
 
-func RoutesProtoCount(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route protocol "+protocol) + " count"
-	return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil)
+func RoutesProtoCount(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route protocol "+protocol) + " count"
+	return RunAndParse(useCache, GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil)
 }
 
-func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route primary protocol "+protocol) + " count"
-	return RunAndParse(GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil)
+func RoutesProtoPrimaryCount(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route primary protocol "+protocol) + " count"
+	return RunAndParse(useCache, GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil)
 }
 
-func PipeRoutesFilteredCount(pipe string, table string, neighborAddress string) (Parsed, bool) {
+func PipeRoutesFilteredCount(useCache bool, pipe string, table string, neighborAddress string) (Parsed, bool) {
 	cmd := "route table " + table + " noexport " + pipe + " where from=" + neighborAddress + " count"
-	return RunAndParse(GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil)
+	return RunAndParse(useCache, GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil)
 }
 
-func PipeRoutesFiltered(pipe string, table string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route table '" + table + "' noexport '" + pipe + "' all")
-	return RunAndParse(GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil)
+func PipeRoutesFiltered(useCache bool, pipe string, table string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route table '"+table+"' noexport '"+pipe+"' all")
+	return RunAndParse(useCache, GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil)
 }
 
-func RoutesFiltered(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route all filtered protocol " + protocol)
-	return RunAndParse(GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil)
+func RoutesFiltered(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route all filtered protocol "+protocol)
+	return RunAndParse(useCache, GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil)
 }
 
-func RoutesExport(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route all export " + protocol)
-	return RunAndParse(GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil)
+func RoutesExport(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route all export "+protocol)
+	return RunAndParse(useCache, GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil)
 }
 
-func RoutesNoExport(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route all noexport " + protocol)
-	return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil)
+func RoutesNoExport(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route all noexport "+protocol)
+	return RunAndParse(useCache, GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil)
 }
 
-func RoutesExportCount(protocol string) (Parsed, bool) {
-	cmd := routeQueryForChannel("route export "+protocol) + " count"
-	return RunAndParse(GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil)
+func RoutesExportCount(useCache bool, protocol string) (Parsed, bool) {
+	cmd := routeQueryForChannel(useCache, "route export "+protocol) + " count"
+	return RunAndParse(useCache, GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil)
 }
 
-func RoutesTable(table string) (Parsed, bool) {
-	return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil)
+func RoutesTable(useCache bool, table string) (Parsed, bool) {
+	return RunAndParse(useCache, GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil)
 }
 
-func RoutesTableFiltered(table string) (Parsed, bool) {
-	return RunAndParse(GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil)
+func RoutesTableFiltered(useCache bool, table string) (Parsed, bool) {
+	return RunAndParse(useCache, GetCacheKey("RoutesTableFiltered", table), "route table "+table+" filtered", parseRoutes, nil)
 }
 
-func RoutesTableCount(table string) (Parsed, bool) {
-	return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil)
+func RoutesTableCount(useCache bool, table string) (Parsed, bool) {
+	return RunAndParse(useCache, GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil)
 }
 
-func RoutesLookupTable(net string, table string) (Parsed, bool) {
-	return RunAndParse(GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil)
+func RoutesLookupTable(useCache bool, net string, table string) (Parsed, bool) {
+	return RunAndParse(useCache, GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil)
 }
 
-func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) {
-	return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil)
+func RoutesLookupProtocol(useCache bool, net string, protocol string) (Parsed, bool) {
+	return RunAndParse(useCache, GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil)
 }
 
-func routeQueryForChannel(cmd string) string {
-	status, _ := Status()
+func routeQueryForChannel(useCache bool, cmd string) string {
+	status, _ := Status(useCache)
 	if IsSpecial(status) {
 		return cmd
 	}
diff --git a/endpoints/config.go b/endpoints/config.go
index f49599d..4e2e370 100644
--- a/endpoints/config.go
+++ b/endpoints/config.go
@@ -4,6 +4,7 @@ package endpoints
 type ServerConfig struct {
 	AllowFrom      []string `toml:"allow_from"`
 	ModulesEnabled []string `toml:"modules_enabled"`
+	AllowUncached  bool     `toml:"allow_uncached"`
 
 	EnableTLS bool   `toml:"enable_tls"`
 	Crt       string `toml:"crt"`
diff --git a/endpoints/endpoint.go b/endpoints/endpoint.go
index 8138f83..3dda116 100644
--- a/endpoints/endpoint.go
+++ b/endpoints/endpoint.go
@@ -14,7 +14,7 @@ import (
 	"github.com/julienschmidt/httprouter"
 )
 
-type endpoint func(*http.Request, httprouter.Params) (bird.Parsed, bool)
+type endpoint func(*http.Request, httprouter.Params, bool) (bird.Parsed, bool)
 
 var Conf ServerConfig
 
@@ -42,6 +42,17 @@ func CheckAccess(req *http.Request) error {
 	return fmt.Errorf("%s is not allowed to access this service.", ip)
 }
 
+func CheckUseCache(req *http.Request) bool {
+	qs := req.URL.Query()
+
+	if Conf.AllowUncached &&
+		len(qs["uncached"]) == 1 && qs["uncached"][0] == "true" {
+		return false
+	}
+
+	return true
+}
+
 func Endpoint(wrapped endpoint) httprouter.Handle {
 	return func(w http.ResponseWriter,
 		r *http.Request,
@@ -54,7 +65,9 @@ func Endpoint(wrapped endpoint) httprouter.Handle {
 		}
 
 		res := make(map[string]interface{})
-		ret, from_cache := wrapped(r, ps)
+
+		useCache := CheckUseCache(r)
+		ret, from_cache := wrapped(r, ps, useCache)
 
 		if reflect.DeepEqual(ret, bird.NilParse) {
 			w.WriteHeader(http.StatusTooManyRequests)
diff --git a/endpoints/protocols.go b/endpoints/protocols.go
index eb23534..f090cbf 100644
--- a/endpoints/protocols.go
+++ b/endpoints/protocols.go
@@ -7,14 +7,14 @@ import (
 	"github.com/julienschmidt/httprouter"
 )
 
-func Protocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.Protocols()
+func Protocols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	return bird.Protocols(useCache)
 }
 
-func Bgp(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.ProtocolsBgp()
+func Bgp(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	return bird.ProtocolsBgp(useCache)
 }
 
-func ProtocolsShort(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.ProtocolsShort()
+func ProtocolsShort(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	return bird.ProtocolsShort(useCache)
 }
diff --git a/endpoints/routes.go b/endpoints/routes.go
index 394ed69..d0359b9 100644
--- a/endpoints/routes.go
+++ b/endpoints/routes.go
@@ -8,34 +8,34 @@ import (
 	"github.com/julienschmidt/httprouter"
 )
 
-func ProtoRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func ProtoRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	protocol, err := ValidateProtocolParam(ps.ByName("protocol"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesProto(protocol)
+	return bird.RoutesProto(useCache, protocol)
 }
 
-func RoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func RoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	protocol, err := ValidateProtocolParam(ps.ByName("protocol"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesFiltered(protocol)
+	return bird.RoutesFiltered(useCache, protocol)
 }
 
-func RoutesNoExport(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func RoutesNoExport(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	protocol, err := ValidateProtocolParam(ps.ByName("protocol"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesNoExport(protocol)
+	return bird.RoutesNoExport(useCache, protocol)
 }
 
-func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func RoutesPrefixed(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	qs := r.URL.Query()
 	prefixl := qs["prefix"]
 	if len(prefixl) != 1 {
@@ -47,28 +47,28 @@ func RoutesPrefixed(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesPrefixed(prefix)
+	return bird.RoutesPrefixed(useCache, prefix)
 }
 
-func TableRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func TableRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	table, err := ValidateProtocolParam(ps.ByName("table"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesTable(table)
+	return bird.RoutesTable(useCache, table)
 }
 
-func TableRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func TableRoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	table, err := ValidateProtocolParam(ps.ByName("table"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesTableFiltered(table)
+	return bird.RoutesTableFiltered(useCache, table)
 }
 
-func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func TableAndPeerRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	table, err := ValidateProtocolParam(ps.ByName("table"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
@@ -79,45 +79,45 @@ func TableAndPeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, boo
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesTableAndPeer(table, peer)
+	return bird.RoutesTableAndPeer(useCache, table, peer)
 }
 
-func ProtoCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func ProtoCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	protocol, err := ValidateProtocolParam(ps.ByName("protocol"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesProtoCount(protocol)
+	return bird.RoutesProtoCount(useCache, protocol)
 }
 
-func ProtoPrimaryCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func ProtoPrimaryCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	protocol, err := ValidateProtocolParam(ps.ByName("protocol"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
-	return bird.RoutesProtoPrimaryCount(protocol)
+	return bird.RoutesProtoPrimaryCount(useCache, protocol)
 }
 
-func TableCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func TableCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	table, err := ValidateProtocolParam(ps.ByName("table"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesTableCount(table)
+	return bird.RoutesTableCount(useCache, table)
 }
 
-func RouteNet(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func RouteNet(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	net, err := ValidatePrefixParam(ps.ByName("net"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesLookupTable(net, "master")
+	return bird.RoutesLookupTable(useCache, net, "master")
 }
 
-func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func RouteNetTable(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	net, err := ValidatePrefixParam(ps.ByName("net"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
@@ -128,10 +128,10 @@ func RouteNetTable(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesLookupTable(net, table)
+	return bird.RoutesLookupTable(useCache, net, table)
 }
 
-func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func PipeRoutesFiltered(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	qs := r.URL.Query()
 
 	if len(qs["table"]) != 1 {
@@ -150,10 +150,10 @@ func PipeRoutesFiltered(r *http.Request, ps httprouter.Params) (bird.Parsed, boo
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.PipeRoutesFiltered(pipe, table)
+	return bird.PipeRoutesFiltered(useCache, pipe, table)
 }
 
-func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	qs := r.URL.Query()
 
 	if len(qs["table"]) != 1 {
@@ -180,14 +180,14 @@ func PipeRoutesFilteredCount(r *http.Request, ps httprouter.Params) (bird.Parsed
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.PipeRoutesFilteredCount(pipe, table, address)
+	return bird.PipeRoutesFilteredCount(useCache, pipe, table, address)
 }
 
-func PeerRoutes(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
+func PeerRoutes(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
 	peer, err := ValidatePrefixParam(ps.ByName("peer"))
 	if err != nil {
 		return bird.Parsed{"error": fmt.Sprintf("%s", err)}, false
 	}
 
-	return bird.RoutesPeer(peer)
+	return bird.RoutesPeer(useCache, peer)
 }
diff --git a/endpoints/status.go b/endpoints/status.go
index fd18eb4..36670e2 100644
--- a/endpoints/status.go
+++ b/endpoints/status.go
@@ -7,6 +7,6 @@ import (
 	"github.com/julienschmidt/httprouter"
 )
 
-func Status(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.Status()
+func Status(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	return bird.Status(useCache)
 }
diff --git a/endpoints/symbols.go b/endpoints/symbols.go
index df9245b..63fbdab 100644
--- a/endpoints/symbols.go
+++ b/endpoints/symbols.go
@@ -7,20 +7,20 @@ import (
 	"github.com/julienschmidt/httprouter"
 )
 
-func Symbols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	return bird.Symbols()
+func Symbols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	return bird.Symbols(useCache)
 }
 
-func SymbolTables(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	val, from_cache := bird.Symbols()
+func SymbolTables(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	val, from_cache := bird.Symbols(useCache)
 	if bird.IsSpecial(val) {
 		return val, from_cache
 	}
 	return bird.Parsed{"symbols": val["symbols"].(bird.Parsed)["routing table"]}, from_cache
 }
 
-func SymbolProtocols(r *http.Request, ps httprouter.Params) (bird.Parsed, bool) {
-	val, from_cache := bird.Symbols()
+func SymbolProtocols(r *http.Request, ps httprouter.Params, useCache bool) (bird.Parsed, bool) {
+	val, from_cache := bird.Symbols(useCache)
 	if bird.IsSpecial(val) {
 		return val, from_cache
 	}
diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf
index 3a5e17a..b15f4dd 100755
--- a/etc/birdwatcher/birdwatcher.conf
+++ b/etc/birdwatcher/birdwatcher.conf
@@ -5,6 +5,8 @@
 [server]
 # Restrict access to certain IPs. Leave empty to allow from all.
 allow_from = []
+# Allow queries that bypass the cache
+allow_uncached = false
 
 # Available modules:
 ## low-level modules (translation from birdc output to JSON objects)

From e0031c61a0f6e6d5a90be5b4cb69fa994e85ac8e Mon Sep 17 00:00:00 2001
From: Patrick Seeburger <info@8bit.de>
Date: Fri, 22 Feb 2019 10:44:13 +0100
Subject: [PATCH 15/20] Change extended communities fomat

Change format for extended BGP communities from (string,int,int) to
(string,string,string) in order to support communities like
(generic, 0x43000000, 0x1)
---
 bird/parser.go                |  4 ++--
 bird/parser_test.go           | 32 +++++++++++++++++---------------
 test/routes_bird1_ipv4.sample |  4 ++--
 test/routes_bird2_ipv4.sample |  4 ++--
 4 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/bird/parser.go b/bird/parser.go
index 074e808..92ed1d9 100644
--- a/bird/parser.go
+++ b/bird/parser.go
@@ -79,7 +79,7 @@ func init() {
 	regex.routes.bgp = regexp.MustCompile(`^\s+BGP.(\w+):\s+(.+)\s*$`)
 	regex.routes.community = regexp.MustCompile(`^\((\d+),\s*(\d+)\)`)
 	regex.routes.largeCommunity = regexp.MustCompile(`^\((\d+),\s*(\d+),\s*(\d+)\)`)
-	regex.routes.extendedCommunity = regexp.MustCompile(`^\(([^,]+),\s*(\d+),\s*(\d+)\)`)
+	regex.routes.extendedCommunity = regexp.MustCompile(`^\(([^,]+),\s*([^,]+),\s*([^,]+)\)`)
 	regex.routes.origin = regexp.MustCompile(`\([^\(]*\)\s*`)
 	regex.routes.prefixBird2 = regexp.MustCompile(`^([0-9a-f\.\:\/]+)?\s+unicast\s+\[([\w\.:]+)\s+([0-9\-\:\s]+)(?:\s+from\s+([0-9a-f\.\:\/]+))?\]\s+(?:(\*)\s+)?\((\d+)(?:\/\d+)?(?:\/[^\)]*)?\).*$`)
 	regex.routes.gatewayBird2 = regexp.MustCompile(`^\s+via\s+([0-9a-f\.\:]+)\s+on\s+([\w\.]+)\s*$`)
@@ -503,7 +503,7 @@ func parseRoutesExtendedCommunities(groups []string, res Parsed) {
 	for _, community := range regex.routes.origin.FindAllString(groups[2], -1) {
 		if regex.routes.extendedCommunity.MatchString(community) {
 			communityGroups := regex.routes.extendedCommunity.FindStringSubmatch(community)
-			communities = append(communities, []interface{}{communityGroups[1], parseInt(communityGroups[2]), parseInt(communityGroups[3])})
+			communities = append(communities, []interface{}{communityGroups[1], communityGroups[2], communityGroups[3]})
 		}
 	}
 
diff --git a/bird/parser_test.go b/bird/parser_test.go
index a4a81f8..82da052 100644
--- a/bird/parser_test.go
+++ b/bird/parser_test.go
@@ -169,7 +169,8 @@ func runTestForIpv4WithFile(file string, t *testing.T) {
 			{9033, 65666, 9},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"rt", int64(48858), int64(50)},
+			[]interface{}{"rt", "42", "1234"},
+			[]interface{}{"generic", "0x43000000", "0x1"},
 		},
 		metric:    100,
 		localPref: "100",
@@ -190,9 +191,9 @@ func runTestForIpv4WithFile(file string, t *testing.T) {
 			{9033, 65666, 9},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"ro", int64(21414), int64(52001)},
-			[]interface{}{"ro", int64(21414), int64(52004)},
-			[]interface{}{"ro", int64(21414), int64(64515)},
+			[]interface{}{"ro", "21414", "52001"},
+			[]interface{}{"ro", "21414", "52004"},
+			[]interface{}{"ro", "21414", "64515"},
 		},
 		metric:    100,
 		localPref: "100",
@@ -213,9 +214,9 @@ func runTestForIpv4WithFile(file string, t *testing.T) {
 			{9033, 65666, 9},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"ro", int64(21414), int64(52001)},
-			[]interface{}{"ro", int64(21414), int64(52004)},
-			[]interface{}{"ro", int64(21414), int64(64515)},
+			[]interface{}{"ro", "21414", "52001"},
+			[]interface{}{"ro", "21414", "52004"},
+			[]interface{}{"ro", "21414", "64515"},
 		},
 		metric:    100,
 		localPref: "100",
@@ -236,7 +237,8 @@ func runTestForIpv4WithFile(file string, t *testing.T) {
 			{9033, 65666, 9},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"rt", int64(48858), int64(50)},
+			[]interface{}{"rt", "42", "1234"},
+			[]interface{}{"generic", "0x43000000", "0x1"},
 		},
 		metric:    100,
 		localPref: "100",
@@ -332,9 +334,9 @@ func runTestForIpv6WithFile(file string, t *testing.T) {
 			{48821, 0, 2100},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"ro", int64(21414), int64(52001)},
-			[]interface{}{"ro", int64(21414), int64(52004)},
-			[]interface{}{"ro", int64(21414), int64(64515)},
+			[]interface{}{"ro", "21414", "52001"},
+			[]interface{}{"ro", "21414", "52004"},
+			[]interface{}{"ro", "21414", "64515"},
 		},
 		metric:    100,
 		localPref: "500",
@@ -355,9 +357,9 @@ func runTestForIpv6WithFile(file string, t *testing.T) {
 			{48821, 0, 3100},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"ro", int64(21414), int64(52001)},
-			[]interface{}{"ro", int64(21414), int64(52004)},
-			[]interface{}{"ro", int64(21414), int64(64515)},
+			[]interface{}{"ro", "21414", "52001"},
+			[]interface{}{"ro", "21414", "52004"},
+			[]interface{}{"ro", "21414", "64515"},
 		},
 		localPref: "100",
 		metric:    100,
@@ -378,7 +380,7 @@ func runTestForIpv6WithFile(file string, t *testing.T) {
 			{48821, 0, 2100},
 		},
 		extendedCommunities: []interface{}{
-			[]interface{}{"unknown 0x4300", int64(0), int64(1)},
+			[]interface{}{"unknown 0x4300", "0", "1"},
 		},
 		metric:    100,
 		localPref: "5000",
diff --git a/test/routes_bird1_ipv4.sample b/test/routes_bird1_ipv4.sample
index e56318c..814ea82 100644
--- a/test/routes_bird1_ipv4.sample
+++ b/test/routes_bird1_ipv4.sample
@@ -8,7 +8,7 @@ BIRD 1.6.3 ready.
 	BGP.community: (0,5464) (0,8339) (0,8741) (0,8823) (0,12387) (0,13101) (0,16097) (0,16316) (0,20546) (0,20686) (0,20723) (0,21083) (0,21385) (0,24940) (0,25504) (0,28876) (0,29545) (0,30058) (0,31103) (0,31400) (0,39090) (0,39392) (0,39912) (0,42473) (0,43957) (0,44453) (0,47297) (0,47692) (0,48200) (0,50629) (0,51191) (0,51839) (0,51852) (0,54113) (0,56719) (0,57957) (0,60517) (0,60574) (0,61303) (0,62297) (0,62336) (0,62359) (33891,33892) (33891,50673) (48793,48793) (50673,500)
 		(65101,11077) (65102,11000) (65103,724) (65104,150)
 	BGP.large_community: (9033, 65666, 12) (9033, 65666, 9)
-	BGP.ext_community: (rt, 48858, 50)
+	BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 
 200.0.0.0/24       via 1.2.3.15 on eno7 [ID8497_AS1339 2017-06-21 08:17:31] * (100) [AS1339i]
 	Type: BGP unicast univ
 	BGP.origin: IGP
@@ -35,4 +35,4 @@ BIRD 1.6.3 ready.
 	BGP.local_pref: 100
 	BGP.community: (65011,3) (9033,3251)
 	BGP.large_community: (9033, 65666, 12) (9033, 65666, 9)
-	BGP.ext_community: (rt, 48858, 50)
+	BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 
diff --git a/test/routes_bird2_ipv4.sample b/test/routes_bird2_ipv4.sample
index a566130..4bd1f5c 100644
--- a/test/routes_bird2_ipv4.sample
+++ b/test/routes_bird2_ipv4.sample
@@ -9,7 +9,7 @@ BIRD 1.6.3 ready.
 	BGP.community: (0,5464) (0,8339) (0,8741) (0,8823) (0,12387) (0,13101) (0,16097) (0,16316) (0,20546) (0,20686) (0,20723) (0,21083) (0,21385) (0,24940) (0,25504) (0,28876) (0,29545) (0,30058) (0,31103) (0,31400) (0,39090) (0,39392) (0,39912) (0,42473) (0,43957) (0,44453) (0,47297) (0,47692) (0,48200) (0,50629) (0,51191) (0,51839) (0,51852) (0,54113) (0,56719) (0,57957) (0,60517) (0,60574) (0,61303) (0,62297) (0,62336) (0,62359) (33891,33892) (33891,50673) (48793,48793) (50673,500)
 		(65101,11077) (65102,11000) (65103,724) (65104,150)
 	BGP.large_community: (9033, 65666, 12) (9033, 65666, 9)
-	BGP.ext_community: (rt, 48858, 50)
+	BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 
 200.0.0.0/24	unicast [ID8497_AS1339 2017-06-21 08:17:31] * (100) [AS1339i]
 	via 1.2.3.15 on eno7 
 	Type: BGP univ
@@ -39,4 +39,4 @@ BIRD 1.6.3 ready.
 	BGP.local_pref: 100
 	BGP.community: (65011,3) (9033,3251)
 	BGP.large_community: (9033, 65666, 12) (9033, 65666, 9)
-	BGP.ext_community: (rt, 48858, 50)
+	BGP.ext_community: (rt, 42, 1234) (generic, 0x43000000, 0x1) 

From 5f28a6d81ff7fe0718cebbee2cc65f47dcac7232 Mon Sep 17 00:00:00 2001
From: Matthias Hannig <mail@matthiashannig.de>
Date: Sun, 5 May 2019 17:35:19 +0200
Subject: [PATCH 16/20] versionbump

---
 VERSION        | 2 +-
 birdwatcher.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/VERSION b/VERSION
index 89c881b..43ded90 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.12.4
+1.13.5
diff --git a/birdwatcher.go b/birdwatcher.go
index e5d8d54..7c43cdf 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -16,7 +16,7 @@ import (
 )
 
 //go:generate versionize
-var VERSION = "1.13.0"
+var VERSION = "1.13.5"
 
 func isModuleEnabled(module string, modulesEnabled []string) bool {
 	for _, enabled := range modulesEnabled {

From 1c7e953589e0c717cacf186ae7ea3c46ce29ccc0 Mon Sep 17 00:00:00 2001
From: Matthias Hannig <mail@matthiashannig.de>
Date: Sun, 5 May 2019 17:44:32 +0200
Subject: [PATCH 17/20] release notes 2.0.0

---
 CHANGELOG      | 17 +++++++++++++++++
 README.md      | 10 ++++++++++
 VERSION        |  2 +-
 birdwatcher.go |  2 +-
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index f091405..d43fc62 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,21 @@
 
+2.0.0 - 
+
+BREAKING CHANGES AHEAD:
+
+In order to ease the load on the routeservers and even further
+reduce the memory footprint, we decided to move the per peer table
+configuration out of the birdwatcher and into alice.
+
+Please be aware, that you need the newest version of alice.
+
+Everything else:
+
+* Improved cach setup for inmemory and redis
+* Improved housekeeping and memory footprint reduction
+* Performance improvements and other good stuff.
+
+
 1.12.4
 
 * Add the ability to switch between redis and the classic
diff --git a/README.md b/README.md
index a3b9100..971115d 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,13 @@ Running `make linux` will create a Linux executable (by default for
 `amd64`, but that is configurable by providing the `ARCH` argument
 to the Makefile).
 
+
+#### 2.0 Breaking Change
+
+The per peer table configuration is no longer done in the birdwatcher,
+but directly in alice.
+
+
 ### BIRD configuration
 
 Birdwatcher parses the output of birdc and expects (for now)
@@ -153,3 +160,6 @@ Initially developed by Daniel and MC from [Netnod](https://www.netnod.se/) in
 two days at the RIPE 73 IXP Tools Hackathon in Madrid, Spain.
 
 Running bird and parsing the results was added by [Veit Heller](https://github.com/hellerve/) on behalf of [ecix](http://ecix.net/).
+
+With major contributions from: Patrick Seeburger and Benedikt Rudolph on behalf of [DE-CIX](https://de-cix.net).
+
diff --git a/VERSION b/VERSION
index 43ded90..227cea2 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.13.5
+2.0.0
diff --git a/birdwatcher.go b/birdwatcher.go
index 7c43cdf..3314906 100644
--- a/birdwatcher.go
+++ b/birdwatcher.go
@@ -16,7 +16,7 @@ import (
 )
 
 //go:generate versionize
-var VERSION = "1.13.5"
+var VERSION = "2.0.0"
 
 func isModuleEnabled(module string, modulesEnabled []string) bool {
 	for _, enabled := range modulesEnabled {

From 5744c7e0033b5b2fe69861222b987abe8b354cd8 Mon Sep 17 00:00:00 2001
From: Matthias Hannig <mail@matthiashannig.de>
Date: Sun, 5 May 2019 18:56:23 +0200
Subject: [PATCH 18/20] fixed version case

---
 endpoints/utils.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/endpoints/utils.go b/endpoints/utils.go
index ce79857..57ae4b8 100644
--- a/endpoints/utils.go
+++ b/endpoints/utils.go
@@ -17,7 +17,7 @@ type CacheStatus struct {
 }
 
 type APIInfo struct {
-	Version         string
+	Version         string      `json:"version"`
 	ResultFromCache bool        `json:"result_from_cache"`
 	CacheStatus     CacheStatus `json:"cache_status"`
 }

From d9b368edf103d64d8955a052a0c5b0578a3c178a Mon Sep 17 00:00:00 2001
From: Matthias Hannig <mail@matthiashannig.de>
Date: Sun, 5 May 2019 19:08:15 +0200
Subject: [PATCH 19/20] reverted change as this breaks alice. TODO: fix this.

---
 endpoints/utils.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/endpoints/utils.go b/endpoints/utils.go
index 57ae4b8..ce79857 100644
--- a/endpoints/utils.go
+++ b/endpoints/utils.go
@@ -17,7 +17,7 @@ type CacheStatus struct {
 }
 
 type APIInfo struct {
-	Version         string      `json:"version"`
+	Version         string
 	ResultFromCache bool        `json:"result_from_cache"`
 	CacheStatus     CacheStatus `json:"cache_status"`
 }

From 574f68799c08c181de17f76425d4489b575f4c36 Mon Sep 17 00:00:00 2001
From: Matthias Hannig <mail@matthiashannig.de>
Date: Wed, 10 Jul 2019 10:02:11 +0200
Subject: [PATCH 20/20] Update README.md

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 971115d..c1dfeff 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,8 @@ of the config.
 ## Installation
 
 You will need to have go installed to build the package.
+Please make sure your go version is `>= 1.9`.
+
 Running `go get github.com/alice-lg/birdwatcher` will give you
 a binary. You might need to cross-compile it for your
 bird-running servive (`GOARCH` and `GOOS` are your friends).