Mirrors/birdwatcher
1
0
Fork 0
mirror of https://github.com/alice-lg/birdwatcher.git synced 2025-03-09 00:00:05 +01:00
Code Issues Releases Wiki Activity

Merge branch 'develop' into master for 1.11.5

Browse source 
Changes since last version:
* Fix testcase for /protocols/bgp
* New birdc query 'RoutesFilteredCount()'
* Queue for birdc commands, prevents running the same birdc command
  multiple times in parallel on concurrent API requests.
* Cache: redesign cache structure, separation of Parsed and Meta cache
  * allows independent cache access
  * implement convenience methods for interaction with the cache
This commit is contained in:
Benedikt Rudolph 2019-02-22 19:05:47 +01:00
commit 85eac133e0
3 changed files with 182 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

270
bird/bird.go
View file

@ -20,22 +20,59 @@ var RateLimitConf struct {
Conf RateLimitConfig
}
var Cache = struct {
type Cache struct {
sync.RWMutex
m map[string]Parsed
}{m: make(map[string]Parsed)}
}
var ParsedCache = Cache{m: make(map[string]Parsed)}
var MetaCache = Cache{m: make(map[string]Parsed)}
var NilParse Parsed = (Parsed)(nil)
var BirdError Parsed = Parsed{"error": "bird unreachable"}
var RunQueue sync.Map
func IsSpecial(ret Parsed) bool {
return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError)
}
func fromCache(key string) (Parsed, bool) {
Cache.RLock()
val, ok := Cache.m[key]
Cache.RUnlock()
// Determines the key in the cache, where the result of specific functions are stored.
// Eliminates the need to know what command was executed by that function.
func GetCacheKey(fname string, fargs ...interface{}) string {
key := strings.ToLower(fname)
for _, arg := range fargs {
switch arg.(type) {
case string:
key += "_" + strings.ToLower(arg.(string))
}
}
return key
}
func (c *Cache) Store(key string, val Parsed) {
var ttl int = 5
if ClientConf.CacheTtl > 0 {
ttl = ClientConf.CacheTtl
}
cachedAt := time.Now().UTC()
cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute)
c.Lock()
// This is not a really ... clean way of doing this.
val["ttl"] = cacheTtl
val["cached_at"] = cachedAt
c.m[key] = val
c.Unlock()
}
func (c *Cache) Get(key string) (Parsed, bool) {
c.RLock()
val, ok := c.m[key]
c.RUnlock()
if !ok {
return NilParse, false
}
@ -48,23 +85,6 @@ func fromCache(key string) (Parsed, bool) {
return val, ok
}
func toCache(key string, val Parsed) {
var ttl int = 5
if ClientConf.CacheTtl > 0 {
ttl = ClientConf.CacheTtl
}
cachedAt := time.Now().UTC()
cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute)
// This is not a really ... clean way of doing this.
val["ttl"] = cacheTtl
val["cached_at"] = cachedAt
Cache.Lock()
Cache.m[key] = val
Cache.Unlock()
}
func Run(args string) (io.Reader, error) {
args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument
argsList := strings.Split(args, " ")
@ -111,69 +131,108 @@ func checkRateLimit() bool {
return true
}
func RunAndParse(cmd string, parser func(io.Reader) Parsed) (Parsed, bool) {
if val, ok := fromCache(cmd); ok {
func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) {
if val, ok := ParsedCache.Get(cmd); ok {
return val, true
}
var wg sync.WaitGroup
wg.Add(1)
if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded {
(*queueGroup.(*sync.WaitGroup)).Wait()
if val, ok := ParsedCache.Get(cmd); ok {
return val, true
} else {
// TODO BirdError should also be signaled somehow
return NilParse, false
}
}
if !checkRateLimit() {
wg.Done()
RunQueue.Delete(cmd)
return NilParse, false
}
out, err := Run(cmd)
if err != nil {
// ignore errors for now
wg.Done()
RunQueue.Delete(cmd)
return BirdError, false
}
parsed := parser(out)
toCache(cmd, parsed)
if updateCache != nil {
updateCache(&parsed)
}
ParsedCache.Store(cmd, parsed)
wg.Done()
RunQueue.Delete(cmd)
return parsed, false
}
func Status() (Parsed, bool) {
birdStatus, from_cache := RunAndParse("status", parseStatus)
if IsSpecial(birdStatus) {
return birdStatus, from_cache
updateParsedCache := func(p *Parsed) {
status := (*p)["status"].(Parsed)
// Last Reconfig Timestamp source:
var lastReconfig string
switch StatusConf.ReconfigTimestampSource {
case "bird":
lastReconfig = status["last_reconfig"].(string)
break
case "config_modified":
lastReconfig = lastReconfigTimestampFromFileStat(
ClientConf.ConfigFilename,
)
case "config_regex":
lastReconfig = lastReconfigTimestampFromFileContent(
ClientConf.ConfigFilename,
StatusConf.ReconfigTimestampMatch,
)
}
status["last_reconfig"] = lastReconfig
// Filter fields
for _, field := range StatusConf.FilterFields {
status[field] = nil
}
}
if from_cache {
return birdStatus, from_cache
}
status := birdStatus["status"].(Parsed)
// Last Reconfig Timestamp source:
var lastReconfig string
switch StatusConf.ReconfigTimestampSource {
case "bird":
lastReconfig = status["last_reconfig"].(string)
break
case "config_modified":
lastReconfig = lastReconfigTimestampFromFileStat(
ClientConf.ConfigFilename,
)
case "config_regex":
lastReconfig = lastReconfigTimestampFromFileContent(
ClientConf.ConfigFilename,
StatusConf.ReconfigTimestampMatch,
)
}
status["last_reconfig"] = lastReconfig
// Filter fields
for _, field := range StatusConf.FilterFields {
status[field] = nil
}
birdStatus["status"] = status
birdStatus, from_cache := RunAndParse(GetCacheKey("Status"), "status", parseStatus, updateParsedCache)
return birdStatus, from_cache
}
func Protocols() (Parsed, bool) {
return RunAndParse("protocols all", parseProtocols)
createMetaCache := func(p *Parsed) {
metaProtocol := Parsed{"protocols": Parsed{"bird_protocol": Parsed{}}}
for key, _ := range (*p)["protocols"].(Parsed) {
parsed := (*p)["protocols"].(Parsed)[key].(Parsed)
protocol := parsed["protocol"].(string)
birdProtocol := parsed["bird_protocol"].(string)
// Check if the structure for the current birdProtocol already exists inside the metaProtocol cache, if not create it (BGP|Pipe|etc)
if _, ok := metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol]; !ok {
metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol] = Parsed{}
}
metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed
}
MetaCache.Store(GetCacheKey("Protocols"), metaProtocol)
}
res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache)
return res, from_cache
}
func ProtocolsBgp() (Parsed, bool) {
@ -182,12 +241,13 @@ func ProtocolsBgp() (Parsed, bool) {
return protocols, from_cache
}
protocolsMeta, _ := MetaCache.Get(GetCacheKey("Protocols"))
metaProtocol := protocolsMeta["protocols"].(Parsed)
bgpProtocols := Parsed{}
for key, protocol := range protocols["protocols"].(Parsed) {
if protocol.(Parsed)["bird_protocol"] == "BGP" {
bgpProtocols[key] = protocol
}
for key, protocol := range metaProtocol["bird_protocol"].(Parsed)["BGP"].(Parsed) {
bgpProtocols[key] = *(protocol.(*Parsed))
}
return Parsed{"protocols": bgpProtocols,
@ -196,82 +256,100 @@ func ProtocolsBgp() (Parsed, bool) {
}
func Symbols() (Parsed, bool) {
return RunAndParse("symbols", parseSymbols)
return RunAndParse(GetCacheKey("Symbols"), "symbols", parseSymbols, nil)
}
func RoutesPrefixed(prefix string) (Parsed, bool) {
cmd := routeQueryForChannel("route " + prefix + " all")
return RunAndParse(cmd, parseRoutes)
return RunAndParse(GetCacheKey("RoutesPrefixed", prefix), cmd, parseRoutes, nil)
}
func RoutesProto(protocol string) (Parsed, bool) {
cmd := routeQueryForChannel("route all protocol " + protocol)
return RunAndParse(cmd, parseRoutes)
return RunAndParse(GetCacheKey("RoutesProto", protocol), cmd, parseRoutes, nil)
}
func RoutesProtoCount(protocol string) (Parsed, bool) {
cmd := routeQueryForChannel("route protocol "+protocol) + " count"
return RunAndParse(cmd, parseRoutesCount)
return RunAndParse(GetCacheKey("RoutesProtoCount", protocol), cmd, parseRoutesCount, nil)
}
func RoutesProtoPrimaryCount(protocol string) (Parsed, bool) {
cmd := routeQueryForChannel("route primary protocol "+protocol) + " count"
return RunAndParse(cmd, parseRoutesCount)
return RunAndParse(GetCacheKey("RoutesProtoPrimaryCount", protocol), cmd, parseRoutesCount, nil)
}
func PipeRoutesFilteredCount(pipe string, table string, neighborAddress string) (Parsed, bool) {
cmd := "route table " + table + " noexport " + pipe + " where from=" + neighborAddress + " count"
return RunAndParse(GetCacheKey("PipeRoutesFilteredCount", table, pipe, neighborAddress), cmd, parseRoutesCount, nil)
}
func PipeRoutesFiltered(pipe string, table string) (Parsed, bool) {
cmd := routeQueryForChannel("route table '" + table + "' noexport '" + pipe + "' all")
return RunAndParse(GetCacheKey("PipeRoutesFiltered", table, pipe), cmd, parseRoutes, nil)
}
func RoutesFiltered(protocol string) (Parsed, bool) {
cmd := routeQueryForChannel("route all filtered protocol " + protocol)
return RunAndParse(cmd, parseRoutes)
return RunAndParse(GetCacheKey("RoutesFiltered", protocol), cmd, parseRoutes, nil)
}
func RoutesExport(protocol string) (Parsed, bool) {
cmd := routeQueryForChannel("route all export " + protocol)
return RunAndParse(cmd, parseRoutes)
return RunAndParse(GetCacheKey("RoutesExport", protocol), cmd, parseRoutes, nil)
}
func RoutesNoExport(protocol string) (Parsed, bool) {
// In case we have a multi table setup, we have to query
// the pipe protocol.
if ParserConf.PerPeerTables &&
strings.HasPrefix(protocol, ParserConf.PeerProtocolPrefix) {
protocolsRes, from_cache := ProtocolsBgp()
if IsSpecial(protocolsRes) {
return protocolsRes, from_cache
}
if _, ok := protocolsRes["protocols"].(Parsed)[protocol]; !ok {
return NilParse, false
}
// Replace prefix
protocol = ParserConf.PipeProtocolPrefix +
protocol[len(ParserConf.PeerProtocolPrefix):]
}
cmd := routeQueryForChannel("route all noexport " + protocol)
return RunAndParse(cmd, parseRoutes)
return RunAndParse(GetCacheKey("RoutesNoExport", protocol), cmd, parseRoutes, nil)
}
func RoutesExportCount(protocol string) (Parsed, bool) {
cmd := routeQueryForChannel("route export "+protocol) + " count"
return RunAndParse(cmd, parseRoutesCount)
return RunAndParse(GetCacheKey("RoutesExportCount", protocol), cmd, parseRoutesCount, nil)
}
func RoutesTable(table string) (Parsed, bool) {
return RunAndParse("route table "+table+" all", parseRoutes)
return RunAndParse(GetCacheKey("RoutesTable", table), "route table "+table+" all", parseRoutes, nil)
}
func RoutesTableCount(table string) (Parsed, bool) {
return RunAndParse("route table "+table+" count", parseRoutesCount)
return RunAndParse(GetCacheKey("RoutesTableCount", table), "route table "+table+" count", parseRoutesCount, nil)
}
func RoutesLookupTable(net string, table string) (Parsed, bool) {
return RunAndParse("route for "+net+" table "+table+" all", parseRoutes)
return RunAndParse(GetCacheKey("RoutesLookupTable", net, table), "route for "+net+" table "+table+" all", parseRoutes, nil)
}
func RoutesLookupProtocol(net string, protocol string) (Parsed, bool) {
return RunAndParse("route for "+net+" protocol "+protocol+" all", parseRoutes)
return RunAndParse(GetCacheKey("RoutesLookupProtocol", net, protocol), "route for "+net+" protocol "+protocol+" all", parseRoutes, nil)
}
func RoutesPeer(peer string) (Parsed, bool) {
cmd := routeQueryForChannel("route export " + peer)
return RunAndParse(cmd, parseRoutes)
return RunAndParse(GetCacheKey("RoutesPeer", peer), cmd, parseRoutes, nil)
}
func RoutesDump() (Parsed, bool) {
// TODO insert hook to update the cache with the route count information
if ParserConf.PerPeerTables {
return RoutesDumpPerPeerTable()
}
@ -280,8 +358,14 @@ func RoutesDump() (Parsed, bool) {
}
func RoutesDumpSingleTable() (Parsed, bool) {
importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes)
filteredRes, _ := RunAndParse(routeQueryForChannel("route all filtered"), parseRoutes)
importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil)
if IsSpecial(importedRes) {
return importedRes, cached
}
filteredRes, cached := RunAndParse(GetCacheKey("RoutesDumpSingleTable", "filtered"), routeQueryForChannel("route all filtered"), parseRoutes, nil)
if IsSpecial(filteredRes) {
return filteredRes, cached
}
imported := importedRes["routes"]
filtered := filteredRes["routes"]
@ -295,12 +379,18 @@ func RoutesDumpSingleTable() (Parsed, bool) {
}
func RoutesDumpPerPeerTable() (Parsed, bool) {
importedRes, cached := RunAndParse(routeQueryForChannel("route all"), parseRoutes)
importedRes, cached := RunAndParse(GetCacheKey("RoutesDumpPerPeerTable", "imported"), routeQueryForChannel("route all"), parseRoutes, nil)
if IsSpecial(importedRes) {
return importedRes, cached
}
imported := importedRes["routes"]
filtered := []Parsed{}
// Get protocols with filtered routes
protocolsRes, _ := ProtocolsBgp()
protocolsRes, cached := ProtocolsBgp()
if IsSpecial(protocolsRes) {
return protocolsRes, cached
}
protocols := protocolsRes["protocols"].(Parsed)
for protocol, details := range protocols {
@ -315,11 +405,7 @@ func RoutesDumpPerPeerTable() (Parsed, bool) {
continue // nothing to do here.
}
// Lookup filtered routes
pfilteredRes, from_cache := RoutesFiltered(protocol)
if reflect.DeepEqual(pfilteredRes, BirdError) {
return pfilteredRes, from_cache
}
pfilteredRes, _ := RoutesFiltered(protocol)
pfiltered, ok := pfilteredRes["routes"].([]Parsed)
if !ok {
continue // something went wrong...
@ -338,6 +424,10 @@ func RoutesDumpPerPeerTable() (Parsed, bool) {
func routeQueryForChannel(cmd string) string {
status, _ := Status()
if IsSpecial(status) {
return cmd
}
birdStatus, ok := status["status"].(Parsed)
if !ok {
return cmd

2
endpoints/endpoint.go
View file

@ -54,8 +54,8 @@ func Endpoint(wrapped endpoint) httprouter.Handle {
}
res := make(map[string]interface{})
ret, from_cache := wrapped(r, ps)
if reflect.DeepEqual(ret, bird.NilParse) {
w.WriteHeader(http.StatusTooManyRequests)
return

1
test/protocols_bgp_pipe.sample
View file

@ -39,4 +39,5 @@ R194_42 BGP T65001_nada_co_ripe up 2018-05-31 15:38:40 Established
Route limit: 710/200000
Hold timer: 151/180
Keepalive timer: 43/60
Last error: Socket: Connection closed