diff --git a/bird/bird.go b/bird/bird.go index 0fb8195..c606cff 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -16,20 +16,20 @@ import ( type Cache interface { Set(key string, val Parsed, ttl int) error Get(key string) (Parsed, error) + Expire() int } var ClientConf BirdConfig var StatusConf StatusConfig var IPVersion = "4" var cache Cache // stores parsed birdc output +var CacheConf CacheConfig var RateLimitConf struct { sync.RWMutex Conf RateLimitConfig } var RunQueue sync.Map // queue birdc commands before execution -var CacheRedis *RedisCache - var NilParse Parsed = (Parsed)(nil) // special Parsed values var BirdError Parsed = Parsed{"error": "bird unreachable"} @@ -40,8 +40,23 @@ func IsSpecial(ret Parsed) bool { // test for special Parsed values // intitialize the Cache once during setup with either a MemoryCache or // RedisCache implementation. // TODO implement singleton pattern -func InitializeCache(c Cache) { - cache = c +func InitializeCache() { + var err error + if CacheConf.UseRedis { + cache, err = NewRedisCache(CacheConf) + if err != nil { + log.Println("Could not initialize redis cache, falling back to memory cache:", err) + } + } else { // initialize the MemoryCache + cache, err = NewMemoryCache() + if err != nil { + log.Fatal("Could not initialize MemoryCache:", err) + } + } +} + +func ExpireCache() int { + return cache.Expire() } /* Convenience method to make new entries in the cache. diff --git a/bird/memory_cache.go b/bird/memory_cache.go index 89384b4..7475e06 100644 --- a/bird/memory_cache.go +++ b/bird/memory_cache.go @@ -59,3 +59,23 @@ func (c *MemoryCache) Set(key string, val Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (c *MemoryCache) Expire() int { + c.Lock() + + expiredKeys := []string{} + for key, _ := range c.m { + ttl, correct := c.m[key]["ttl"].(time.Time) + if !correct || ttl.Before(time.Now()) { + expiredKeys = append(expiredKeys, key) + } + } + + for _, key := range expiredKeys { + delete(c.m, key) + } + + c.Unlock() + + return len(expiredKeys) +} diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go index b4a6a78..4026fae 100644 --- a/bird/memory_cache_test.go +++ b/bird/memory_cache_test.go @@ -26,6 +26,7 @@ func Test_MemoryCacheAccess(t *testing.T) { t.Error(err) } + cache.Expire() t.Log(parsed) } diff --git a/bird/redis_cache.go b/bird/redis_cache.go index f20f7e8..faf672b 100644 --- a/bird/redis_cache.go +++ b/bird/redis_cache.go @@ -3,6 +3,7 @@ package bird import ( "encoding/json" "errors" + "log" "time" "github.com/go-redis/redis" @@ -74,3 +75,8 @@ func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (self *RedisCache) Expire() int { + log.Printf("Cannot expire entries in RedisCache backend, redis does this automatically") + return 0 +} diff --git a/birdwatcher.go b/birdwatcher.go index 5a3fee1..4e60212 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -178,21 +178,8 @@ func main() { bird.RateLimitConf.Conf = conf.Ratelimit bird.RateLimitConf.Unlock() bird.ParserConf = conf.Parser - - var cache bird.Cache - if conf.Cache.UseRedis { - cache, err = bird.NewRedisCache(conf.Cache) - if err != nil { - log.Fatal("Could not initialize redis cache, falling back to memory cache:", err) - } - } else { // initialize the MemoryCache - cache, err = bird.NewMemoryCache() - if err != nil { - log.Fatal("Could not initialize MemoryCache:", err) - } else { - bird.InitializeCache(cache) - } - } + bird.CacheConf = conf.Cache + bird.InitializeCache() endpoints.Conf = conf.Server @@ -205,6 +192,8 @@ func main() { myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime)) mylogger := &MyLogger{myquerylog} + go Housekeeping(conf.Housekeeping, !(bird.CacheConf.UseRedis)) // expire caches only for MemoryCache + if conf.Server.EnableTLS { if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 { log.Fatalln("You have enabled TLS support but not specified both a .crt and a .key file in the config.") diff --git a/config.go b/config.go index a722a35..12e2325 100644 --- a/config.go +++ b/config.go @@ -17,12 +17,13 @@ import ( type Config struct { Server endpoints.ServerConfig - Ratelimit bird.RateLimitConfig - Status bird.StatusConfig - Bird bird.BirdConfig - Bird6 bird.BirdConfig - Parser bird.ParserConfig - Cache bird.CacheConfig + Ratelimit bird.RateLimitConfig + Status bird.StatusConfig + Bird bird.BirdConfig + Bird6 bird.BirdConfig + Parser bird.ParserConfig + Cache bird.CacheConfig + Housekeeping HousekeepingConfig } // Try to load configfiles as specified in the files diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index 1b7356a..ac83885 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -78,12 +78,14 @@ ttl = 5 # time to live (in minutes) for caching of cli output # Remove fields e.g. interface filter_fields = [] -# Enable support for multitable configurations -per_peer_tables = true -peer_protocol_prefix = 'ID' -pipe_protocol_prefix = 'P' - [cache] -use_redis = false +use_redis = false # if not using redis cache, activate housekeeping to save memory! redis_server = "myredis:6379" redis_db = 0 + +# Housekeeping expires old cache entries (memory cache backend) and performs a GC/SCVG run if configured. +[housekeeping] +# Interval for the housekeeping routine in minutes +interval = 5 +# Try to release memory via a forced GC/SCVG run on every housekeeping run +force_release_memory = true diff --git a/housekeeping.go b/housekeeping.go new file mode 100644 index 0000000..3c38db3 --- /dev/null +++ b/housekeeping.go @@ -0,0 +1,42 @@ +package main + +import ( + "log" + "runtime/debug" + "time" + + "github.com/alice-lg/birdwatcher/bird" +) + +type HousekeepingConfig struct { + Interval int `toml:"interval"` + ForceReleaseMemory bool `toml:"force_release_memory"` +} + +// This is used to run regular housekeeping tasks, currently expiring old +// Cache entries to release memory +func Housekeeping(config HousekeepingConfig, expireCaches bool) { + for { + if config.Interval > 0 { + time.Sleep(time.Duration(config.Interval) * time.Minute) + } else { + time.Sleep(5 * time.Minute) + } + + log.Println("Housekeeping started") + + if (bird.ClientConf.CacheTtl > 0) && expireCaches { + // Expire the caches + log.Println("Expiring MemoryCache") + + count := bird.ExpireCache() + log.Println("Expired", count, "entries (MemoryCache)") + } + + if config.ForceReleaseMemory { + // Trigger a GC and SCVG run + log.Println("Freeing memory") + debug.FreeOSMemory() + } + } +}