From e6ed0cb9012f00b3eb43da63aa4628b7293c4fbe Mon Sep 17 00:00:00 2001 From: Benedikt Rudolph Date: Wed, 20 Feb 2019 11:17:13 +0100 Subject: [PATCH] Refactor housekeeping and memory cache * run Expire() only on MemoryCaches * make initialization of the cache look pretty --- bird/bird.go | 43 ++++++++++++++------------------ bird/memory_cache.go | 20 +++++++++++++++ bird/memory_cache_test.go | 1 + bird/redis_cache.go | 6 +++++ birdwatcher.go | 19 +++----------- etc/birdwatcher/birdwatcher.conf | 3 ++- housekeeping.go | 16 +++++------- 7 files changed, 57 insertions(+), 51 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 5b584fa..c606cff 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -16,20 +16,20 @@ import ( type Cache interface { Set(key string, val Parsed, ttl int) error Get(key string) (Parsed, error) + Expire() int } var ClientConf BirdConfig var StatusConf StatusConfig var IPVersion = "4" var cache Cache // stores parsed birdc output +var CacheConf CacheConfig var RateLimitConf struct { sync.RWMutex Conf RateLimitConfig } var RunQueue sync.Map // queue birdc commands before execution -var CacheRedis *RedisCache - var NilParse Parsed = (Parsed)(nil) // special Parsed values var BirdError Parsed = Parsed{"error": "bird unreachable"} @@ -40,8 +40,23 @@ func IsSpecial(ret Parsed) bool { // test for special Parsed values // intitialize the Cache once during setup with either a MemoryCache or // RedisCache implementation. // TODO implement singleton pattern -func InitializeCache(c Cache) { - cache = c +func InitializeCache() { + var err error + if CacheConf.UseRedis { + cache, err = NewRedisCache(CacheConf) + if err != nil { + log.Println("Could not initialize redis cache, falling back to memory cache:", err) + } + } else { // initialize the MemoryCache + cache, err = NewMemoryCache() + if err != nil { + log.Fatal("Could not initialize MemoryCache:", err) + } + } +} + +func ExpireCache() int { + return cache.Expire() } /* Convenience method to make new entries in the cache. @@ -98,26 +113,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string { return key } -func (c *MemoryCache) Expire() int { - c.Lock() - - expiredKeys := []string{} - for key, _ := range c.m { - ttl, correct := c.m[key]["ttl"].(time.Time) - if !correct || ttl.Before(time.Now()) { - expiredKeys = append(expiredKeys, key) - } - } - - for _, key := range expiredKeys { - delete(c.m, key) - } - - c.Unlock() - - return len(expiredKeys) -} - func Run(args string) (io.Reader, error) { args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument argsList := strings.Split(args, " ") diff --git a/bird/memory_cache.go b/bird/memory_cache.go index 89384b4..7475e06 100644 --- a/bird/memory_cache.go +++ b/bird/memory_cache.go @@ -59,3 +59,23 @@ func (c *MemoryCache) Set(key string, val Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (c *MemoryCache) Expire() int { + c.Lock() + + expiredKeys := []string{} + for key, _ := range c.m { + ttl, correct := c.m[key]["ttl"].(time.Time) + if !correct || ttl.Before(time.Now()) { + expiredKeys = append(expiredKeys, key) + } + } + + for _, key := range expiredKeys { + delete(c.m, key) + } + + c.Unlock() + + return len(expiredKeys) +} diff --git a/bird/memory_cache_test.go b/bird/memory_cache_test.go index b4a6a78..4026fae 100644 --- a/bird/memory_cache_test.go +++ b/bird/memory_cache_test.go @@ -26,6 +26,7 @@ func Test_MemoryCacheAccess(t *testing.T) { t.Error(err) } + cache.Expire() t.Log(parsed) } diff --git a/bird/redis_cache.go b/bird/redis_cache.go index f20f7e8..faf672b 100644 --- a/bird/redis_cache.go +++ b/bird/redis_cache.go @@ -3,6 +3,7 @@ package bird import ( "encoding/json" "errors" + "log" "time" "github.com/go-redis/redis" @@ -74,3 +75,8 @@ func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error { return errors.New("Negative TTL value for key" + key) } } + +func (self *RedisCache) Expire() int { + log.Printf("Cannot expire entries in RedisCache backend, redis does this automatically") + return 0 +} diff --git a/birdwatcher.go b/birdwatcher.go index a1ebaf5..4e60212 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -178,21 +178,8 @@ func main() { bird.RateLimitConf.Conf = conf.Ratelimit bird.RateLimitConf.Unlock() bird.ParserConf = conf.Parser - - var cache bird.Cache - if conf.Cache.UseRedis { - cache, err = bird.NewRedisCache(conf.Cache) - if err != nil { - log.Fatal("Could not initialize redis cache, falling back to memory cache:", err) - } - } else { // initialize the MemoryCache - cache, err = bird.NewMemoryCache() - if err != nil { - log.Fatal("Could not initialize MemoryCache:", err) - } else { - bird.InitializeCache(cache) - } - } + bird.CacheConf = conf.Cache + bird.InitializeCache() endpoints.Conf = conf.Server @@ -205,7 +192,7 @@ func main() { myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime)) mylogger := &MyLogger{myquerylog} - go Housekeeping(conf.Housekeeping) + go Housekeeping(conf.Housekeeping, !(bird.CacheConf.UseRedis)) // expire caches only for MemoryCache if conf.Server.EnableTLS { if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 { diff --git a/etc/birdwatcher/birdwatcher.conf b/etc/birdwatcher/birdwatcher.conf index cea27d8..ac83885 100755 --- a/etc/birdwatcher/birdwatcher.conf +++ b/etc/birdwatcher/birdwatcher.conf @@ -79,10 +79,11 @@ ttl = 5 # time to live (in minutes) for caching of cli output filter_fields = [] [cache] -use_redis = false +use_redis = false # if not using redis cache, activate housekeeping to save memory! redis_server = "myredis:6379" redis_db = 0 +# Housekeeping expires old cache entries (memory cache backend) and performs a GC/SCVG run if configured. [housekeeping] # Interval for the housekeeping routine in minutes interval = 5 diff --git a/housekeeping.go b/housekeeping.go index b194b39..3c38db3 100644 --- a/housekeeping.go +++ b/housekeeping.go @@ -1,14 +1,13 @@ package main import ( - "time" "log" "runtime/debug" + "time" "github.com/alice-lg/birdwatcher/bird" ) - type HousekeepingConfig struct { Interval int `toml:"interval"` ForceReleaseMemory bool `toml:"force_release_memory"` @@ -16,7 +15,7 @@ type HousekeepingConfig struct { // This is used to run regular housekeeping tasks, currently expiring old // Cache entries to release memory -func Housekeeping(config HousekeepingConfig) { +func Housekeeping(config HousekeepingConfig, expireCaches bool) { for { if config.Interval > 0 { time.Sleep(time.Duration(config.Interval) * time.Minute) @@ -26,15 +25,12 @@ func Housekeeping(config HousekeepingConfig) { log.Println("Housekeeping started") - if bird.ClientConf.CacheTtl > 0 { + if (bird.ClientConf.CacheTtl > 0) && expireCaches { // Expire the caches - log.Println("Expiring caches") + log.Println("Expiring MemoryCache") - count := bird.ParsedCache.Expire() - log.Println("Expired", count, "entries (ParsedCache)") - - count = bird.MetaCache.Expire() - log.Println("Expired", count, "entries (MetaCache)") + count := bird.ExpireCache() + log.Println("Expired", count, "entries (MemoryCache)") } if config.ForceReleaseMemory {