Refactor housekeeping and memory cache

* run Expire() only on MemoryCaches
* make initialization of the cache look pretty

bird/bird.go

@@ -16,20 +16,20 @@ import (
 type Cache interface {
 	Set(key string, val Parsed, ttl int) error
 	Get(key string) (Parsed, error)
+	Expire() int
 }
 
 var ClientConf BirdConfig
 var StatusConf StatusConfig
 var IPVersion = "4"
 var cache Cache // stores parsed birdc output
+var CacheConf CacheConfig
 var RateLimitConf struct {
 	sync.RWMutex
 	Conf RateLimitConfig
 }
 var RunQueue sync.Map // queue birdc commands before execution
 
-var CacheRedis *RedisCache
-
 var NilParse Parsed = (Parsed)(nil) // special Parsed values
 var BirdError Parsed = Parsed{"error": "bird unreachable"}
 
@@ -40,8 +40,23 @@ func IsSpecial(ret Parsed) bool { // test for special Parsed values
 // intitialize the Cache once during setup with either a MemoryCache or
 // RedisCache implementation.
 // TODO implement singleton pattern
-func InitializeCache(c Cache) {
-	cache = c
+func InitializeCache() {
+	var err error
+	if CacheConf.UseRedis {
+		cache, err = NewRedisCache(CacheConf)
+		if err != nil {
+			log.Println("Could not initialize redis cache, falling back to memory cache:", err)
+		}
+	} else { // initialize the MemoryCache
+		cache, err = NewMemoryCache()
+		if err != nil {
+			log.Fatal("Could not initialize MemoryCache:", err)
+		}
+	}
+}
+
+func ExpireCache() int {
+	return cache.Expire()
 }
 
 /* Convenience method to make new entries in the cache.
@@ -98,26 +113,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string {
 	return key
 }
 
-func (c *MemoryCache) Expire() int {
-	c.Lock()
-
-	expiredKeys := []string{}
-	for key, _ := range c.m {
-		ttl, correct := c.m[key]["ttl"].(time.Time)
-		if !correct || ttl.Before(time.Now()) {
-			expiredKeys = append(expiredKeys, key)
-		}
-	}
-
-	for _, key := range expiredKeys {
-		delete(c.m, key)
-	}
-
-	c.Unlock()
-
-	return len(expiredKeys)
-}
-
 func Run(args string) (io.Reader, error) {
 	args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument
 	argsList := strings.Split(args, " ")

bird/memory_cache.go

@@ -59,3 +59,23 @@ func (c *MemoryCache) Set(key string, val Parsed, ttl int) error {
 		return errors.New("Negative TTL value for key" + key)
 	}
 }
+
+func (c *MemoryCache) Expire() int {
+	c.Lock()
+
+	expiredKeys := []string{}
+	for key, _ := range c.m {
+		ttl, correct := c.m[key]["ttl"].(time.Time)
+		if !correct || ttl.Before(time.Now()) {
+			expiredKeys = append(expiredKeys, key)
+		}
+	}
+
+	for _, key := range expiredKeys {
+		delete(c.m, key)
+	}
+
+	c.Unlock()
+
+	return len(expiredKeys)
+}

bird/memory_cache_test.go

@@ -26,6 +26,7 @@ func Test_MemoryCacheAccess(t *testing.T) {
 		t.Error(err)
 	}
 
+	cache.Expire()
 	t.Log(parsed)
 }
 

bird/redis_cache.go

@@ -3,6 +3,7 @@ package bird
 import (
 	"encoding/json"
 	"errors"
+	"log"
 	"time"
 
 	"github.com/go-redis/redis"
@@ -74,3 +75,8 @@ func (self *RedisCache) Set(key string, parsed Parsed, ttl int) error {
 		return errors.New("Negative TTL value for key" + key)
 	}
 }
+
+func (self *RedisCache) Expire() int {
+	log.Printf("Cannot expire entries in RedisCache backend, redis does this automatically")
+	return 0
+}

birdwatcher.go

@@ -178,21 +178,8 @@ func main() {
 	bird.RateLimitConf.Conf = conf.Ratelimit
 	bird.RateLimitConf.Unlock()
 	bird.ParserConf = conf.Parser
-
-	var cache bird.Cache
-	if conf.Cache.UseRedis {
-		cache, err = bird.NewRedisCache(conf.Cache)
-		if err != nil {
-			log.Fatal("Could not initialize redis cache, falling back to memory cache:", err)
-		}
-	} else { // initialize the MemoryCache
-		cache, err = bird.NewMemoryCache()
-		if err != nil {
-			log.Fatal("Could not initialize MemoryCache:", err)
-		} else {
-			bird.InitializeCache(cache)
-		}
-	}
+	bird.CacheConf = conf.Cache
+	bird.InitializeCache()
 
 	endpoints.Conf = conf.Server
 
@@ -205,7 +192,7 @@ func main() {
 	myquerylog.SetFlags(myquerylog.Flags() &^ (log.Ldate | log.Ltime))
 	mylogger := &MyLogger{myquerylog}
 
-	go Housekeeping(conf.Housekeeping)
+	go Housekeeping(conf.Housekeeping, !(bird.CacheConf.UseRedis)) // expire caches only for MemoryCache
 
 	if conf.Server.EnableTLS {
 		if len(conf.Server.Crt) == 0 || len(conf.Server.Key) == 0 {

etc/birdwatcher/birdwatcher.conf

@@ -79,10 +79,11 @@ ttl = 5 # time to live (in minutes) for caching of cli output
 filter_fields = []
 
 [cache]
-use_redis = false 
+use_redis = false # if not using redis cache, activate housekeeping to save memory! 
 redis_server = "myredis:6379"
 redis_db = 0
 
+# Housekeeping expires old cache entries (memory cache backend) and performs a GC/SCVG run if configured.
 [housekeeping]
 # Interval for the housekeeping routine in minutes
 interval = 5

housekeeping.go

@@ -1,14 +1,13 @@
 package main
 
 import (
-	"time"
 	"log"
 	"runtime/debug"
+	"time"
 
 	"github.com/alice-lg/birdwatcher/bird"
 )
 
-
 type HousekeepingConfig struct {
 	Interval           int  `toml:"interval"`
 	ForceReleaseMemory bool `toml:"force_release_memory"`
@@ -16,7 +15,7 @@ type HousekeepingConfig struct {
 
 // This is used to run regular housekeeping tasks, currently expiring old
 // Cache entries to release memory
-func Housekeeping(config HousekeepingConfig) {
+func Housekeeping(config HousekeepingConfig, expireCaches bool) {
 	for {
 		if config.Interval > 0 {
 			time.Sleep(time.Duration(config.Interval) * time.Minute)
@@ -26,15 +25,12 @@ func Housekeeping(config HousekeepingConfig) {
 
 		log.Println("Housekeeping started")
 
-		if bird.ClientConf.CacheTtl > 0 {
+		if (bird.ClientConf.CacheTtl > 0) && expireCaches {
 			// Expire the caches
-			log.Println("Expiring caches")
+			log.Println("Expiring MemoryCache")
 
-			count := bird.ParsedCache.Expire()
-			log.Println("Expired", count, "entries (ParsedCache)")
-
-			count = bird.MetaCache.Expire()
-			log.Println("Expired", count, "entries (MetaCache)")
+			count := bird.ExpireCache()
+			log.Println("Expired", count, "entries (MemoryCache)")
 		}
 
 		if config.ForceReleaseMemory {