Merge branch 'feature/cache-backends' into develop

Add support for various cache backends in anticipation of the merge
with upstream/master that has the redis backend.

bird/bird.go

@@ -3,6 +3,7 @@ package bird
 import (
 	"bytes"
 	"io"
+	"log"
 	"reflect"
 	"strconv"
 	"strings"
@@ -12,31 +13,70 @@ import (
 	"os/exec"
 )
 
+type Cache interface {
+	Set(key string, val Parsed, ttl int) error
+	Get(key string) (Parsed, error)
+}
+
 var ClientConf BirdConfig
 var StatusConf StatusConfig
 var IPVersion = "4"
+var cache Cache // stores parsed birdc output
 var RateLimitConf struct {
 	sync.RWMutex
 	Conf RateLimitConfig
 }
+var RunQueue sync.Map // queue birdc commands before execution
 
-type Cache struct {
-	sync.RWMutex
-	m map[string]Parsed
-}
-
-var ParsedCache = Cache{m: make(map[string]Parsed)}
-var MetaCache = Cache{m: make(map[string]Parsed)}
-
-var NilParse Parsed = (Parsed)(nil)
+var NilParse Parsed = (Parsed)(nil) // special Parsed values
 var BirdError Parsed = Parsed{"error": "bird unreachable"}
 
-var RunQueue sync.Map
-
-func IsSpecial(ret Parsed) bool {
+func IsSpecial(ret Parsed) bool { // test for special Parsed values
 	return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError)
 }
 
+// intitialize the Cache once during setup with either a MemoryCache or
+// RedisCache implementation.
+// TODO implement singleton pattern
+func InitializeCache(c Cache) {
+	cache = c
+}
+
+/* Convenience method to make new entries in the cache.
+ * Abstracts over the specific caching implementation and the ability to set
+ * individual TTL values for entries. Always use the default TTL value from the
+ * config.
+ */
+func toCache(key string, val Parsed) bool {
+	var ttl int
+	if ClientConf.CacheTtl > 0 {
+		ttl = ClientConf.CacheTtl
+	} else {
+		ttl = 5 // five minutes
+	}
+
+	if err := cache.Set(key, val, ttl); err == nil {
+		return true
+	} else {
+		log.Println(err)
+		return false
+	}
+}
+
+/* Convenience method to retrieve entries from the cache.
+ * Abstracts over the specific caching implementations.
+ */
+func fromCache(key string) (Parsed, bool) {
+	val, err := cache.Get(key)
+	if err == nil {
+		return val, true
+	} else {
+		return val, false
+	}
+	//DEBUG log.Println(err)
+
+}
+
 // Determines the key in the cache, where the result of specific functions are stored.
 // Eliminates the need to know what command was executed by that function.
 func GetCacheKey(fname string, fargs ...interface{}) string {
@@ -52,39 +92,6 @@ func GetCacheKey(fname string, fargs ...interface{}) string {
 	return key
 }
 
-func (c *Cache) Store(key string, val Parsed) {
-	var ttl int = 5
-	if ClientConf.CacheTtl > 0 {
-		ttl = ClientConf.CacheTtl
-	}
-	cachedAt := time.Now().UTC()
-	cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute)
-
-	c.Lock()
-	// This is not a really ... clean way of doing this.
-	val["ttl"] = cacheTtl
-	val["cached_at"] = cachedAt
-
-	c.m[key] = val
-	c.Unlock()
-}
-
-func (c *Cache) Get(key string) (Parsed, bool) {
-	c.RLock()
-	val, ok := c.m[key]
-	c.RUnlock()
-	if !ok {
-		return NilParse, false
-	}
-
-	ttl, correct := val["ttl"].(time.Time)
-	if !correct || ttl.Before(time.Now()) {
-		return NilParse, false
-	}
-
-	return val, ok
-}
-
 func Run(args string) (io.Reader, error) {
 	args = "-r " + "show " + args // enforce birdc in restricted mode with "-r" argument
 	argsList := strings.Split(args, " ")
@@ -132,7 +139,7 @@ func checkRateLimit() bool {
 }
 
 func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCache func(*Parsed)) (Parsed, bool) {
-	if val, ok := ParsedCache.Get(cmd); ok {
+	if val, ok := fromCache(cmd); ok {
 		return val, true
 	}
 
@@ -141,7 +148,7 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa
 	if queueGroup, queueLoaded := RunQueue.LoadOrStore(cmd, &wg); queueLoaded {
 		(*queueGroup.(*sync.WaitGroup)).Wait()
 
-		if val, ok := ParsedCache.Get(cmd); ok {
+		if val, ok := fromCache(cmd); ok {
 			return val, true
 		} else {
 			// TODO BirdError should also be signaled somehow
@@ -169,7 +176,7 @@ func RunAndParse(key string, cmd string, parser func(io.Reader) Parsed, updateCa
 		updateCache(&parsed)
 	}
 
-	ParsedCache.Store(cmd, parsed)
+	toCache(cmd, parsed)
 
 	wg.Done()
 
@@ -228,10 +235,10 @@ func Protocols() (Parsed, bool) {
 			metaProtocol["protocols"].(Parsed)["bird_protocol"].(Parsed)[birdProtocol].(Parsed)[protocol] = &parsed
 		}
 
-		MetaCache.Store(GetCacheKey("Protocols"), metaProtocol)
+		toCache(GetCacheKey("metaProtocol"), metaProtocol)
 	}
 
-	res, from_cache := RunAndParse(GetCacheKey("Protocols"), "protocols all", parseProtocols, createMetaCache)
+	res, from_cache := RunAndParse(GetCacheKey("metaProtocol"), "protocols all", parseProtocols, createMetaCache)
 	return res, from_cache
 }
 
@@ -241,7 +248,7 @@ func ProtocolsBgp() (Parsed, bool) {
 		return protocols, from_cache
 	}
 
-	protocolsMeta, _ := MetaCache.Get(GetCacheKey("Protocols"))
+	protocolsMeta, _ := fromCache(GetCacheKey("metaProtocol"))
 	metaProtocol := protocolsMeta["protocols"].(Parsed)
 
 	bgpProtocols := Parsed{}

bird/memory_cache.go

@@ -0,0 +1,61 @@
+package bird
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+// Implementation of the MemoryCache backend.
+
+type MemoryCache struct {
+	sync.RWMutex
+	m map[string]Parsed
+}
+
+func NewMemoryCache() (*MemoryCache, error) {
+	var cache *MemoryCache
+	cache = &MemoryCache{m: make(map[string]Parsed)}
+	return cache, nil
+}
+
+func (c *MemoryCache) Get(key string) (Parsed, error) {
+	c.RLock()
+	val, ok := c.m[key]
+	c.RUnlock()
+	if !ok { // cache miss
+		return NilParse, errors.New("Failed to retrive key '" + key + "' from MemoryCache.")
+	}
+
+	ttl, correct := val["ttl"].(time.Time)
+	if !correct {
+		return NilParse, errors.New("Invalid TTL value for key '" + key + "'")
+	}
+
+	if ttl.Before(time.Now()) {
+		return val, errors.New("TTL expired for key '" + key + "'") // TTL expired
+	} else {
+		return val, nil // cache hit
+	}
+}
+
+func (c *MemoryCache) Set(key string, val Parsed, ttl int) error {
+	switch {
+	case ttl == 0:
+		return nil // do not cache
+	case ttl > 0:
+		cachedAt := time.Now().UTC()
+		cacheTtl := cachedAt.Add(time.Duration(ttl) * time.Minute)
+
+		c.Lock()
+		// This is not a really ... clean way of doing this.
+		val["ttl"] = cacheTtl
+		val["cached_at"] = cachedAt
+
+		c.m[key] = val
+		c.Unlock()
+		return nil
+	default: // ttl negative - invalid
+		return errors.New("Negative TTL value for key" + key)
+	}
+}

bird/memory_cache_test.go

@@ -0,0 +1,74 @@
+package bird
+
+import (
+	"testing"
+)
+
+func Test_MemoryCacheAccess(t *testing.T) {
+
+	cache, err := NewMemoryCache()
+
+	parsed := Parsed{
+		"foo": 23,
+		"bar": 42,
+		"baz": true,
+	}
+
+	t.Log("Setting memory cache...")
+	err = cache.Set("testkey", parsed, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log("Fetching from memory cache...")
+	parsed, err = cache.Get("testkey")
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log(parsed)
+}
+
+func Test_MemoryCacheAccessKeyMissing(t *testing.T) {
+
+	cache, err := NewMemoryCache()
+
+	parsed, err := cache.Get("test_missing_key")
+	if !IsSpecial(parsed) {
+		t.Error(err)
+	}
+	t.Log("Cache error:", err)
+	t.Log(parsed)
+}
+
+func Test_MemoryCacheRoutes(t *testing.T) {
+	f, err := openFile("routes_bird1_ipv4.sample")
+	if err != nil {
+		t.Error(err)
+	}
+	defer f.Close()
+
+	parsed := parseRoutes(f)
+	_, ok := parsed["routes"].([]Parsed)
+	if !ok {
+		t.Fatal("Error getting routes")
+	}
+
+	cache, err := NewMemoryCache()
+
+	err = cache.Set("routes_protocol_test", parsed, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	parsed, err = cache.Get("routes_protocol_test")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	routes, ok := parsed["routes"].([]Parsed)
+	if !ok {
+		t.Error("Error getting routes")
+	}
+	t.Log("Retrieved routes:", len(routes))
+}

birdwatcher.go

@@ -163,6 +163,14 @@ func main() {
 	bird.RateLimitConf.Conf = conf.Ratelimit
 	bird.RateLimitConf.Unlock()
 	bird.ParserConf = conf.Parser
+
+	var cache bird.Cache
+	cache, err = bird.NewMemoryCache() // initialze the MemoryCache
+	if err != nil {
+		log.Fatal("Could not initialize MemoryCache:", err)
+	}
+	bird.InitializeCache(cache)
+
 	endpoints.Conf = conf.Server
 
 	// Make server