Merge redis cache from 'upstream/master' into develop

Leave redis cache the way it is for now. Rework in future commits.

bird/bird.go

@@ -28,6 +28,7 @@ var RateLimitConf struct {
 }
 var RunQueue sync.Map // queue birdc commands before execution
 
+var CacheRedis *RedisCache
 var NilParse Parsed = (Parsed)(nil) // special Parsed values
 var BirdError Parsed = Parsed{"error": "bird unreachable"}
 

bird/config.go

@@ -28,3 +28,10 @@ type RateLimitConfig struct {
 	Max     int `toml:"requests_per_minute"`
 	Enabled bool
 }
+
+type CacheConfig struct {
+	UseRedis      bool   `toml:"use_redis"`
+	RedisServer   string `toml:"redis_server"`
+	RedisPassword string `toml:"redis_password"`
+	RedisDb       int    `toml:"redis_db"`
+}

bird/redis_cache.go

@@ -0,0 +1,53 @@
+package bird
+
+import (
+	"encoding/json"
+	"github.com/go-redis/redis"
+	"time"
+)
+
+type RedisCache struct {
+	client *redis.Client
+}
+
+func NewRedisCache(config CacheConfig) (*RedisCache, error) {
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     config.RedisServer,
+		Password: config.RedisPassword,
+		DB:       config.RedisDb,
+	})
+
+	_, err := client.Ping().Result()
+	if err != nil {
+		return nil, err
+	}
+
+	cache := &RedisCache{
+		client: client,
+	}
+
+	return cache, nil
+}
+
+func (self *RedisCache) Get(key string) (Parsed, error) {
+	data, err := self.client.Get(key).Result()
+	if err != nil {
+		return NilParse, err
+	}
+
+	parsed := Parsed{}
+	err = json.Unmarshal([]byte(data), &parsed)
+
+	return parsed, err
+}
+
+func (self *RedisCache) Set(key string, parsed Parsed) error {
+	payload, err := json.Marshal(parsed)
+	if err != nil {
+		return err
+	}
+
+	_, err = self.client.Set(key, payload, time.Minute*5).Result()
+	return err
+}

bird/redis_cache_test.go

@@ -0,0 +1,100 @@
+package bird
+
+import (
+	"testing"
+)
+
+func Test_RedisCacheAccess(t *testing.T) {
+
+	cache, err := NewRedisCache(CacheConfig{
+		RedisServer: "localhost:6379",
+	})
+
+	if err != nil {
+		t.Log("Redis server not available:", err)
+		t.Log("Skipping redis tests.")
+		return
+	}
+
+	parsed := Parsed{
+		"foo": 23,
+		"bar": 42,
+		"baz": true,
+	}
+
+	t.Log("Setting redis cache...")
+	err = cache.Set("testkey", parsed)
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log("Fetching from redis...")
+	parsed, err = cache.Get("testkey")
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log(parsed)
+}
+
+func Test_RedisCacheAccessKeyMissing(t *testing.T) {
+
+	cache, err := NewRedisCache(CacheConfig{
+		RedisServer: "localhost:6379",
+	})
+
+	if err != nil {
+		t.Log("Redis server not available:", err)
+		t.Log("Skipping redis tests.")
+		return
+	}
+
+	parsed, err := cache.Get("test_missing_key")
+	if err == nil {
+		t.Error(err)
+	}
+	t.Log("Cache error:", err)
+	t.Log(parsed)
+}
+
+func Test_RedisCacheRoutes(t *testing.T) {
+	f, err := openFile("routes_bird1_ipv4.sample")
+	if err != nil {
+		t.Error(err)
+	}
+	defer f.Close()
+
+	parsed := parseRoutes(f)
+	_, ok := parsed["routes"].([]Parsed)
+	if !ok {
+		t.Fatal("Error getting routes")
+	}
+
+	cache, err := NewRedisCache(CacheConfig{
+		RedisServer: "localhost:6379",
+	})
+
+	if err != nil {
+		t.Log("Redis server not available:", err)
+		t.Log("Skipping redis tests.")
+		return
+	}
+
+	err = cache.Set("routes_protocol_test", parsed)
+	if err != nil {
+		t.Error(err)
+	}
+
+	parsed, err = cache.Get("routes_protocol_test")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	routes, ok := parsed["routes"].([]interface{})
+	if !ok {
+		t.Error("Error getting routes")
+	}
+
+	t.Log("Retrieved routes:", len(routes))
+}

birdwatcher.go

@@ -104,6 +104,13 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) {
 		log.Println("        AllowFrom:", strings.Join(conf.Server.AllowFrom, ", "))
 	}
 
+	if conf.Cache.UseRedis {
+		log.Println("    Caching backend: REDIS")
+		log.Println("       Using server:", conf.Cache.RedisServer)
+	} else {
+		log.Println("    Caching backend: MEMORY")
+	}
+
 	log.Println("   ModulesEnabled:")
 	for _, m := range conf.Server.ModulesEnabled {
 		log.Println("       -", m)
@@ -165,11 +172,19 @@ func main() {
 	bird.ParserConf = conf.Parser
 
 	var cache bird.Cache
-	cache, err = bird.NewMemoryCache() // initialze the MemoryCache
-	if err != nil {
-		log.Fatal("Could not initialize MemoryCache:", err)
+	if conf.Cache.UseRedis {
+		bird.CacheRedis, err = bird.NewRedisCache(conf.Cache)
+		if err != nil {
+			log.Fatal("Could not initialize redis cache, falling back to MemoryCache:", err)
+		}
+	} else { // initialze the MemoryCache
+		cache, err = bird.NewMemoryCache()
+		if err != nil {
+			log.Fatal("Could not initialize MemoryCache:", err)
+		} else {
+			bird.InitializeCache(cache)
+		}
 	}
-	bird.InitializeCache(cache)
 
 	endpoints.Conf = conf.Server
 

config.go

@@ -22,6 +22,7 @@ type Config struct {
 	Bird      bird.BirdConfig
 	Bird6     bird.BirdConfig
 	Parser    bird.ParserConfig
+	Cache     bird.CacheConfig
 }
 
 // Try to load configfiles as specified in the files

etc/birdwatcher/birdwatcher.conf

@@ -73,3 +73,8 @@ filter_fields = []
 per_peer_tables = true
 peer_protocol_prefix = 'ID'
 pipe_protocol_prefix = 'P'
+
+[cache]
+use_redis = false 
+redis_server = "myredis:6379"
+redis_db = 0