From 1518f64cd49628a4c46478201dcd8fb0f9f118e7 Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 17:15:17 +0100 Subject: [PATCH 1/7] added redis cache helper --- bird/redis_cache.go | 53 ++++++++++++++++++++++++++++++++++++ bird/redis_cache_test.go | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 bird/redis_cache.go create mode 100644 bird/redis_cache_test.go diff --git a/bird/redis_cache.go b/bird/redis_cache.go new file mode 100644 index 0000000..6335b8b --- /dev/null +++ b/bird/redis_cache.go @@ -0,0 +1,53 @@ +package bird + +import ( + "encoding/json" + "github.com/go-redis/redis" + "time" +) + +type RedisCache struct { + client *redis.Client +} + +func NewRedisCache(config CacheConfig) (*RedisCache, error) { + + client := redis.NewClient(&redis.Options{ + Addr: config.RedisServer, + Password: config.RedisPassword, + DB: config.RedisDb, + }) + + _, err := client.Ping().Result() + if err != nil { + return nil, err + } + + cache := &RedisCache{ + client: client, + } + + return cache, nil +} + +func (self *RedisCache) Get(key string) (Parsed, error) { + data, err := self.client.Get(key).Result() + if err != nil { + return NilParse, err + } + + parsed := Parsed{} + err = json.Unmarshal([]byte(data), &parsed) + + return parsed, err +} + +func (self *RedisCache) Set(key string, parsed Parsed) error { + payload, err := json.Marshal(parsed) + if err != nil { + return err + } + + _, err = self.client.Set(key, payload, time.Minute*5).Result() + return err +} diff --git a/bird/redis_cache_test.go b/bird/redis_cache_test.go new file mode 100644 index 0000000..e54c546 --- /dev/null +++ b/bird/redis_cache_test.go @@ -0,0 +1,58 @@ +package bird + +import ( + "testing" +) + +func Test_RedisCacheAccess(t *testing.T) { + + cache, err := NewRedisCache(CacheConfig{ + RedisServer: "localhost:6379", + }) + + if err != nil { + t.Log("Redis server not available:", err) + t.Log("Skipping redis tests.") + return + } + + parsed := Parsed{ + "foo": 23, + "bar": 42, + "baz": true, + } + + t.Log("Setting redis cache...") + err = cache.Set("testkey", parsed) + if err != nil { + t.Error(err) + } + + t.Log("Fetching from redis...") + parsed, err = cache.Get("testkey") + if err != nil { + t.Error(err) + } + + t.Log(parsed) +} + +func Test_RedisCacheAccessKeyMissing(t *testing.T) { + + cache, err := NewRedisCache(CacheConfig{ + RedisServer: "localhost:6379", + }) + + if err != nil { + t.Log("Redis server not available:", err) + t.Log("Skipping redis tests.") + return + } + + parsed, err := cache.Get("test_missing_key") + if err == nil { + t.Error(err) + } + t.Log("Cache error:", err) + t.Log(parsed) +} From 52f5e9506ad010536df990fd0f35f2237106ad6b Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 17:15:54 +0100 Subject: [PATCH 2/7] use redis caching when available --- bird/bird.go | 59 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/bird/bird.go b/bird/bird.go index 2e7b9b9..f93c182 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -3,6 +3,7 @@ package bird import ( "bytes" "io" + "log" "reflect" "strconv" "strings" @@ -20,11 +21,13 @@ var RateLimitConf struct { Conf RateLimitConfig } -var Cache = struct { +var CacheMap = struct { sync.RWMutex m map[string]Parsed }{m: make(map[string]Parsed)} +var CacheRedis *RedisCache + var NilParse Parsed = (Parsed)(nil) var BirdError Parsed = Parsed{"error": "bird unreachable"} @@ -32,10 +35,10 @@ func isSpecial(ret Parsed) bool { return reflect.DeepEqual(ret, NilParse) || reflect.DeepEqual(ret, BirdError) } -func fromCache(key string) (Parsed, bool) { - Cache.RLock() - val, ok := Cache.m[key] - Cache.RUnlock() +func fromCacheMemory(key string) (Parsed, bool) { + CacheMap.RLock() + val, ok := CacheMap.m[key] + CacheMap.RUnlock() if !ok { return NilParse, false } @@ -48,11 +51,49 @@ func fromCache(key string) (Parsed, bool) { return val, ok } -func toCache(key string, val Parsed) { +func fromCacheRedis(key string) (Parsed, bool) { + val, err := CacheRedis.Get(key) + if err != nil { + return NilParse, false + } + + ttl, correct := val["ttl"].(time.Time) + if !correct || ttl.Before(time.Now()) { + return NilParse, false + } + + return val, true +} + +func fromCache(key string) (Parsed, bool) { + if CacheRedis == nil { + return fromCacheMemory(key) + } + + return fromCacheRedis(key) +} + +func toCacheMemory(key string, val Parsed) { val["ttl"] = time.Now().Add(5 * time.Minute) - Cache.Lock() - Cache.m[key] = val - Cache.Unlock() + CacheMap.Lock() + CacheMap.m[key] = val + CacheMap.Unlock() +} + +func toCacheRedis(key string, val Parsed) { + val["ttl"] = time.Now().Add(5 * time.Minute) + err := CacheRedis.Set(key, val) + if err != nil { + log.Println("Could not set cache for key:", key, "Error:", err) + } +} + +func toCache(key string, val Parsed) { + if CacheRedis == nil { + toCacheMemory(key, val) + } else { + toCacheRedis(key, val) + } } func Run(args string) (io.Reader, error) { From 8552547afae8b26b698d64a9024b1d110f5fb1fa Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 17:16:19 +0100 Subject: [PATCH 3/7] configure redis caching --- config.go | 1 + etc/ecix/birdwatcher.conf | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/config.go b/config.go index d4e5fda..a722a35 100644 --- a/config.go +++ b/config.go @@ -22,6 +22,7 @@ type Config struct { Bird bird.BirdConfig Bird6 bird.BirdConfig Parser bird.ParserConfig + Cache bird.CacheConfig } // Try to load configfiles as specified in the files diff --git a/etc/ecix/birdwatcher.conf b/etc/ecix/birdwatcher.conf index 717ac85..61e3e3e 100644 --- a/etc/ecix/birdwatcher.conf +++ b/etc/ecix/birdwatcher.conf @@ -61,3 +61,7 @@ per_peer_tables = true peer_protocol_prefix = 'ID' pipe_protocol_prefix = 'P' +[cache] +use_redis = true +redis_server = "localhost:6379" +redis_db = 0 From 15f2561c87d7e6c5404ae40f33b373f17ce22370 Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 17:16:35 +0100 Subject: [PATCH 4/7] use redis cache if configured --- bird/config.go | 7 +++++++ birdwatcher.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/bird/config.go b/bird/config.go index bfcdaed..03849a5 100644 --- a/bird/config.go +++ b/bird/config.go @@ -27,3 +27,10 @@ type RateLimitConfig struct { Max int `toml:"requests_per_minute"` Enabled bool } + +type CacheConfig struct { + UseRedis bool `toml:"use_redis"` + RedisServer string `toml:"redis_server"` + RedisPassword string `toml:"redis_password"` + RedisDb int `toml:"redis_db"` +} diff --git a/birdwatcher.go b/birdwatcher.go index 358b3d3..355dab9 100644 --- a/birdwatcher.go +++ b/birdwatcher.go @@ -97,6 +97,13 @@ func PrintServiceInfo(conf *Config, birdConf bird.BirdConfig) { log.Println(" AllowFrom:", strings.Join(conf.Server.AllowFrom, ", ")) } + if conf.Cache.UseRedis { + log.Println(" Caching backend: REDIS") + log.Println(" Using server:", conf.Cache.RedisServer) + } else { + log.Println(" Caching backend: MEMORY") + } + log.Println(" ModulesEnabled:") for _, m := range conf.Server.ModulesEnabled { log.Println(" -", m) @@ -133,6 +140,14 @@ func main() { bird.StatusConf = conf.Status bird.RateLimitConf.Conf = conf.Ratelimit bird.ParserConf = conf.Parser + + if conf.Cache.UseRedis { + bird.CacheRedis, err = bird.NewRedisCache(conf.Cache) + if err != nil { + log.Fatal("Could not initialize redis cache:", err) + } + } + endpoints.Conf = conf.Server // Make server From f0d98a69ef9d054e1beca8c15de08d94b4743802 Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 17:17:52 +0100 Subject: [PATCH 5/7] updated example config --- etc/ecix/birdwatcher.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/ecix/birdwatcher.conf b/etc/ecix/birdwatcher.conf index 61e3e3e..9da5156 100644 --- a/etc/ecix/birdwatcher.conf +++ b/etc/ecix/birdwatcher.conf @@ -62,6 +62,6 @@ peer_protocol_prefix = 'ID' pipe_protocol_prefix = 'P' [cache] -use_redis = true -redis_server = "localhost:6379" +use_redis = false +redis_server = "myredis:6379" redis_db = 0 From 1ee674068864a416acd673ee32248b57fc31fb07 Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 19:01:05 +0100 Subject: [PATCH 6/7] improved caching tests --- bird/redis_cache_test.go | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/bird/redis_cache_test.go b/bird/redis_cache_test.go index e54c546..a8b6103 100644 --- a/bird/redis_cache_test.go +++ b/bird/redis_cache_test.go @@ -56,3 +56,45 @@ func Test_RedisCacheAccessKeyMissing(t *testing.T) { t.Log("Cache error:", err) t.Log(parsed) } + +func Test_RedisCacheRoutes(t *testing.T) { + f, err := openFile("routes_bird1_ipv4.sample") + if err != nil { + t.Error(err) + } + defer f.Close() + + parsed := parseRoutes(f) + _, ok := parsed["routes"].([]Parsed) + if !ok { + t.Fatal("Error getting routes") + } + + cache, err := NewRedisCache(CacheConfig{ + RedisServer: "localhost:6379", + }) + + if err != nil { + t.Log("Redis server not available:", err) + t.Log("Skipping redis tests.") + return + } + + err = cache.Set("routes_protocol_test", parsed) + if err != nil { + t.Error(err) + } + + parsed, err = cache.Get("routes_protocol_test") + if err != nil { + t.Error(err) + return + } + + routes, ok := parsed["routes"].([]interface{}) + if !ok { + t.Error("Error getting routes") + } + + t.Log("Retrieved routes:", len(routes)) +} From 837c97da13db14f77d29038ea831d722f49e4471 Mon Sep 17 00:00:00 2001 From: Matthias Hannig Date: Thu, 24 Jan 2019 19:01:41 +0100 Subject: [PATCH 7/7] prefix cache key for use with birdc and birdc6 --- bird/bird.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bird/bird.go b/bird/bird.go index f93c182..32424d5 100644 --- a/bird/bird.go +++ b/bird/bird.go @@ -52,6 +52,7 @@ func fromCacheMemory(key string) (Parsed, bool) { } func fromCacheRedis(key string) (Parsed, bool) { + key = "B" + IPVersion + "_" + key val, err := CacheRedis.Get(key) if err != nil { return NilParse, false @@ -81,6 +82,7 @@ func toCacheMemory(key string, val Parsed) { } func toCacheRedis(key string, val Parsed) { + key = "B" + IPVersion + "_" + key val["ttl"] = time.Now().Add(5 * time.Minute) err := CacheRedis.Set(key, val) if err != nil {