From b2efc9f51c5f2ccff75b0e367302294ca01064dc Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 8 Sep 2021 17:55:12 -0300 Subject: [PATCH] Make redis conn timeout and keep alive configurable (#1968) * Make redis conn timeout and keep alive configurable * Document new configs * Correct config name --- changes/redis-timeout-configurable | 1 + cmd/fleet/serve.go | 9 +++++++- docs/2-Deploying/2-Configuration.md | 26 ++++++++++++++++++++++ server/config/config.go | 10 +++++++-- server/datastore/redis/redis.go | 12 +++++----- server/datastore/redis/redis_test.go | 2 +- server/live_query/redis_live_query_test.go | 2 +- server/pubsub/testing_utils.go | 3 ++- server/sso/session_store_test.go | 2 +- 9 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 changes/redis-timeout-configurable diff --git a/changes/redis-timeout-configurable b/changes/redis-timeout-configurable new file mode 100644 index 0000000000..c7546919ad --- /dev/null +++ b/changes/redis-timeout-configurable @@ -0,0 +1 @@ +* Add fleet serve config to change the redis connection timeout and keep alive interval. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 6d2fcc2d1a..91b403ac98 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -200,7 +200,14 @@ the way that the Fleet server works. } } - redisPool, err := redis.NewRedisPool(config.Redis.Address, config.Redis.Password, config.Redis.Database, config.Redis.UseTLS) + redisPool, err := redis.NewRedisPool( + config.Redis.Address, + config.Redis.Password, + config.Redis.Database, + config.Redis.UseTLS, + config.Redis.ConnectTimeout, + config.Redis.KeepAlive, + ) if err != nil { initFatal(err, "initialize Redis") } diff --git a/docs/2-Deploying/2-Configuration.md b/docs/2-Deploying/2-Configuration.md index 5db2078d4f..3a095e96ae 100644 --- a/docs/2-Deploying/2-Configuration.md +++ b/docs/2-Deploying/2-Configuration.md @@ -356,6 +356,32 @@ Whether or not to duplicate Live Query results to another Redis channel named `L duplicate_results: true ``` +###### redis_connect_timeout + +Timeout for redis connection. + +- Default value: 5s +- Environment variable: `FLEET_REDIS_CONNECT_TIMEOUT` +- Config file format: + + ``` + redis: + connect_timeout: 10s + ``` + +###### redis_keep_alive + +Interval between keep alive probes. + +- Default value: 10s +- Environment variable: `FLEET_REDIS_KEEP_ALIVE` +- Config file format: + + ``` + redis: + keep_alive: 30s + ``` + ##### Server ###### server_address diff --git a/server/config/config.go b/server/config/config.go index 4fa91c9b08..7c770a266c 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -40,8 +40,10 @@ type RedisConfig struct { Address string Password string Database int - UseTLS bool `yaml:"use_tls"` - DuplicateResults bool `yaml:"duplicate_results"` + UseTLS bool `yaml:"use_tls"` + DuplicateResults bool `yaml:"duplicate_results"` + ConnectTimeout time.Duration `yaml:"connect_timeout"` + KeepAlive time.Duration `yaml:"keep_alive"` } const ( @@ -238,6 +240,8 @@ func (man Manager) addConfigs() { "Redis server database number") man.addConfigBool("redis.use_tls", false, "Redis server enable TLS") man.addConfigBool("redis.duplicate_results", false, "Duplicate Live Query results to another Redis channel") + man.addConfigDuration("redis.connect_timeout", 5*time.Second, "Timeout at connection time") + man.addConfigDuration("redis.keep_alive", 10*time.Second, "Interval between keep alive probes") // Server man.addConfigString("server.address", "0.0.0.0:8080", @@ -415,6 +419,8 @@ func (man Manager) LoadConfig() FleetConfig { Database: man.getConfigInt("redis.database"), UseTLS: man.getConfigBool("redis.use_tls"), DuplicateResults: man.getConfigBool("redis.duplicate_results"), + ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), + KeepAlive: man.getConfigDuration("redis.keep_alive"), }, Server: ServerConfig{ Address: man.getConfigString("server.address"), diff --git a/server/datastore/redis/redis.go b/server/datastore/redis/redis.go index 8f1c6dee23..d9046af6e8 100644 --- a/server/datastore/redis/redis.go +++ b/server/datastore/redis/redis.go @@ -25,8 +25,10 @@ func (p *standalonePool) Stats() map[string]redis.PoolStats { // NewRedisPool creates a Redis connection pool using the provided server // address, password and database. -func NewRedisPool(server, password string, database int, useTLS bool) (fleet.RedisPool, error) { - cluster := newCluster(server, password, database, useTLS) +func NewRedisPool( + server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration, +) (fleet.RedisPool, error) { + cluster := newCluster(server, password, database, useTLS, connTimeout, keepAlive) if err := cluster.Refresh(); err != nil { if isClusterDisabled(err) || isClusterCommandUnknown(err) { // not a Redis Cluster setup, use a standalone Redis pool @@ -70,7 +72,7 @@ func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { return fn(conn) } -func newCluster(server, password string, database int, useTLS bool) *redisc.Cluster { +func newCluster(server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration) *redisc.Cluster { return &redisc.Cluster{ StartupNodes: []string{server}, CreatePool: func(server string, opts ...redis.DialOption) (*redis.Pool, error) { @@ -83,8 +85,8 @@ func newCluster(server, password string, database int, useTLS bool) *redisc.Clus server, redis.DialDatabase(database), redis.DialUseTLS(useTLS), - redis.DialConnectTimeout(5*time.Second), - redis.DialKeepAlive(10*time.Second), + redis.DialConnectTimeout(connTimeout), + redis.DialKeepAlive(keepAlive), // Read/Write timeouts not set here because we may see results // only rarely on the pub/sub channel. ) diff --git a/server/datastore/redis/redis_test.go b/server/datastore/redis/redis_test.go index 4552228364..ad3da7d6b5 100644 --- a/server/datastore/redis/redis_test.go +++ b/server/datastore/redis/redis_test.go @@ -74,7 +74,7 @@ func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardo } addr += port - pool, err := NewRedisPool(addr, password, database, useTLS) + pool, err := NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) conn := pool.Get() diff --git a/server/live_query/redis_live_query_test.go b/server/live_query/redis_live_query_test.go index 52990c14ee..bdabe609f2 100644 --- a/server/live_query/redis_live_query_test.go +++ b/server/live_query/redis_live_query_test.go @@ -111,7 +111,7 @@ func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, tea } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS) + pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) store = NewRedisLiveQuery(pool) diff --git a/server/pubsub/testing_utils.go b/server/pubsub/testing_utils.go index 70f71003e7..2b8a02cc41 100644 --- a/server/pubsub/testing_utils.go +++ b/server/pubsub/testing_utils.go @@ -2,6 +2,7 @@ package pubsub import ( "testing" + "time" "github.com/fleetdm/fleet/v4/server/datastore/redis" redigo "github.com/gomodule/redigo/redis" @@ -22,7 +23,7 @@ func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, te } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS) + pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) store = NewRedisQueryResults(pool, dupResults) diff --git a/server/sso/session_store_test.go b/server/sso/session_store_test.go index 5c98656b4e..45b46af866 100644 --- a/server/sso/session_store_test.go +++ b/server/sso/session_store_test.go @@ -25,7 +25,7 @@ func newPool(t *testing.T, cluster bool) fleet.RedisPool { } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS) + pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) require.NoError(t, err) conn := pool.Get() defer conn.Close()