Make redis conn timeout and keep alive configurable (#1968)

* Make redis conn timeout and keep alive configurable

* Document new configs

* Correct config name

changes/redis-timeout-configurable

@@ -0,0 +1 @@
+* Add fleet serve config to change the redis connection timeout and keep alive interval.

cmd/fleet/serve.go

@@ -200,7 +200,14 @@ the way that the Fleet server works.
 				}
 			}
 
-			redisPool, err := redis.NewRedisPool(config.Redis.Address, config.Redis.Password, config.Redis.Database, config.Redis.UseTLS)
+			redisPool, err := redis.NewRedisPool(
+				config.Redis.Address,
+				config.Redis.Password,
+				config.Redis.Database,
+				config.Redis.UseTLS,
+				config.Redis.ConnectTimeout,
+				config.Redis.KeepAlive,
+			)
 			if err != nil {
 				initFatal(err, "initialize Redis")
 			}

docs/2-Deploying/2-Configuration.md

@@ -356,6 +356,32 @@ Whether or not to duplicate Live Query results to another Redis channel named `L
     duplicate_results: true
   ```
 
+###### redis_connect_timeout
+
+Timeout for redis connection. 
+
+- Default value: 5s
+- Environment variable: `FLEET_REDIS_CONNECT_TIMEOUT`
+- Config file format:
+
+  ```
+  redis:
+    connect_timeout: 10s
+  ```
+
+###### redis_keep_alive
+
+Interval between keep alive probes.
+
+- Default value: 10s
+- Environment variable: `FLEET_REDIS_KEEP_ALIVE`
+- Config file format:
+
+  ```
+  redis:
+    keep_alive: 30s
+  ```
+
 ##### Server
 
 ###### server_address

server/config/config.go

@@ -40,8 +40,10 @@ type RedisConfig struct {
 	Address          string
 	Password         string
 	Database         int
-	UseTLS           bool `yaml:"use_tls"`
-	DuplicateResults bool `yaml:"duplicate_results"`
+	UseTLS           bool          `yaml:"use_tls"`
+	DuplicateResults bool          `yaml:"duplicate_results"`
+	ConnectTimeout   time.Duration `yaml:"connect_timeout"`
+	KeepAlive        time.Duration `yaml:"keep_alive"`
 }
 
 const (
@@ -238,6 +240,8 @@ func (man Manager) addConfigs() {
 		"Redis server database number")
 	man.addConfigBool("redis.use_tls", false, "Redis server enable TLS")
 	man.addConfigBool("redis.duplicate_results", false, "Duplicate Live Query results to another Redis channel")
+	man.addConfigDuration("redis.connect_timeout", 5*time.Second, "Timeout at connection time")
+	man.addConfigDuration("redis.keep_alive", 10*time.Second, "Interval between keep alive probes")
 
 	// Server
 	man.addConfigString("server.address", "0.0.0.0:8080",
@@ -415,6 +419,8 @@ func (man Manager) LoadConfig() FleetConfig {
 			Database:         man.getConfigInt("redis.database"),
 			UseTLS:           man.getConfigBool("redis.use_tls"),
 			DuplicateResults: man.getConfigBool("redis.duplicate_results"),
+			ConnectTimeout:   man.getConfigDuration("redis.connect_timeout"),
+			KeepAlive:        man.getConfigDuration("redis.keep_alive"),
 		},
 		Server: ServerConfig{
 			Address:    man.getConfigString("server.address"),

server/datastore/redis/redis.go

@@ -25,8 +25,10 @@ func (p *standalonePool) Stats() map[string]redis.PoolStats {
 
 // NewRedisPool creates a Redis connection pool using the provided server
 // address, password and database.
-func NewRedisPool(server, password string, database int, useTLS bool) (fleet.RedisPool, error) {
-	cluster := newCluster(server, password, database, useTLS)
+func NewRedisPool(
+	server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration,
+) (fleet.RedisPool, error) {
+	cluster := newCluster(server, password, database, useTLS, connTimeout, keepAlive)
 	if err := cluster.Refresh(); err != nil {
 		if isClusterDisabled(err) || isClusterCommandUnknown(err) {
 			// not a Redis Cluster setup, use a standalone Redis pool
@@ -70,7 +72,7 @@ func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error {
 	return fn(conn)
 }
 
-func newCluster(server, password string, database int, useTLS bool) *redisc.Cluster {
+func newCluster(server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration) *redisc.Cluster {
 	return &redisc.Cluster{
 		StartupNodes: []string{server},
 		CreatePool: func(server string, opts ...redis.DialOption) (*redis.Pool, error) {
@@ -83,8 +85,8 @@ func newCluster(server, password string, database int, useTLS bool) *redisc.Clus
 						server,
 						redis.DialDatabase(database),
 						redis.DialUseTLS(useTLS),
-						redis.DialConnectTimeout(5*time.Second),
-						redis.DialKeepAlive(10*time.Second),
+						redis.DialConnectTimeout(connTimeout),
+						redis.DialKeepAlive(keepAlive),
 						// Read/Write timeouts not set here because we may see results
 						// only rarely on the pub/sub channel.
 					)

server/datastore/redis/redis_test.go

@@ -74,7 +74,7 @@ func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardo
 	}
 	addr += port
 
-	pool, err := NewRedisPool(addr, password, database, useTLS)
+	pool, err := NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second)
 	require.NoError(t, err)
 
 	conn := pool.Get()

server/live_query/redis_live_query_test.go

@@ -111,7 +111,7 @@ func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, tea
 	}
 	addr += port
 
-	pool, err := redis.NewRedisPool(addr, password, database, useTLS)
+	pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second)
 	require.NoError(t, err)
 	store = NewRedisLiveQuery(pool)
 

server/pubsub/testing_utils.go

@@ -2,6 +2,7 @@ package pubsub
 
 import (
 	"testing"
+	"time"
 
 	"github.com/fleetdm/fleet/v4/server/datastore/redis"
 	redigo "github.com/gomodule/redigo/redis"
@@ -22,7 +23,7 @@ func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, te
 	}
 	addr += port
 
-	pool, err := redis.NewRedisPool(addr, password, database, useTLS)
+	pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second)
 	require.NoError(t, err)
 	store = NewRedisQueryResults(pool, dupResults)
 

server/sso/session_store_test.go

@@ -25,7 +25,7 @@ func newPool(t *testing.T, cluster bool) fleet.RedisPool {
 		}
 		addr += port
 
-		pool, err := redis.NewRedisPool(addr, password, database, useTLS)
+		pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second)
 		require.NoError(t, err)
 		conn := pool.Get()
 		defer conn.Close()