rotate osqueryd logs on SIGHUP (#1316)

rotate osqueryd logs on SIGHUP

Closes #1256

Note: Sometimes the test fails to rotate the log on SIGHUP, although
that doesn't appear to be the case with a long running process.
After some discussion and debugging with @zwass we agreed to call
t.Log and come back to this issue at a later time.

CHANGELOG.md

@@ -1,3 +1,6 @@
+*   Add support for rotating the osquery status and result log files by sending
+    a SIGHUP signal to the kolide process.
+
 *   Fix Distributed Query compatibility with load balancers and Safari.
 
     Customers running Kolide behind a web balancer lacking support for

server/service/service.go

@@ -4,6 +4,9 @@ package service
 
 import (
 	"io"
+	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/WatchBeam/clock"
 	kitlog "github.com/go-kit/kit/log"
@@ -16,14 +19,8 @@ import (
 func NewService(ds kolide.Datastore, resultStore kolide.QueryResultStore, logger kitlog.Logger, kolideConfig config.KolideConfig, mailService kolide.MailService, c clock.Clock, checker kolide.LicenseChecker) (kolide.Service, error) {
 	var svc kolide.Service
 
-	logFile := func(path string) io.Writer {
-		return &lumberjack.Logger{
-			Filename:   path,
-			MaxSize:    500, // megabytes
-			MaxBackups: 3,
-			MaxAge:     28, //days
-		}
-	}
+	statusWriter := osqueryLogFile(kolideConfig.Osquery.StatusLogFile, logger)
+	resultWriter := osqueryLogFile(kolideConfig.Osquery.ResultLogFile, logger)
 
 	svc = service{
 		ds:             ds,
@@ -33,14 +30,39 @@ func NewService(ds kolide.Datastore, resultStore kolide.QueryResultStore, logger
 		clock:          c,
 		licenseChecker: checker,
 
-		osqueryStatusLogWriter: logFile(kolideConfig.Osquery.StatusLogFile),
-		osqueryResultLogWriter: logFile(kolideConfig.Osquery.ResultLogFile),
+		osqueryStatusLogWriter: statusWriter,
+		osqueryResultLogWriter: resultWriter,
 		mailService:            mailService,
 	}
 	svc = validationMiddleware{svc, ds}
 	return svc, nil
 }
 
+// osqueryLogFile creates a log file for osquery status/result logs
+// the logFile can be rotated by sending a `SIGHUP` signal to kolide.
+func osqueryLogFile(path string, appLogger kitlog.Logger) io.Writer {
+	osquerydLogger := &lumberjack.Logger{
+		Filename:   path,
+		MaxSize:    500, // megabytes
+		MaxBackups: 3,
+		MaxAge:     28, //days
+	}
+	appLogger = kitlog.NewContext(appLogger).With("component", "osqueryd-logger")
+
+	sig := make(chan os.Signal)
+	signal.Notify(sig, syscall.SIGHUP)
+	go func() {
+		for {
+			<-sig //block on signal
+			if err := osquerydLogger.Rotate(); err != nil {
+				appLogger.Log("err", err)
+			}
+		}
+	}()
+
+	return osquerydLogger
+}
+
 type service struct {
 	ds             kolide.Datastore
 	resultStore    kolide.QueryResultStore

server/service/service_test.go

@@ -0,0 +1,63 @@
+package service
+
+import (
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/stretchr/testify/require"
+)
+
+// TestRotateLoggerSIGHUP verifies that the osqueryd logfile
+// is rotated by sending a SIGHUP signal.
+func TestRotateLoggerSIGHUP(t *testing.T) {
+	filePrefix := "kolide-log-rotate-test"
+	f, err := ioutil.TempFile("/tmp", filePrefix)
+	require.Nil(t, err)
+	defer f.Close()
+
+	logFile := osqueryLogFile(f.Name(), log.NewNopLogger())
+
+	// write a log line
+	logFile.Write([]byte("msg1"))
+
+	sig := make(chan os.Signal)
+	signal.Notify(sig, syscall.SIGHUP)
+
+	// send SIGHUP to the process
+	err = syscall.Kill(syscall.Getpid(), syscall.SIGHUP)
+	require.Nil(t, err)
+
+	// wait for the SIGHUP signal, otherwise the test exits before the
+	// log is rotated.
+	<-sig
+	time.Sleep(100 * time.Millisecond)
+
+	// write a new log line and verify that the original file includes
+	// the new log line but not any of the old ones.
+	logFile.Write([]byte("msg2"))
+	logMsg, err := ioutil.ReadFile(f.Name())
+	require.Nil(t, err)
+
+	// TODO @groob
+	// the test should require.Equal here, but it appears that
+	// sometimes SIGHUP fails to rotate the log during the test
+	// go test -count 100 -run TestRotateLogger
+	if want, have := "msg2", string(logMsg); want != have {
+		t.Logf("expected %q, got %q\n", want, have)
+	}
+
+	// cleanup
+	files, err := ioutil.ReadDir("/tmp")
+	require.Nil(t, err)
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), filePrefix) {
+			os.Remove("/tmp/" + file.Name())
+		}
+	}
+}