From 6d328ed70c1175156bfa2e36a25f82fa339908e7 Mon Sep 17 00:00:00 2001 From: Victor Vrantchan Date: Tue, 12 Dec 2017 10:43:33 -0500 Subject: [PATCH] Write the raw status log to the writer (#1666) Instead of trying to decode and re-encode status logs, we now write them directly as they come in. This change prevents future changes to the osquery status log file format (addition and deletion of fields ) from affecting Fleet. A similar change was implemented in #1636 for result logs. Closes #1664 --- server/kolide/osquery.go | 12 +----------- server/launcher/launcher.go | 22 ++-------------------- server/launcher/launcher_test.go | 2 +- server/mock/service_osquery.go | 4 ++-- server/service/endpoint_osquery.go | 2 +- server/service/logging_osquery.go | 2 +- server/service/service_osquery.go | 5 ++--- server/service/service_osquery_test.go | 2 +- 8 files changed, 11 insertions(+), 40 deletions(-) diff --git a/server/kolide/osquery.go b/server/kolide/osquery.go index 54a81f8aa7..16ef4f3e4e 100644 --- a/server/kolide/osquery.go +++ b/server/kolide/osquery.go @@ -18,7 +18,7 @@ type OsqueryService interface { // feature. GetDistributedQueries(ctx context.Context) (queries map[string]string, accelerate uint, err error) SubmitDistributedQueryResults(ctx context.Context, results OsqueryDistributedQueryResults, statuses map[string]string) (err error) - SubmitStatusLogs(ctx context.Context, logs []OsqueryStatusLog) (err error) + SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) (err error) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error) } @@ -71,13 +71,3 @@ type OsqueryConfig struct { // FIM (File Integrity Monitoring) FilePaths FIMSections `json:"file_paths,omitempty"` } - -// OsqueryStatusLog is the format of an osquery status log. -type OsqueryStatusLog struct { - Severity string `json:"severity"` - Filename string `json:"filename"` - Line string `json:"line"` - Message string `json:"message"` - Version string `json:"version"` - Decorations map[string]string `json:"decorations"` -} diff --git a/server/launcher/launcher.go b/server/launcher/launcher.go index 1ea0f8d777..ca59e78d80 100644 --- a/server/launcher/launcher.go +++ b/server/launcher/launcher.go @@ -87,28 +87,10 @@ func (svc *launcherWrapper) PublishLogs(ctx context.Context, nodeKey string, log switch logType { case logger.LogTypeStatus: - var statuses []kolide.OsqueryStatusLog + var statuses []json.RawMessage for _, log := range logs { - // StatusLog handles osquery logging messages - var statusLog = struct { - Severity string `json:"s"` - Filename string `json:"f"` - Line string `json:"i"` - Message string `json:"m"` - }{} - - if err := json.Unmarshal([]byte(log), &statusLog); err != nil { - return "", "", false, errors.Wrap(err, "decode status log from launcher") - } - - statuses = append(statuses, kolide.OsqueryStatusLog{ - Severity: statusLog.Severity, - Filename: statusLog.Filename, - Line: statusLog.Line, - Message: statusLog.Message, - }) + statuses = append(statuses, []byte(log)) } - err = svc.tls.SubmitStatusLogs(newCtx, statuses) return "", "", false, errors.Wrap(err, "submit status logs from launcher") case logger.LogTypeSnapshot, logger.LogTypeString: diff --git a/server/launcher/launcher_test.go b/server/launcher/launcher_test.go index cdddf5571a..b84dfb8405 100644 --- a/server/launcher/launcher_test.go +++ b/server/launcher/launcher_test.go @@ -146,7 +146,7 @@ func newTLSService(t *testing.T) *mock.TLSService { return }, - SubmitStatusLogsFunc: func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) { + SubmitStatusLogsFunc: func(ctx context.Context, logs []json.RawMessage) (err error) { return }, SubmitResultLogsFunc: func(ctx context.Context, logs []json.RawMessage) (err error) { diff --git a/server/mock/service_osquery.go b/server/mock/service_osquery.go index 578baccf81..cb97a46bcb 100644 --- a/server/mock/service_osquery.go +++ b/server/mock/service_osquery.go @@ -21,7 +21,7 @@ type GetDistributedQueriesFunc func(ctx context.Context) (queries map[string]str type SubmitDistributedQueryResultsFunc func(ctx context.Context, results kolide.OsqueryDistributedQueryResults, statuses map[string]string) (err error) -type SubmitStatusLogsFunc func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) +type SubmitStatusLogsFunc func(ctx context.Context, logs []json.RawMessage) (err error) type SubmitResultLogsFunc func(ctx context.Context, logs []json.RawMessage) (err error) @@ -73,7 +73,7 @@ func (s *TLSService) SubmitDistributedQueryResults(ctx context.Context, results return s.SubmitDistributedQueryResultsFunc(ctx, results, statuses) } -func (s *TLSService) SubmitStatusLogs(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) { +func (s *TLSService) SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) (err error) { s.SubmitStatusLogsFuncInvoked = true return s.SubmitStatusLogsFunc(ctx, logs) } diff --git a/server/service/endpoint_osquery.go b/server/service/endpoint_osquery.go index 5519e258fa..c3c75e7622 100644 --- a/server/service/endpoint_osquery.go +++ b/server/service/endpoint_osquery.go @@ -136,7 +136,7 @@ func makeSubmitLogsEndpoint(svc kolide.Service) endpoint.Endpoint { var err error switch req.LogType { case "status": - var statuses []kolide.OsqueryStatusLog + var statuses []json.RawMessage if err := json.Unmarshal(req.Data, &statuses); err != nil { err = osqueryError{message: "unmarshalling status logs: " + err.Error()} break diff --git a/server/service/logging_osquery.go b/server/service/logging_osquery.go index dccda32b1d..089316774b 100644 --- a/server/service/logging_osquery.go +++ b/server/service/logging_osquery.go @@ -105,7 +105,7 @@ func (mw loggingMiddleware) SubmitDistributedQueryResults(ctx context.Context, r return err } -func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []kolide.OsqueryStatusLog) error { +func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) error { var ( err error ) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index ca5069753c..fd6ecf6a46 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -220,10 +220,9 @@ type flusher interface { Flush() error } -func (svc service) SubmitStatusLogs(ctx context.Context, logs []kolide.OsqueryStatusLog) error { +func (svc service) SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) error { for _, log := range logs { - err := json.NewEncoder(svc.osqueryStatusLogWriter).Encode(log) - if err != nil { + if _, err := svc.osqueryStatusLogWriter.Write(append(log, '\n')); err != nil { return osqueryError{message: "error writing status log: " + err.Error()} } } diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 0f188477c0..8570f0bfc1 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -118,7 +118,7 @@ func TestSubmitStatusLogs(t *testing.T) { } logJSON := fmt.Sprintf("[%s]", strings.Join(logs, ",")) - var status []kolide.OsqueryStatusLog + var status []json.RawMessage err = json.Unmarshal([]byte(logJSON), &status) require.Nil(t, err)