From 8291119067cc95c84f5b47af9ed1423a81555808 Mon Sep 17 00:00:00 2001 From: Victor Vrantchan Date: Sat, 18 Nov 2017 19:59:32 -0500 Subject: [PATCH] use json.RawMessage for result logs (#1636) Initially fleet decoded the incoming JSON sent to the log endpoint. Then the log event would be written to a log writer by calling json.Encoder{}.Encode. Re-encoding logs is lossy; whenever a new field is sent by osqueryd we don't keep up with them. Instead of caring about the content of the OsqueryResultLog, fleet will now write all log results exactly as sent to the server by osqueryd. Closes #1632 Closes #1615 --- CHANGELOG.md | 4 ++++ server/kolide/osquery.go | 18 ++---------------- server/launcher/launcher.go | 8 ++------ server/launcher/launcher_test.go | 3 ++- server/mock/service_osquery.go | 5 +++-- server/service/endpoint_osquery.go | 2 +- server/service/logging_osquery.go | 3 ++- server/service/service_osquery.go | 5 ++--- server/service/service_osquery_test.go | 5 ++++- 9 files changed, 22 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c91fd76ad..666e00e679 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Kolide Fleet 1.0.6 (TBD, 2017) + +* Fixed bugs that caused logs to sometimes be ommited from the logwriter. (#1636, #1617) + * Fixed a bug where SAML client would create too many HTTP connections. (#1587) * Fixed bug in which default query was run instead of entered query. (#1611) diff --git a/server/kolide/osquery.go b/server/kolide/osquery.go index ec519114e8..54a81f8aa7 100644 --- a/server/kolide/osquery.go +++ b/server/kolide/osquery.go @@ -2,6 +2,7 @@ package kolide import ( "context" + "encoding/json" ) type OsqueryService interface { @@ -18,7 +19,7 @@ type OsqueryService interface { GetDistributedQueries(ctx context.Context) (queries map[string]string, accelerate uint, err error) SubmitDistributedQueryResults(ctx context.Context, results OsqueryDistributedQueryResults, statuses map[string]string) (err error) SubmitStatusLogs(ctx context.Context, logs []OsqueryStatusLog) (err error) - SubmitResultLogs(ctx context.Context, logs []OsqueryResultLog) (err error) + SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error) } // OsqueryDistributedQueryResults represents the format of the results of an @@ -71,21 +72,6 @@ type OsqueryConfig struct { FilePaths FIMSections `json:"file_paths,omitempty"` } -// OsqueryResultLog is the format of an osquery result log (ie: a differential -// or snapshot query). -type OsqueryResultLog struct { - Name string `json:"name"` - HostIdentifier string `json:"hostIdentifier"` - UnixTime string `json:"unixTime"` - CalendarTime string `json:"calendarTime"` - // Columns stores the columns of differential queries - Columns map[string]string `json:"columns,omitempty"` - // Snapshot stores the rows and columns of snapshot queries - Snapshot []map[string]string `json:"snapshot,omitempty"` - Action string `json:"action"` - Decorations map[string]string `json:"decorations"` -} - // OsqueryStatusLog is the format of an osquery status log. type OsqueryStatusLog struct { Severity string `json:"severity"` diff --git a/server/launcher/launcher.go b/server/launcher/launcher.go index 35274d52d1..1ea0f8d777 100644 --- a/server/launcher/launcher.go +++ b/server/launcher/launcher.go @@ -112,13 +112,9 @@ func (svc *launcherWrapper) PublishLogs(ctx context.Context, nodeKey string, log err = svc.tls.SubmitStatusLogs(newCtx, statuses) return "", "", false, errors.Wrap(err, "submit status logs from launcher") case logger.LogTypeSnapshot, logger.LogTypeString: - var results []kolide.OsqueryResultLog + var results []json.RawMessage for _, log := range logs { - var result kolide.OsqueryResultLog - if err := json.Unmarshal([]byte(log), &result); err != nil { - return "", "", false, errors.Wrap(err, "unmarshaling result log") - } - results = append(results, result) + results = append(results, []byte(log)) } err = svc.tls.SubmitResultLogs(newCtx, results) return "", "", false, errors.Wrap(err, "submit result logs from launcher") diff --git a/server/launcher/launcher_test.go b/server/launcher/launcher_test.go index ab2c3b9514..cdddf5571a 100644 --- a/server/launcher/launcher_test.go +++ b/server/launcher/launcher_test.go @@ -2,6 +2,7 @@ package launcher import ( "context" + "encoding/json" "testing" "github.com/go-kit/kit/log" @@ -148,7 +149,7 @@ func newTLSService(t *testing.T) *mock.TLSService { SubmitStatusLogsFunc: func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) { return }, - SubmitResultLogsFunc: func(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) { + SubmitResultLogsFunc: func(ctx context.Context, logs []json.RawMessage) (err error) { return }, } diff --git a/server/mock/service_osquery.go b/server/mock/service_osquery.go index 50f3c9806d..578baccf81 100644 --- a/server/mock/service_osquery.go +++ b/server/mock/service_osquery.go @@ -4,6 +4,7 @@ package mock import ( "context" + "encoding/json" "github.com/kolide/fleet/server/kolide" ) @@ -22,7 +23,7 @@ type SubmitDistributedQueryResultsFunc func(ctx context.Context, results kolide. type SubmitStatusLogsFunc func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) -type SubmitResultLogsFunc func(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) +type SubmitResultLogsFunc func(ctx context.Context, logs []json.RawMessage) (err error) type TLSService struct { EnrollAgentFunc EnrollAgentFunc @@ -77,7 +78,7 @@ func (s *TLSService) SubmitStatusLogs(ctx context.Context, logs []kolide.Osquery return s.SubmitStatusLogsFunc(ctx, logs) } -func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) { +func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error) { s.SubmitResultLogsFuncInvoked = true return s.SubmitResultLogsFunc(ctx, logs) } diff --git a/server/service/endpoint_osquery.go b/server/service/endpoint_osquery.go index 95e7e9b787..5519e258fa 100644 --- a/server/service/endpoint_osquery.go +++ b/server/service/endpoint_osquery.go @@ -148,7 +148,7 @@ func makeSubmitLogsEndpoint(svc kolide.Service) endpoint.Endpoint { } case "result": - var results []kolide.OsqueryResultLog + var results []json.RawMessage if err := json.Unmarshal(req.Data, &results); err != nil { err = osqueryError{message: "unmarshalling result logs: " + err.Error()} break diff --git a/server/service/logging_osquery.go b/server/service/logging_osquery.go index 6e5449cd0a..ca4e642ffb 100644 --- a/server/service/logging_osquery.go +++ b/server/service/logging_osquery.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "time" kithttp "github.com/go-kit/kit/transport/http" @@ -121,7 +122,7 @@ func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []kolide. return err } -func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error { +func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error { var ( err error ) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index e207e44955..dfdce1ee82 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -236,10 +236,9 @@ func (svc service) SubmitStatusLogs(ctx context.Context, logs []kolide.OsquerySt return nil } -func (svc service) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error { +func (svc service) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error { for _, log := range logs { - err := json.NewEncoder(svc.osqueryResultLogWriter).Encode(log) - if err != nil { + if _, err := svc.osqueryResultLogWriter.Write(append(log, '\n')); err != nil { return osqueryError{message: "error writing result log: " + err.Error()} } } diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index a44457c37a..0f188477c0 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -159,10 +159,13 @@ func TestSubmitResultLogs(t *testing.T) { `{"name":"system_info","hostIdentifier":"some_uuid","calendarTime":"Fri Sep 30 17:55:15 2016 UTC","unixTime":"1475258115","decorations":{"host_uuid":"some_uuid","username":"zwass"},"columns":{"cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz","hostname":"hostimus","physical_memory":"17179869184"},"action":"added"}`, `{"name":"encrypted","hostIdentifier":"some_uuid","calendarTime":"Fri Sep 30 21:19:15 2016 UTC","unixTime":"1475270355","decorations":{"host_uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB","username":"zwass"},"columns":{"encrypted":"1","name":"\/dev\/disk1","type":"AES-XTS","uid":"","user_uuid":"","uuid":"some_uuid"},"action":"added"}`, `{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"time","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":"1484078931","decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`, + `{"diffResults":{"removed":[{"address":"127.0.0.1","hostnames":"kl.groob.io"}],"added":""},"name":"pack\/test\/hosts","hostIdentifier":"FA01680E-98CA-5557-8F59-7716ECFEE964","calendarTime":"Sun Nov 19 00:02:08 2017 UTC","unixTime":"1511049728","epoch":"0","counter":"10","decorations":{"host_uuid":"FA01680E-98CA-5557-8F59-7716ECFEE964","hostname":"kl.groob.io"}}`, + // fleet will accept anything in the "data" field of an log request. + `{"unknown":{"foo": [] }}`, } logJSON := fmt.Sprintf("[%s]", strings.Join(logs, ",")) - var results []kolide.OsqueryResultLog + var results []json.RawMessage err = json.Unmarshal([]byte(logJSON), &results) require.Nil(t, err)