From 1e92f8fcfd4b267178ef8cd41fc41846b3503e2b Mon Sep 17 00:00:00 2001 From: John Murphy Date: Thu, 17 Aug 2017 16:43:02 -0500 Subject: [PATCH] gRPC Server for Launcher (#1544) Add a gRPC server that will interact with osquery through Launcher. This endpoint will expose the osquery configuration suitable for use via the Launcher plugin, and collect log and query results. --- cli/serve.go | 12 +- docs/third-party/licenses.md | 4 + glide.lock | 49 ++- glide.yaml | 17 +- server/launcher/auth_middleware.go | 98 ++++++ server/launcher/auth_middleware_test.go | 76 +++++ server/launcher/binding.go | 183 +++++++++++ server/launcher/binding_test.go | 391 +++++++++++++++++++++++ server/launcher/context.go | 33 ++ server/launcher/logging_middleware.go | 100 ++++++ server/launcher/server.go | 41 +++ tools/lint_license/license_settings.yaml | 2 +- 12 files changed, 993 insertions(+), 13 deletions(-) create mode 100644 server/launcher/auth_middleware.go create mode 100644 server/launcher/auth_middleware_test.go create mode 100644 server/launcher/binding.go create mode 100644 server/launcher/binding_test.go create mode 100644 server/launcher/context.go create mode 100644 server/launcher/logging_middleware.go create mode 100644 server/launcher/server.go diff --git a/cli/serve.go b/cli/serve.go index 5d15e6f6f1..ad2b0381c6 100644 --- a/cli/serve.go +++ b/cli/serve.go @@ -19,6 +19,7 @@ import ( "github.com/kolide/fleet/server/config" "github.com/kolide/fleet/server/datastore/mysql" "github.com/kolide/fleet/server/kolide" + "github.com/kolide/fleet/server/launcher" "github.com/kolide/fleet/server/license" "github.com/kolide/fleet/server/mail" "github.com/kolide/fleet/server/pubsub" @@ -144,6 +145,9 @@ the way that the kolide server works. if err != nil { initFatal(err, "initializing service") } + // Instantiate a gRPC service to handle launcher requests. + launcher := launcher.New(svc, logger) + defer launcher.GracefulStop() go func() { ticker := time.NewTicker(1 * time.Hour) @@ -201,12 +205,14 @@ the way that the kolide server works. } r := http.NewServeMux() + r.Handle("/healthz", prometheus.InstrumentHandler("healthz", healthz(httpLogger, healthCheckers))) r.Handle("/version", prometheus.InstrumentHandler("version", version.Handler())) r.Handle("/assets/", prometheus.InstrumentHandler("static_assets", service.ServeStaticAssets("/assets/"))) r.Handle("/metrics", prometheus.InstrumentHandler("metrics", promhttp.Handler())) r.Handle("/api/", apiHandler) r.Handle("/", frontendHandler) + if path, ok := os.LookupEnv("KOLIDE_TEST_PAGE_PATH"); ok { // test that we can load this _, err := ioutil.ReadFile(path) @@ -237,7 +243,7 @@ the way that the kolide server works. srv := &http.Server{ Addr: config.Server.Address, - Handler: r, + Handler: launcher.Handler(r), ReadTimeout: 25 * time.Second, WriteTimeout: 40 * time.Second, ReadHeaderTimeout: 5 * time.Second, @@ -308,7 +314,9 @@ func healthz(logger kitlog.Logger, deps map[string]interface{}) http.HandlerFunc // profile is 'modern'. // See https://wiki.mozilla.org/Security/Server_Side_TLS func getTLSConfig(profile string) *tls.Config { - cfg := tls.Config{PreferServerCipherSuites: true} + cfg := tls.Config{ + PreferServerCipherSuites: true, + } switch profile { case config.TLSProfileModern: diff --git a/docs/third-party/licenses.md b/docs/third-party/licenses.md index 34eba4204e..92deafc01f 100644 --- a/docs/third-party/licenses.md +++ b/docs/third-party/licenses.md @@ -390,6 +390,7 @@ Third-Party Licenses | [github.com/jordan-wright/email](https://github.com/jordan-wright/email) | [MIT](https://opensource.org/licenses/MIT) | | [github.com/kolide/agent-api](https://github.com/kolide/agent-api) | [Kolide](https://kolide.com) | | [github.com/kolide/goose](https://github.com/kolide/goose) | [MIT](https://opensource.org/licenses/MIT) | +| [github.com/kolide/osquery-go](https://github.com/kolide/osquery-go) | [MIT](https://opensource.org/licenses/MIT) | | [github.com/kr/logfmt](https://github.com/kr/logfmt) | [MIT](https://opensource.org/licenses/MIT) | | [github.com/magiconair/properties](https://github.com/magiconair/properties) | [FreeBSD](https://opensource.org/licenses/BSD-2-Clause) | | [github.com/matttproud/golang_protobuf_extensions](https://github.com/matttproud/golang_protobuf_extensions) | [Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0) | @@ -411,6 +412,7 @@ Third-Party Licenses | [github.com/spf13/jwalterweatherman](https://github.com/spf13/jwalterweatherman) | [MIT](https://opensource.org/licenses/MIT) | | [github.com/spf13/pflag](https://github.com/spf13/pflag) | [NewBSD](https://opensource.org/licenses/BSD-3-Clause) | | [github.com/spf13/viper](https://github.com/spf13/viper) | [MIT](https://opensource.org/licenses/MIT) | +| [github.com/stretchr/objx](https://github.com/stretchr/objx) | [MIT](https://opensource.org/licenses/MIT) | | [github.com/stretchr/testify](https://github.com/stretchr/testify) | [MIT](https://opensource.org/licenses/MIT) | | [glob](https://www.npmjs.com/package/glob) | [ISC](https://opensource.org/licenses/ISC) | | [glob-base](https://www.npmjs.com/package/glob-base) | [MIT](https://opensource.org/licenses/MIT) | @@ -424,6 +426,8 @@ Third-Party Licenses | [golang.org/x/sys](https://golang.org/x/sys) | [NewBSD](https://opensource.org/licenses/BSD-3-Clause) | | [golang.org/x/text](https://golang.org/x/text) | [NewBSD](https://opensource.org/licenses/BSD-3-Clause) | | [gonzales-pe](https://www.npmjs.com/package/gonzales-pe) | [MIT](https://opensource.org/licenses/MIT) | +| [google.golang.org/genproto](https://google.golang.org/genproto) | [Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [google.golang.org/grpc](https://google.golang.org/grpc) | [Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0) | | [gopkg.in/alecthomas/kingpin.v2](https://gopkg.in/alecthomas/kingpin.v2) | [MIT](https://opensource.org/licenses/MIT) | | [gopkg.in/go-playground/validator.v8](https://gopkg.in/go-playground/validator.v8) | [MIT](https://opensource.org/licenses/MIT) | | [gopkg.in/natefinch/lumberjack.v2](https://gopkg.in/natefinch/lumberjack.v2) | [MIT](https://opensource.org/licenses/MIT) | diff --git a/glide.lock b/glide.lock index 4512e59c62..578cdbe06e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: a437c9ca450b1cec9e2b7ad815179956279cb459a594d3c6fb80937f11ddede7 -updated: 2017-08-10T12:15:43.00289563-07:00 +hash: a580dd48772ffed8168170ef7c7874e1dbe74f2288d44613378367e05d8ec722 +updated: 2017-08-13T10:48:17.061574863-05:00 imports: - name: github.com/alecthomas/template version: a0175ee3bccc567396460bf5acd36800cb10c49c @@ -53,6 +53,7 @@ imports: version: c3cefd437628a0b7d31b34fe44b3a7a540e98527 subpackages: - proto + - ptypes/any - name: github.com/gorilla/context version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 - name: github.com/gorilla/mux @@ -89,6 +90,11 @@ imports: repo: git@github.com:kolide/agent-api.git - name: github.com/kolide/goose version: 4a7848793d4402d338de853019ad72b9a4b3e68e +- name: github.com/kolide/osquery-go + version: 5419715d060ff8c2e1b7f6ff5a8127ac2ce3fbdc + subpackages: + - plugin/distributed + - plugin/logger - name: github.com/kr/logfmt version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 - name: github.com/magiconair/properties @@ -104,7 +110,7 @@ imports: - name: github.com/pelletier/go-toml version: 69d355db5304c0f7f809a2edc054553e7142f016 - name: github.com/pkg/errors - version: c605e284fe17294bda444b34710735b29d1a9d90 + version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib version: 792786c7400a136282c1664665ae0a8db921c6c2 subpackages: @@ -153,10 +159,13 @@ imports: version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 - name: github.com/spf13/viper version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 +- name: github.com/stretchr/objx + version: cbeaeb16a013161a98496fad62933b1d21786672 - name: github.com/stretchr/testify - version: 890a5c3458b43e6104ff5da8dfa139d013d77544 + version: 05e8a0eda380579888eb53c394909df027f06991 subpackages: - assert + - mock - require - name: github.com/VividCortex/mysqlerr version: 6c6b55f8796f578c870b7e19bafb16103bc40095 @@ -168,19 +177,47 @@ imports: - bcrypt - blowfish - name: golang.org/x/net - version: 4d38db76854b199960801a1734443fd02870d7e1 + version: f5079bd7f6f74e23c4d65efa0f4ce14cbd6a3c0f subpackages: - context - context/ctxhttp + - http2 + - http2/hpack + - idna + - internal/timeseries + - lex/httplex + - trace - name: golang.org/x/sys version: a646d33e2ee3172a661fc09bca23bb4889a41bc8 subpackages: - unix - name: golang.org/x/text - version: b19bf474d317b857955b12035d2c5acb57ce8b01 + version: 3bd178b88a8180be2df394a1fbb81313916f0e7b subpackages: + - secure/bidirule - transform + - unicode/bidi - unicode/norm +- name: google.golang.org/genproto + version: 09f6ed296fc66555a25fe4ce95173148778dfa85 + subpackages: + - googleapis/rpc/status +- name: google.golang.org/grpc + version: b8669c35455183da6d5c474ea6e72fbf55183274 + subpackages: + - codes + - credentials + - grpclb/grpc_lb_v1 + - grpclog + - internal + - keepalive + - metadata + - naming + - peer + - stats + - status + - tap + - transport - name: gopkg.in/alecthomas/kingpin.v2 version: 1087e65c9441605df944fb12c33f0fe7072d18ca - name: gopkg.in/go-playground/validator.v8 diff --git a/glide.yaml b/glide.yaml index b264a4597b..85b9515e95 100644 --- a/glide.yaml +++ b/glide.yaml @@ -23,10 +23,6 @@ import: subpackages: - bcrypt - blowfish -- package: golang.org/x/net - version: 4d38db76854b199960801a1734443fd02870d7e1 - subpackages: - - context - package: golang.org/x/sys version: a646d33e2ee3172a661fc09bca23bb4889a41bc8 subpackages: @@ -52,6 +48,9 @@ import: - prometheus - package: github.com/patrickmn/sortutil - package: github.com/stretchr/testify + version: 05e8a0eda380579888eb53c394909df027f06991 + subpackages: + - mock - package: github.com/garyburd/redigo version: ^1.0.0 subpackages: @@ -70,5 +69,15 @@ import: - package: github.com/spf13/cast version: ~1.0.0 - package: github.com/russellhaering/gosaml2 +- package: google.golang.org/grpc + version: ^1.5.1 - package: github.com/kolide/agent-api repo: git@github.com:kolide/agent-api.git +- package: github.com/kolide/osquery-go + subpackages: + - plugin/distributed + - plugin/logger +- package: google.golang.org/genproto + subpackages: + - googleapis/rpc/status +- package: github.com/stretchr/objx diff --git a/server/launcher/auth_middleware.go b/server/launcher/auth_middleware.go new file mode 100644 index 0000000000..c046587e83 --- /dev/null +++ b/server/launcher/auth_middleware.go @@ -0,0 +1,98 @@ +package launcher + +import ( + pb "github.com/kolide/agent-api" + "github.com/kolide/fleet/server/kolide" + + "golang.org/x/net/context" +) + +type authMiddleware struct { + svc kolide.OsqueryService + next pb.ApiServer +} + +func newAuthMiddleware(svc kolide.OsqueryService) func(svc pb.ApiServer) pb.ApiServer { + return func(next pb.ApiServer) pb.ApiServer { + return authMiddleware{ + svc: svc, + next: next, + } + } +} + +func (s authMiddleware) RequestEnrollment(ctx context.Context, req *pb.EnrollmentRequest) (*pb.EnrollmentResponse, error) { + return s.next.RequestEnrollment(ctx, req) +} + +func (s authMiddleware) RequestConfig(ctx context.Context, req *pb.AgentApiRequest) (*pb.ConfigResponse, error) { + authCtx, auth, err := s.authenticateHost(ctx, req.NodeKey) + if err != nil { + return nil, err + } + if auth.nodeInvalid { + return &pb.ConfigResponse{NodeInvalid: auth.nodeInvalid, ErrorCode: auth.errorCode}, nil + } + return s.next.RequestConfig(authCtx, req) +} + +func (s authMiddleware) RequestQueries(ctx context.Context, req *pb.AgentApiRequest) (resp *pb.QueryCollection, err error) { + authCtx, auth, err := s.authenticateHost(ctx, req.NodeKey) + if err != nil { + return nil, err + } + if auth.nodeInvalid { + return &pb.QueryCollection{NodeInvalid: auth.nodeInvalid, ErrorCode: auth.errorCode}, nil + } + return s.next.RequestQueries(authCtx, req) +} + +func (s authMiddleware) PublishLogs(ctx context.Context, req *pb.LogCollection) (resp *pb.AgentApiResponse, err error) { + authCtx, auth, err := s.authenticateHost(ctx, req.NodeKey) + if err != nil { + return nil, err + } + if auth.nodeInvalid { + return &pb.AgentApiResponse{NodeInvalid: auth.nodeInvalid, ErrorCode: auth.errorCode}, nil + } + return s.next.PublishLogs(authCtx, req) +} + +func (s authMiddleware) PublishResults(ctx context.Context, req *pb.ResultCollection) (resp *pb.AgentApiResponse, err error) { + authCtx, auth, err := s.authenticateHost(ctx, req.NodeKey) + if err != nil { + return nil, err + } + if auth.nodeInvalid { + return &pb.AgentApiResponse{NodeInvalid: auth.nodeInvalid, ErrorCode: auth.errorCode}, nil + } + return s.next.PublishResults(authCtx, req) +} + +func (s authMiddleware) HotConfigure(req *pb.AgentApiRequest, svr pb.Api_HotConfigureServer) (err error) { + if _, _, err := s.authenticateHost(context.Background(), req.NodeKey); err != nil { + return err + } + return s.next.HotConfigure(req, svr) +} + +func (s authMiddleware) HotlineBling(svr pb.Api_HotlineBlingServer) (err error) { + return s.next.HotlineBling(svr) +} + +type auth struct { + nodeInvalid bool + errorCode string + host *kolide.Host +} + +func (s authMiddleware) authenticateHost(ctx context.Context, nodeKey string) (context.Context, *auth, error) { + host, err := s.svc.AuthenticateHost(newCtx(ctx), nodeKey) + if err != nil { + if errEnroll, ok := err.(enrollmentError); ok { + return ctx, &auth{nodeInvalid: errEnroll.NodeInvalid(), errorCode: errEnroll.Error()}, nil + } + return nil, nil, err + } + return withHost(ctx, *host), &auth{host: host}, nil +} diff --git a/server/launcher/auth_middleware_test.go b/server/launcher/auth_middleware_test.go new file mode 100644 index 0000000000..c7ff21e1b8 --- /dev/null +++ b/server/launcher/auth_middleware_test.go @@ -0,0 +1,76 @@ +package launcher + +import ( + "context" + "testing" + + pb "github.com/kolide/agent-api" + "github.com/kolide/fleet/server/contexts/host" + "github.com/kolide/fleet/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +var authTestHost = &kolide.Host{HostName: "jimmy"} +var nullHost *kolide.Host + +func TestAuthRequestConfig(t *testing.T) { + mockSvc := new(mockOsqueryService) + mockSvc.On( + "AuthenticateHost", + oldContext, + "nodekey", + ).Return( + authTestHost, + nil, + ) + mockSvc.On( + "GetClientConfig", + mock.MatchedBy(func(ctx context.Context) bool { + if h, ok := host.FromContext(ctx); ok { + return h.HostName == authTestHost.HostName + } + return false + }), + ).Return( + &kolide.OsqueryConfig{}, + nil, + ) + svr := newAuthMiddleware(mockSvc)(&agentBinding{mockSvc}) + resp, err := svr.RequestConfig(oldContext, &pb.AgentApiRequest{NodeKey: "nodekey"}) + mockSvc.AssertExpectations(t) + require.Nil(t, err) + require.NotNil(t, resp) +} + +func TestAuthFailRequestConfig(t *testing.T) { + cxtMatcher := mock.MatchedBy(func(ctx context.Context) bool { + if h, ok := hostFromContext(ctx); ok { + return h.HostName == authTestHost.HostName + } + return false + }) + mockSvc := new(mockOsqueryService) + mockSvc.On( + "AuthenticateHost", + oldContext, + "nodekey", + ).Return( + nullHost, + &mockEnrollError{}, + ) + mockSvc.On( + "GetClientConfig", + cxtMatcher, + ).Return( + &kolide.OsqueryConfig{}, + nil, + ) + svr := newAuthMiddleware(mockSvc)(&agentBinding{mockSvc}) + resp, err := svr.RequestConfig(oldContext, &pb.AgentApiRequest{NodeKey: "nodekey"}) + mockSvc.AssertNotCalled(t, "GetClientConfig", newContext) + require.Nil(t, err) + require.NotNil(t, resp) + assert.True(t, resp.NodeInvalid) +} diff --git a/server/launcher/binding.go b/server/launcher/binding.go new file mode 100644 index 0000000000..8f699bc401 --- /dev/null +++ b/server/launcher/binding.go @@ -0,0 +1,183 @@ +package launcher + +import ( + "bytes" + "encoding/json" + "strconv" + + pb "github.com/kolide/agent-api" + "github.com/kolide/fleet/server/kolide" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +var errNotImplmented = errors.New("not implemented") + +// agentBinding implements ApiClient interface and maps gRPC domain functions to the application. +type agentBinding struct { + service kolide.OsqueryService +} + +func newAgentBinding(svc kolide.OsqueryService) pb.ApiServer { + return &agentBinding{ + service: svc, + } +} + +type enrollmentError interface { + NodeInvalid() bool + Error() string +} + +// Attempt to enroll a host with kolide/cloud +func (b *agentBinding) RequestEnrollment(ctx context.Context, req *pb.EnrollmentRequest) (*pb.EnrollmentResponse, error) { + var resp pb.EnrollmentResponse + nodeKey, err := b.service.EnrollAgent(newCtx(ctx), req.EnrollSecret, req.HostIdentifier) + if err != nil { + if errEnroll, ok := err.(enrollmentError); ok { + resp.NodeInvalid = errEnroll.NodeInvalid() + resp.ErrorCode = errEnroll.Error() + return &resp, nil + } + return nil, err + } + resp.NodeKey = nodeKey + return &resp, nil +} + +// RequestConfig requests an updated configuration +func (b *agentBinding) RequestConfig(ctx context.Context, req *pb.AgentApiRequest) (*pb.ConfigResponse, error) { + config, err := b.service.GetClientConfig(newCtx(ctx)) + if err != nil { + return nil, err + } + // Launcher manages plugins so remove them from configuration if they exist. + for _, optionName := range []string{"distributed_plugin", "logger_plugin"} { + if _, ok := config.Options[optionName]; ok { + delete(config.Options, optionName) + } + } + var writer bytes.Buffer + if err = json.NewEncoder(&writer).Encode(config); err != nil { + return nil, err + } + return &pb.ConfigResponse{ConfigJsonBlob: writer.String()}, nil +} + +// RequestQueries request/pull distributed queries +func (b *agentBinding) RequestQueries(ctx context.Context, _ *pb.AgentApiRequest) (*pb.QueryCollection, error) { + queryMap, _, err := b.service.GetDistributedQueries(newCtx(ctx)) + if err != nil { + return nil, err + } + var result pb.QueryCollection + for id, query := range queryMap { + result.Queries = append(result.Queries, &pb.QueryCollection_Query{Id: id, Query: query}) + } + return &result, nil +} + +// StatusLog handles osquery logging messages +type StatusLog struct { + Severity string `json:"s"` + Filename string `json:"f"` + Line string `json:"i"` + Message string `json:"m"` +} + +// convert the json from grpc client to an object suitable +// for consumption by fleet +func toKolideLog(jsn string) (*kolide.OsqueryStatusLog, error) { + var status StatusLog + err := json.NewDecoder(bytes.NewBufferString(jsn)).Decode(&status) + if err != nil { + return nil, err + } + result := &kolide.OsqueryStatusLog{ + Severity: status.Severity, + Filename: status.Filename, + Line: status.Line, + Message: status.Message, + } + return result, nil +} + +// PublishLogs publish logs from osqueryd +func (b *agentBinding) PublishLogs(ctx context.Context, coll *pb.LogCollection) (*pb.AgentApiResponse, error) { + handler := func(_ context.Context, _ *pb.LogCollection) error { return nil } + switch coll.LogType { + case pb.LogCollection_RESULT: + handler = b.handleResultLogs + case pb.LogCollection_STATUS: + handler = b.handleStatusLogs + } + if err := handler(ctx, coll); err != nil { + return nil, err + } + return &pb.AgentApiResponse{}, nil +} + +func (b *agentBinding) handleResultLogs(ctx context.Context, coll *pb.LogCollection) error { + var results []kolide.OsqueryResultLog + for _, log := range coll.Logs { + var result kolide.OsqueryResultLog + if err := json.Unmarshal([]byte(log.Data), &result); err != nil { + return errors.Wrap(err, "unmarshaling result log") + } + results = append(results, result) + } + if err := b.service.SubmitResultLogs(newCtx(ctx), results); err != nil { + return errors.Wrap(err, "submitting status logs") + } + return nil +} + +func (b *agentBinding) handleStatusLogs(ctx context.Context, coll *pb.LogCollection) error { + var statuses []kolide.OsqueryStatusLog + for _, record := range coll.Logs { + status, err := toKolideLog(record.Data) + if err != nil { + return errors.Wrap(err, "decoding status log") + } + statuses = append(statuses, *status) + } + if err := b.service.SubmitStatusLogs(newCtx(ctx), statuses); err != nil { + return errors.Wrap(err, "submitting status logs") + } + return nil +} + +// PublishResults publish distributed query results +func (b *agentBinding) PublishResults(ctx context.Context, coll *pb.ResultCollection) (*pb.AgentApiResponse, error) { + results := kolide.OsqueryDistributedQueryResults{} + statuses := map[string]string{} + for _, result := range coll.Results { + statuses[result.Id] = strconv.Itoa(int(result.Status)) + rows := []map[string]string{} + for _, row := range result.Rows { + cols := map[string]string{} + for _, colVal := range row.Columns { + cols[colVal.Name] = colVal.Value + } + if len(cols) == 0 { + continue + } + rows = append(rows, cols) + } + results[result.Id] = rows + } + if err := b.service.SubmitDistributedQueryResults(newCtx(ctx), results, statuses); err != nil { + return nil, errors.Wrap(err, "submitting distributed query results") + } + return &pb.AgentApiResponse{}, nil +} + +// HotConfigure pushed configurations +func (b *agentBinding) HotConfigure(in *pb.AgentApiRequest, svr pb.Api_HotConfigureServer) error { + return errNotImplmented +} + +// HotlineBling this would be live query push to agent +func (b *agentBinding) HotlineBling(svr pb.Api_HotlineBlingServer) error { + return errNotImplmented +} diff --git a/server/launcher/binding_test.go b/server/launcher/binding_test.go new file mode 100644 index 0000000000..9aea14c58a --- /dev/null +++ b/server/launcher/binding_test.go @@ -0,0 +1,391 @@ +package launcher + +import ( + newctx "context" + "errors" + "testing" + + pb "github.com/kolide/agent-api" + "github.com/kolide/fleet/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +type mockEnrollError struct{} + +func (ee *mockEnrollError) NodeInvalid() bool { return true } +func (ee *mockEnrollError) Error() string { return "enroll failed" } + +type mockOsqueryService struct { + mock.Mock +} + +func (m *mockOsqueryService) EnrollAgent(ctx newctx.Context, enrollSecret, hostIdentifier string) (string, error) { + args := m.Called(ctx, enrollSecret, hostIdentifier) + return args.String(0), args.Error(1) +} + +func (m *mockOsqueryService) AuthenticateHost(ctx newctx.Context, nodeKey string) (*kolide.Host, error) { + args := m.Called(ctx, nodeKey) + return args.Get(0).(*kolide.Host), args.Error(1) +} +func (m *mockOsqueryService) GetClientConfig(ctx newctx.Context) (*kolide.OsqueryConfig, error) { + args := m.Called(ctx) + return args.Get(0).(*kolide.OsqueryConfig), args.Error(1) + +} +func (m *mockOsqueryService) GetDistributedQueries(ctx newctx.Context) (map[string]string, uint, error) { + args := m.Called(ctx) + return args.Get(0).(map[string]string), args.Get(1).(uint), args.Error(2) +} +func (m *mockOsqueryService) SubmitDistributedQueryResults(ctx newctx.Context, results kolide.OsqueryDistributedQueryResults, statuses map[string]string) error { + args := m.Called(ctx, results, statuses) + return args.Error(0) +} +func (m *mockOsqueryService) SubmitStatusLogs(ctx newctx.Context, logs []kolide.OsqueryStatusLog) error { + args := m.Called(ctx, logs) + return args.Error(0) +} +func (m *mockOsqueryService) SubmitResultLogs(ctx newctx.Context, logs []kolide.OsqueryResultLog) error { + args := m.Called(ctx, logs) + return args.Error(0) +} + +var oldContext = context.Background() +var newContext = newctx.Background() +var errMockEnrollError = &mockEnrollError{} +var errTestError = errors.New("test error") + +func TestRequestEnrollementHappyPath(t *testing.T) { + request := &pb.EnrollmentRequest{ + EnrollSecret: "supersecret", + HostIdentifier: "somehost", + } + + mockSvc := new(mockOsqueryService) + mockSvc.On( + "EnrollAgent", + newctx.Background(), + request.EnrollSecret, + request.HostIdentifier, + ).Return( + "nodekey", + nil, + ) + agent := agentBinding{ + service: mockSvc, + } + + resp, err := agent.RequestEnrollment(oldContext, request) + mockSvc.AssertExpectations(t) + assert.Nil(t, err) + assert.Equal(t, "nodekey", resp.NodeKey) + assert.False(t, resp.NodeInvalid) +} + +func TestRequestEnrollmentFailed(t *testing.T) { + request := &pb.EnrollmentRequest{ + EnrollSecret: "supersecret", + HostIdentifier: "somehost", + } + + mockSvc := new(mockOsqueryService) + mockSvc.On( + "EnrollAgent", + newctx.Background(), + request.EnrollSecret, + request.HostIdentifier, + ).Return( + "", + errMockEnrollError, + ) + agent := agentBinding{ + service: mockSvc, + } + + resp, err := agent.RequestEnrollment(oldContext, request) + mockSvc.AssertExpectations(t) + assert.Nil(t, err) + assert.True(t, resp.NodeInvalid) +} + +func TestRequestEnrollmentError(t *testing.T) { + request := &pb.EnrollmentRequest{ + EnrollSecret: "supersecret", + HostIdentifier: "somehost", + } + + mockSvc := new(mockOsqueryService) + mockSvc.On( + "EnrollAgent", + newctx.Background(), + request.EnrollSecret, + request.HostIdentifier, + ).Return( + "", + errTestError, + ) + agent := agentBinding{ + service: mockSvc, + } + + _, err := agent.RequestEnrollment(oldContext, request) + mockSvc.AssertExpectations(t) + require.NotNil(t, err) + assert.Equal(t, errTestError, err) +} + +func TestRequestConfigHappyPath(t *testing.T) { + request := &pb.AgentApiRequest{ + NodeKey: "nodekey", + } + + mockSvc := new(mockOsqueryService) + mockSvc.On( + "GetClientConfig", + newctx.Background(), + ).Return( + &kolide.OsqueryConfig{ + Options: map[string]interface{}{ + "option1": "optionval", + "distributed_plugin": "tls", + }, + Decorators: kolide.Decorators{ + Load: []string{ + "SELECT * FROM users u JOIN groups g WHERE u.gid = g.gid", + }, + }, + }, + nil, + ) + agent := agentBinding{ + service: mockSvc, + } + + resp, err := agent.RequestConfig(oldContext, request) + mockSvc.AssertExpectations(t) + assert.Nil(t, err) + // verify distributed_plugin was removed + expectedJSON := "{\"options\":{\"option1\":\"optionval\"},\"decorators\":{\"load\":[\"SELECT * FROM users u JOIN groups g WHERE u.gid = g.gid\"]}}\n" + require.NotNil(t, resp) + assert.Equal(t, expectedJSON, resp.ConfigJsonBlob) + assert.False(t, resp.NodeInvalid) +} + +func TestRequestConfigError(t *testing.T) { + request := &pb.AgentApiRequest{ + NodeKey: "nodekey", + } + var nilConfig *kolide.OsqueryConfig + + mockSvc := new(mockOsqueryService) + mockSvc.On( + "GetClientConfig", + newctx.Background(), + ).Return( + nilConfig, + errTestError, + ) + agent := agentBinding{ + service: mockSvc, + } + + resp, err := agent.RequestConfig(oldContext, request) + mockSvc.AssertExpectations(t) + require.NotNil(t, err) + assert.Equal(t, errTestError, err) + assert.Nil(t, resp) +} + +func TestRequestQueriesHappyPath(t *testing.T) { + mockSvc := new(mockOsqueryService) + mockSvc.On( + "GetDistributedQueries", + newctx.Background(), + ).Return( + map[string]string{ + "query1": "select * from foo;", + }, + uint(0), + nil, + ) + agent := agentBinding{mockSvc} + qc, err := agent.RequestQueries(oldContext, nil) + mockSvc.AssertExpectations(t) + require.Nil(t, err) + require.NotNil(t, qc) + assert.Len(t, qc.Queries, 1) +} + +func TestToKolideLog(t *testing.T) { + jsn := "{\"s\":\"0\",\"f\":\"scheduler.cpp\",\"i\":\"73\",\"m\":\"Executing scheduled query pack\\/xxx\\/services: select name, port, protocol from etc_services;\",\"h\":\"DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C\",\"c\":\"Fri Aug 11 22:32:27 2017 UTC\",\"u\":\"1502490747\"}" + sl, err := toKolideLog(jsn) + require.Nil(t, err, "unexpected error") + assert.Equal(t, "0", sl.Severity, "severity mismatch") + assert.Equal(t, "scheduler.cpp", sl.Filename, "file name mismatch") + assert.Equal(t, "73", sl.Line, "line number mismatch") + malformedJSON := "{\"s\":\"0,\"f\":\"scheduler.cpp\",\"i\":\"73\",\"m\":\"Executing scheduled query pack\\/xxx\\/services: select name, port, protocol from etc_services;\",\"h\":\"DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C\",\"c\":\"Fri Aug 11 22:32:27 2017 UTC\",\"u\":\"1502490747\"}" + sl, err = toKolideLog(malformedJSON) + assert.NotNil(t, err, "malformed json should have erred") + assert.Nil(t, sl, "result should be nil on err") +} + +func TestPublishStatusLogs(t *testing.T) { + statusJSON := "{\"s\":\"0\",\"f\":\"scheduler.cpp\",\"i\":\"73\",\"m\":\"Executing scheduled query pack\\/xxx\\/services: select name, port, protocol from etc_services;\",\"h\":\"DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C\",\"c\":\"Fri Aug 11 22:32:27 2017 UTC\",\"u\":\"1502490747\"}" + statusLogCol := &pb.LogCollection{ + LogType: pb.LogCollection_STATUS, + Logs: []*pb.LogCollection_Log{ + &pb.LogCollection_Log{ + Data: statusJSON, + }, + }, + } + statuses := []kolide.OsqueryStatusLog{ + kolide.OsqueryStatusLog{ + Severity: "0", + Filename: "scheduler.cpp", + Line: "73", + Message: `Executing scheduled query pack/xxx/services: select name, port, protocol from etc_services;`, + }, + } + mockSvc := new(mockOsqueryService) + mockSvc.On( + "SubmitStatusLogs", + newContext, + statuses, + ).Return( + nil, + ) + agent := agentBinding{mockSvc} + resp, err := agent.PublishLogs(oldContext, statusLogCol) + mockSvc.AssertExpectations(t) + mockSvc.AssertCalled(t, "SubmitStatusLogs", newContext, statuses) + require.Nil(t, err) + require.NotNil(t, resp) + assert.False(t, resp.NodeInvalid) +} + +func TestPublishStatusLogsUnhandledLogType(t *testing.T) { + statusJSON := "{\"s\":\"0\",\"f\":\"scheduler.cpp\",\"i\":\"73\",\"m\":\"Executing scheduled query pack\\/xxx\\/services: select name, port, protocol from etc_services;\",\"h\":\"DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C\",\"c\":\"Fri Aug 11 22:32:27 2017 UTC\",\"u\":\"1502490747\"}" + statusLogCol := &pb.LogCollection{ + LogType: pb.LogCollection_AGENT, + Logs: []*pb.LogCollection_Log{ + &pb.LogCollection_Log{ + Data: statusJSON, + }, + }, + } + statuses := []kolide.OsqueryStatusLog{ + kolide.OsqueryStatusLog{ + Severity: "0", + Filename: "scheduler.cpp", + Line: "73", + Message: `Executing scheduled query pack/xxx/services: select name, port, protocol from etc_services;`, + }, + } + mockSvc := new(mockOsqueryService) + mockSvc.On( + "SubmitStatusLogs", + newContext, + statuses, + ).Return( + nil, + ) + agent := agentBinding{mockSvc} + resp, err := agent.PublishLogs(oldContext, statusLogCol) + mockSvc.AssertNotCalled(t, "SubmitStatusLogs", newContext, statuses) + require.Nil(t, err) + require.NotNil(t, resp) + assert.False(t, resp.NodeInvalid) +} + +func TestPublishResultLogs(t *testing.T) { + resultJSON := "{\"name\":\"pack\\/xxx\\/services\",\"hostIdentifier\":\"DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C\",\"calendarTime\":\"Fri Aug 11 22:16:45 2017 UTC\",\"unixTime\":\"1502489805\",\"decorations\":{\"host_uuid\":\"DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C\",\"hostname\":\"Johns-MacBook-Pro.local\"},\"columns\":{\"name\":\"ms-dotnetster\",\"port\":\"3126\",\"protocol\":\"udp\"},\"action\":\"added\"}" + resultLogColl := &pb.LogCollection{ + LogType: pb.LogCollection_RESULT, + Logs: []*pb.LogCollection_Log{ + &pb.LogCollection_Log{ + Data: resultJSON, + }, + }, + } + results := []kolide.OsqueryResultLog{ + kolide.OsqueryResultLog{ + Name: "pack/xxx/services", + HostIdentifier: "DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C", + UnixTime: "1502489805", + CalendarTime: "Fri Aug 11 22:16:45 2017 UTC", + Columns: map[string]string{ + "name": "ms-dotnetster", + "port": "3126", + "protocol": "udp", + }, + Action: "added", + Decorations: map[string]string{ + "host_uuid": "DE56C776-2F5A-56DF-81C7-F64EE1BBEC8C", + "hostname": "Johns-MacBook-Pro.local", + }, + }, + } + mockSvc := new(mockOsqueryService) + mockSvc.On( + "SubmitResultLogs", + newContext, + results, + ).Return( + nil, + ) + agent := agentBinding{mockSvc} + resp, err := agent.PublishLogs(oldContext, resultLogColl) + mockSvc.AssertExpectations(t) + require.Nil(t, err) + require.NotNil(t, resp) +} + +func TestPublishResults(t *testing.T) { + coll := &pb.ResultCollection{ + NodeKey: "somekey", + Results: []*pb.ResultCollection_Result{ + &pb.ResultCollection_Result{ + Id: "myquery", + Status: 0, + Rows: []*pb.ResultCollection_Result_ResultRow{ + &pb.ResultCollection_Result_ResultRow{ + Columns: []*pb.ResultCollection_Result_ResultRow_Column{ + &pb.ResultCollection_Result_ResultRow_Column{ + Name: "aColumn", + Value: "aValue", + }, + }, + }, + }, + }, + }, + } + results := kolide.OsqueryDistributedQueryResults{ + "myquery": []map[string]string{ + map[string]string{ + "aColumn": "aValue", + }, + }, + } + statuses := map[string]string{ + "myquery": "0", + } + mockSvc := new(mockOsqueryService) + mockSvc.On( + "SubmitDistributedQueryResults", + newContext, + results, + statuses, + ).Return( + nil, + ) + agent := agentBinding{mockSvc} + resp, err := agent.PublishResults(oldContext, coll) + mockSvc.AssertExpectations(t) + require.Nil(t, err) + require.NotNil(t, resp) +} diff --git a/server/launcher/context.go b/server/launcher/context.go new file mode 100644 index 0000000000..a39ea79a8c --- /dev/null +++ b/server/launcher/context.go @@ -0,0 +1,33 @@ +package launcher + +import ( + newcontext "context" + + "github.com/kolide/fleet/server/contexts/host" + "github.com/kolide/fleet/server/kolide" + old "golang.org/x/net/context" +) + +type contextKey int + +const hostKey contextKey = 0 + +// newCtx is used to map the old golang.com/net/context which we are forced to use +// because our generated gRPC code uses it, to the new stdlib context, which is used +// by the Fleet application. +func newCtx(ctx old.Context) newcontext.Context { + if h, ok := ctx.Value(hostKey).(kolide.Host); ok { + return host.NewContext(newcontext.Background(), h) + } + return newcontext.Background() +} + +// withHost creates a golang.org/x/net/context containing a host +func withHost(ctx old.Context, h kolide.Host) old.Context { + return old.WithValue(ctx, hostKey, h) +} + +func hostFromContext(ctx old.Context) (kolide.Host, bool) { + h, ok := ctx.Value(hostKey).(kolide.Host) + return h, ok +} diff --git a/server/launcher/logging_middleware.go b/server/launcher/logging_middleware.go new file mode 100644 index 0000000000..eeb3dc67a0 --- /dev/null +++ b/server/launcher/logging_middleware.go @@ -0,0 +1,100 @@ +package launcher + +import ( + "time" + + kitlog "github.com/go-kit/kit/log" + pb "github.com/kolide/agent-api" + "golang.org/x/net/context" +) + +type loggingMiddleware struct { + logger kitlog.Logger + next pb.ApiServer +} + +func newLoggingMiddleware(logger kitlog.Logger) func(svc pb.ApiServer) pb.ApiServer { + return func(next pb.ApiServer) pb.ApiServer { + return loggingMiddleware{ + logger: kitlog.With(logger, "component", "gRPC Launcher"), + next: next, + } + } +} + +func (s loggingMiddleware) RequestEnrollment(ctx context.Context, req *pb.EnrollmentRequest) (resp *pb.EnrollmentResponse, err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "RequestEnrollment", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.RequestEnrollment(ctx, req) +} + +func (s loggingMiddleware) RequestConfig(ctx context.Context, req *pb.AgentApiRequest) (resp *pb.ConfigResponse, err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "RequestConfig", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.RequestConfig(ctx, req) +} + +func (s loggingMiddleware) RequestQueries(ctx context.Context, req *pb.AgentApiRequest) (resp *pb.QueryCollection, err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "RequestQueries", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.RequestQueries(ctx, req) +} + +func (s loggingMiddleware) PublishLogs(ctx context.Context, req *pb.LogCollection) (resp *pb.AgentApiResponse, err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "PublishLogs", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.PublishLogs(ctx, req) +} + +func (s loggingMiddleware) PublishResults(ctx context.Context, req *pb.ResultCollection) (resp *pb.AgentApiResponse, err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "PublishResults", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.PublishResults(ctx, req) +} + +func (s loggingMiddleware) HotConfigure(req *pb.AgentApiRequest, svr pb.Api_HotConfigureServer) (err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "HotConfigure", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.HotConfigure(req, svr) +} + +func (s loggingMiddleware) HotlineBling(svr pb.Api_HotlineBlingServer) (err error) { + defer func(begin time.Time) { + s.logger.Log( + "method", "HotlineBling", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + return s.next.HotlineBling(svr) +} diff --git a/server/launcher/server.go b/server/launcher/server.go new file mode 100644 index 0000000000..139f40870f --- /dev/null +++ b/server/launcher/server.go @@ -0,0 +1,41 @@ +// Package launcher provides a gRPC server to handle launcher requests. +package launcher + +import ( + "net/http" + "strings" + + kitlog "github.com/go-kit/kit/log" + pb "github.com/kolide/agent-api" + "github.com/kolide/fleet/server/kolide" + grpc "google.golang.org/grpc" +) + +// Handler extends the grpc.Server, providing Handler that allows us to serve +// both gRPC and http traffic. +type Handler struct { + *grpc.Server +} + +// New creates a gRPC server to handler remote requests from launcher. +func New(svc kolide.OsqueryService, logger kitlog.Logger, opts ...grpc.ServerOption) *Handler { + binding := newAgentBinding(svc) + binding = newAuthMiddleware(svc)(binding) + binding = newLoggingMiddleware(logger)(binding) + + server := grpc.NewServer(opts...) + pb.RegisterApiServer(server, binding) + return &Handler{server} +} + +// Handler will route gRPC traffic to the gRPC server, other http traffic +// will be routed to normal http handler functions. +func (hgprc *Handler) Handler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") { + hgprc.ServeHTTP(w, r) + } else { + h.ServeHTTP(w, r) + } + }) +} diff --git a/tools/lint_license/license_settings.yaml b/tools/lint_license/license_settings.yaml index 8d6213d92c..4643cfc194 100644 --- a/tools/lint_license/license_settings.yaml +++ b/tools/lint_license/license_settings.yaml @@ -135,4 +135,4 @@ overrides: vendor/github.com/go-sql-driver/mysql: MPL-2.0 vendor/github.com/kolide/agent-api: - Kolide \ No newline at end of file + Kolide