From ff92f6749a03ee83fde3722798445002bc32f32e Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Tue, 6 Sep 2016 17:28:07 -0400 Subject: [PATCH] Osqueryd Service (#119) * host service stuff * working on the osquery service api * log status and results logs --- datastore/datastore_test.go | 4 +- datastore/gorm.go | 52 ++++++++++++-- kolide/datastore.go | 1 + kolide/hosts.go | 45 ++++++++++++ kolide/osquery.go | 44 ++++++------ kolide/service.go | 1 + server/service_hosts.go | 92 ++++++++++++++++++++++++ server/service_hosts_test.go | 124 +++++++++++++++++++++++++++++++++ server/service_osquery.go | 26 +++++-- server/service_osquery_test.go | 72 +++++++++++++++++++ 10 files changed, 428 insertions(+), 33 deletions(-) create mode 100644 kolide/hosts.go create mode 100644 server/service_hosts.go create mode 100644 server/service_hosts_test.go create mode 100644 server/service_osquery_test.go diff --git a/datastore/datastore_test.go b/datastore/datastore_test.go index 639cc0028d..461bdde714 100644 --- a/datastore/datastore_test.go +++ b/datastore/datastore_test.go @@ -90,7 +90,7 @@ var enrollTests = []struct { }, } -func testEnrollHost(t *testing.T, db kolide.OsqueryStore) { +func testEnrollHost(t *testing.T, db kolide.HostStore) { var hosts []*kolide.Host for i, tt := range enrollTests { h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.ip, tt.platform, tt.nodeKeySize) @@ -145,7 +145,7 @@ func testEnrollHost(t *testing.T, db kolide.OsqueryStore) { } -func testAuthenticateHost(t *testing.T, db kolide.OsqueryStore) { +func testAuthenticateHost(t *testing.T, db kolide.HostStore) { for i, tt := range enrollTests { h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.ip, tt.platform, tt.nodeKeySize) if err != nil { diff --git a/datastore/gorm.go b/datastore/gorm.go index b0834b0e4c..015fb5d69d 100644 --- a/datastore/gorm.go +++ b/datastore/gorm.go @@ -175,6 +175,44 @@ func (orm gormDB) SaveHost(host *kolide.Host) error { return nil } +func (orm gormDB) DeleteHost(host *kolide.Host) error { + return orm.DB.Delete(host).Error +} + +func (orm gormDB) Host(id uint) (*kolide.Host, error) { + host := &kolide.Host{ + ID: id, + } + err := orm.DB.Where(host).First(host).Error + if err != nil { + return nil, err + } + return host, nil +} + +func (orm gormDB) Hosts() ([]*kolide.Host, error) { + var hosts []*kolide.Host + err := orm.DB.Find(&hosts).Error + if err != nil { + return nil, err + } + return hosts, nil +} + +func (orm gormDB) NewHost(host *kolide.Host) (*kolide.Host, error) { + if host == nil { + return nil, errors.New( + "error creating host", + "nil pointer passed to NewHost", + ) + } + err := orm.DB.Create(host).Error + if err != nil { + return nil, err + } + return host, err +} + func (orm gormDB) MarkHostSeen(host *kolide.Host, t time.Time) error { updateTime := time.Now() err := orm.DB.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", t, host.NodeKey).Error @@ -476,13 +514,13 @@ SELECT q.differential, q.platform, q.version -FROM - queries q -JOIN - pack_queries pq -ON - pq.query_id = q.id -AND +FROM + queries q +JOIN + pack_queries pq +ON + pq.query_id = q.id +AND pq.pack_id = ?; `, pack.ID).Rows() if err != nil && err != gorm.ErrRecordNotFound { diff --git a/kolide/datastore.go b/kolide/datastore.go index 956ef61b18..a122c12210 100644 --- a/kolide/datastore.go +++ b/kolide/datastore.go @@ -6,6 +6,7 @@ type Datastore interface { QueryStore PackStore OsqueryStore + HostStore EmailStore SessionStore Name() string diff --git a/kolide/hosts.go b/kolide/hosts.go new file mode 100644 index 0000000000..51cf978e25 --- /dev/null +++ b/kolide/hosts.go @@ -0,0 +1,45 @@ +package kolide + +import ( + "time" + + "golang.org/x/net/context" +) + +type HostStore interface { + NewHost(host *Host) (*Host, error) + SaveHost(host *Host) error + DeleteHost(host *Host) error + Host(id uint) (*Host, error) + Hosts() ([]*Host, error) + EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*Host, error) + AuthenticateHost(nodeKey string) (*Host, error) + MarkHostSeen(host *Host, t time.Time) error +} + +type HostService interface { + GetAllHosts(ctx context.Context) ([]*Host, error) + GetHost(ctx context.Context, id uint) (*Host, error) + NewHost(ctx context.Context, p HostPayload) (*Host, error) + ModifyHost(ctx context.Context, id uint, p HostPayload) (*Host, error) + DeleteHost(ctx context.Context, id uint) error +} + +type HostPayload struct { + NodeKey *string + HostName *string + UUID *string + IPAddress *string + Platform *string +} + +type Host struct { + ID uint `gorm:"primary_key"` + CreatedAt time.Time + UpdatedAt time.Time + NodeKey string `gorm:"unique_index:idx_host_unique_nodekey"` + HostName string + UUID string `gorm:"unique_index:idx_host_unique_uuid"` + IPAddress string + Platform string +} diff --git a/kolide/osquery.go b/kolide/osquery.go index 65f4ab6c23..2fda20b3ac 100644 --- a/kolide/osquery.go +++ b/kolide/osquery.go @@ -8,11 +8,6 @@ import ( ) type OsqueryStore interface { - EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*Host, error) - AuthenticateHost(nodeKey string) (*Host, error) - SaveHost(host *Host) error - MarkHostSeen(host *Host, t time.Time) error - LabelQueriesForHost(host *Host, cutoff time.Time) (map[string]string, error) RecordLabelQueryExecutions(host *Host, results map[string]bool, t time.Time) error NewLabel(label *Label) error @@ -20,30 +15,39 @@ type OsqueryStore interface { type OsqueryService interface { EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier string) (string, error) - GetClientConfig(ctx context.Context, action string, data *json.RawMessage) (*OsqueryConfig, error) - Log(ctx context.Context, logType string, data *json.RawMessage) error + GetClientConfig(ctx context.Context, action string, data json.RawMessage) (OsqueryConfig, error) GetDistributedQueries(ctx context.Context) (map[string]string, error) - LogDistributedQueryResults(ctx context.Context, queries map[string][]map[string]string) error + SubmitDistributedQueryResults(ctx context.Context, results OsqueryDistributedQueryResults) error + SubmitStatusLogs(ctx context.Context, logs []OsqueryResultLog) error + SubmitResultsLogs(ctx context.Context, logs []OsqueryStatusLog) error } -type Host struct { - ID uint `gorm:"primary_key"` - CreatedAt time.Time - UpdatedAt time.Time - NodeKey string `gorm:"unique_index:idx_host_unique_nodekey"` - HostName string - UUID string `gorm:"unique_index:idx_host_unique_uuid"` - IPAddress string - Platform string -} +type OsqueryDistributedQueryResults map[string][]map[string]string type OsqueryConfig struct { Packs []Pack Schedule []Query - Options map[string]interface{} } -// TODO: move this to just use OsqueryServerStore.LabelQueriesForHot +type OsqueryResultLog struct { + Name string `json:"name"` + HostIdentifier string `json:"hostIdentifier"` + UnixTime string `json:"unixTime"` + CalendarTime string `json:"calendarTime"` + Columns map[string]string `json:"columns"` + Action string `json:"action"` +} + +type OsqueryStatusLog struct { + Severity string `json:"severity"` + Filename string `json:"filename"` + Line string `json:"line"` + Message string `json:"message"` + Version string `json:"version"` + Decorations map[string]string `json:"decorations"` +} + +// TODO: move this to just use LabelQueriesForHot // LabelQueriesForHost calculates the appropriate update cutoff (given // interval) and uses the datastore to retrieve the label queries for the // provided host. diff --git a/kolide/service.go b/kolide/service.go index 8261f915b1..90a0bb04b0 100644 --- a/kolide/service.go +++ b/kolide/service.go @@ -7,4 +7,5 @@ type Service interface { PackService QueryService OsqueryService + HostService } diff --git a/server/service_hosts.go b/server/service_hosts.go new file mode 100644 index 0000000000..1c16cfa4b6 --- /dev/null +++ b/server/service_hosts.go @@ -0,0 +1,92 @@ +package server + +import ( + "github.com/kolide/kolide-ose/kolide" + "golang.org/x/net/context" +) + +func (svc service) GetAllHosts(ctx context.Context) ([]*kolide.Host, error) { + return svc.ds.Hosts() +} + +func (svc service) GetHost(ctx context.Context, id uint) (*kolide.Host, error) { + return svc.ds.Host(id) +} + +func (svc service) NewHost(ctx context.Context, p kolide.HostPayload) (*kolide.Host, error) { + var host kolide.Host + + if p.HostName != nil { + host.HostName = *p.HostName + } + + if p.IPAddress != nil { + host.IPAddress = *p.IPAddress + } + + if p.NodeKey != nil { + host.NodeKey = *p.NodeKey + } + + if p.Platform != nil { + host.Platform = *p.Platform + } + + if p.UUID != nil { + host.UUID = *p.UUID + } + + _, err := svc.ds.NewHost(&host) + if err != nil { + return nil, err + } + return &host, nil +} + +func (svc service) ModifyHost(ctx context.Context, id uint, p kolide.HostPayload) (*kolide.Host, error) { + host, err := svc.ds.Host(id) + if err != nil { + return nil, err + } + + if p.HostName != nil { + host.HostName = *p.HostName + } + + if p.IPAddress != nil { + host.IPAddress = *p.IPAddress + } + + if p.NodeKey != nil { + host.NodeKey = *p.NodeKey + } + + if p.Platform != nil { + host.Platform = *p.Platform + } + + if p.UUID != nil { + host.UUID = *p.UUID + } + + err = svc.ds.SaveHost(host) + if err != nil { + return nil, err + } + + return host, nil +} + +func (svc service) DeleteHost(ctx context.Context, id uint) error { + host, err := svc.ds.Host(id) + if err != nil { + return err + } + + err = svc.ds.DeleteHost(host) + if err != nil { + return err + } + + return nil +} diff --git a/server/service_hosts_test.go b/server/service_hosts_test.go new file mode 100644 index 0000000000..efbe824ab2 --- /dev/null +++ b/server/service_hosts_test.go @@ -0,0 +1,124 @@ +package server + +import ( + "testing" + + "github.com/kolide/kolide-ose/datastore" + "github.com/kolide/kolide-ose/kolide" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +func TestGetAllHosts(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + svc, err := NewService(testConfig(ds)) + assert.Nil(t, err) + + ctx := context.Background() + + hosts, err := svc.GetAllHosts(ctx) + assert.Nil(t, err) + assert.Len(t, hosts, 0) + + _, err = ds.NewHost(&kolide.Host{ + HostName: "foo", + }) + assert.Nil(t, err) + + hosts, err = svc.GetAllHosts(ctx) + assert.Nil(t, err) + assert.Len(t, hosts, 1) +} + +func TestGetHost(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + svc, err := NewService(testConfig(ds)) + assert.Nil(t, err) + + ctx := context.Background() + + host, err := ds.NewHost(&kolide.Host{ + HostName: "foo", + }) + assert.Nil(t, err) + assert.NotZero(t, host.ID) + + hostVerify, err := svc.GetHost(ctx, host.ID) + assert.Nil(t, err) + + assert.Equal(t, host.ID, hostVerify.ID) +} + +func TestNewHost(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + svc, err := NewService(testConfig(ds)) + assert.Nil(t, err) + + ctx := context.Background() + + hostname := "foo" + _, err = svc.NewHost(ctx, kolide.HostPayload{ + HostName: &hostname, + }) + + assert.Nil(t, err) + + hosts, err := ds.Hosts() + assert.Nil(t, err) + assert.Len(t, hosts, 1) +} + +func TestModifyHost(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + svc, err := NewService(testConfig(ds)) + assert.Nil(t, err) + + ctx := context.Background() + + host, err := ds.NewHost(&kolide.Host{ + HostName: "foo", + }) + assert.Nil(t, err) + assert.NotZero(t, host.ID) + + newHostname := "bar" + hostVerify, err := svc.ModifyHost(ctx, host.ID, kolide.HostPayload{ + HostName: &newHostname, + }) + assert.Nil(t, err) + + assert.Equal(t, host.ID, hostVerify.ID) + assert.Equal(t, "bar", hostVerify.HostName) +} + +func TestDeleteHost(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + svc, err := NewService(testConfig(ds)) + assert.Nil(t, err) + + ctx := context.Background() + + host, err := ds.NewHost(&kolide.Host{ + HostName: "foo", + }) + assert.Nil(t, err) + assert.NotZero(t, host.ID) + + err = svc.DeleteHost(ctx, host.ID) + assert.Nil(t, err) + + hosts, err := ds.Hosts() + assert.Nil(t, err) + assert.Len(t, hosts, 0) + +} diff --git a/server/service_osquery.go b/server/service_osquery.go index 56f789fa1a..d1d6492884 100644 --- a/server/service_osquery.go +++ b/server/service_osquery.go @@ -3,6 +3,7 @@ package server import ( "encoding/json" "fmt" + "net/http" "github.com/kolide/kolide-ose/errors" "github.com/kolide/kolide-ose/kolide" @@ -25,11 +26,28 @@ func (svc service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier return host.NodeKey, nil } -func (svc service) GetClientConfig(ctx context.Context, action string, data *json.RawMessage) (*kolide.OsqueryConfig, error) { - return nil, nil +func (svc service) GetClientConfig(ctx context.Context, action string, data json.RawMessage) (kolide.OsqueryConfig, error) { + var config kolide.OsqueryConfig + return config, nil } -func (svc service) Log(ctx context.Context, logType string, data *json.RawMessage) error { +func (svc service) SubmitStatusLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error { + for _, log := range logs { + err := json.NewEncoder(svc.osqueryStatusLogWriter).Encode(log) + if err != nil { + return errors.NewFromError(err, http.StatusInternalServerError, "error writing status log") + } + } + return nil +} + +func (svc service) SubmitResultsLogs(ctx context.Context, logs []kolide.OsqueryStatusLog) error { + for _, log := range logs { + err := json.NewEncoder(svc.osqueryResultsLogWriter).Encode(log) + if err != nil { + return errors.NewFromError(err, http.StatusInternalServerError, "error writing result log") + } + } return nil } @@ -41,6 +59,6 @@ func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string return queries, nil } -func (svc service) LogDistributedQueryResults(ctx context.Context, queries map[string][]map[string]string) error { +func (svc service) SubmitDistributedQueryResults(ctx context.Context, results kolide.OsqueryDistributedQueryResults) error { return nil } diff --git a/server/service_osquery_test.go b/server/service_osquery_test.go new file mode 100644 index 0000000000..bfa4c9d9c1 --- /dev/null +++ b/server/service_osquery_test.go @@ -0,0 +1,72 @@ +package server + +import ( + "context" + "testing" + + kitlog "github.com/go-kit/kit/log" + "github.com/kolide/kolide-ose/datastore" + "github.com/stretchr/testify/assert" +) + +func TestEnrollAgent(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + config := ServiceConfig{ + Datastore: ds, + Logger: kitlog.NewNopLogger(), + BcryptCost: 12, + SaltKeySize: 24, + SessionCookieName: "KolideSession", + OsqueryNodeKeySize: 12, + OsqueryEnrollSecret: "foobar", + } + svc, err := NewService(config) + assert.Nil(t, err) + + ctx := context.Background() + + hosts, err := ds.Hosts() + assert.Nil(t, err) + assert.Len(t, hosts, 0) + + nodeKey, err := svc.EnrollAgent(ctx, "foobar", "host123") + assert.Nil(t, err) + assert.NotEmpty(t, nodeKey) + + hosts, err = ds.Hosts() + assert.Nil(t, err) + assert.Len(t, hosts, 1) +} + +func TestEnrollAgentIncorrectEnrollSecret(t *testing.T) { + ds, err := datastore.New("gorm-sqlite3", ":memory:") + assert.Nil(t, err) + + config := ServiceConfig{ + Datastore: ds, + Logger: kitlog.NewNopLogger(), + BcryptCost: 12, + SaltKeySize: 24, + SessionCookieName: "KolideSession", + OsqueryNodeKeySize: 12, + OsqueryEnrollSecret: "foobar", + } + svc, err := NewService(config) + assert.Nil(t, err) + + ctx := context.Background() + + hosts, err := ds.Hosts() + assert.Nil(t, err) + assert.Len(t, hosts, 0) + + nodeKey, err := svc.EnrollAgent(ctx, "not_correct", "host123") + assert.NotNil(t, err) + assert.Empty(t, nodeKey) + + hosts, err = ds.Hosts() + assert.Nil(t, err) + assert.Len(t, hosts, 0) +}