diff --git a/docs/api/README.md b/docs/api/README.md index 81f8d65094..1d7b182bff 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -35,3 +35,57 @@ Queries, packs, scheduled queries, labels, invites, users, sessions all behave t All of these objects are put together and distributed to the appropriate osquery agents at the appropriate time. At this time, the best source of truth for the API is the [HTTP handler file](https://github.com/kolide/kolide/blob/master/server/service/handler.go) in the Go application. The REST API is exposed via a transport layer on top of an RPC service which is implemented using a micro-service library called [Go Kit](https://github.com/go-kit/kit). If using the Kolide API is important to you right now, being familiar with Go Kit would definitely be helpful. Like it was said above, we have plans to include richer API documentation in the near future, so stay tuned. If using this API is important to you, please contact us at [support@kolide.co](mailto:support@kolide.co) and tell us, so that we can prioritize creating stable API documentation. + +### Osquery Configuration Import + +You can load packs, queries and other settings from an existing [Osquery configuration file](https://osquery.readthedocs.io/en/stable/deployment/configuration/) by importing the file into Kolide. This can be done posting the stringified contents of the Osquery configuration to the following Kolide endpoint: +``` +// POST body the value of "config" is JSON that has been converted to a string + +{ + "config": "{\"options\":null,\"schedule\":null,\"packs\":{ ... +} + +// POST endpoint + +/api/v1/kolide/osquery/config/import +``` +We provide [a utility program](https://github.com/kolide/configimporter) that will import the configuration automatically. +If you opt to manually import your Osquery configuration you will need to include the contents of externally +referenced packs in your main Osquery configuration file before posting it to Kolide. If you reference packs +in a file like the example below, you will need to get the pack from `external_pack.conf` +and include it in the main configuration. +``` +// Configuration referencing external pack + +{ + "packs": { + "external_pack": "/path/to/external_pack.conf", + "internal_stuff": { + [...] + } + } +} +``` +``` +// Edited configuration containing the internal pack + +{ + "packs": { + "external_pack": { + "shard": "10", + "queries": { + "suid_bins": { + "query": "select * from suid_bins;", + "interval": "3600" + } + } + } + "internal_stuff": { + [...] + } + } +} +``` +Once the configuration file and all the external packs it references are consolidated, post the stringified contents of the configuration +file to Kolide. diff --git a/server/kolide/import_config_test.go b/server/kolide/import_config_test.go index 3234b99428..eac80d4946 100644 --- a/server/kolide/import_config_test.go +++ b/server/kolide/import_config_test.go @@ -10,6 +10,56 @@ import ( "github.com/stretchr/testify/require" ) +func TestConfigUnmarshalling(t *testing.T) { + contents := ` + { + "options":null, + "schedule":null, + "packs":{ + "internal_stuff":{ + "discovery":["select pid from processes where name = 'ldap';"], + "platform":"linux", + "queries":{ + "active_directory":{ + "description":"Check each user's active directory cached settings.", + "interval":"1200", + "query":"select * from ad_config;" + } + }, + "version":"1.5.2" + }, + "testing":{ + "queries":{ + "suid_bins":{ + "interval":"3600", + "query":"select * from suid_bins;" + } + }, + "shard":"10" + } + }, + "file_paths":null, + "yara":null, + "prometheus_targets":null, + "decorators":null + } + ` + + conf := ImportConfig{ + Packs: make(PackNameMap), + ExternalPacks: make(PackNameToPackDetails), + } + + err := json.Unmarshal([]byte(contents), &conf) + assert.Nil(t, err) + require.NotNil(t, conf.Packs["testing"]) + // platform is not defined in the testing pack, so, per osquery docs + // we default to 'all' platforms + details, ok := conf.Packs["testing"].(PackDetails) + require.True(t, ok) + assert.Equal(t, "", details.Platform) +} + func TestIntervalUnmarshal(t *testing.T) { scenarios := []struct { name string diff --git a/server/kolide/import_config_unmarshaler.go b/server/kolide/import_config_unmarshaler.go index 53898da006..50aaf264c1 100644 --- a/server/kolide/import_config_unmarshaler.go +++ b/server/kolide/import_config_unmarshaler.go @@ -2,8 +2,10 @@ package kolide import ( "encoding/json" - "errors" "strconv" + + "github.com/pkg/errors" + "github.com/spf13/cast" ) var wrongTypeError = errors.New("argument missing or unexpected type") @@ -29,7 +31,7 @@ func (pnm PackNameMap) UnmarshalJSON(b []byte) error { } pnm[key] = val default: - return errors.New("can't unmarshal json") + return errors.Errorf("can't unmarshal %s %v", key, val) } } return nil @@ -69,14 +71,6 @@ func uintptr(v interface{}) (*OsQueryConfigInt, error) { return &i, nil } -// Use this when we expext a string value, in this case nil is an error -func toString(v interface{}) (string, error) { - if s, ok := v.(string); ok { - return s, nil - } - return "", wrongTypeError -} - func unmarshalPackDetails(v map[string]interface{}) (PackDetails, error) { var result PackDetails queries, err := unmarshalQueryDetails(v["queries"]) @@ -87,10 +81,7 @@ func unmarshalPackDetails(v map[string]interface{}) (PackDetails, error) { if err != nil { return result, err } - platform, err := toString(v["platform"]) - if err != nil { - return result, err - } + platform := cast.ToString(v["platform"]) shard, err := uintptr(v["shard"]) if err != nil { return result, err @@ -99,7 +90,6 @@ func unmarshalPackDetails(v map[string]interface{}) (PackDetails, error) { if err != nil { return result, err } - result = PackDetails{ Queries: queries, Shard: shard, @@ -120,7 +110,7 @@ func unmarshalDiscovery(val interface{}) ([]string, error) { return result, wrongTypeError } for _, val := range v { - query, err := toString(val) + query, err := cast.ToStringE(val) if err != nil { return result, err } @@ -154,7 +144,7 @@ func unmarshalQueryDetail(val interface{}) (QueryDetails, error) { if err != nil { return result, err } - query, err := toString(v["query"]) + query, err := cast.ToStringE(v["query"]) if err != nil { return result, err } diff --git a/server/service/endpoint_import_config_test.go b/server/service/endpoint_import_config_test.go index 4a779e9ade..2f27b402aa 100644 --- a/server/service/endpoint_import_config_test.go +++ b/server/service/endpoint_import_config_test.go @@ -36,6 +36,31 @@ func testImportConfigWithGlob(t *testing.T, r *testResource) { assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.PacksSection].ImportCount) } +func testImportConfigWithInvalidPlatform(t *testing.T, r *testResource) { + testJSON := ` +{ + "config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"foo\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"glob\":[\"globpack\"],\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}", + "external_pack_configs": { + "external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}", + "globpack": "{\"discovery\":[\"select pid from processes where name = 'zip';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from other;\",\"interval\":1200,\"description\":\"Check other.\"}}}" + }, + "glob_pack_names": ["globpack"] +} +` + buff := bytes.NewBufferString(testJSON) + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + var v mockValidationError + err = json.NewDecoder(resp.Body).Decode(&v) + require.Nil(t, err) + require.Len(t, v.Errors, 1) + assert.Equal(t, "'foo' is not a valid platform", v.Errors[0].Reason) +} + func testImportConfigWithMissingGlob(t *testing.T, r *testResource) { testJSON := ` { diff --git a/server/service/endpoint_test.go b/server/service/endpoint_test.go index 23adac75dd..c6f0a970e7 100644 --- a/server/service/endpoint_test.go +++ b/server/service/endpoint_test.go @@ -123,6 +123,7 @@ var testFunctions = [...]func(*testing.T, *testResource){ testNewDecoratorFailValidation, testDeleteDecorator, testModifyDecoratorNoChanges, + testImportConfigWithInvalidPlatform, } func TestEndpoints(t *testing.T) { diff --git a/server/service/validation_import_config.go b/server/service/validation_import_config.go index 53be38624e..78fb2d8958 100644 --- a/server/service/validation_import_config.go +++ b/server/service/validation_import_config.go @@ -93,8 +93,10 @@ func (vm validationMiddleware) validatePacks(cfg *kolide.ImportConfig, argErrs * } // make sure that each glob pack has JSON content for _, p := range cfg.GlobPackNames { - if _, ok := cfg.ExternalPacks[p]; !ok { + if pd, ok := cfg.ExternalPacks[p]; !ok { argErrs.Appendf("external_packs", "missing content for '%s'", p) + } else { + vm.validatePackContents(p, pd, argErrs) } } continue @@ -102,12 +104,24 @@ func (vm validationMiddleware) validatePacks(cfg *kolide.ImportConfig, argErrs * // if value is a string we expect a file path, in this case, the user has to supply the // contents of said file which we store in ExternalPacks, if it's not there we need to // raise an error - switch pack.(type) { + switch val := pack.(type) { case string: - if _, ok := cfg.ExternalPacks[packName]; !ok { + if pd, ok := cfg.ExternalPacks[packName]; !ok { argErrs.Appendf("external_packs", "missing content for '%s'", packName) + } else { + vm.validatePackContents(packName, pd, argErrs) } + case kolide.PackDetails: + vm.validatePackContents(packName, val, argErrs) } } } } + +func (vm validationMiddleware) validatePackContents(packName string, pack kolide.PackDetails, argErrs *invalidArgumentError) { + switch pack.Platform { + case "", "darwin", "freebsd", "windows", "linux", "any", "all": + default: + argErrs.Appendf("pack", "'%s' is not a valid platform", pack.Platform) + } +}