From 2795a61b3cfd6209488fd69b2e85ddcaef73cd38 Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Mon, 7 May 2018 17:50:20 -0600 Subject: [PATCH] Working prototype of fleetctl apply (#1762) --- cmd/fleetctl/apply.go | 160 ++++++++++++++++++ cmd/fleetctl/fleetctl.go | 6 + docs/cli/README.md | 36 ++-- examples/config-many-files/config.yml | 4 +- examples/config-many-files/labels.yml | 28 +-- .../packs/osquery-monitoring.yml | 4 +- examples/config-many-files/queries.yml | 62 +++---- examples/config-single-file.yml | 97 ++++++----- 8 files changed, 281 insertions(+), 116 deletions(-) create mode 100644 cmd/fleetctl/apply.go diff --git a/cmd/fleetctl/apply.go b/cmd/fleetctl/apply.go new file mode 100644 index 0000000000..990e5da40b --- /dev/null +++ b/cmd/fleetctl/apply.go @@ -0,0 +1,160 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "strings" + + "github.com/ghodss/yaml" + "github.com/kolide/fleet/server/kolide" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +type specMetadata struct { + Kind string `json:"kind"` + Version string `json:"apiVersion"` + Spec json.RawMessage `json:"spec"` +} + +type specGroup struct { + Queries []*kolide.QuerySpec + Packs []*kolide.PackSpec + Labels []*kolide.LabelSpec + Options *kolide.OptionsSpec +} + +func specGroupFromBytes(b []byte) (*specGroup, error) { + specs := &specGroup{ + Queries: []*kolide.QuerySpec{}, + Packs: []*kolide.PackSpec{}, + Labels: []*kolide.LabelSpec{}, + } + + for _, spec := range strings.Split(string(b), "---") { + if strings.TrimSpace(spec) == "" { + continue + } + + var s specMetadata + if err := yaml.Unmarshal([]byte(spec), &s); err != nil { + return nil, err + } + + if s.Spec == nil { + return nil, errors.Errorf("no spec field on %q document", s.Kind) + } + + switch strings.ToLower(s.Kind) { + case "query": + var querySpec *kolide.QuerySpec + if err := yaml.Unmarshal(s.Spec, &querySpec); err != nil { + return nil, errors.Wrap(err, "unmarshaling query spec") + } + specs.Queries = append(specs.Queries, querySpec) + + case "pack": + var packSpec *kolide.PackSpec + if err := yaml.Unmarshal(s.Spec, &packSpec); err != nil { + return nil, errors.Wrap(err, "unmarshaling pack spec") + } + specs.Packs = append(specs.Packs, packSpec) + + case "label": + var labelSpec *kolide.LabelSpec + if err := yaml.Unmarshal(s.Spec, &labelSpec); err != nil { + return nil, errors.Wrap(err, "unmarshaling label spec") + } + specs.Labels = append(specs.Labels, labelSpec) + + case "options": + if specs.Options != nil { + return nil, errors.New("options defined twice in the same file") + } + + var optionSpec *kolide.OptionsSpec + if err := yaml.Unmarshal(s.Spec, &optionSpec); err != nil { + return nil, errors.Wrap(err, "unmarshaling option spec") + } + specs.Options = optionSpec + + default: + return nil, errors.Errorf("unknown kind %q", s.Kind) + } + } + + return specs, nil +} + +func applyCommand() cli.Command { + var ( + flFilename string + flDebug bool + ) + return cli.Command{ + Name: "apply", + Usage: "Apply files to declaratively manage osquery configurations", + UsageText: `fleetctl apply [options]`, + Flags: []cli.Flag{ + configFlag(), + contextFlag(), + cli.StringFlag{ + Name: "f", + EnvVar: "FILENAME", + Value: "", + Destination: &flFilename, + Usage: "A file to apply", + }, + cli.BoolFlag{ + Name: "debug", + EnvVar: "DEBUG", + Destination: &flDebug, + Usage: "Whether or not to enable debug logging", + }, + }, + Action: func(c *cli.Context) error { + if flFilename == "" { + return errors.New("-f must be specified") + } + + b, err := ioutil.ReadFile(flFilename) + if err != nil { + return err + } + + fleet, err := clientFromCLI(c) + if err != nil { + return err + } + + specs, err := specGroupFromBytes(b) + if err != nil { + return err + } + + if len(specs.Queries) > 0 { + if err := fleet.ApplyQuerySpecs(specs.Queries); err != nil { + return errors.Wrap(err, "applying queries") + } + fmt.Printf("[+] applied %d queries\n", len(specs.Queries)) + } + + if len(specs.Labels) > 0 { + if err := fleet.ApplyLabelSpecs(specs.Labels); err != nil { + return errors.Wrap(err, "applying labels") + } + fmt.Printf("[+] applied %d labels\n", len(specs.Labels)) + } + + if len(specs.Packs) > 0 { + if err := fleet.ApplyPackSpecs(specs.Packs); err != nil { + return errors.Wrap(err, "applying packs") + } + fmt.Printf("[+] applied %d packs\n", len(specs.Packs)) + } + + return nil + }, + } +} diff --git a/cmd/fleetctl/fleetctl.go b/cmd/fleetctl/fleetctl.go index 5bd4a8cb66..cc3c6552b3 100644 --- a/cmd/fleetctl/fleetctl.go +++ b/cmd/fleetctl/fleetctl.go @@ -22,9 +22,15 @@ func main() { } app.Commands = []cli.Command{ + applyCommand(), setupCommand(), loginCommand(), logoutCommand(), + cli.Command{ + Name: "get", + Usage: "Get/list resources", + Subcommands: []cli.Command{}, + }, cli.Command{ Name: "config", Usage: "Modify how and which Fleet server to connect to", diff --git a/docs/cli/README.md b/docs/cli/README.md index a3ec43c443..038ba46419 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -95,8 +95,8 @@ All of these files can be concatenated together into [one file](../../examples/c The following file describes configuration options passed to the osquery instance. All other configuration data will be over-written by the application of this file. ```yaml -apiVersion: kolide.com/v1alpha1 -kind: OsqueryOptions +apiVersion: v1 +kind: options spec: config: options: @@ -170,14 +170,14 @@ spec: The following file describes the labels which hosts should be automatically grouped into. The label resource should reference the query by name. Both of these resources can be included in the same file as such: ```yaml -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: slack_not_running query: slack_not_running --- apiVersion: kolide.com/v1/alpha1 -kind: OsqueryQuery +kind: query spec: name: slack_not_running query: > @@ -194,8 +194,8 @@ spec: For especially long or complex queries, you may want to define one query in one file. Continued edits and applications to this file will update the query as long as the `metadata.name` does not change. If you want to change the name of a query, you must first create a new query with the new name and then delete the query with the old name. Make sure the old query name is not defined in any packs before deleting it or an error will occur. ```yaml -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: docker_processes descriptions: The docker containers processes that are running on a system. @@ -207,11 +207,11 @@ spec: - darwin ``` -To define multiple queries in a file, concatenate multiple `OsqueryQuery` resources together in a single file with `---`. For example, consider a file that you might store at `queries/osquery_monitoring.yml`: +To define multiple queries in a file, concatenate multiple `query` resources together in a single file with `---`. For example, consider a file that you might store at `queries/osquery_monitoring.yml`: ```yaml -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_version description: The version of the Launcher and Osquery process @@ -220,22 +220,22 @@ spec: launcher: 0.3.0 osquery: 2.9.0 --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_schedule description: Report performance stats for each file in the query schedule. query: select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed from osquery_schedule; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_info description: A heartbeat counter that reports general performance (CPU, memory) and version. query: select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_events description: Report event publisher health and track event counters. @@ -247,8 +247,8 @@ spec: To define query packs, reference queries defined elsewhere by name. This is why the "name" of a query is so important. You can define many of these packs in many files. ```yaml -apiVersion: kolide.com/v1alpha1 -kind: OsqueryPack +apiVersion: v1 +kind: pack spec: name: osquery_monitoring targets: diff --git a/examples/config-many-files/config.yml b/examples/config-many-files/config.yml index 412c3c733c..39349e068f 100644 --- a/examples/config-many-files/config.yml +++ b/examples/config-many-files/config.yml @@ -1,6 +1,6 @@ --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryOptions +apiVersion: v1 +kind: options spec: config: options: diff --git a/examples/config-many-files/labels.yml b/examples/config-many-files/labels.yml index c06b2e7fe4..be22c2dcca 100644 --- a/examples/config-many-files/labels.yml +++ b/examples/config-many-files/labels.yml @@ -1,42 +1,42 @@ --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: all_hosts query: always_true --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: macs query: darwin_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: ubuntu query: ubuntu_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: centos query: centos_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: windows query: windows_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: query: pending_updates platforms: - darwin --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: query: slack_not_running diff --git a/examples/config-many-files/packs/osquery-monitoring.yml b/examples/config-many-files/packs/osquery-monitoring.yml index 52af588a4c..3cae39c726 100644 --- a/examples/config-many-files/packs/osquery-monitoring.yml +++ b/examples/config-many-files/packs/osquery-monitoring.yml @@ -1,6 +1,6 @@ --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryPack +apiVersion: v1 +kind: pack spec: name: osquery_monitoring targets: diff --git a/examples/config-many-files/queries.yml b/examples/config-many-files/queries.yml index b03f1267d3..da81cb751e 100644 --- a/examples/config-many-files/queries.yml +++ b/examples/config-many-files/queries.yml @@ -1,6 +1,6 @@ --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_version description: The version of the Launcher and Osquery process @@ -9,30 +9,30 @@ spec: launcher: 0.3.0 osquery: 2.9.0 --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_schedule description: Report performance stats for each file in the query schedule. query: select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed from osquery_schedule; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_info description: A heartbeat counter that reports general performance (CPU, memory) and version. query: select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_events description: Report event publisher health and track event counters. query: select name, publisher, type, subscriptions, events, active from osquery_events; -apiVersion: kolide.com/v1alpha1 +apiVersion: v1 --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: docker_processes descriptions: The docker containers processes that are running on a system. @@ -43,38 +43,38 @@ spec: - linux - darwin --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: hostname query: select hostname from system_info; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: uuid query: select uuid from osquery_info; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: instance_id query: select instance_id from system_info; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: always_true query: select 1; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: pending_updates query: SELECT value from plist where path = "/Library/Preferences/ManagedInstalls.plist" and key = "PendingUpdateCount" and value > "0"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: slack_not_running query: > @@ -85,26 +85,26 @@ spec: WHERE name LIKE "%Slack%" ); --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: centos_hosts query: select 1 from os_version where platform = "centos"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: ubuntu_hosts query: select 1 from os_version where platform = "ubuntu"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: windows_hosts query: select 1 from os_version where platform = "windows"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: darwin_hosts query: select 1 from os_version where platform = "darwin"; diff --git a/examples/config-single-file.yml b/examples/config-single-file.yml index c2f73b41bc..ee800c5b37 100644 --- a/examples/config-single-file.yml +++ b/examples/config-single-file.yml @@ -1,6 +1,6 @@ --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryOptions +apiVersion: v1 +kind: options spec: config: options: @@ -67,50 +67,50 @@ spec: interval: 3600: "SELECT total_seconds AS uptime FROM uptime" --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: all_hosts query: always_true --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: macs query: darwin_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: ubuntu query: ubuntu_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: centos query: centos_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: name: windows query: windows_hosts --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: query: pending_updates platforms: - darwin --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryLabel +apiVersion: v1 +kind: label spec: query: slack_not_running --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryPack +apiVersion: v1 +kind: pack spec: name: osquery_monitoring targets: @@ -134,8 +134,8 @@ spec: interval: 600 removed: false --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_version description: The version of the Launcher and Osquery process @@ -144,30 +144,29 @@ spec: launcher: 0.3.0 osquery: 2.9.0 --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_schedule description: Report performance stats for each file in the query schedule. query: select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed from osquery_schedule; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_info description: A heartbeat counter that reports general performance (CPU, memory) and version. query: select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: osquery_events description: Report event publisher health and track event counters. query: select name, publisher, type, subscriptions, events, active from osquery_events; -apiVersion: kolide.com/v1alpha1 --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: docker_processes descriptions: The docker containers processes that are running on a system. @@ -178,38 +177,38 @@ spec: - linux - darwin --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: hostname query: select hostname from system_info; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: uuid query: select uuid from osquery_info; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: instance_id query: select instance_id from system_info; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: always_true query: select 1; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: pending_updates query: SELECT value from plist where path = "/Library/Preferences/ManagedInstalls.plist" and key = "PendingUpdateCount" and value > "0"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: slack_not_running query: > @@ -220,26 +219,26 @@ spec: WHERE name LIKE "%Slack%" ); --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: centos_hosts query: select 1 from os_version where platform = "centos"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: ubuntu_hosts query: select 1 from os_version where platform = "ubuntu"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: windows_hosts query: select 1 from os_version where platform = "windows"; --- -apiVersion: kolide.com/v1alpha1 -kind: OsqueryQuery +apiVersion: v1 +kind: query spec: name: darwin_hosts query: select 1 from os_version where platform = "darwin";