From a7f15010542f4a5fc8c3dea038cc2d4054a032cd Mon Sep 17 00:00:00 2001 From: Jacob Shandling <61553566+jacobshandling@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:39:20 -0700 Subject: [PATCH] Update `fleetctl convert` for schedulable queries (#13125) --- changes/12657-update-fleetctl-convert | 1 + cmd/fleetctl/convert.go | 126 +++++--- cmd/fleetctl/convert_test.go | 2 +- cmd/fleetctl/testdata/convert_input.conf | 210 +++++++++++-- cmd/fleetctl/testdata/convert_output.yml | 363 +++++++++++++++++++---- 5 files changed, 563 insertions(+), 139 deletions(-) create mode 100644 changes/12657-update-fleetctl-convert diff --git a/changes/12657-update-fleetctl-convert b/changes/12657-update-fleetctl-convert new file mode 100644 index 0000000000..55959efee9 --- /dev/null +++ b/changes/12657-update-fleetctl-convert @@ -0,0 +1 @@ +- Update `fleetctl convert` to convert packs to the new combined schedule and query format diff --git a/cmd/fleetctl/convert.go b/cmd/fleetctl/convert.go index 6de76d838b..600295d3c9 100644 --- a/cmd/fleetctl/convert.go +++ b/cmd/fleetctl/convert.go @@ -19,15 +19,60 @@ import ( "github.com/urfave/cli/v2" ) +// mappings based on https://github.com/osquery/osquery/blob/b87a4b5f1567415a72acd5ecd0e9e7ab75754959/tools/codegen/genwebsitejson.py#L38C18-L38C18 +var platformMapping = map[string][]string{ + "darwin": {"darwin"}, + "linux": {"linux"}, + "windows": {"windows"}, + "chrome": {"chrome"}, + "specs": {"darwin", "linux", "windows"}, + "utility": {"darwin", "linux", "windows"}, + "yara": {"darwin", "linux", "windows"}, + "smart": {"darwin", "linux"}, + "kernel": {"darwin"}, + "linwin": {"linux", "windows"}, + "macwin": {"darwin", "windows"}, + "posix": {"darwin", "linux"}, + "sleuthkit": {"darwin", "linux"}, + "any": {""}, + "all": {""}, + "": {""}, +} + +func convertPlatforms(platformsIn string) (string, error) { + splitPlatformsIn := strings.Split(platformsIn, ",") + + // validate and convert each substring + mapped := map[string]struct{}{} // use a set to dedupe + for _, substring := range splitPlatformsIn { + mappedSubstring, ok := platformMapping[substring] + // validate substring + if !ok { + return "", fmt.Errorf("unsupported platform: %s", substring) + } + for _, p := range mappedSubstring { + mapped[p] = struct{}{} + } + } + + // convert set to slice + result := make([]string, 0, len(mapped)) + + for p := range mapped { + result = append(result, p) + } + + // sort for deterministic output + sort.Strings(result) + + resultString := strings.Join(result, ",") + + return resultString, nil +} + func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spec.Group, error) { specs := &spec.Group{ Queries: []*fleet.QuerySpec{}, - Packs: []*fleet.PackSpec{}, - Labels: []*fleet.LabelSpec{}, - } - - pack := &fleet.PackSpec{ - Name: name, } // this ensures order is consistent in output @@ -41,12 +86,8 @@ func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spe for _, name := range keys { query := inputPack.Queries[name] - spec := &fleet.QuerySpec{ - Name: name, - Description: query.Description, - Query: query.Query, - } + // get the interval as uint from a variety of possible types interval := uint(0) switch i := query.Interval.(type) { case string: @@ -61,21 +102,33 @@ func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spe interval = uint(i) } - specs.Queries = append(specs.Queries, spec) - pack.Queries = append(pack.Queries, fleet.PackSpecQuery{ - Name: name, - QueryName: name, - Interval: interval, - Description: query.Description, - Snapshot: query.Snapshot, - Removed: query.Removed, - Shard: query.Shard, - Platform: query.Platform, - Version: query.Version, - }) - } + // handle nil query.Platform + var queryPlatforms string + if query.Platform != nil { + queryPlatforms = *query.Platform + } + convertedPlatforms, err := convertPlatforms(queryPlatforms) + if err != nil { + return nil, err + } - specs.Packs = append(specs.Packs, pack) + // handle nil query.Version + var minOsqueryVersion string + if query.Version != nil { + minOsqueryVersion = *query.Version + } + + spec := &fleet.QuerySpec{ + Name: name, + Description: query.Description, + Query: query.Query, + Interval: interval, + Platform: convertedPlatforms, + MinOsqueryVersion: minOsqueryVersion, + } + + specs.Queries = append(specs.Queries, spec) + } return specs, nil } @@ -87,7 +140,7 @@ func convertCommand() *cli.Command { ) return &cli.Command{ Name: "convert", - Usage: "Convert osquery packs into decomposed fleet configs", + Usage: "Convert osquery packs into Fleet queries", UsageText: `fleetctl convert [options]`, Flags: []cli.Flag{ configFlag(), @@ -151,27 +204,6 @@ func convertCommand() *cli.Command { w = file } - for _, pack := range specs.Packs { - specBytes, err := json.Marshal(pack) - if err != nil { - return err - } - - meta := spec.Metadata{ - Kind: fleet.PackKind, - Version: fleet.ApiVersion, - Spec: specBytes, - } - - out, err := yaml.Marshal(meta) - if err != nil { - return err - } - - fmt.Fprintln(w, "---") - fmt.Fprint(w, string(out)) - } - for _, query := range specs.Queries { specBytes, err := json.Marshal(query) if err != nil { diff --git a/cmd/fleetctl/convert_test.go b/cmd/fleetctl/convert_test.go index 6bfd0a4d2c..6f21660d4d 100644 --- a/cmd/fleetctl/convert_test.go +++ b/cmd/fleetctl/convert_test.go @@ -64,5 +64,5 @@ func TestConvertFileStdout(t *testing.T) { os.Stdout = oldStdout w.Close() out, _ := ioutil.ReadAll(r) - require.Equal(t, string(expected), string(out)) + require.YAMLEq(t, string(expected), string(out)) } diff --git a/cmd/fleetctl/testdata/convert_input.conf b/cmd/fleetctl/testdata/convert_input.conf index c211fa7cf2..44716abb27 100644 --- a/cmd/fleetctl/testdata/convert_input.conf +++ b/cmd/fleetctl/testdata/convert_input.conf @@ -1,44 +1,194 @@ { "queries": { "launchd": { - "query" : "select * from launchd;", - "interval" : "3600", - "platform" : "darwin", - "version" : "1.4.5", - "description" : "Retrieves all the daemons that will run in the start of the target OSX system.", - "value" : "Identify malware that uses this persistence mechanism to launch at system boot" + "query": "select * from launchd;", + "interval": "3600", + "platform": "darwin", + "version": "1.4.5", + "description": "Retrieves all the daemons that will run in the start of the target OSX system.", + "value": "Identify malware that uses this persistence mechanism to launch at system boot" }, - "disk_encryption": { - "query" : "select * from disk_encryption;", - "interval" : "86400", + "disk_encryption (posix)": { + "query": "select * from disk_encryption;", + "interval": "86400", "platform": "posix", - "version" : "1.4.5", - "description" : "Retrieves the current disk encryption status for the target system.", - "value" : "Identifies a system potentially vulnerable to disk cloning." + "version": "1.4.5", + "description": "Retrieves the current disk encryption status for the target system.", + "value": "Identifies a system potentially vulnerable to disk cloning." + }, + "disk_encryption (darwin,linux)": { + "query": "select * from disk_encryption;", + "interval": "300", + "platform": "darwin,linux", + "version": "1.4.5", + "description": "Retrieves the current disk encryption status for the target system.", + "value": "Identifies a system potentially vulnerable to disk cloning." }, "iptables": { - "query" : "select * from iptables;", - "interval" : "3600", - "platform" : "linux", - "version" : "1.4.5", - "description" : "Retrieves the current filters and chains per filter in the target system.", - "value" : "Verify firewall settings are as restrictive as you need. Identify unwanted firewall holes made by malware or humans" + "query": "select * from iptables;", + "interval": "3600", + "platform": "linux", + "version": "1.4.5", + "description": "Retrieves the current filters and chains per filter in the target system.", + "value": "Verify firewall settings are as restrictive as you need. Identify unwanted firewall holes made by malware or humans" }, "app_schemes": { - "query" : "select * from app_schemes;", - "interval" : "86400", - "platform" : "darwin", - "version" : "1.4.7", - "description" : "Retrieves the list of application scheme/protocol-based IPC handlers.", - "value" : "Post-priori hijack detection, detect potential sensitive information leakage." + "query": "select * from app_schemes;", + "interval": "86400", + "platform": "darwin", + "version": "1.4.7", + "description": "Retrieves the list of application scheme/protocol-based IPC handlers.", + "value": "Post-priori hijack detection, detect potential sensitive information leakage." }, "sandboxes": { - "query" : "select * from sandboxes;", - "interval" : "86400", - "platform" : "darwin", - "version" : "1.4.7", - "description" : "Lists the application bundle that owns a sandbox label.", - "value" : "Post-priori hijack detection, detect potential sensitive information leakage." + "query": "select * from sandboxes;", + "interval": "86400", + "platform": "darwin", + "version": "1.4.7", + "description": "Lists the application bundle that owns a sandbox label.", + "value": "Post-priori hijack detection, detect potential sensitive information leakage." + }, + "disk_info": { + "query": "select * from disk_info;", + "interval": "86400", + "platform": "chrome,windows", + "version": "1.4.7", + "description": "Retrieve basic information about the physical disks of a system.", + "value": "Identify scary possibilities with disks." + }, + "listening_ports (specs)": { + "query": "select * from listening_ports;", + "interval": "3600", + "platform": "specs", + "version": "1.4.7", + "description": "Retrieves the list of listening ports.", + "value": "Identify unwanted open ports." + }, + "listening_ports (utility)": { + "query": "select * from listening_ports;", + "interval": "3600", + "platform": "utility", + "version": "1.4.7", + "description": "Retrieves the list of listening ports.", + "value": "Identify unwanted open ports." + }, + "yara (yara)": { + "query": "select * from yara;", + "interval": "0", + "platform": "yara", + "version": "1.4.7", + "description": "Triggers one-off YARA query for files at the specified path. Requires one of sig_group, sigfile, or sigrule.", + "value": "TBD" + }, + "ulimit_info (smart)": { + "query": "select * from ulimit_info;", + "interval": "300", + "platform": "smart", + "version": "1.4.7", + "description": "System resource usage limits.", + "value": "Identify potential resource exhaustion attacks." + }, + "uptime (kernel)": { + "query": "select * from uptime;", + "interval": "600", + "platform": "kernel", + "version": "1.4.7", + "description": "System uptime.", + "value": "Identify systems that have been rebooted recently." + }, + "uptime (linwin)": { + "query": "select * from uptime;", + "interval": "600", + "platform": "linwin", + "version": "1.4.7", + "description": "System uptime.", + "value": "Identify systems that have been rebooted recently." + }, + "uptime (macwin)": { + "query": "select * from uptime;", + "interval": "600", + "platform": "macwin", + "version": "1.4.7", + "description": "System uptime.", + "value": "Identify systems that have been rebooted recently." + }, + "uptime (sleuthkit)": { + "query": "select * from uptime;", + "interval": "600", + "platform": "sleuthkit", + "version": "1.4.7", + "description": "System uptime.", + "value": "Identify systems that have been rebooted recently." + }, + "windows crashes": { + "query": "select * from windows_crashes;", + "interval": "3600", + "platform": "windows", + "version": "1.4.7", + "description": "Extracted information from Windows crash logs (Minidumps).", + "value": "Identify systems that have been rebooted recently." + }, + "user groups (any)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "any", + "version": "1.4.7", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (missing platform)": { + "query": "select * from user_groups;", + "interval": "3600", + "version": "1.4.7", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (missing version)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "darwin", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (all)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "all", + "version": "1.4.7", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (empty string platform, empty string version)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "", + "version": "", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (darwin,linux)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "darwin,linux", + "version": "1.4.7", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (linux,darwin)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "linux,darwin", + "version": "1.4.7", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." + }, + "user groups (windows,chrome)": { + "query": "select * from user_groups;", + "interval": "3600", + "platform": "windows,chrome", + "version": "1.4.7", + "description": "List of all user groups.", + "value": "Identify unwanted user groups." } } } diff --git a/cmd/fleetctl/testdata/convert_output.yml b/cmd/fleetctl/testdata/convert_output.yml index a360959840..b09a8a0847 100644 --- a/cmd/fleetctl/testdata/convert_output.yml +++ b/cmd/fleetctl/testdata/convert_output.yml @@ -1,57 +1,15 @@ --- apiVersion: v1 -kind: pack -spec: - disabled: false - name: convert_input - queries: - - description: Retrieves the list of application scheme/protocol-based IPC handlers. - interval: 86400 - name: app_schemes - platform: darwin - query: app_schemes - version: 1.4.7 - - description: Retrieves the current disk encryption status for the target system. - interval: 86400 - name: disk_encryption - platform: posix - query: disk_encryption - version: 1.4.5 - - description: Retrieves the current filters and chains per filter in the target - system. - interval: 3600 - name: iptables - platform: linux - query: iptables - version: 1.4.5 - - description: Retrieves all the daemons that will run in the start of the target - OSX system. - interval: 3600 - name: launchd - platform: darwin - query: launchd - version: 1.4.5 - - description: Lists the application bundle that owns a sandbox label. - interval: 86400 - name: sandboxes - platform: darwin - query: sandboxes - version: 1.4.7 - targets: - labels: null - teams: null ---- -apiVersion: v1 kind: query spec: automations_enabled: false description: Retrieves the list of application scheme/protocol-based IPC handlers. - interval: 0 + interval: 86400 logging: "" - min_osquery_version: "" + min_osquery_version: 1.4.7 name: app_schemes observer_can_run: false - platform: "" + platform: darwin query: select * from app_schemes; team: "" --- @@ -60,12 +18,12 @@ kind: query spec: automations_enabled: false description: Retrieves the current disk encryption status for the target system. - interval: 0 + interval: 86400 logging: "" - min_osquery_version: "" - name: disk_encryption + min_osquery_version: 1.4.5 + name: disk_encryption (posix) observer_can_run: false - platform: "" + platform: darwin,linux query: select * from disk_encryption; team: "" --- @@ -73,13 +31,41 @@ apiVersion: v1 kind: query spec: automations_enabled: false - description: Retrieves the current filters and chains per filter in the target system. - interval: 0 + description: Retrieves the current disk encryption status for the target system. + interval: 300 logging: "" - min_osquery_version: "" + min_osquery_version: 1.4.5 + name: disk_encryption (darwin,linux) + observer_can_run: false + platform: darwin,linux + query: select * from disk_encryption; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: Retrieve basic information about the physical disks of a system. + interval: 86400 + logging: "" + min_osquery_version: 1.4.7 + name: disk_info + observer_can_run: false + platform: chrome,windows + query: select * from disk_info; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: Retrieves the current filters and chains per filter in the target system. + interval: 3600 + logging: "" + min_osquery_version: 1.4.5 name: iptables observer_can_run: false - platform: "" + platform: linux query: select * from iptables; team: "" --- @@ -87,14 +73,15 @@ apiVersion: v1 kind: query spec: automations_enabled: false - description: Retrieves all the daemons that will run in the start of the target + description: + Retrieves all the daemons that will run in the start of the target OSX system. - interval: 0 + interval: 3600 logging: "" - min_osquery_version: "" + min_osquery_version: 1.4.5 name: launchd observer_can_run: false - platform: "" + platform: darwin query: select * from launchd; team: "" --- @@ -102,12 +89,266 @@ apiVersion: v1 kind: query spec: automations_enabled: false - description: Lists the application bundle that owns a sandbox label. - interval: 0 + description: Retrieves the list of listening ports. + interval: 3600 logging: "" - min_osquery_version: "" + min_osquery_version: 1.4.7 + name: listening_ports (specs) + observer_can_run: false + platform: darwin,linux,windows + query: select * from listening_ports; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: Retrieves the list of listening ports. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: listening_ports (utility) + observer_can_run: false + platform: darwin,linux,windows + query: select * from listening_ports; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: Lists the application bundle that owns a sandbox label. + interval: 86400 + logging: "" + min_osquery_version: 1.4.7 name: sandboxes observer_can_run: false - platform: "" + platform: darwin query: select * from sandboxes; team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: System resource usage limits. + interval: 300 + logging: "" + min_osquery_version: 1.4.7 + name: ulimit_info (smart) + observer_can_run: false + platform: darwin,linux + query: select * from ulimit_info; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: System uptime. + interval: 600 + logging: "" + min_osquery_version: 1.4.7 + name: uptime (kernel) + observer_can_run: false + platform: darwin + query: select * from uptime; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: System uptime. + interval: 600 + logging: "" + min_osquery_version: 1.4.7 + name: uptime (linwin) + observer_can_run: false + platform: linux,windows + query: select * from uptime; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: System uptime. + interval: 600 + logging: "" + min_osquery_version: 1.4.7 + name: uptime (macwin) + observer_can_run: false + platform: darwin,windows + query: select * from uptime; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: System uptime. + interval: 600 + logging: "" + min_osquery_version: 1.4.7 + name: uptime (sleuthkit) + observer_can_run: false + platform: darwin,linux + query: select * from uptime; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: Lists the application bundle that owns a sandbox label. + interval: 86400 + logging: "" + min_osquery_version: 1.4.7 + name: sandboxes + observer_can_run: false + platform: darwin + query: select * from sandboxes; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: user groups (all) + observer_can_run: false + platform: "" + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: user groups (any) + observer_can_run: false + platform: "" + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: user groups (darwin,linux) + observer_can_run: false + platform: darwin,linux + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: "" + name: user groups (empty string platform, empty string version) + observer_can_run: false + platform: "" + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: user groups (linux,darwin) + observer_can_run: false + platform: darwin,linux + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: user groups (missing platform) + observer_can_run: false + platform: "" + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: "" + name: user groups (missing version) + observer_can_run: false + platform: darwin + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: List of all user groups. + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: user groups (windows,chrome) + observer_can_run: false + platform: chrome,windows + query: select * from user_groups; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: Extracted information from Windows crash logs (Minidumps). + interval: 3600 + logging: "" + min_osquery_version: 1.4.7 + name: windows crashes + observer_can_run: false + platform: windows + query: select * from windows_crashes; + team: "" +--- +apiVersion: v1 +kind: query +spec: + automations_enabled: false + description: + Triggers one-off YARA query for files at the specified path. Requires + one of sig_group, sigfile, or sigrule. + interval: 0 + logging: "" + min_osquery_version: 1.4.7 + name: yara (yara) + observer_can_run: false + platform: darwin,linux,windows + query: select * from yara; + team: ""