From b3ca45564a0073f94c327b1c012bfc9c5cb2ed42 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Wed, 12 Nov 2025 09:33:18 -0800 Subject: [PATCH] Add `yaml_to_json` and `file_contents` tables to fleetd (#35297) **Related issue:** Resolves #35548 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually (QAed on macOS) ## fleetd/orbit/Fleet Desktop - [x] Verified compatibility with the latest released version of Fleet (see [Must rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md)) - [x] Verified that fleetd runs on macOS, Linux and Windows (Not manually checked, but this change should not impact it) --- orbit/changes/35050-yaml-to-json-table | 2 + orbit/pkg/table/extension.go | 5 + orbit/pkg/table/filecontents/file_contents.go | 91 +++++++++++++ .../table/filecontents/file_contents_test.go | 75 +++++++++++ orbit/pkg/table/yaml_to_json/yaml_to_json.go | 41 ++++++ .../table/yaml_to_json/yaml_to_json_test.go | 121 ++++++++++++++++++ schema/osquery_fleet_schema.json | 59 ++++++++- schema/tables/file_contents.yml | 23 ++++ schema/tables/file_lines.yml | 4 +- schema/tables/yaml_to_json.yml | 18 +++ 10 files changed, 435 insertions(+), 4 deletions(-) create mode 100644 orbit/changes/35050-yaml-to-json-table create mode 100644 orbit/pkg/table/filecontents/file_contents.go create mode 100644 orbit/pkg/table/filecontents/file_contents_test.go create mode 100644 orbit/pkg/table/yaml_to_json/yaml_to_json.go create mode 100644 orbit/pkg/table/yaml_to_json/yaml_to_json_test.go create mode 100644 schema/tables/file_contents.yml create mode 100644 schema/tables/yaml_to_json.yml diff --git a/orbit/changes/35050-yaml-to-json-table b/orbit/changes/35050-yaml-to-json-table new file mode 100644 index 0000000000..4d32edfd2c --- /dev/null +++ b/orbit/changes/35050-yaml-to-json-table @@ -0,0 +1,2 @@ +* Add `yaml_to_json` table for converting YAML in input to JSON in output. +* Add `file_contents` table for retrieving contents of a file. This table is like `file_lines` but returns the full file contents in a single row rather than a separate row for each line. \ No newline at end of file diff --git a/orbit/pkg/table/extension.go b/orbit/pkg/table/extension.go index 7455a6515c..b22f28bc53 100644 --- a/orbit/pkg/table/extension.go +++ b/orbit/pkg/table/extension.go @@ -10,10 +10,12 @@ import ( "github.com/fleetdm/fleet/v4/orbit/pkg/table/cryptoinfotable" "github.com/fleetdm/fleet/v4/orbit/pkg/table/dataflattentable" + "github.com/fleetdm/fleet/v4/orbit/pkg/table/filecontents" "github.com/fleetdm/fleet/v4/orbit/pkg/table/firefox_preferences" "github.com/fleetdm/fleet/v4/orbit/pkg/table/fleetd_logs" "github.com/fleetdm/fleet/v4/orbit/pkg/table/mcp_listening_servers" "github.com/fleetdm/fleet/v4/orbit/pkg/table/sntp_request" + "github.com/fleetdm/fleet/v4/orbit/pkg/table/yaml_to_json" "github.com/macadmins/osquery-extension/tables/chromeuserprofiles" "github.com/macadmins/osquery-extension/tables/fileline" "github.com/macadmins/osquery-extension/tables/puppet" @@ -140,6 +142,7 @@ func OrbitDefaultTables(opts PluginOpts) []osquery.OsqueryPlugin { table.NewPlugin("puppet_state", puppet.PuppetStateColumns(), puppet.PuppetStateGenerate), table.NewPlugin("google_chrome_profiles", chromeuserprofiles.GoogleChromeProfilesColumns(), chromeuserprofiles.GoogleChromeProfilesGenerate), table.NewPlugin("file_lines", fileline.FileLineColumns(), fileline.FileLineGenerate), + table.NewPlugin("file_contents", filecontents.Columns(), filecontents.Generate), // Orbit extensions. table.NewPlugin("sntp_request", sntp_request.Columns(), sntp_request.GenerateFunc), @@ -165,6 +168,8 @@ func OrbitDefaultTables(opts PluginOpts) []osquery.OsqueryPlugin { return mcp_listening_servers.Generate(ctx, queryContext, opts.Socket) }, ), + + table.NewPlugin("yaml_to_json", yaml_to_json.Columns(), yaml_to_json.GenerateFunc), } return plugins } diff --git a/orbit/pkg/table/filecontents/file_contents.go b/orbit/pkg/table/filecontents/file_contents.go new file mode 100644 index 0000000000..6a159bff43 --- /dev/null +++ b/orbit/pkg/table/filecontents/file_contents.go @@ -0,0 +1,91 @@ +package filecontents + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/osquery/osquery-go/plugin/table" +) + +const ( + columnPath = "path" + columnContents = "contents" +) + +// Columns returns the schema for the file_contents table. +func Columns() []table.ColumnDefinition { + return []table.ColumnDefinition{ + table.TextColumn(columnPath), + table.TextColumn(columnContents), + } +} + +func Generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + path := "" + wildcard := false + + if constraintList, present := queryContext.Constraints[columnPath]; present { + // 'path' is in the where clause + for _, constraint := range constraintList.Constraints { + // LIKE + if constraint.Operator == table.OperatorLike { + path = constraint.Expression + wildcard = true + } + // = + if constraint.Operator == table.OperatorEquals { + path = constraint.Expression + wildcard = false + } + } + } + var results []map[string]string + output, err := processFile(path, wildcard) + if err != nil { + return results, err + } + + for _, item := range output { + results = append(results, map[string]string{ + columnContents: item.Contents, + columnPath: item.Path, + }) + } + + return results, nil +} + +type fileContents struct { + Contents string + Path string +} + +func processFile(path string, wildcard bool) ([]fileContents, error) { + var output []fileContents + + if wildcard { + replacedPath := strings.ReplaceAll(path, "%", "*") + + files, err := filepath.Glob(replacedPath) + if err != nil { + return nil, err + } + for _, file := range files { + contents, err := os.ReadFile(file) + if err != nil { + return nil, err + } + output = append(output, fileContents{Path: file, Contents: string(contents)}) + } + } else { + contents, err := os.ReadFile(path) + if err != nil { + return nil, err + } + output = append(output, fileContents{Path: path, Contents: string(contents)}) + } + + return output, nil +} diff --git a/orbit/pkg/table/filecontents/file_contents_test.go b/orbit/pkg/table/filecontents/file_contents_test.go new file mode 100644 index 0000000000..24cc29b571 --- /dev/null +++ b/orbit/pkg/table/filecontents/file_contents_test.go @@ -0,0 +1,75 @@ +package filecontents + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/osquery/osquery-go/plugin/table" + "github.com/stretchr/testify/require" +) + +func TestGenerateWithExactPath(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + defer os.RemoveAll(dir) + + path := filepath.Join(dir, "example.txt") + require.NoError(t, os.WriteFile(path, []byte("hello\nworld\n"), 0o600)) + + rows, err := Generate(context.Background(), table.QueryContext{ + Constraints: map[string]table.ConstraintList{ + columnPath: { + Constraints: []table.Constraint{{ + Expression: path, + Operator: table.OperatorEquals, + }}, + }, + }, + }) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, path, rows[0][columnPath]) + require.Equal(t, "hello\nworld\n", rows[0][columnContents]) +} + +func TestGenerateWithWildcard(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + defer os.RemoveAll(dir) + + paths := []string{ + filepath.Join(dir, "foo.txt"), + filepath.Join(dir, "bar.txt"), + } + + for _, path := range paths { + require.NoError(t, os.WriteFile(path, []byte(filepath.Base(path)+"\n"+filepath.Base(path)+"\n"), 0o600)) + } + + rows, err := Generate(context.Background(), table.QueryContext{ + Constraints: map[string]table.ConstraintList{ + columnPath: { + Constraints: []table.Constraint{{ + Expression: filepath.Join(dir, "%.txt"), + Operator: table.OperatorLike, + }}, + }, + }, + }) + require.NoError(t, err) + require.Len(t, rows, len(paths)) + + got := make(map[string]string, len(rows)) + for _, row := range rows { + got[row[columnPath]] = row[columnContents] + } + + for _, path := range paths { + require.Contains(t, got, path) + require.Equal(t, filepath.Base(path)+"\n"+filepath.Base(path)+"\n", got[path]) + } +} diff --git a/orbit/pkg/table/yaml_to_json/yaml_to_json.go b/orbit/pkg/table/yaml_to_json/yaml_to_json.go new file mode 100644 index 0000000000..e77f81d0c0 --- /dev/null +++ b/orbit/pkg/table/yaml_to_json/yaml_to_json.go @@ -0,0 +1,41 @@ +package yaml_to_json + +import ( + "context" + "errors" + "fmt" + + "github.com/ghodss/yaml" + "github.com/osquery/osquery-go/plugin/table" +) + +func Columns() []table.ColumnDefinition { + return []table.ColumnDefinition{ + table.TextColumn("yaml"), + table.TextColumn("json"), + } +} + +func GenerateFunc(_ context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + yamlContent := "" + if constraints, ok := queryContext.Constraints["yaml"]; ok { + for _, constraint := range constraints.Constraints { + if constraint.Operator == table.OperatorEquals { + yamlContent = constraint.Expression + } + } + } + if yamlContent == "" { + return nil, errors.New("missing yaml column constraint; e.g. WHERE yaml = 'key: value'") + } + + jsonData, err := yaml.YAMLToJSON([]byte(yamlContent)) + if err != nil { + return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err) + } + + return []map[string]string{{ + "yaml": yamlContent, + "json": string(jsonData), + }}, nil +} diff --git a/orbit/pkg/table/yaml_to_json/yaml_to_json_test.go b/orbit/pkg/table/yaml_to_json/yaml_to_json_test.go new file mode 100644 index 0000000000..5d023cb22b --- /dev/null +++ b/orbit/pkg/table/yaml_to_json/yaml_to_json_test.go @@ -0,0 +1,121 @@ +package yaml_to_json + +import ( + "context" + "encoding/json" + "testing" + + "github.com/osquery/osquery-go/plugin/table" + "github.com/stretchr/testify/require" +) + +func TestGenerateFunc(t *testing.T) { + tests := []struct { + name string + yamlContent string + wantErr bool + validateJSON bool + }{ + { + name: "simple YAML", + yamlContent: `name: test +version: 1.0 +features: + - feature1 + - feature2`, + wantErr: false, + validateJSON: true, + }, + { + name: "complex nested YAML", + yamlContent: `server: + host: localhost + port: 8080 + ssl: + enabled: true + cert: /path/to/cert +database: + type: postgres + connection: + host: db.example.com + port: 5432`, + wantErr: false, + validateJSON: true, + }, + { + name: "YAML with different types", + yamlContent: `string: hello +number: 42 +float: 3.14 +bool: true +null_value: null`, + wantErr: false, + validateJSON: true, + }, + { + name: "invalid YAML", + yamlContent: "{ invalid: [[[", + wantErr: true, + }, + { + name: "simple key-value", + yamlContent: "key: value", + wantErr: false, + validateJSON: true, + }, + { + name: "array at root", + yamlContent: `- item1 +- item2 +- item3`, + wantErr: false, + validateJSON: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + queryContext := table.QueryContext{ + Constraints: map[string]table.ConstraintList{ + "yaml": { + Constraints: []table.Constraint{ + { + Operator: table.OperatorEquals, + Expression: tt.yamlContent, + }, + }, + }, + }, + } + + results, err := GenerateFunc(context.Background(), queryContext) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + require.Len(t, results, 1) + require.Equal(t, tt.yamlContent, results[0]["yaml"]) + + if tt.validateJSON { + // Validate that the output is valid JSON + var jsonData interface{} + err := json.Unmarshal([]byte(results[0]["json"]), &jsonData) + require.NoError(t, err, "output should be valid JSON") + } + }) + } +} + +func TestGenerateFuncMissingYamlConstraint(t *testing.T) { + // Test without yaml constraint + queryContext := table.QueryContext{ + Constraints: map[string]table.ConstraintList{}, + } + + results, err := GenerateFunc(context.Background(), queryContext) + require.Error(t, err) + require.Contains(t, err.Error(), "missing yaml column constraint") + require.Nil(t, results) +} diff --git a/schema/osquery_fleet_schema.json b/schema/osquery_fleet_schema.json index 2f67b0306b..2a9579a437 100644 --- a/schema/osquery_fleet_schema.json +++ b/schema/osquery_fleet_schema.json @@ -11096,6 +11096,34 @@ ], "fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/file.yml" }, + { + "name": "file_contents", + "notes": "See also `file_lines` for reading a file line by line.", + "description": "Allows reading an arbitrary file. The entire contents of the file are returned as a single row.", + "platforms": [ + "darwin", + "windows", + "linux" + ], + "evented": false, + "examples": "Output the content of `/etc/hosts`. \n\n```\nSELECT * FROM file_contents WHERE path='/etc/hosts';\n```", + "columns": [ + { + "name": "path", + "description": "Path of the file to read.", + "required": true, + "type": "text" + }, + { + "name": "contents", + "description": "Contents of the file", + "required": false, + "type": "text" + } + ], + "url": "https://fleetdm.com/tables/file_contents", + "fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/file_contents.yml" + }, { "name": "file_events", "description": "Track time/action changes to files specified in configuration data.", @@ -11276,8 +11304,8 @@ }, { "name": "file_lines", - "notes": "This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension).", - "description": "Allows reading an arbitrary file.", + "notes": "See also `file_contents` for reading the entire contents of a file as a single row. This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension).", + "description": "Allows reading an arbitrary file. Each line of the file is returned as a separate row.", "platforms": [ "darwin", "windows", @@ -31592,6 +31620,33 @@ ], "fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/xprotect_reports.yml" }, + { + "name": "yaml_to_json", + "platforms": [ + "darwin", + "windows", + "linux" + ], + "description": "Converts YAML content to JSON format.", + "columns": [ + { + "name": "yaml", + "type": "text", + "required": true, + "description": "YAML content to convert to JSON." + }, + { + "name": "json", + "type": "text", + "required": false, + "description": "JSON representation of the YAML content." + } + ], + "notes": "This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).", + "evented": false, + "url": "https://fleetdm.com/tables/yaml_to_json", + "fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/yaml_to_json.yml" + }, { "name": "yara", "description": "Triggers one-off YARA query for files at the specified path. Requires one of `sig_group`, `sigfile`, or `sigrule`.", diff --git a/schema/tables/file_contents.yml b/schema/tables/file_contents.yml new file mode 100644 index 0000000000..8e4ac40ea7 --- /dev/null +++ b/schema/tables/file_contents.yml @@ -0,0 +1,23 @@ +name: file_contents +notes: See also `file_lines` for reading a file line by line. +description: Allows reading an arbitrary file. The entire contents of the file are returned as a single row. +platforms: + - darwin + - windows + - linux +evented: false +examples: |- + Output the content of `/etc/hosts`. + + ``` + SELECT * FROM file_contents WHERE path='/etc/hosts'; + ``` +columns: + - name: path + description: Path of the file to read. + required: true + type: text + - name: contents + description: Contents of the file + required: false + type: text \ No newline at end of file diff --git a/schema/tables/file_lines.yml b/schema/tables/file_lines.yml index 91f5bdd679..ee6c0ce0d3 100644 --- a/schema/tables/file_lines.yml +++ b/schema/tables/file_lines.yml @@ -1,6 +1,6 @@ name: file_lines -notes: This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension). -description: Allows reading an arbitrary file. +notes: See also `file_contents` for reading the entire contents of a file as a single row. This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension). +description: Allows reading an arbitrary file. Each line of the file is returned as a separate row. platforms: - darwin - windows diff --git a/schema/tables/yaml_to_json.yml b/schema/tables/yaml_to_json.yml new file mode 100644 index 0000000000..79c2a53439 --- /dev/null +++ b/schema/tables/yaml_to_json.yml @@ -0,0 +1,18 @@ +name: yaml_to_json +platforms: + - darwin + - windows + - linux +description: Converts YAML content to JSON format. +columns: + - name: yaml + type: text + required: true + description: YAML content to convert to JSON. + - name: json + type: text + required: false + description: JSON representation of the YAML content. +notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)). +evented: false +