Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 18 Packages Projects Releases Wiki Activity

Add yaml_to_json and file_contents tables to fleetd (#35297)

Browse Source 
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #35548 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Added/updated automated tests

- [x] QA'd all new/changed functionality manually (QAed on macOS)

## fleetd/orbit/Fleet Desktop

- [x] Verified compatibility with the latest released version of Fleet
(see [Must
rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md))
- [x] Verified that fleetd runs on macOS, Linux and Windows (Not
manually checked, but this change should not impact it)
This commit is contained in:
Zach Wasserman
2025-11-12 09:33:18 -08:00
committed by GitHub
parent 1c09d241c0
commit b3ca45564a
10 changed files with 435 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orbit/changes/35050-yaml-to-json-table
+2
View File
@@ -0,0 +1,2 @@
* Add `yaml_to_json` table for converting YAML in input to JSON in output.
* Add `file_contents` table for retrieving contents of a file. This table is like `file_lines` but returns the full file contents in a single row rather than a separate row for each line.
orbit/pkg/table/extension.go
+5
View File
@@ -10,10 +10,12 @@ import (
"github.com/fleetdm/fleet/v4/orbit/pkg/table/cryptoinfotable"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/dataflattentable"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/filecontents"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/firefox_preferences"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/fleetd_logs"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/mcp_listening_servers"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/sntp_request"
"github.com/fleetdm/fleet/v4/orbit/pkg/table/yaml_to_json"
"github.com/macadmins/osquery-extension/tables/chromeuserprofiles"
"github.com/macadmins/osquery-extension/tables/fileline"
"github.com/macadmins/osquery-extension/tables/puppet"
@@ -140,6 +142,7 @@ func OrbitDefaultTables(opts PluginOpts) []osquery.OsqueryPlugin {
table.NewPlugin("puppet_state", puppet.PuppetStateColumns(), puppet.PuppetStateGenerate),
table.NewPlugin("google_chrome_profiles", chromeuserprofiles.GoogleChromeProfilesColumns(), chromeuserprofiles.GoogleChromeProfilesGenerate),
table.NewPlugin("file_lines", fileline.FileLineColumns(), fileline.FileLineGenerate),
table.NewPlugin("file_contents", filecontents.Columns(), filecontents.Generate),
// Orbit extensions.
table.NewPlugin("sntp_request", sntp_request.Columns(), sntp_request.GenerateFunc),
@@ -165,6 +168,8 @@ func OrbitDefaultTables(opts PluginOpts) []osquery.OsqueryPlugin {
return mcp_listening_servers.Generate(ctx, queryContext, opts.Socket)
},
),
table.NewPlugin("yaml_to_json", yaml_to_json.Columns(), yaml_to_json.GenerateFunc),
}
return plugins
}
orbit/pkg/table/filecontents/file_contents.go
+91
View File
@@ -0,0 +1,91 @@
package filecontents
import (
"context"
"os"
"path/filepath"
"strings"
"github.com/osquery/osquery-go/plugin/table"
)
const (
columnPath = "path"
columnContents = "contents"
)
// Columns returns the schema for the file_contents table.
func Columns() []table.ColumnDefinition {
return []table.ColumnDefinition{
table.TextColumn(columnPath),
table.TextColumn(columnContents),
}
}
func Generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) {
path := ""
wildcard := false
if constraintList, present := queryContext.Constraints[columnPath]; present {
// 'path' is in the where clause
for _, constraint := range constraintList.Constraints {
// LIKE
if constraint.Operator == table.OperatorLike {
path = constraint.Expression
wildcard = true
}
// =
if constraint.Operator == table.OperatorEquals {
path = constraint.Expression
wildcard = false
}
}
}
var results []map[string]string
output, err := processFile(path, wildcard)
if err != nil {
return results, err
}
for _, item := range output {
results = append(results, map[string]string{
columnContents: item.Contents,
columnPath: item.Path,
})
}
return results, nil
}
type fileContents struct {
Contents string
Path string
}
func processFile(path string, wildcard bool) ([]fileContents, error) {
var output []fileContents
if wildcard {
replacedPath := strings.ReplaceAll(path, "%", "*")
files, err := filepath.Glob(replacedPath)
if err != nil {
return nil, err
}
for _, file := range files {
contents, err := os.ReadFile(file)
if err != nil {
return nil, err
}
output = append(output, fileContents{Path: file, Contents: string(contents)})
}
} else {
contents, err := os.ReadFile(path)
if err != nil {
return nil, err
}
output = append(output, fileContents{Path: path, Contents: string(contents)})
}
return output, nil
}
orbit/pkg/table/filecontents/file_contents_test.go
+75
View File
@@ -0,0 +1,75 @@
package filecontents
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/osquery/osquery-go/plugin/table"
"github.com/stretchr/testify/require"
)
func TestGenerateWithExactPath(t *testing.T) {
t.Parallel()
dir := t.TempDir()
defer os.RemoveAll(dir)
path := filepath.Join(dir, "example.txt")
require.NoError(t, os.WriteFile(path, []byte("hello\nworld\n"), 0o600))
rows, err := Generate(context.Background(), table.QueryContext{
Constraints: map[string]table.ConstraintList{
columnPath: {
Constraints: []table.Constraint{{
Expression: path,
Operator: table.OperatorEquals,
}},
},
},
})
require.NoError(t, err)
require.Len(t, rows, 1)
require.Equal(t, path, rows[0][columnPath])
require.Equal(t, "hello\nworld\n", rows[0][columnContents])
}
func TestGenerateWithWildcard(t *testing.T) {
t.Parallel()
dir := t.TempDir()
defer os.RemoveAll(dir)
paths := []string{
filepath.Join(dir, "foo.txt"),
filepath.Join(dir, "bar.txt"),
}
for _, path := range paths {
require.NoError(t, os.WriteFile(path, []byte(filepath.Base(path)+"\n"+filepath.Base(path)+"\n"), 0o600))
}
rows, err := Generate(context.Background(), table.QueryContext{
Constraints: map[string]table.ConstraintList{
columnPath: {
Constraints: []table.Constraint{{
Expression: filepath.Join(dir, "%.txt"),
Operator: table.OperatorLike,
}},
},
},
})
require.NoError(t, err)
require.Len(t, rows, len(paths))
got := make(map[string]string, len(rows))
for _, row := range rows {
got[row[columnPath]] = row[columnContents]
}
for _, path := range paths {
require.Contains(t, got, path)
require.Equal(t, filepath.Base(path)+"\n"+filepath.Base(path)+"\n", got[path])
}
}
orbit/pkg/table/yaml_to_json/yaml_to_json.go
+41
View File
@@ -0,0 +1,41 @@
package yaml_to_json
import (
"context"
"errors"
"fmt"
"github.com/ghodss/yaml"
"github.com/osquery/osquery-go/plugin/table"
)
func Columns() []table.ColumnDefinition {
return []table.ColumnDefinition{
table.TextColumn("yaml"),
table.TextColumn("json"),
}
}
func GenerateFunc(_ context.Context, queryContext table.QueryContext) ([]map[string]string, error) {
yamlContent := ""
if constraints, ok := queryContext.Constraints["yaml"]; ok {
for _, constraint := range constraints.Constraints {
if constraint.Operator == table.OperatorEquals {
yamlContent = constraint.Expression
}
}
}
if yamlContent == "" {
return nil, errors.New("missing yaml column constraint; e.g. WHERE yaml = 'key: value'")
}
jsonData, err := yaml.YAMLToJSON([]byte(yamlContent))
if err != nil {
return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err)
}
return []map[string]string{{
"yaml": yamlContent,
"json": string(jsonData),
}}, nil
}
orbit/pkg/table/yaml_to_json/yaml_to_json_test.go
+121
View File
@@ -0,0 +1,121 @@
package yaml_to_json
import (
"context"
"encoding/json"
"testing"
"github.com/osquery/osquery-go/plugin/table"
"github.com/stretchr/testify/require"
)
func TestGenerateFunc(t *testing.T) {
tests := []struct {
name string
yamlContent string
wantErr bool
validateJSON bool
}{
{
name: "simple YAML",
yamlContent: `name: test
version: 1.0
features:
- feature1
- feature2`,
wantErr: false,
validateJSON: true,
},
{
name: "complex nested YAML",
yamlContent: `server:
host: localhost
port: 8080
ssl:
enabled: true
cert: /path/to/cert
database:
type: postgres
connection:
host: db.example.com
port: 5432`,
wantErr: false,
validateJSON: true,
},
{
name: "YAML with different types",
yamlContent: `string: hello
number: 42
float: 3.14
bool: true
null_value: null`,
wantErr: false,
validateJSON: true,
},
{
name: "invalid YAML",
yamlContent: "{ invalid: [[[",
wantErr: true,
},
{
name: "simple key-value",
yamlContent: "key: value",
wantErr: false,
validateJSON: true,
},
{
name: "array at root",
yamlContent: `- item1
- item2
- item3`,
wantErr: false,
validateJSON: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
queryContext := table.QueryContext{
Constraints: map[string]table.ConstraintList{
"yaml": {
Constraints: []table.Constraint{
{
Operator: table.OperatorEquals,
Expression: tt.yamlContent,
},
},
},
},
}
results, err := GenerateFunc(context.Background(), queryContext)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Len(t, results, 1)
require.Equal(t, tt.yamlContent, results[0]["yaml"])
if tt.validateJSON {
// Validate that the output is valid JSON
var jsonData interface{}
err := json.Unmarshal([]byte(results[0]["json"]), &jsonData)
require.NoError(t, err, "output should be valid JSON")
}
})
}
}
func TestGenerateFuncMissingYamlConstraint(t *testing.T) {
// Test without yaml constraint
queryContext := table.QueryContext{
Constraints: map[string]table.ConstraintList{},
}
results, err := GenerateFunc(context.Background(), queryContext)
require.Error(t, err)
require.Contains(t, err.Error(), "missing yaml column constraint")
require.Nil(t, results)
}
schema/osquery_fleet_schema.json
+57 -2
View File
@@ -11096,6 +11096,34 @@
],
"fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/file.yml"
},
{
"name": "file_contents",
"notes": "See also `file_lines` for reading a file line by line.",
"description": "Allows reading an arbitrary file. The entire contents of the file are returned as a single row.",
"platforms": [
"darwin",
"windows",
"linux"
],
"evented": false,
"examples": "Output the content of `/etc/hosts`. \n\n```\nSELECT * FROM file_contents WHERE path='/etc/hosts';\n```",
"columns": [
{
"name": "path",
"description": "Path of the file to read.",
"required": true,
"type": "text"
},
{
"name": "contents",
"description": "Contents of the file",
"required": false,
"type": "text"
}
],
"url": "https://fleetdm.com/tables/file_contents",
"fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/file_contents.yml"
},
{
"name": "file_events",
"description": "Track time/action changes to files specified in configuration data.",
@@ -11276,8 +11304,8 @@
},
{
"name": "file_lines",
"notes": "This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension).",
"description": "Allows reading an arbitrary file.",
"notes": "See also `file_contents` for reading the entire contents of a file as a single row. This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension).",
"description": "Allows reading an arbitrary file. Each line of the file is returned as a separate row.",
"platforms": [
"darwin",
"windows",
@@ -31592,6 +31620,33 @@
],
"fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/xprotect_reports.yml"
},
{
"name": "yaml_to_json",
"platforms": [
"darwin",
"windows",
"linux"
],
"description": "Converts YAML content to JSON format.",
"columns": [
{
"name": "yaml",
"type": "text",
"required": true,
"description": "YAML content to convert to JSON."
},
{
"name": "json",
"type": "text",
"required": false,
"description": "JSON representation of the YAML content."
}
],
"notes": "This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).",
"evented": false,
"url": "https://fleetdm.com/tables/yaml_to_json",
"fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/yaml_to_json.yml"
},
{
"name": "yara",
"description": "Triggers one-off YARA query for files at the specified path. Requires one of `sig_group`, `sigfile`, or `sigrule`.",
schema/tables/file_contents.yml
+23
View File
@@ -0,0 +1,23 @@
name: file_contents
notes: See also `file_lines` for reading a file line by line.
description: Allows reading an arbitrary file. The entire contents of the file are returned as a single row.
platforms:
- darwin
- windows
- linux
evented: false
examples: |-
Output the content of `/etc/hosts`.
```
SELECT * FROM file_contents WHERE path='/etc/hosts';
```
columns:
- name: path
description: Path of the file to read.
required: true
type: text
- name: contents
description: Contents of the file
required: false
type: text
schema/tables/file_lines.yml
+2 -2
View File
@@ -1,6 +1,6 @@
name: file_lines
notes: This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension).
description: Allows reading an arbitrary file.
notes: See also `file_contents` for reading the entire contents of a file as a single row. This table is from the [Mac Admins osquery extension](https://github.com/macadmins/osquery-extension).
description: Allows reading an arbitrary file. Each line of the file is returned as a separate row.
platforms:
- darwin
- windows
schema/tables/yaml_to_json.yml
+18
View File
@@ -0,0 +1,18 @@
name: yaml_to_json
platforms:
- darwin
- windows
- linux
description: Converts YAML content to JSON format.
columns:
- name: yaml
type: text
required: true
description: YAML content to convert to JSON.
- name: json
type: text
required: false
description: JSON representation of the YAML content.
notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false