diff --git a/changes/40302-support-glob-for-scripts b/changes/40302-support-glob-for-scripts new file mode 100644 index 0000000000..6161280074 --- /dev/null +++ b/changes/40302-support-glob-for-scripts @@ -0,0 +1 @@ +- Add support for the `paths:` key for scripts in GitOps fleet files. diff --git a/go.mod b/go.mod index 4a64151dc0..bae31f5e2f 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/beevik/etree v1.5.0 github.com/beevik/ntp v0.3.0 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/boltdb/bolt v1.3.1 github.com/briandowns/spinner v1.23.1 github.com/cavaliergopher/rpm v1.2.0 diff --git a/go.sum b/go.sum index 8fb5a542d1..24884d2f05 100644 --- a/go.sum +++ b/go.sum @@ -171,6 +171,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650= diff --git a/pkg/spec/gitops.go b/pkg/spec/gitops.go index c384cac7a8..8c15bfe0a2 100644 --- a/pkg/spec/gitops.go +++ b/pkg/spec/gitops.go @@ -12,6 +12,7 @@ import ( "strings" "unicode" + "github.com/bmatcuk/doublestar/v4" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/ghodss/yaml" @@ -138,7 +139,8 @@ func YamlUnmarshal(yamlBytes []byte, out any) error { } type BaseItem struct { - Path *string `json:"path"` + Path *string `json:"path"` + Paths *string `json:"paths"` } type GitOpsControls struct { @@ -373,7 +375,7 @@ func GitOpsFromFile(filePath, baseDir string, appConfig *fleet.EnrichedAppConfig multiError = parseLabels(top, result, baseDir, filePath, multiError) } // Get other top-level entities. - multiError = parseControls(top, result, multiError, filePath) + multiError = parseControls(top, result, multiError, filePath, logFn) multiError = parseAgentOptions(top, result, baseDir, logFn, filePath, multiError) multiError = parseQueries(top, result, baseDir, logFn, filePath, multiError) @@ -738,7 +740,7 @@ func parseAgentOptions(top map[string]json.RawMessage, result *GitOps, baseDir s return multiError } -func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *multierror.Error, yamlFilename string) *multierror.Error { +func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *multierror.Error, yamlFilename string, logFn Logf) *multierror.Error { controlsRaw, ok := top["controls"] if !ok { // Nothing to do, return. @@ -757,17 +759,14 @@ func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *m } controlsDir := filepath.Dir(controlsFilePath) - result.Controls.Scripts, err = resolveScriptPaths(result.Controls.Scripts, controlsDir) - if err != nil { - return multierror.Append(multiError, fmt.Errorf("failed to parse scripts list in %s: %v", controlsFilePath, err)) + var scriptErrs []error + result.Controls.Scripts, scriptErrs = resolveScriptPaths(result.Controls.Scripts, controlsDir, logFn) + for _, err := range scriptErrs { + multiError = multierror.Append(multiError, fmt.Errorf("failed to parse scripts list in %s: %v", controlsFilePath, err)) } // Find Fleet secrets in scripts. for _, script := range result.Controls.Scripts { - if script.Path == nil { - // This should never happen because we checked for missing paths above (with code added in https://github.com/fleetdm/fleet/pull/24639). - return multierror.Append(multiError, errors.New("controls.scripts.path is missing")) - } fileBytes, err := os.ReadFile(*script.Path) if err != nil { return multierror.Append(multiError, fmt.Errorf("failed to read scripts file %s: %v", *script.Path, err)) @@ -934,19 +933,155 @@ func resolveAndUpdateProfilePathToAbsolute(controlsDir string, profile *fleet.MD return nil } -func resolveScriptPaths(input []BaseItem, baseDir string) ([]BaseItem, error) { - var resolved []BaseItem - for _, item := range input { - if item.Path == nil { - return nil, errors.New(`script entry was specified without a path; check for a stray "-" in your scripts list`) - } +// defaultAllowedExtensions is the default set of file extensions allowed for +// glob expansion (YAML files). Entity types that need different extensions +// (e.g. scripts) should override this in their GlobExpandOptions. +var defaultAllowedExtensions = map[string]bool{ + ".yml": true, + ".yaml": true, +} - resolvedPath := resolveApplyRelativePath(baseDir, *item.Path) - item.Path = &resolvedPath - resolved = append(resolved, item) +// allowedScriptExtensions is the set of file extensions allowed for scripts. +var allowedScriptExtensions = map[string]bool{ + ".sh": true, + ".ps1": true, +} + +// GlobExpandOptions configures how flattenBaseItems expands glob patterns. +type GlobExpandOptions struct { + // AllowedExtensions filters glob results to only these extensions. + // Files with other extensions are skipped with a warning. + // Defaults to {".yml", ".yaml"} if nil. + AllowedExtensions map[string]bool + // RequireUniqueBasenames, if true, returns an error when two items resolve to the + // same filename (filepath.Base). + RequireUniqueBasenames bool + // Optional function to log warnings (e.g. about files skipped due to extension mismatch). + LogFn Logf +} + +func (o *GlobExpandOptions) setDefaults() { + if o.AllowedExtensions == nil { + o.AllowedExtensions = defaultAllowedExtensions + } + if o.LogFn == nil { + o.LogFn = func(_ string, _ ...any) {} + } +} + +// containsGlobMeta returns true if the string contains glob metacharacters. +func containsGlobMeta(s string) bool { + return strings.ContainsAny(s, "*?[{") +} + +// expandGlobPattern expands a glob pattern relative to baseDir and returns +// all of the matching files with allowed extensions. +func expandGlobPattern(pattern string, baseDir string, entityType string, opts GlobExpandOptions) ([]string, error) { + absPattern := resolveApplyRelativePath(baseDir, pattern) + matches, err := doublestar.FilepathGlob(absPattern) + if err != nil { + return nil, fmt.Errorf("invalid glob pattern %q: %w", pattern, err) } - return resolved, nil + var result []string + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + return nil, fmt.Errorf("failed to stat %s: %w", match, err) + } + if info.IsDir() { + continue + } + ext := strings.ToLower(filepath.Ext(match)) + if !opts.AllowedExtensions[ext] { + opts.LogFn("[!] glob pattern %q matched non-%s file %q, skipping\n", pattern, entityType, match) + continue + } + result = append(result, match) + } + + slices.Sort(result) + return result, nil +} + +// flattenBaseItems validates path/paths fields on each item, expands glob +// patterns in "paths" entries, and returns a flat list where every item has only +// Path set (resolved to an absolute path). Errors are collected rather than +// returned early, so callers get all problems in one pass. +func flattenBaseItems(input []BaseItem, baseDir string, entityType string, opts GlobExpandOptions) ([]BaseItem, []error) { + opts.setDefaults() + var result []BaseItem + var errs []error + seenBasenames := make(map[string]string) // basename -> source (path or pattern) + + for _, item := range input { + hasPath := item.Path != nil + hasPaths := item.Paths != nil + + switch { + case hasPath && hasPaths: + errs = append(errs, fmt.Errorf(`%s entry cannot have both "path" and "paths" fields`, entityType)) + continue + // Inline item (no file reference) — pass through unchanged. + case !hasPath && !hasPaths: + errs = append(errs, fmt.Errorf(`%s entry has no "path" or "paths" field; check for a stray "-" in the list`, entityType)) + continue + // Single path -- resolve to absolute path and add to result. + case hasPath: + if containsGlobMeta(*item.Path) { + errs = append(errs, fmt.Errorf(`%s "path" %q contains glob characters; use "paths" for glob patterns`, entityType, *item.Path)) + continue + } + resolved := resolveApplyRelativePath(baseDir, *item.Path) + // Check for duplicate filenames if requested. + if opts.RequireUniqueBasenames { + base := filepath.Base(resolved) + if existing, ok := seenBasenames[base]; ok { + errs = append(errs, fmt.Errorf("duplicate %s basename %q (from %q and %q)", entityType, base, existing, *item.Path)) + continue + } + seenBasenames[base] = *item.Path + } + result = append(result, BaseItem{Path: &resolved}) + // Glob -- expand and add files to result. + case hasPaths: + if !containsGlobMeta(*item.Paths) { + errs = append(errs, fmt.Errorf(`%s "paths" %q does not contain glob characters; use "path" for a specific file`, entityType, *item.Paths)) + continue + } + expanded, err := expandGlobPattern(*item.Paths, baseDir, entityType, opts) + if err != nil { + errs = append(errs, err) + continue + } + if len(expanded) == 0 { + opts.LogFn("[!] glob pattern %q matched no %s files\n", *item.Paths, entityType) + continue + } + for _, p := range expanded { + // Check for duplicate filenames if requested. + if opts.RequireUniqueBasenames { + base := filepath.Base(p) + if existing, ok := seenBasenames[base]; ok { + errs = append(errs, fmt.Errorf("duplicate %s basename %q (from %q and %q)", entityType, base, existing, *item.Paths)) + continue + } + seenBasenames[base] = *item.Paths + } + result = append(result, BaseItem{Path: &p}) + } + } + } + + return result, errs +} + +func resolveScriptPaths(input []BaseItem, baseDir string, logFn Logf) ([]BaseItem, []error) { + return flattenBaseItems(input, baseDir, "script", GlobExpandOptions{ + AllowedExtensions: allowedScriptExtensions, + RequireUniqueBasenames: true, + LogFn: logFn, + }) } func parseLabels(top map[string]json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error { diff --git a/pkg/spec/gitops_test.go b/pkg/spec/gitops_test.go index 6d0d1fcf2e..79524d3d14 100644 --- a/pkg/spec/gitops_test.go +++ b/pkg/spec/gitops_test.go @@ -10,6 +10,7 @@ import ( "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1664,3 +1665,251 @@ policies: [] assert.NotNil(t, gitops.TeamSettings["webhook_settings"]) }) } + +func TestContainsGlobMeta(t *testing.T) { + t.Parallel() + tests := []struct { + input string + want bool + }{ + {"./scripts/foo.sh", false}, + {"./scripts/*.sh", true}, + {"./scripts/**/*.sh", true}, + {"./scripts/[abc].sh", true}, + {"./scripts/{a,b}.sh", true}, + {"./scripts/foo?.sh", true}, + {"", false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, containsGlobMeta(tt.input), "containsGlobMeta(%q)", tt.input) + } +} + +func TestResolveScriptPathsGlob(t *testing.T) { + t.Parallel() + + // requireErrorContains is a helper that asserts at least one error contains substr. + requireErrorContains := func(t *testing.T, errs []error, substr string) { + t.Helper() + require.NotEmpty(t, errs, "expected errors but got none") + var found bool + for _, err := range errs { + if strings.Contains(err.Error(), substr) { + found = true + break + } + } + assert.True(t, found, "expected an error containing %q, got: %v", substr, errs) + } + + t.Run("basic_glob", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "c.ps1"), []byte("# powershell"), 0o644)) + + items := []BaseItem{{Paths: ptr.String("*.sh")}} + result, errs := resolveScriptPaths(items, dir, nopLogf) + require.Empty(t, errs) + require.Len(t, result, 2) + assert.Equal(t, filepath.Join(dir, "a.sh"), *result[0].Path) + assert.Equal(t, filepath.Join(dir, "b.sh"), *result[1].Path) + // Paths field should not be set on expanded items + assert.Nil(t, result[0].Paths) + assert.Nil(t, result[1].Paths) + }) + + t.Run("recursive_glob", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + subdir := filepath.Join(dir, "sub") + require.NoError(t, os.MkdirAll(subdir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "top.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(subdir, "nested.sh"), []byte("#!/bin/bash"), 0o644)) + + items := []BaseItem{{Paths: ptr.String("**/*.sh")}} + result, errs := resolveScriptPaths(items, dir, nopLogf) + require.Empty(t, errs) + require.Len(t, result, 2) + // Results are sorted + assert.Equal(t, filepath.Join(subdir, "nested.sh"), *result[0].Path) + assert.Equal(t, filepath.Join(dir, "top.sh"), *result[1].Path) + }) + + t.Run("mixed_path_and_paths", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "single.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "glob1.ps1"), []byte("# ps1"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "glob2.ps1"), []byte("# ps1"), 0o644)) + + items := []BaseItem{ + {Path: ptr.String("single.sh")}, + {Paths: ptr.String("*.ps1")}, + } + result, errs := resolveScriptPaths(items, dir, nopLogf) + require.Empty(t, errs) + require.Len(t, result, 3) + assert.Equal(t, filepath.Join(dir, "single.sh"), *result[0].Path) + assert.Equal(t, filepath.Join(dir, "glob1.ps1"), *result[1].Path) + assert.Equal(t, filepath.Join(dir, "glob2.ps1"), *result[2].Path) + }) + + t.Run("paths_without_glob_error", func(t *testing.T) { + t.Parallel() + items := []BaseItem{{Paths: ptr.String("scripts/foo.sh")}} + _, errs := resolveScriptPaths(items, "/tmp", nopLogf) + requireErrorContains(t, errs, `does not contain glob characters`) + }) + + t.Run("path_with_glob_error", func(t *testing.T) { + t.Parallel() + items := []BaseItem{{Path: ptr.String("scripts/*.sh")}} + _, errs := resolveScriptPaths(items, "/tmp", nopLogf) + requireErrorContains(t, errs, `contains glob characters`) + }) + + t.Run("both_path_and_paths_error", func(t *testing.T) { + t.Parallel() + items := []BaseItem{{Path: ptr.String("foo.sh"), Paths: ptr.String("*.sh")}} + _, errs := resolveScriptPaths(items, "/tmp", nopLogf) + requireErrorContains(t, errs, `cannot have both "path" and "paths"`) + }) + + t.Run("neither_path_nor_paths_error", func(t *testing.T) { + t.Parallel() + items := []BaseItem{{}} + _, errs := resolveScriptPaths(items, "/tmp", nopLogf) + requireErrorContains(t, errs, `no "path" or "paths" field`) + }) + + t.Run("no_matches_warning", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + var warnings []string + logFn := func(format string, args ...any) { + warnings = append(warnings, fmt.Sprintf(format, args...)) + } + items := []BaseItem{{Paths: ptr.String("*.sh")}} + result, errs := resolveScriptPaths(items, dir, logFn) + require.Empty(t, errs) + assert.Empty(t, result) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0], "matched no script") + }) + + t.Run("duplicate_basenames_error", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + sub1 := filepath.Join(dir, "sub1") + sub2 := filepath.Join(dir, "sub2") + require.NoError(t, os.MkdirAll(sub1, 0o755)) + require.NoError(t, os.MkdirAll(sub2, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(sub1, "dup.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(sub2, "dup.sh"), []byte("#!/bin/bash"), 0o644)) + + items := []BaseItem{{Paths: ptr.String("**/*.sh")}} + _, errs := resolveScriptPaths(items, dir, nopLogf) + requireErrorContains(t, errs, "duplicate script basename") + }) + + t.Run("duplicate_basenames_across_items_error", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + require.NoError(t, os.MkdirAll(sub, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "script.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(sub, "script.sh"), []byte("#!/bin/bash"), 0o644)) + + items := []BaseItem{ + {Path: ptr.String("script.sh")}, + {Paths: ptr.String("sub/*.sh")}, + } + _, errs := resolveScriptPaths(items, dir, nopLogf) + requireErrorContains(t, errs, "duplicate script basename") + }) + + t.Run("non_script_files_skipped_with_warning", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "good.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "bad.txt"), []byte("text"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "bad.py"), []byte("python"), 0o644)) + + var warnings []string + logFn := func(format string, args ...any) { + warnings = append(warnings, fmt.Sprintf(format, args...)) + } + + items := []BaseItem{{Paths: ptr.String("*")}} + result, errs := resolveScriptPaths(items, dir, logFn) + require.Empty(t, errs) + require.Len(t, result, 1) + assert.Equal(t, filepath.Join(dir, "good.sh"), *result[0].Path) + assert.Len(t, warnings, 2) + }) + + // Results are only sorted for the sake of tests, + // but having an explicit test protects against regression. + t.Run("results_sorted", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "z.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.sh"), []byte("#!/bin/bash"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "m.sh"), []byte("#!/bin/bash"), 0o644)) + + items := []BaseItem{{Paths: ptr.String("*.sh")}} + result, errs := resolveScriptPaths(items, dir, nopLogf) + require.Empty(t, errs) + require.Len(t, result, 3) + assert.Equal(t, filepath.Join(dir, "a.sh"), *result[0].Path) + assert.Equal(t, filepath.Join(dir, "m.sh"), *result[1].Path) + assert.Equal(t, filepath.Join(dir, "z.sh"), *result[2].Path) + }) + + t.Run("multiple_errors_collected", func(t *testing.T) { + t.Parallel() + items := []BaseItem{{}, {Path: ptr.String("scripts/*.sh")}, {Paths: ptr.String("noglob.sh")}} + _, errs := resolveScriptPaths(items, "", nil) + require.Len(t, errs, 3) + assert.Contains(t, errs[0].Error(), `no "path" or "paths"`) + assert.Contains(t, errs[1].Error(), `contains glob characters`) + assert.Contains(t, errs[2].Error(), `does not contain glob characters`) + }) +} + +func TestGitOpsGlobScripts(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + scriptsDir := filepath.Join(dir, "scripts") + require.NoError(t, os.MkdirAll(scriptsDir, 0o755)) + scriptsSubDir := filepath.Join(scriptsDir, "sub") + require.NoError(t, os.MkdirAll(scriptsSubDir, 0o755)) + + // Create script files + require.NoError(t, os.WriteFile(filepath.Join(scriptsDir, "alpha.sh"), []byte("#!/bin/bash\necho alpha"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(scriptsDir, "beta.sh"), []byte("#!/bin/bash\necho beta"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(scriptsDir, "gamma.ps1"), []byte("Write-Host gamma"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(scriptsSubDir, "delta.sh"), []byte("nada"), 0o644)) + + // Write a gitops YAML file that uses paths: glob + config := getGlobalConfig([]string{"controls"}) + config += `controls: + scripts: + - paths: scripts/*.sh + - path: scripts/gamma.ps1 +` + yamlPath := filepath.Join(dir, "gitops.yml") + require.NoError(t, os.WriteFile(yamlPath, []byte(config), 0o644)) + + result, err := GitOpsFromFile(yamlPath, dir, nil, nopLogf) + require.NoError(t, err) + require.Len(t, result.Controls.Scripts, 3) + + // Glob results come first (sorted), then the explicit path + assert.Equal(t, filepath.Join(scriptsDir, "alpha.sh"), *result.Controls.Scripts[0].Path) + assert.Equal(t, filepath.Join(scriptsDir, "beta.sh"), *result.Controls.Scripts[1].Path) + assert.Equal(t, filepath.Join(scriptsDir, "gamma.ps1"), *result.Controls.Scripts[2].Path) +}