Support globs in script paths in GitOps (#40799)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #40302

# Details

This PR adds support for a `paths:` key for scripts declared under
`controls:` in a GitOps fleet file. If supplied, `paths:` must contain a
"glob" expression (as [supported by the doublestar
package](https://github.com/bmatcuk/doublestar?tab=readme-ov-file#patterns)).
The existing `path:` key still works but may not contain glob
expressions. When a `paths:` key is encountered, we expand it and add
all matching valid (as in, `.sh` or `.ps1`) files to the set of script
files to process.

Subsequent PRs will add this functionality to other entities that use
`path:` (such as reports and policies).

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [X] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

## Testing

- [X] Added/updated automated tests
- [X] QA'd all new/changed functionality manually
Tried with various combinations of `*` and `**` in gitops runs, and
mixing of `path:` and `paths:`
This commit is contained in:
Scott Gress
2026-03-02 22:18:36 -06:00
committed by GitHub
parent 7064e4356a
commit 2c56b89072
5 changed files with 408 additions and 20 deletions
+1
View File
@@ -0,0 +1 @@
- Add support for the `paths:` key for scripts in GitOps fleet files.
+1
View File
@@ -34,6 +34,7 @@ require (
github.com/beevik/etree v1.5.0
github.com/beevik/ntp v0.3.0
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/boltdb/bolt v1.3.1
github.com/briandowns/spinner v1.23.1
github.com/cavaliergopher/rpm v1.2.0
+2
View File
@@ -171,6 +171,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650=
+155 -20
View File
@@ -12,6 +12,7 @@ import (
"strings"
"unicode"
"github.com/bmatcuk/doublestar/v4"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/ghodss/yaml"
@@ -138,7 +139,8 @@ func YamlUnmarshal(yamlBytes []byte, out any) error {
}
type BaseItem struct {
Path *string `json:"path"`
Path *string `json:"path"`
Paths *string `json:"paths"`
}
type GitOpsControls struct {
@@ -373,7 +375,7 @@ func GitOpsFromFile(filePath, baseDir string, appConfig *fleet.EnrichedAppConfig
multiError = parseLabels(top, result, baseDir, filePath, multiError)
}
// Get other top-level entities.
multiError = parseControls(top, result, multiError, filePath)
multiError = parseControls(top, result, multiError, filePath, logFn)
multiError = parseAgentOptions(top, result, baseDir, logFn, filePath, multiError)
multiError = parseQueries(top, result, baseDir, logFn, filePath, multiError)
@@ -738,7 +740,7 @@ func parseAgentOptions(top map[string]json.RawMessage, result *GitOps, baseDir s
return multiError
}
func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *multierror.Error, yamlFilename string) *multierror.Error {
func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *multierror.Error, yamlFilename string, logFn Logf) *multierror.Error {
controlsRaw, ok := top["controls"]
if !ok {
// Nothing to do, return.
@@ -757,17 +759,14 @@ func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *m
}
controlsDir := filepath.Dir(controlsFilePath)
result.Controls.Scripts, err = resolveScriptPaths(result.Controls.Scripts, controlsDir)
if err != nil {
return multierror.Append(multiError, fmt.Errorf("failed to parse scripts list in %s: %v", controlsFilePath, err))
var scriptErrs []error
result.Controls.Scripts, scriptErrs = resolveScriptPaths(result.Controls.Scripts, controlsDir, logFn)
for _, err := range scriptErrs {
multiError = multierror.Append(multiError, fmt.Errorf("failed to parse scripts list in %s: %v", controlsFilePath, err))
}
// Find Fleet secrets in scripts.
for _, script := range result.Controls.Scripts {
if script.Path == nil {
// This should never happen because we checked for missing paths above (with code added in https://github.com/fleetdm/fleet/pull/24639).
return multierror.Append(multiError, errors.New("controls.scripts.path is missing"))
}
fileBytes, err := os.ReadFile(*script.Path)
if err != nil {
return multierror.Append(multiError, fmt.Errorf("failed to read scripts file %s: %v", *script.Path, err))
@@ -934,19 +933,155 @@ func resolveAndUpdateProfilePathToAbsolute(controlsDir string, profile *fleet.MD
return nil
}
func resolveScriptPaths(input []BaseItem, baseDir string) ([]BaseItem, error) {
var resolved []BaseItem
for _, item := range input {
if item.Path == nil {
return nil, errors.New(`script entry was specified without a path; check for a stray "-" in your scripts list`)
}
// defaultAllowedExtensions is the default set of file extensions allowed for
// glob expansion (YAML files). Entity types that need different extensions
// (e.g. scripts) should override this in their GlobExpandOptions.
var defaultAllowedExtensions = map[string]bool{
".yml": true,
".yaml": true,
}
resolvedPath := resolveApplyRelativePath(baseDir, *item.Path)
item.Path = &resolvedPath
resolved = append(resolved, item)
// allowedScriptExtensions is the set of file extensions allowed for scripts.
var allowedScriptExtensions = map[string]bool{
".sh": true,
".ps1": true,
}
// GlobExpandOptions configures how flattenBaseItems expands glob patterns.
type GlobExpandOptions struct {
// AllowedExtensions filters glob results to only these extensions.
// Files with other extensions are skipped with a warning.
// Defaults to {".yml", ".yaml"} if nil.
AllowedExtensions map[string]bool
// RequireUniqueBasenames, if true, returns an error when two items resolve to the
// same filename (filepath.Base).
RequireUniqueBasenames bool
// Optional function to log warnings (e.g. about files skipped due to extension mismatch).
LogFn Logf
}
func (o *GlobExpandOptions) setDefaults() {
if o.AllowedExtensions == nil {
o.AllowedExtensions = defaultAllowedExtensions
}
if o.LogFn == nil {
o.LogFn = func(_ string, _ ...any) {}
}
}
// containsGlobMeta returns true if the string contains glob metacharacters.
func containsGlobMeta(s string) bool {
return strings.ContainsAny(s, "*?[{")
}
// expandGlobPattern expands a glob pattern relative to baseDir and returns
// all of the matching files with allowed extensions.
func expandGlobPattern(pattern string, baseDir string, entityType string, opts GlobExpandOptions) ([]string, error) {
absPattern := resolveApplyRelativePath(baseDir, pattern)
matches, err := doublestar.FilepathGlob(absPattern)
if err != nil {
return nil, fmt.Errorf("invalid glob pattern %q: %w", pattern, err)
}
return resolved, nil
var result []string
for _, match := range matches {
info, err := os.Stat(match)
if err != nil {
return nil, fmt.Errorf("failed to stat %s: %w", match, err)
}
if info.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(match))
if !opts.AllowedExtensions[ext] {
opts.LogFn("[!] glob pattern %q matched non-%s file %q, skipping\n", pattern, entityType, match)
continue
}
result = append(result, match)
}
slices.Sort(result)
return result, nil
}
// flattenBaseItems validates path/paths fields on each item, expands glob
// patterns in "paths" entries, and returns a flat list where every item has only
// Path set (resolved to an absolute path). Errors are collected rather than
// returned early, so callers get all problems in one pass.
func flattenBaseItems(input []BaseItem, baseDir string, entityType string, opts GlobExpandOptions) ([]BaseItem, []error) {
opts.setDefaults()
var result []BaseItem
var errs []error
seenBasenames := make(map[string]string) // basename -> source (path or pattern)
for _, item := range input {
hasPath := item.Path != nil
hasPaths := item.Paths != nil
switch {
case hasPath && hasPaths:
errs = append(errs, fmt.Errorf(`%s entry cannot have both "path" and "paths" fields`, entityType))
continue
// Inline item (no file reference) — pass through unchanged.
case !hasPath && !hasPaths:
errs = append(errs, fmt.Errorf(`%s entry has no "path" or "paths" field; check for a stray "-" in the list`, entityType))
continue
// Single path -- resolve to absolute path and add to result.
case hasPath:
if containsGlobMeta(*item.Path) {
errs = append(errs, fmt.Errorf(`%s "path" %q contains glob characters; use "paths" for glob patterns`, entityType, *item.Path))
continue
}
resolved := resolveApplyRelativePath(baseDir, *item.Path)
// Check for duplicate filenames if requested.
if opts.RequireUniqueBasenames {
base := filepath.Base(resolved)
if existing, ok := seenBasenames[base]; ok {
errs = append(errs, fmt.Errorf("duplicate %s basename %q (from %q and %q)", entityType, base, existing, *item.Path))
continue
}
seenBasenames[base] = *item.Path
}
result = append(result, BaseItem{Path: &resolved})
// Glob -- expand and add files to result.
case hasPaths:
if !containsGlobMeta(*item.Paths) {
errs = append(errs, fmt.Errorf(`%s "paths" %q does not contain glob characters; use "path" for a specific file`, entityType, *item.Paths))
continue
}
expanded, err := expandGlobPattern(*item.Paths, baseDir, entityType, opts)
if err != nil {
errs = append(errs, err)
continue
}
if len(expanded) == 0 {
opts.LogFn("[!] glob pattern %q matched no %s files\n", *item.Paths, entityType)
continue
}
for _, p := range expanded {
// Check for duplicate filenames if requested.
if opts.RequireUniqueBasenames {
base := filepath.Base(p)
if existing, ok := seenBasenames[base]; ok {
errs = append(errs, fmt.Errorf("duplicate %s basename %q (from %q and %q)", entityType, base, existing, *item.Paths))
continue
}
seenBasenames[base] = *item.Paths
}
result = append(result, BaseItem{Path: &p})
}
}
}
return result, errs
}
func resolveScriptPaths(input []BaseItem, baseDir string, logFn Logf) ([]BaseItem, []error) {
return flattenBaseItems(input, baseDir, "script", GlobExpandOptions{
AllowedExtensions: allowedScriptExtensions,
RequireUniqueBasenames: true,
LogFn: logFn,
})
}
func parseLabels(top map[string]json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error {
+249
View File
@@ -10,6 +10,7 @@ import (
"github.com/fleetdm/fleet/v4/pkg/file"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -1664,3 +1665,251 @@ policies: []
assert.NotNil(t, gitops.TeamSettings["webhook_settings"])
})
}
func TestContainsGlobMeta(t *testing.T) {
t.Parallel()
tests := []struct {
input string
want bool
}{
{"./scripts/foo.sh", false},
{"./scripts/*.sh", true},
{"./scripts/**/*.sh", true},
{"./scripts/[abc].sh", true},
{"./scripts/{a,b}.sh", true},
{"./scripts/foo?.sh", true},
{"", false},
}
for _, tt := range tests {
assert.Equal(t, tt.want, containsGlobMeta(tt.input), "containsGlobMeta(%q)", tt.input)
}
}
func TestResolveScriptPathsGlob(t *testing.T) {
t.Parallel()
// requireErrorContains is a helper that asserts at least one error contains substr.
requireErrorContains := func(t *testing.T, errs []error, substr string) {
t.Helper()
require.NotEmpty(t, errs, "expected errors but got none")
var found bool
for _, err := range errs {
if strings.Contains(err.Error(), substr) {
found = true
break
}
}
assert.True(t, found, "expected an error containing %q, got: %v", substr, errs)
}
t.Run("basic_glob", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "a.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "b.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "c.ps1"), []byte("# powershell"), 0o644))
items := []BaseItem{{Paths: ptr.String("*.sh")}}
result, errs := resolveScriptPaths(items, dir, nopLogf)
require.Empty(t, errs)
require.Len(t, result, 2)
assert.Equal(t, filepath.Join(dir, "a.sh"), *result[0].Path)
assert.Equal(t, filepath.Join(dir, "b.sh"), *result[1].Path)
// Paths field should not be set on expanded items
assert.Nil(t, result[0].Paths)
assert.Nil(t, result[1].Paths)
})
t.Run("recursive_glob", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
subdir := filepath.Join(dir, "sub")
require.NoError(t, os.MkdirAll(subdir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "top.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(subdir, "nested.sh"), []byte("#!/bin/bash"), 0o644))
items := []BaseItem{{Paths: ptr.String("**/*.sh")}}
result, errs := resolveScriptPaths(items, dir, nopLogf)
require.Empty(t, errs)
require.Len(t, result, 2)
// Results are sorted
assert.Equal(t, filepath.Join(subdir, "nested.sh"), *result[0].Path)
assert.Equal(t, filepath.Join(dir, "top.sh"), *result[1].Path)
})
t.Run("mixed_path_and_paths", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "single.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "glob1.ps1"), []byte("# ps1"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "glob2.ps1"), []byte("# ps1"), 0o644))
items := []BaseItem{
{Path: ptr.String("single.sh")},
{Paths: ptr.String("*.ps1")},
}
result, errs := resolveScriptPaths(items, dir, nopLogf)
require.Empty(t, errs)
require.Len(t, result, 3)
assert.Equal(t, filepath.Join(dir, "single.sh"), *result[0].Path)
assert.Equal(t, filepath.Join(dir, "glob1.ps1"), *result[1].Path)
assert.Equal(t, filepath.Join(dir, "glob2.ps1"), *result[2].Path)
})
t.Run("paths_without_glob_error", func(t *testing.T) {
t.Parallel()
items := []BaseItem{{Paths: ptr.String("scripts/foo.sh")}}
_, errs := resolveScriptPaths(items, "/tmp", nopLogf)
requireErrorContains(t, errs, `does not contain glob characters`)
})
t.Run("path_with_glob_error", func(t *testing.T) {
t.Parallel()
items := []BaseItem{{Path: ptr.String("scripts/*.sh")}}
_, errs := resolveScriptPaths(items, "/tmp", nopLogf)
requireErrorContains(t, errs, `contains glob characters`)
})
t.Run("both_path_and_paths_error", func(t *testing.T) {
t.Parallel()
items := []BaseItem{{Path: ptr.String("foo.sh"), Paths: ptr.String("*.sh")}}
_, errs := resolveScriptPaths(items, "/tmp", nopLogf)
requireErrorContains(t, errs, `cannot have both "path" and "paths"`)
})
t.Run("neither_path_nor_paths_error", func(t *testing.T) {
t.Parallel()
items := []BaseItem{{}}
_, errs := resolveScriptPaths(items, "/tmp", nopLogf)
requireErrorContains(t, errs, `no "path" or "paths" field`)
})
t.Run("no_matches_warning", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
var warnings []string
logFn := func(format string, args ...any) {
warnings = append(warnings, fmt.Sprintf(format, args...))
}
items := []BaseItem{{Paths: ptr.String("*.sh")}}
result, errs := resolveScriptPaths(items, dir, logFn)
require.Empty(t, errs)
assert.Empty(t, result)
require.Len(t, warnings, 1)
assert.Contains(t, warnings[0], "matched no script")
})
t.Run("duplicate_basenames_error", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
sub1 := filepath.Join(dir, "sub1")
sub2 := filepath.Join(dir, "sub2")
require.NoError(t, os.MkdirAll(sub1, 0o755))
require.NoError(t, os.MkdirAll(sub2, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(sub1, "dup.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(sub2, "dup.sh"), []byte("#!/bin/bash"), 0o644))
items := []BaseItem{{Paths: ptr.String("**/*.sh")}}
_, errs := resolveScriptPaths(items, dir, nopLogf)
requireErrorContains(t, errs, "duplicate script basename")
})
t.Run("duplicate_basenames_across_items_error", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
sub := filepath.Join(dir, "sub")
require.NoError(t, os.MkdirAll(sub, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "script.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(sub, "script.sh"), []byte("#!/bin/bash"), 0o644))
items := []BaseItem{
{Path: ptr.String("script.sh")},
{Paths: ptr.String("sub/*.sh")},
}
_, errs := resolveScriptPaths(items, dir, nopLogf)
requireErrorContains(t, errs, "duplicate script basename")
})
t.Run("non_script_files_skipped_with_warning", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "good.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "bad.txt"), []byte("text"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "bad.py"), []byte("python"), 0o644))
var warnings []string
logFn := func(format string, args ...any) {
warnings = append(warnings, fmt.Sprintf(format, args...))
}
items := []BaseItem{{Paths: ptr.String("*")}}
result, errs := resolveScriptPaths(items, dir, logFn)
require.Empty(t, errs)
require.Len(t, result, 1)
assert.Equal(t, filepath.Join(dir, "good.sh"), *result[0].Path)
assert.Len(t, warnings, 2)
})
// Results are only sorted for the sake of tests,
// but having an explicit test protects against regression.
t.Run("results_sorted", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "z.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "a.sh"), []byte("#!/bin/bash"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "m.sh"), []byte("#!/bin/bash"), 0o644))
items := []BaseItem{{Paths: ptr.String("*.sh")}}
result, errs := resolveScriptPaths(items, dir, nopLogf)
require.Empty(t, errs)
require.Len(t, result, 3)
assert.Equal(t, filepath.Join(dir, "a.sh"), *result[0].Path)
assert.Equal(t, filepath.Join(dir, "m.sh"), *result[1].Path)
assert.Equal(t, filepath.Join(dir, "z.sh"), *result[2].Path)
})
t.Run("multiple_errors_collected", func(t *testing.T) {
t.Parallel()
items := []BaseItem{{}, {Path: ptr.String("scripts/*.sh")}, {Paths: ptr.String("noglob.sh")}}
_, errs := resolveScriptPaths(items, "", nil)
require.Len(t, errs, 3)
assert.Contains(t, errs[0].Error(), `no "path" or "paths"`)
assert.Contains(t, errs[1].Error(), `contains glob characters`)
assert.Contains(t, errs[2].Error(), `does not contain glob characters`)
})
}
func TestGitOpsGlobScripts(t *testing.T) {
t.Parallel()
dir := t.TempDir()
scriptsDir := filepath.Join(dir, "scripts")
require.NoError(t, os.MkdirAll(scriptsDir, 0o755))
scriptsSubDir := filepath.Join(scriptsDir, "sub")
require.NoError(t, os.MkdirAll(scriptsSubDir, 0o755))
// Create script files
require.NoError(t, os.WriteFile(filepath.Join(scriptsDir, "alpha.sh"), []byte("#!/bin/bash\necho alpha"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(scriptsDir, "beta.sh"), []byte("#!/bin/bash\necho beta"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(scriptsDir, "gamma.ps1"), []byte("Write-Host gamma"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(scriptsSubDir, "delta.sh"), []byte("nada"), 0o644))
// Write a gitops YAML file that uses paths: glob
config := getGlobalConfig([]string{"controls"})
config += `controls:
scripts:
- paths: scripts/*.sh
- path: scripts/gamma.ps1
`
yamlPath := filepath.Join(dir, "gitops.yml")
require.NoError(t, os.WriteFile(yamlPath, []byte(config), 0o644))
result, err := GitOpsFromFile(yamlPath, dir, nil, nopLogf)
require.NoError(t, err)
require.Len(t, result.Controls.Scripts, 3)
// Glob results come first (sorted), then the explicit path
assert.Equal(t, filepath.Join(scriptsDir, "alpha.sh"), *result.Controls.Scripts[0].Path)
assert.Equal(t, filepath.Join(scriptsDir, "beta.sh"), *result.Controls.Scripts[1].Path)
assert.Equal(t, filepath.Join(scriptsDir, "gamma.ps1"), *result.Controls.Scripts[2].Path)
}