diff --git a/.github/workflows/fleetctl-preview-latest.yml b/.github/workflows/fleetctl-preview-latest.yml index bce78a4b9c..d00d55b6a2 100644 --- a/.github/workflows/fleetctl-preview-latest.yml +++ b/.github/workflows/fleetctl-preview-latest.yml @@ -1,7 +1,7 @@ name: Test latest changes in fleetctl preview # Tests the `fleetctl preview` command with latest changes in fleetctl and -# docs/01-Using-Fleet/standard-query-library/standard-query-library.yml +# docs/01-Using-Fleet/starter-library/starter-library.yml on: push: @@ -16,7 +16,7 @@ on: - 'server/context/**.go' - 'orbit/**.go' - 'ee/fleetctl/**.go' - - 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml' + - 'docs/01-Using-Fleet/starter-library/starter-library.yml' - '.github/workflows/fleetctl-preview-latest.yml' - 'tools/osquery/in-a-box' pull_request: @@ -27,7 +27,7 @@ on: - 'server/context/**.go' - 'orbit/**.go' - 'ee/fleetctl/**.go' - - 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml' + - 'docs/01-Using-Fleet/starter-library/starter-library.yml' - '.github/workflows/fleetctl-preview-latest.yml' - 'tools/osquery/in-a-box' workflow_dispatch: # Manual @@ -78,8 +78,8 @@ jobs: run: | ./build/fleetctl preview \ --disable-open-browser \ - --preview-config-path ./tools/osquery/in-a-box \ - --std-query-lib-file-path $(pwd)/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml + --starter-library-file-path $(pwd)/docs/01-Using-Fleet/starter-library/starter-library.yml \ + --preview-config-path ./tools/osquery/in-a-box sleep 10 ./build/fleetctl get hosts | tee hosts.txt [ $( cat hosts.txt | grep online | wc -l) -eq 8 ] diff --git a/cmd/fleetctl/fleetctl/preview.go b/cmd/fleetctl/fleetctl/preview.go index b181e1b226..249d7a241e 100644 --- a/cmd/fleetctl/fleetctl/preview.go +++ b/cmd/fleetctl/fleetctl/preview.go @@ -24,10 +24,10 @@ import ( "github.com/fleetdm/fleet/v4/orbit/pkg/update" "github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/pkg/open" - "github.com/fleetdm/fleet/v4/pkg/spec" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/service" + kitlog "github.com/go-kit/log" "github.com/google/go-github/v37/github" "github.com/mitchellh/go-ps" "github.com/urfave/cli/v2" @@ -36,7 +36,6 @@ import ( type dockerComposeVersion int const ( - standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml" licenseKeyFlagName = "license-key" tagFlagName = "tag" previewConfigFlagName = "preview-config" @@ -45,7 +44,7 @@ const ( osquerydChannel = "osqueryd-channel" updateURL = "update-url" updateRootKeys = "update-roots" - stdQueryLibFilePath = "std-query-lib-file-path" + starterLibraryFilePath = "starter-library-file-path" previewConfigPathFlagName = "preview-config-path" disableOpenBrowser = "disable-open-browser" @@ -145,8 +144,8 @@ Use the stop and reset subcommands to manage the server and dependencies once st Value: "", }, &cli.StringFlag{ - Name: stdQueryLibFilePath, - Usage: "Use custom standard query library yml file (used for development/testing)", + Name: starterLibraryFilePath, + Usage: "Use custom starter library yml file (used for development/testing)", Value: "", }, &cli.StringFlag{ @@ -370,37 +369,17 @@ Use the stop and reset subcommands to manage the server and dependencies once st } client.SetToken(token) - fmt.Println("Loading standard query library...") - var buf []byte - if fp := c.String(stdQueryLibFilePath); fp != "" { - var err error - buf, err = os.ReadFile(fp) - if err != nil { - return fmt.Errorf("failed to read standard query library file %q: %w", fp, err) - } - } else { - var err error - buf, err = downloadStandardQueryLibrary() - if err != nil { - return fmt.Errorf("failed to download standard query library: %w", err) - } - } - - specs, err := spec.GroupFromBytes(buf) - if err != nil { - return err - } - logf := func(format string, a ...interface{}) { - fmt.Fprintf(c.App.Writer, format, a...) - } - // this only applies standard queries, the base directory is not used, - // so pass in the current working directory. - teamsSoftwareInstallers := make(map[string][]fleet.SoftwarePackageResponse) - teamsScripts := make(map[string][]fleet.ScriptResponse) - teamsVPPApps := make(map[string][]fleet.VPPAppResponse) - _, _, _, _, err = client.ApplyGroup(c.Context, false, specs, ".", logf, nil, fleet.ApplyClientSpecOptions{}, teamsSoftwareInstallers, teamsVPPApps, teamsScripts) - if err != nil { - return err + fmt.Println("Loading starter library...") + if err := service.ApplyStarterLibrary( + c.Context, + address, + token, + kitlog.NewLogfmtLogger(os.Stderr), + fleethttp.NewClient, + service.NewClient, + nil, // No mock ApplyGroup for production code + ); err != nil { + return fmt.Errorf("failed to apply starter library: %w", err) } // disable analytics collection and enable software inventory for preview @@ -574,21 +553,6 @@ func downloadFromFleetRepo( return nil } -func downloadStandardQueryLibrary() ([]byte, error) { - resp, err := http.Get(standardQueryLibraryUrl) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("status: %d", resp.StatusCode) - } - buf, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read response body: %w", err) - } - return buf, nil -} - func waitStartup() error { retryStrategy := backoff.NewExponentialBackOff() retryStrategy.MaxInterval = 1 * time.Second diff --git a/cmd/fleetctl/integrationtest/preview/preview_test.go b/cmd/fleetctl/integrationtest/preview/preview_test.go index e81187b91f..ae9e5188ea 100644 --- a/cmd/fleetctl/integrationtest/preview/preview_test.go +++ b/cmd/fleetctl/integrationtest/preview/preview_test.go @@ -1,10 +1,8 @@ package preview import ( - "bytes" "os/exec" "path/filepath" - "regexp" "strings" "testing" @@ -30,10 +28,8 @@ func TestIntegrationsPreview(t *testing.T) { require.Equal(t, "", fleetctl.RunAppForTest(t, []string{"preview", "--config", configPath, "stop"})) }) - var output *bytes.Buffer require.NoError(t, nettest.RunWithNetRetry(t, func() error { - var err error - output, err = fleetctl.RunAppNoChecks([]string{ + _, err := fleetctl.RunAppNoChecks([]string{ "preview", "--config", configPath, "--preview-config-path", filepath.Join(gitRootPath(t), "tools", "osquery", "in-a-box"), @@ -43,17 +39,12 @@ func TestIntegrationsPreview(t *testing.T) { return err })) - queriesRe := regexp.MustCompile(`applied ([0-9]+) queries`) - policiesRe := regexp.MustCompile(`applied ([0-9]+) policies`) - require.True(t, queriesRe.MatchString(output.String())) - require.True(t, policiesRe.MatchString(output.String())) - // run some sanity checks on the preview environment - // standard queries must have been loaded + // starter library queries must have been loaded queries := fleetctl.RunAppForTest(t, []string{"get", "queries", "--config", configPath, "--json"}) n := strings.Count(queries, `"kind":"query"`) - require.Greater(t, n, 10) + require.Greater(t, n, 0) // app configuration must disable analytics appConf := fleetctl.RunAppForTest(t, []string{"get", "config", "--include-server-config", "--config", configPath, "--yaml"}) diff --git a/server/service/endpoint_setup.go b/server/service/endpoint_setup.go index 300aa7dd1b..807b6428d5 100644 --- a/server/service/endpoint_setup.go +++ b/server/service/endpoint_setup.go @@ -95,7 +95,7 @@ func makeSetupEndpoint(svc fleet.Service, logger kitlog.Logger) endpoint.Endpoin // Apply starter library using the admin token we just created if req.ServerURL != nil { - if err := applyStarterLibrary( + if err := ApplyStarterLibrary( ctx, *req.ServerURL, session.Key, @@ -121,11 +121,11 @@ func makeSetupEndpoint(svc fleet.Service, logger kitlog.Logger) endpoint.Endpoin } } -// applyStarterLibrary downloads the starter library from GitHub +// ApplyStarterLibrary downloads the starter library from GitHub // and applies it to the Fleet server using an authenticated client. // TODO: Move the apply starter library logic to use the serve command as an entry point to simplify and leverage the entire fleet.Service. // Entry point: https://github.com/fleetdm/fleet/blob/2dfadc0971c6ba45c19dad2f5f1f4cd0f1b89b20/cmd/fleet/serve.go#L1099-L1100 -func applyStarterLibrary( +func ApplyStarterLibrary( ctx context.Context, serverURL string, token string, @@ -176,12 +176,12 @@ func applyStarterLibrary( } // Find all script references in the YAML and download them - scriptNames := extractScriptNames(specs) + scriptNames := ExtractScriptNames(specs) level.Debug(logger).Log("msg", "Found script references in starter library", "count", len(scriptNames)) // Download scripts and update references in specs if len(scriptNames) > 0 { - err = downloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, logger) + err = DownloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, logger) if err != nil { return fmt.Errorf("failed to download and update scripts: %w", err) } @@ -194,6 +194,40 @@ func applyStarterLibrary( } client.SetToken(token) + // Always check if license is free and skip teams for free licenses + appConfig, err := client.GetAppConfig() + if err != nil { + level.Debug(logger).Log("msg", "Error getting app config", "err", err) + // Continue even if there's an error getting the app config + } else if appConfig.License == nil || !appConfig.License.IsPremium() { + // Remove teams from specs to avoid applying them + level.Debug(logger).Log("msg", "Free license detected, skipping teams and team-related content in starter library") + specs.Teams = nil + + // Filter out policies that reference teams + if specs.Policies != nil { + var filteredPolicies []*fleet.PolicySpec + for _, policy := range specs.Policies { + // Keep only policies that don't reference a team + if policy.Team == "" { + filteredPolicies = append(filteredPolicies, policy) + } + } + specs.Policies = filteredPolicies + } + + // Note: QuerySpec doesn't have a Team field, so we can't filter queries by team + + // Remove scripts from AppConfig if present + if specs.AppConfig != nil { + appConfigMap, ok := specs.AppConfig.(map[string]interface{}) + if ok { + // Remove scripts from AppConfig + delete(appConfigMap, "scripts") + } + } + } + // Log function for ApplyGroup (minimal logging) logf := func(format string, a ...interface{}) {} @@ -207,7 +241,7 @@ func applyStarterLibrary( ctx, false, specs, - ".", + tempDir, logf, nil, fleet.ApplyClientSpecOptions{}, @@ -231,8 +265,8 @@ func applyStarterLibrary( return nil } -// extractScriptNames extracts all script names from the specs -func extractScriptNames(specs *spec.Group) []string { +// ExtractScriptNames extracts all script names from the specs +func ExtractScriptNames(specs *spec.Group) []string { var scriptNames []string scriptMap := make(map[string]bool) // Use a map to deduplicate script names @@ -256,8 +290,8 @@ func extractScriptNames(specs *spec.Group) []string { return scriptNames } -// downloadAndUpdateScripts downloads scripts from URLs and updates the specs to reference local files -func downloadAndUpdateScripts(ctx context.Context, specs *spec.Group, scriptNames []string, tempDir string, logger kitlog.Logger) error { +// DownloadAndUpdateScripts downloads scripts from URLs and updates the specs to reference local files +func DownloadAndUpdateScripts(ctx context.Context, specs *spec.Group, scriptNames []string, tempDir string, logger kitlog.Logger) error { // Create a single HTTP client to be reused for all requests httpClient := fleethttp.NewClient(fleethttp.WithTimeout(5 * time.Second)) @@ -315,6 +349,49 @@ func downloadAndUpdateScripts(ctx context.Context, specs *spec.Group, scriptName } } + // Read script contents and store them in memory + scriptContents := make(map[string][]byte, len(scriptNames)) + for _, scriptName := range scriptNames { + localPath := scriptPaths[scriptName] + content, err := os.ReadFile(localPath) + if err != nil { + return fmt.Errorf("failed to read script %s from local file: %w", scriptName, err) + } + scriptContents[scriptName] = content + } + + // Extract scripts from AppConfig if present + appConfigScripts := extractAppCfgScripts(specs.AppConfig) + if appConfigScripts != nil { + // Replace script paths with actual script contents + appScripts := make([]string, 0, len(appConfigScripts)) + for _, scriptPath := range appConfigScripts { + if content, exists := scriptContents[scriptPath]; exists { + // Create a temporary file with the script content + tempFile, err := os.CreateTemp(tempDir, "script-*") + if err != nil { + return fmt.Errorf("failed to create temporary script file: %w", err) + } + if _, err := tempFile.Write(content); err != nil { + tempFile.Close() + return fmt.Errorf("failed to write script content to temporary file: %w", err) + } + tempFile.Close() + + // Add the temporary file path to the list + appScripts = append(appScripts, tempFile.Name()) + } else { + // Keep the original path if it's not one of our downloaded scripts + appScripts = append(appScripts, scriptPath) + } + } + + // Update the AppConfig with the new script paths + if specs.AppConfig != nil { + specs.AppConfig.(map[string]interface{})["scripts"] = appScripts + } + } + // Update script references in the specs to point to local files for i, teamRaw := range specs.Teams { var teamData map[string]interface{} diff --git a/server/service/endpoint_setup_test.go b/server/service/endpoint_setup_test.go index 4fec38b891..56e2866ee6 100644 --- a/server/service/endpoint_setup_test.go +++ b/server/service/endpoint_setup_test.go @@ -1,12 +1,14 @@ package service import ( + "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "strings" @@ -15,6 +17,7 @@ import ( "github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/pkg/spec" + "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -113,7 +116,7 @@ func TestExtractScriptNames(t *testing.T) { specs := &spec.Group{Teams: teams} // Call the function - scriptNames := extractScriptNames(specs) + scriptNames := ExtractScriptNames(specs) // Verify the results assert.Len(t, scriptNames, len(tt.expected)) @@ -188,7 +191,7 @@ func TestDownloadAndUpdateScripts(t *testing.T) { } // Call the actual production function - err = downloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger()) + err = DownloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger()) require.NoError(t, err) // Verify the scripts were downloaded @@ -285,7 +288,7 @@ func TestDownloadAndUpdateScriptsWithInvalidPaths(t *testing.T) { } // Call the actual production function - err = downloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger()) + err = DownloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger()) require.Error(t, err) assert.Contains(t, err.Error(), tt.errorMsg) }) @@ -408,7 +411,7 @@ func TestDownloadAndUpdateScriptsTimeout(t *testing.T) { defer cancel() // Call the actual production function - err = downloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, kitlog.NewNopLogger()) + err = DownloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, kitlog.NewNopLogger()) if tt.expectError { require.Error(t, err) @@ -471,7 +474,7 @@ func TestApplyStarterLibraryWithMockClient(t *testing.T) { } // Call the function under test - testErr := applyStarterLibrary( + testErr := ApplyStarterLibrary( context.Background(), "https://example.com", "test-token", @@ -565,7 +568,7 @@ func TestApplyStarterLibraryWithMalformedYAML(t *testing.T) { }() // Call the function under test - testErr := applyStarterLibrary( + testErr := ApplyStarterLibrary( context.Background(), "https://example.com", "test-token", @@ -585,3 +588,137 @@ func TestApplyStarterLibraryWithMalformedYAML(t *testing.T) { // If we reach here, no panic occurred and the setup flow was not interrupted } + +func TestApplyStarterLibraryWithFreeLicense(t *testing.T) { + // Read the real production starter library YAML file + starterLibraryPath := "../../docs/01-Using-Fleet/starter-library/starter-library.yml" + starterLibraryContent, err := os.ReadFile(starterLibraryPath) + require.NoError(t, err, "Should be able to read starter library YAML file") + + // Create mock HTTP client for downloading the starter library and scripts + mockRT := &testRoundTripper2{ + calls: []string{}, + RoundTripFunc: func(req *http.Request) (*http.Response, error) { + switch { + case req.URL.String() == starterLibraryURL: + // Return the real starter library content + return createTestResponse(200, string(starterLibraryContent)), nil + case strings.Contains(req.URL.String(), "uninstall-fleetd"): + // Return a simple script for any script URL + return createTestResponse(200, "#!/bin/bash\necho ok"), nil + default: + // For any other URL, return a 404 + return createTestResponse(404, "Not found"), nil + } + }, + } + + httpClientFactory := func(opts ...fleethttp.ClientOpt) *http.Client { + client := fleethttp.NewClient(opts...) + client.Transport = mockRT + return client + } + + // Create a mock client that returns a free license + // Create a properly structured EnrichedAppConfig + mockEnrichedAppConfig := &fleet.EnrichedAppConfig{} + // Set the License field using json marshaling/unmarshaling to bypass unexported field access + configJSON := []byte(`{"license":{"tier":"free"}}`) + if err := json.Unmarshal(configJSON, mockEnrichedAppConfig); err != nil { + t.Fatal("Failed to unmarshal mock config:", err) + } + + // Create a mock client factory + clientFactory := func(serverURL string, insecureSkipVerify bool, rootCA, urlPrefix string, options ...ClientOption) (*Client, error) { + mockClient := &Client{} + + // Override the baseClient with a mock implementation + // Create a mock HTTP client + mockHTTPClient := &mockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Mock the GetAppConfig response + if req.URL.Path == "/api/v1/fleet/config" && req.Method == http.MethodGet { + respBody, _ := json.Marshal(mockEnrichedAppConfig) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(respBody)), + Header: make(http.Header), + }, nil + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("{}")), + Header: make(http.Header), + }, nil + }, + } + + // Set up the baseClient with the mock HTTP client and a valid baseURL + baseURL, _ := url.Parse(serverURL) + mockClient.baseClient = &baseClient{ + http: mockHTTPClient, + baseURL: baseURL, + } + + return mockClient, nil + } + + // Track if ApplyGroup was called and capture the specs + applyGroupCalled := false + var capturedSpecs *spec.Group + // Create a mock ApplyGroup function + mockApplyGroup := func(ctx context.Context, specs *spec.Group) error { + applyGroupCalled = true + capturedSpecs = specs + return nil + } + + // Call the function under test - teams will be skipped automatically for free license + testErr := ApplyStarterLibrary( + context.Background(), + "https://example.com", + "test-token", + kitlog.NewNopLogger(), + httpClientFactory, + clientFactory, + mockApplyGroup, + ) + + // Verify results + require.NoError(t, testErr) + assert.True(t, applyGroupCalled, "ApplyGroup should have been called") + + // Verify that the specs were correctly parsed + require.NotNil(t, capturedSpecs, "Specs should not be nil") + + // Verify that teams were removed + require.Empty(t, capturedSpecs.Teams, "Teams should be empty for free license") + + // Verify that policies referencing teams were filtered out + if capturedSpecs.Policies != nil { + for _, policy := range capturedSpecs.Policies { + assert.Empty(t, policy.Team, "Policies should not reference teams for free license") + } + } + + // Verify that scripts were removed from AppConfig + if capturedSpecs.AppConfig != nil { + appConfigMap, ok := capturedSpecs.AppConfig.(map[string]interface{}) + if ok { + _, hasScripts := appConfigMap["scripts"] + assert.False(t, hasScripts, "AppConfig should not contain scripts for free license") + } + } + + // Verify that the starter library URL was requested + assert.Contains(t, mockRT.calls, starterLibraryURL, "The starter library URL should have been requested") +} + +// mockHTTPClient is a mock implementation of the http.Client +type mockHTTPClient struct { + DoFunc func(req *http.Request) (*http.Response, error) +} + +func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { + return m.DoFunc(req) +}