Apply starter library during for fleetctl preview server (#30519)

.github/workflows/fleetctl-preview-latest.yml

@@ -1,7 +1,7 @@
 name: Test latest changes in fleetctl preview
 
 # Tests the `fleetctl preview` command with latest changes in fleetctl and
-# docs/01-Using-Fleet/standard-query-library/standard-query-library.yml
+# docs/01-Using-Fleet/starter-library/starter-library.yml
 
 on:
   push:
@@ -16,7 +16,7 @@ on:
       - 'server/context/**.go'
       - 'orbit/**.go'
       - 'ee/fleetctl/**.go'
-      - 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml'
+      - 'docs/01-Using-Fleet/starter-library/starter-library.yml'
       - '.github/workflows/fleetctl-preview-latest.yml'
       - 'tools/osquery/in-a-box'
   pull_request:
@@ -27,7 +27,7 @@ on:
       - 'server/context/**.go'
       - 'orbit/**.go'
       - 'ee/fleetctl/**.go'
-      - 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml'
+      - 'docs/01-Using-Fleet/starter-library/starter-library.yml'
       - '.github/workflows/fleetctl-preview-latest.yml'
       - 'tools/osquery/in-a-box'
   workflow_dispatch: # Manual
@@ -78,8 +78,8 @@ jobs:
       run: |
         ./build/fleetctl preview \
           --disable-open-browser \
-          --preview-config-path ./tools/osquery/in-a-box \
-          --std-query-lib-file-path $(pwd)/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml
+          --starter-library-file-path $(pwd)/docs/01-Using-Fleet/starter-library/starter-library.yml \
+          --preview-config-path ./tools/osquery/in-a-box
         sleep 10
         ./build/fleetctl get hosts | tee hosts.txt
         [ $( cat hosts.txt | grep online | wc -l) -eq 8 ]

cmd/fleetctl/fleetctl/preview.go

@@ -24,10 +24,10 @@ import (
 	"github.com/fleetdm/fleet/v4/orbit/pkg/update"
 	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
 	"github.com/fleetdm/fleet/v4/pkg/open"
-	"github.com/fleetdm/fleet/v4/pkg/spec"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/service"
+	kitlog "github.com/go-kit/log"
 	"github.com/google/go-github/v37/github"
 	"github.com/mitchellh/go-ps"
 	"github.com/urfave/cli/v2"
@@ -36,7 +36,6 @@ import (
 type dockerComposeVersion int
 
 const (
-	standardQueryLibraryUrl   = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml"
 	licenseKeyFlagName        = "license-key"
 	tagFlagName               = "tag"
 	previewConfigFlagName     = "preview-config"
@@ -45,7 +44,7 @@ const (
 	osquerydChannel           = "osqueryd-channel"
 	updateURL                 = "update-url"
 	updateRootKeys            = "update-roots"
-	stdQueryLibFilePath       = "std-query-lib-file-path"
+	starterLibraryFilePath    = "starter-library-file-path"
 	previewConfigPathFlagName = "preview-config-path"
 	disableOpenBrowser        = "disable-open-browser"
 
@@ -145,8 +144,8 @@ Use the stop and reset subcommands to manage the server and dependencies once st
 				Value: "",
 			},
 			&cli.StringFlag{
-				Name:  stdQueryLibFilePath,
-				Usage: "Use custom standard query library yml file (used for development/testing)",
+				Name:  starterLibraryFilePath,
+				Usage: "Use custom starter library yml file (used for development/testing)",
 				Value: "",
 			},
 			&cli.StringFlag{
@@ -370,37 +369,17 @@ Use the stop and reset subcommands to manage the server and dependencies once st
 			}
 			client.SetToken(token)
 
-			fmt.Println("Loading standard query library...")
-			var buf []byte
-			if fp := c.String(stdQueryLibFilePath); fp != "" {
-				var err error
-				buf, err = os.ReadFile(fp)
-				if err != nil {
-					return fmt.Errorf("failed to read standard query library file %q: %w", fp, err)
-				}
-			} else {
-				var err error
-				buf, err = downloadStandardQueryLibrary()
-				if err != nil {
-					return fmt.Errorf("failed to download standard query library: %w", err)
-				}
-			}
-
-			specs, err := spec.GroupFromBytes(buf)
-			if err != nil {
-				return err
-			}
-			logf := func(format string, a ...interface{}) {
-				fmt.Fprintf(c.App.Writer, format, a...)
-			}
-			// this only applies standard queries, the base directory is not used,
-			// so pass in the current working directory.
-			teamsSoftwareInstallers := make(map[string][]fleet.SoftwarePackageResponse)
-			teamsScripts := make(map[string][]fleet.ScriptResponse)
-			teamsVPPApps := make(map[string][]fleet.VPPAppResponse)
-			_, _, _, _, err = client.ApplyGroup(c.Context, false, specs, ".", logf, nil, fleet.ApplyClientSpecOptions{}, teamsSoftwareInstallers, teamsVPPApps, teamsScripts)
-			if err != nil {
-				return err
+			fmt.Println("Loading starter library...")
+			if err := service.ApplyStarterLibrary(
+				c.Context,
+				address,
+				token,
+				kitlog.NewLogfmtLogger(os.Stderr),
+				fleethttp.NewClient,
+				service.NewClient,
+				nil, // No mock ApplyGroup for production code
+			); err != nil {
+				return fmt.Errorf("failed to apply starter library: %w", err)
 			}
 
 			// disable analytics collection and enable software inventory for preview
@@ -574,21 +553,6 @@ func downloadFromFleetRepo(
 	return nil
 }
 
-func downloadStandardQueryLibrary() ([]byte, error) {
-	resp, err := http.Get(standardQueryLibraryUrl)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("status: %d", resp.StatusCode)
-	}
-	buf, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("read response body: %w", err)
-	}
-	return buf, nil
-}
-
 func waitStartup() error {
 	retryStrategy := backoff.NewExponentialBackOff()
 	retryStrategy.MaxInterval = 1 * time.Second

cmd/fleetctl/integrationtest/preview/preview_test.go

@@ -1,10 +1,8 @@
 package preview
 
 import (
-	"bytes"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"testing"
 
@@ -30,10 +28,8 @@ func TestIntegrationsPreview(t *testing.T) {
 		require.Equal(t, "", fleetctl.RunAppForTest(t, []string{"preview", "--config", configPath, "stop"}))
 	})
 
-	var output *bytes.Buffer
 	require.NoError(t, nettest.RunWithNetRetry(t, func() error {
-		var err error
-		output, err = fleetctl.RunAppNoChecks([]string{
+		_, err := fleetctl.RunAppNoChecks([]string{
 			"preview",
 			"--config", configPath,
 			"--preview-config-path", filepath.Join(gitRootPath(t), "tools", "osquery", "in-a-box"),
@@ -43,17 +39,12 @@ func TestIntegrationsPreview(t *testing.T) {
 		return err
 	}))
 
-	queriesRe := regexp.MustCompile(`applied ([0-9]+) queries`)
-	policiesRe := regexp.MustCompile(`applied ([0-9]+) policies`)
-	require.True(t, queriesRe.MatchString(output.String()))
-	require.True(t, policiesRe.MatchString(output.String()))
-
 	// run some sanity checks on the preview environment
 
-	// standard queries must have been loaded
+	// starter library queries must have been loaded
 	queries := fleetctl.RunAppForTest(t, []string{"get", "queries", "--config", configPath, "--json"})
 	n := strings.Count(queries, `"kind":"query"`)
-	require.Greater(t, n, 10)
+	require.Greater(t, n, 0)
 
 	// app configuration must disable analytics
 	appConf := fleetctl.RunAppForTest(t, []string{"get", "config", "--include-server-config", "--config", configPath, "--yaml"})

server/service/endpoint_setup.go

@@ -95,7 +95,7 @@ func makeSetupEndpoint(svc fleet.Service, logger kitlog.Logger) endpoint.Endpoin
 
 			// Apply starter library using the admin token we just created
 			if req.ServerURL != nil {
-				if err := applyStarterLibrary(
+				if err := ApplyStarterLibrary(
 					ctx,
 					*req.ServerURL,
 					session.Key,
@@ -121,11 +121,11 @@ func makeSetupEndpoint(svc fleet.Service, logger kitlog.Logger) endpoint.Endpoin
 	}
 }
 
-// applyStarterLibrary downloads the starter library from GitHub
+// ApplyStarterLibrary downloads the starter library from GitHub
 // and applies it to the Fleet server using an authenticated client.
 // TODO: Move the apply starter library logic to use the serve command as an entry point to simplify and leverage the entire fleet.Service.
 // Entry point: https://github.com/fleetdm/fleet/blob/2dfadc0971c6ba45c19dad2f5f1f4cd0f1b89b20/cmd/fleet/serve.go#L1099-L1100
-func applyStarterLibrary(
+func ApplyStarterLibrary(
 	ctx context.Context,
 	serverURL string,
 	token string,
@@ -176,12 +176,12 @@ func applyStarterLibrary(
 	}
 
 	// Find all script references in the YAML and download them
-	scriptNames := extractScriptNames(specs)
+	scriptNames := ExtractScriptNames(specs)
 	level.Debug(logger).Log("msg", "Found script references in starter library", "count", len(scriptNames))
 
 	// Download scripts and update references in specs
 	if len(scriptNames) > 0 {
-		err = downloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, logger)
+		err = DownloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, logger)
 		if err != nil {
 			return fmt.Errorf("failed to download and update scripts: %w", err)
 		}
@@ -194,6 +194,40 @@ func applyStarterLibrary(
 	}
 	client.SetToken(token)
 
+	// Always check if license is free and skip teams for free licenses
+	appConfig, err := client.GetAppConfig()
+	if err != nil {
+		level.Debug(logger).Log("msg", "Error getting app config", "err", err)
+		// Continue even if there's an error getting the app config
+	} else if appConfig.License == nil || !appConfig.License.IsPremium() {
+		// Remove teams from specs to avoid applying them
+		level.Debug(logger).Log("msg", "Free license detected, skipping teams and team-related content in starter library")
+		specs.Teams = nil
+
+		// Filter out policies that reference teams
+		if specs.Policies != nil {
+			var filteredPolicies []*fleet.PolicySpec
+			for _, policy := range specs.Policies {
+				// Keep only policies that don't reference a team
+				if policy.Team == "" {
+					filteredPolicies = append(filteredPolicies, policy)
+				}
+			}
+			specs.Policies = filteredPolicies
+		}
+
+		// Note: QuerySpec doesn't have a Team field, so we can't filter queries by team
+
+		// Remove scripts from AppConfig if present
+		if specs.AppConfig != nil {
+			appConfigMap, ok := specs.AppConfig.(map[string]interface{})
+			if ok {
+				// Remove scripts from AppConfig
+				delete(appConfigMap, "scripts")
+			}
+		}
+	}
+
 	// Log function for ApplyGroup (minimal logging)
 	logf := func(format string, a ...interface{}) {}
 
@@ -207,7 +241,7 @@ func applyStarterLibrary(
 			ctx,
 			false,
 			specs,
-			".",
+			tempDir,
 			logf,
 			nil,
 			fleet.ApplyClientSpecOptions{},
@@ -231,8 +265,8 @@ func applyStarterLibrary(
 	return nil
 }
 
-// extractScriptNames extracts all script names from the specs
-func extractScriptNames(specs *spec.Group) []string {
+// ExtractScriptNames extracts all script names from the specs
+func ExtractScriptNames(specs *spec.Group) []string {
 	var scriptNames []string
 	scriptMap := make(map[string]bool) // Use a map to deduplicate script names
 
@@ -256,8 +290,8 @@ func extractScriptNames(specs *spec.Group) []string {
 	return scriptNames
 }
 
-// downloadAndUpdateScripts downloads scripts from URLs and updates the specs to reference local files
-func downloadAndUpdateScripts(ctx context.Context, specs *spec.Group, scriptNames []string, tempDir string, logger kitlog.Logger) error {
+// DownloadAndUpdateScripts downloads scripts from URLs and updates the specs to reference local files
+func DownloadAndUpdateScripts(ctx context.Context, specs *spec.Group, scriptNames []string, tempDir string, logger kitlog.Logger) error {
 	// Create a single HTTP client to be reused for all requests
 	httpClient := fleethttp.NewClient(fleethttp.WithTimeout(5 * time.Second))
 
@@ -315,6 +349,49 @@ func downloadAndUpdateScripts(ctx context.Context, specs *spec.Group, scriptName
 		}
 	}
 
+	// Read script contents and store them in memory
+	scriptContents := make(map[string][]byte, len(scriptNames))
+	for _, scriptName := range scriptNames {
+		localPath := scriptPaths[scriptName]
+		content, err := os.ReadFile(localPath)
+		if err != nil {
+			return fmt.Errorf("failed to read script %s from local file: %w", scriptName, err)
+		}
+		scriptContents[scriptName] = content
+	}
+
+	// Extract scripts from AppConfig if present
+	appConfigScripts := extractAppCfgScripts(specs.AppConfig)
+	if appConfigScripts != nil {
+		// Replace script paths with actual script contents
+		appScripts := make([]string, 0, len(appConfigScripts))
+		for _, scriptPath := range appConfigScripts {
+			if content, exists := scriptContents[scriptPath]; exists {
+				// Create a temporary file with the script content
+				tempFile, err := os.CreateTemp(tempDir, "script-*")
+				if err != nil {
+					return fmt.Errorf("failed to create temporary script file: %w", err)
+				}
+				if _, err := tempFile.Write(content); err != nil {
+					tempFile.Close()
+					return fmt.Errorf("failed to write script content to temporary file: %w", err)
+				}
+				tempFile.Close()
+
+				// Add the temporary file path to the list
+				appScripts = append(appScripts, tempFile.Name())
+			} else {
+				// Keep the original path if it's not one of our downloaded scripts
+				appScripts = append(appScripts, scriptPath)
+			}
+		}
+
+		// Update the AppConfig with the new script paths
+		if specs.AppConfig != nil {
+			specs.AppConfig.(map[string]interface{})["scripts"] = appScripts
+		}
+	}
+
 	// Update script references in the specs to point to local files
 	for i, teamRaw := range specs.Teams {
 		var teamData map[string]interface{}

server/service/endpoint_setup_test.go

@@ -1,12 +1,14 @@
 package service
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -15,6 +17,7 @@ import (
 
 	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
 	"github.com/fleetdm/fleet/v4/pkg/spec"
+	"github.com/fleetdm/fleet/v4/server/fleet"
 	kitlog "github.com/go-kit/log"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -113,7 +116,7 @@ func TestExtractScriptNames(t *testing.T) {
 			specs := &spec.Group{Teams: teams}
 
 			// Call the function
-			scriptNames := extractScriptNames(specs)
+			scriptNames := ExtractScriptNames(specs)
 
 			// Verify the results
 			assert.Len(t, scriptNames, len(tt.expected))
@@ -188,7 +191,7 @@ func TestDownloadAndUpdateScripts(t *testing.T) {
 			}
 
 			// Call the actual production function
-			err = downloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger())
+			err = DownloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger())
 			require.NoError(t, err)
 
 			// Verify the scripts were downloaded
@@ -285,7 +288,7 @@ func TestDownloadAndUpdateScriptsWithInvalidPaths(t *testing.T) {
 			}
 
 			// Call the actual production function
-			err = downloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger())
+			err = DownloadAndUpdateScripts(context.Background(), specs, tt.scriptNames, tempDir, kitlog.NewNopLogger())
 			require.Error(t, err)
 			assert.Contains(t, err.Error(), tt.errorMsg)
 		})
@@ -408,7 +411,7 @@ func TestDownloadAndUpdateScriptsTimeout(t *testing.T) {
 			defer cancel()
 
 			// Call the actual production function
-			err = downloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, kitlog.NewNopLogger())
+			err = DownloadAndUpdateScripts(ctx, specs, scriptNames, tempDir, kitlog.NewNopLogger())
 
 			if tt.expectError {
 				require.Error(t, err)
@@ -471,7 +474,7 @@ func TestApplyStarterLibraryWithMockClient(t *testing.T) {
 	}
 
 	// Call the function under test
-	testErr := applyStarterLibrary(
+	testErr := ApplyStarterLibrary(
 		context.Background(),
 		"https://example.com",
 		"test-token",
@@ -565,7 +568,7 @@ func TestApplyStarterLibraryWithMalformedYAML(t *testing.T) {
 	}()
 
 	// Call the function under test
-	testErr := applyStarterLibrary(
+	testErr := ApplyStarterLibrary(
 		context.Background(),
 		"https://example.com",
 		"test-token",
@@ -585,3 +588,137 @@ func TestApplyStarterLibraryWithMalformedYAML(t *testing.T) {
 
 	// If we reach here, no panic occurred and the setup flow was not interrupted
 }
+
+func TestApplyStarterLibraryWithFreeLicense(t *testing.T) {
+	// Read the real production starter library YAML file
+	starterLibraryPath := "../../docs/01-Using-Fleet/starter-library/starter-library.yml"
+	starterLibraryContent, err := os.ReadFile(starterLibraryPath)
+	require.NoError(t, err, "Should be able to read starter library YAML file")
+
+	// Create mock HTTP client for downloading the starter library and scripts
+	mockRT := &testRoundTripper2{
+		calls: []string{},
+		RoundTripFunc: func(req *http.Request) (*http.Response, error) {
+			switch {
+			case req.URL.String() == starterLibraryURL:
+				// Return the real starter library content
+				return createTestResponse(200, string(starterLibraryContent)), nil
+			case strings.Contains(req.URL.String(), "uninstall-fleetd"):
+				// Return a simple script for any script URL
+				return createTestResponse(200, "#!/bin/bash\necho ok"), nil
+			default:
+				// For any other URL, return a 404
+				return createTestResponse(404, "Not found"), nil
+			}
+		},
+	}
+
+	httpClientFactory := func(opts ...fleethttp.ClientOpt) *http.Client {
+		client := fleethttp.NewClient(opts...)
+		client.Transport = mockRT
+		return client
+	}
+
+	// Create a mock client that returns a free license
+	// Create a properly structured EnrichedAppConfig
+	mockEnrichedAppConfig := &fleet.EnrichedAppConfig{}
+	// Set the License field using json marshaling/unmarshaling to bypass unexported field access
+	configJSON := []byte(`{"license":{"tier":"free"}}`)
+	if err := json.Unmarshal(configJSON, mockEnrichedAppConfig); err != nil {
+		t.Fatal("Failed to unmarshal mock config:", err)
+	}
+
+	// Create a mock client factory
+	clientFactory := func(serverURL string, insecureSkipVerify bool, rootCA, urlPrefix string, options ...ClientOption) (*Client, error) {
+		mockClient := &Client{}
+
+		// Override the baseClient with a mock implementation
+		// Create a mock HTTP client
+		mockHTTPClient := &mockHTTPClient{
+			DoFunc: func(req *http.Request) (*http.Response, error) {
+				// Mock the GetAppConfig response
+				if req.URL.Path == "/api/v1/fleet/config" && req.Method == http.MethodGet {
+					respBody, _ := json.Marshal(mockEnrichedAppConfig)
+					return &http.Response{
+						StatusCode: http.StatusOK,
+						Body:       io.NopCloser(bytes.NewBuffer(respBody)),
+						Header:     make(http.Header),
+					}, nil
+				}
+				return &http.Response{
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(strings.NewReader("{}")),
+					Header:     make(http.Header),
+				}, nil
+			},
+		}
+
+		// Set up the baseClient with the mock HTTP client and a valid baseURL
+		baseURL, _ := url.Parse(serverURL)
+		mockClient.baseClient = &baseClient{
+			http:    mockHTTPClient,
+			baseURL: baseURL,
+		}
+
+		return mockClient, nil
+	}
+
+	// Track if ApplyGroup was called and capture the specs
+	applyGroupCalled := false
+	var capturedSpecs *spec.Group
+	// Create a mock ApplyGroup function
+	mockApplyGroup := func(ctx context.Context, specs *spec.Group) error {
+		applyGroupCalled = true
+		capturedSpecs = specs
+		return nil
+	}
+
+	// Call the function under test - teams will be skipped automatically for free license
+	testErr := ApplyStarterLibrary(
+		context.Background(),
+		"https://example.com",
+		"test-token",
+		kitlog.NewNopLogger(),
+		httpClientFactory,
+		clientFactory,
+		mockApplyGroup,
+	)
+
+	// Verify results
+	require.NoError(t, testErr)
+	assert.True(t, applyGroupCalled, "ApplyGroup should have been called")
+
+	// Verify that the specs were correctly parsed
+	require.NotNil(t, capturedSpecs, "Specs should not be nil")
+
+	// Verify that teams were removed
+	require.Empty(t, capturedSpecs.Teams, "Teams should be empty for free license")
+
+	// Verify that policies referencing teams were filtered out
+	if capturedSpecs.Policies != nil {
+		for _, policy := range capturedSpecs.Policies {
+			assert.Empty(t, policy.Team, "Policies should not reference teams for free license")
+		}
+	}
+
+	// Verify that scripts were removed from AppConfig
+	if capturedSpecs.AppConfig != nil {
+		appConfigMap, ok := capturedSpecs.AppConfig.(map[string]interface{})
+		if ok {
+			_, hasScripts := appConfigMap["scripts"]
+			assert.False(t, hasScripts, "AppConfig should not contain scripts for free license")
+		}
+	}
+
+	// Verify that the starter library URL was requested
+	assert.Contains(t, mockRT.calls, starterLibraryURL, "The starter library URL should have been requested")
+}
+
+// mockHTTPClient is a mock implementation of the http.Client
+type mockHTTPClient struct {
+	DoFunc func(req *http.Request) (*http.Response, error)
+}
+
+func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	return m.DoFunc(req)
+}