diff --git a/server/datastore/mysql/maintained_apps.go b/server/datastore/mysql/maintained_apps.go index f8b7c4445a..07e1a69285 100644 --- a/server/datastore/mysql/maintained_apps.go +++ b/server/datastore/mysql/maintained_apps.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "net/http" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" @@ -92,21 +91,6 @@ func (ds *Datastore) GetMaintainedAppByID(ctx context.Context, appID uint, teamI return &app, nil } -// NoMaintainedAppsInDatabase is the error type for no Fleet Maintained Apps in the database -type NoMaintainedAppsInDatabase struct { - fleet.ErrorWithUUID -} - -// Error implements the error interface. -func (e *NoMaintainedAppsInDatabase) Error() string { - return `Fleet was unable to ingest the maintained apps list. Run fleetctl trigger name=maintained_apps to try repopulating the apps list.` -} - -// StatusCode implements the go-kit http StatusCoder interface. -func (e *NoMaintainedAppsInDatabase) StatusCode() int { - return http.StatusNotFound -} - func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamID *uint, opt fleet.ListOptions) ([]fleet.MaintainedApp, *fleet.PaginationMetadata, error) { stmt := `SELECT fma.id, fma.name, fma.platform, fma.slug, ` var args []any @@ -141,7 +125,7 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI } if totalCount == 0 { - return nil, nil, &NoMaintainedAppsInDatabase{} + return nil, nil, &fleet.NoMaintainedAppsInDatabaseError{} } } @@ -149,7 +133,7 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI var avail []fleet.MaintainedApp if err := sqlx.SelectContext(ctx, ds.reader(ctx), &avail, stmtPaged, args...); err != nil { - return nil, nil, ctxerr.Wrap(ctx, err, "selecting available fleet managed apps") + return nil, nil, ctxerr.Wrap(ctx, err, "selecting available fleet maintained apps") } meta := &fleet.PaginationMetadata{HasPreviousResults: opt.Page > 0, TotalResults: uint(filteredCount)} //nolint:gosec // dismiss G115 @@ -160,3 +144,26 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI return avail, meta, nil } + +func (ds *Datastore) ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error { + stmt := `DELETE FROM fleet_maintained_apps WHERE slug NOT IN (?)` + + var err error + var args []any + switch len(slugsToKeep) { + case 0: + stmt = `DELETE FROM fleet_maintained_apps` + default: + stmt, args, err = sqlx.In(stmt, slugsToKeep) + if err != nil { + return ctxerr.Wrap(ctx, err, "building sqlx.In statement for clearing removed maintained apps") + } + } + + _, err = ds.writer(ctx).ExecContext(ctx, stmt, args...) + if err != nil { + return ctxerr.Wrap(ctx, err, "clearing removed maintained apps") + } + + return nil +} diff --git a/server/datastore/mysql/maintained_apps_test.go b/server/datastore/mysql/maintained_apps_test.go index 2ceb11a04a..aea9077e16 100644 --- a/server/datastore/mysql/maintained_apps_test.go +++ b/server/datastore/mysql/maintained_apps_test.go @@ -22,6 +22,7 @@ func TestMaintainedApps(t *testing.T) { {"UpsertMaintainedApps", testUpsertMaintainedApps}, {"Sync", testSync}, {"ListAndGetAvailableApps", testListAndGetAvailableApps}, + {"SyncAndRemoveApps", testSyncAndRemoveApps}, } for _, c := range cases { @@ -425,3 +426,7 @@ func testListAndGetAvailableApps(t *testing.T, ds *Datastore) { maintained3.TitleID = nil require.Equal(t, maintained3, gotApp) } + +func testSyncAndRemoveApps(t *testing.T, ds *Datastore) { + maintained_apps.SyncAndRemoveApps(t, ds) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 481f121a22..1950fdbf0e 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1961,6 +1961,10 @@ type Datastore interface { // if a team is specified. ListAvailableFleetMaintainedApps(ctx context.Context, teamID *uint, opt ListOptions) ([]MaintainedApp, *PaginationMetadata, error) + // ClearRemovedFleetMaintainedApps deletes all Fleet-maintained apps that are not in the given + // set of slugs. + ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error + // GetMaintainedAppByID gets a Fleet-maintained app by its ID, including software title ID if // either the maintained app or a custom package/VPP app for the same app is installed on the specified team, // if a team is specified. diff --git a/server/fleet/maintained_apps.go b/server/fleet/maintained_apps.go index 25b7455fb5..0c375de99f 100644 --- a/server/fleet/maintained_apps.go +++ b/server/fleet/maintained_apps.go @@ -1,5 +1,7 @@ package fleet +import "net/http" + // MaintainedApp represents an app in the Fleet library of maintained apps, // as stored in the fleet_library_apps table. type MaintainedApp struct { @@ -37,3 +39,22 @@ func (s *MaintainedApp) BundleIdentifier() string { func (s *MaintainedApp) AuthzType() string { return "maintained_app" } + +// NoMaintainedAppsInDatabaseError is the error type for no Fleet Maintained Apps in the database +type NoMaintainedAppsInDatabaseError struct { + ErrorWithUUID +} + +// Error implements the error interface. +func (e *NoMaintainedAppsInDatabaseError) Error() string { + return `Fleet was unable to ingest the maintained apps list. Run fleetctl trigger name=maintained_apps to try repopulating the apps list.` +} + +// StatusCode implements the go-kit http StatusCoder interface. +func (e *NoMaintainedAppsInDatabaseError) StatusCode() int { + return http.StatusNotFound +} + +func (e *NoMaintainedAppsInDatabaseError) Is(target error) bool { + return target.Error() == e.Error() +} diff --git a/server/mdm/maintainedapps/sync.go b/server/mdm/maintainedapps/sync.go index 525dc8529a..51279b8cf8 100644 --- a/server/mdm/maintainedapps/sync.go +++ b/server/mdm/maintainedapps/sync.go @@ -73,10 +73,14 @@ func Refresh(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) erro return ctxerr.Wrap(ctx, err, "unmarshal apps list") } if appsList.Version != 2 { - return ctxerr.Errorf(ctx, "apps list is an incompatible version") + return ctxerr.New(ctx, "apps list is an incompatible version") } + var gotApps []string + for _, app := range appsList.Apps { + gotApps = append(gotApps, app.Slug) + if app.UniqueIdentifier == "" { app.UniqueIdentifier = app.Name } @@ -91,6 +95,11 @@ func Refresh(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) erro } } + // remove apps that were removed upstream + if err := ds.ClearRemovedFleetMaintainedApps(ctx, gotApps); err != nil { + return ctxerr.Wrap(ctx, err, "clear removed maintained apps during refresh") + } + return nil } diff --git a/server/mdm/maintainedapps/testing_utils.go b/server/mdm/maintainedapps/testing_utils.go index b5fa34d2e7..0ab3fd4fca 100644 --- a/server/mdm/maintainedapps/testing_utils.go +++ b/server/mdm/maintainedapps/testing_utils.go @@ -72,3 +72,63 @@ func ExpectedAppSlugs(t *testing.T) []string { } return slugs } + +func SyncAndRemoveApps(t *testing.T, ds fleet.Datastore) { + _, filename, _, _ := runtime.Caller(0) + base := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(filename)))) + outputsDir := filepath.Join(base, "ee/maintained-apps/outputs") + + b, err := os.ReadFile(filepath.Join(outputsDir, "apps.json")) + require.NoError(t, err) + var appsFile AppsList + require.NoError(t, json.Unmarshal(b, &appsFile)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := json.Marshal(&appsFile) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + _, _ = w.Write(data) + })) + defer srv.Close() + + // not using t.Setenv because we want the env var to be unset on return of + // this call + os.Setenv("FLEET_DEV_MAINTAINED_APPS_BASE_URL", srv.URL) + defer os.Unsetenv("FLEET_DEV_MAINTAINED_APPS_BASE_URL") + + err = Refresh(context.Background(), ds, log.NewNopLogger()) + require.NoError(t, err) + + originalApps, _, err := ds.ListAvailableFleetMaintainedApps(context.Background(), nil, fleet.ListOptions{}) + require.NoError(t, err) + + require.Equal(t, len(appsFile.Apps), len(originalApps)) + + // Modify the apps list to simulate removing an app from upstream + removedApp := appsFile.Apps[0] + appsFile.Apps = appsFile.Apps[1:] + + err = Refresh(context.Background(), ds, log.NewNopLogger()) + require.NoError(t, err) + + modifiedApps, _, err := ds.ListAvailableFleetMaintainedApps(context.Background(), nil, fleet.ListOptions{}) + require.NoError(t, err) + + require.Equal(t, len(appsFile.Apps), len(modifiedApps)) + require.Equal(t, len(originalApps)-1, len(modifiedApps)) + for _, a := range modifiedApps { + require.NotEqual(t, removedApp.Slug, a.Slug) + } + + // remove all apps from upstream. + appsFile.Apps = []appListing{} + + err = Refresh(context.Background(), ds, log.NewNopLogger()) + require.NoError(t, err) + + modifiedApps, _, err = ds.ListAvailableFleetMaintainedApps(context.Background(), nil, fleet.ListOptions{}) + require.ErrorIs(t, err, &fleet.NoMaintainedAppsInDatabaseError{}) + require.Empty(t, modifiedApps) +} diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 89b5cba7cf..9f66733605 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -1234,6 +1234,8 @@ type MaybeUpdateSetupExperienceVPPStatusFunc func(ctx context.Context, hostUUID type ListAvailableFleetMaintainedAppsFunc func(ctx context.Context, teamID *uint, opt fleet.ListOptions) ([]fleet.MaintainedApp, *fleet.PaginationMetadata, error) +type ClearRemovedFleetMaintainedAppsFunc func(ctx context.Context, slugsToKeep []string) error + type GetMaintainedAppByIDFunc func(ctx context.Context, appID uint, teamID *uint) (*fleet.MaintainedApp, error) type UpsertMaintainedAppFunc func(ctx context.Context, app *fleet.MaintainedApp) (*fleet.MaintainedApp, error) @@ -3101,6 +3103,9 @@ type DataStore struct { ListAvailableFleetMaintainedAppsFunc ListAvailableFleetMaintainedAppsFunc ListAvailableFleetMaintainedAppsFuncInvoked bool + ClearRemovedFleetMaintainedAppsFunc ClearRemovedFleetMaintainedAppsFunc + ClearRemovedFleetMaintainedAppsFuncInvoked bool + GetMaintainedAppByIDFunc GetMaintainedAppByIDFunc GetMaintainedAppByIDFuncInvoked bool @@ -7418,6 +7423,13 @@ func (s *DataStore) ListAvailableFleetMaintainedApps(ctx context.Context, teamID return s.ListAvailableFleetMaintainedAppsFunc(ctx, teamID, opt) } +func (s *DataStore) ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error { + s.mu.Lock() + s.ClearRemovedFleetMaintainedAppsFuncInvoked = true + s.mu.Unlock() + return s.ClearRemovedFleetMaintainedAppsFunc(ctx, slugsToKeep) +} + func (s *DataStore) GetMaintainedAppByID(ctx context.Context, appID uint, teamID *uint) (*fleet.MaintainedApp, error) { s.mu.Lock() s.GetMaintainedAppByIDFuncInvoked = true