Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Update FMA refreshing logic to remove apps that were removed upstream (#27594)

Browse Source 
> No issue, we noticed this while testing FMA for Windows

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Added/updated automated tests
- [x] Manual QA for all new/changed functionality
This commit is contained in:
Jahziel Villasana-Espinoza
2025-03-28 14:04:08 -04:00
committed by GitHub
parent c769e50cd3
commit 2802cf57d5
7 changed files with 137 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/datastore/mysql/maintained_apps.go
+25 -18
View File
@@ -5,7 +5,6 @@ import (
"database/sql"
"errors"
"fmt"
"net/http"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
@@ -92,21 +91,6 @@ func (ds *Datastore) GetMaintainedAppByID(ctx context.Context, appID uint, teamI
return &app, nil
}
// NoMaintainedAppsInDatabase is the error type for no Fleet Maintained Apps in the database
type NoMaintainedAppsInDatabase struct {
fleet.ErrorWithUUID
}
// Error implements the error interface.
func (e *NoMaintainedAppsInDatabase) Error() string {
return `Fleet was unable to ingest the maintained apps list. Run fleetctl trigger name=maintained_apps to try repopulating the apps list.`
}
// StatusCode implements the go-kit http StatusCoder interface.
func (e *NoMaintainedAppsInDatabase) StatusCode() int {
return http.StatusNotFound
}
func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamID *uint, opt fleet.ListOptions) ([]fleet.MaintainedApp, *fleet.PaginationMetadata, error) {
stmt := `SELECT fma.id, fma.name, fma.platform, fma.slug, `
var args []any
@@ -141,7 +125,7 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI
}
if totalCount == 0 {
return nil, nil, &NoMaintainedAppsInDatabase{}
return nil, nil, &fleet.NoMaintainedAppsInDatabaseError{}
}
}
@@ -149,7 +133,7 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI
var avail []fleet.MaintainedApp
if err := sqlx.SelectContext(ctx, ds.reader(ctx), &avail, stmtPaged, args...); err != nil {
return nil, nil, ctxerr.Wrap(ctx, err, "selecting available fleet managed apps")
return nil, nil, ctxerr.Wrap(ctx, err, "selecting available fleet maintained apps")
}
meta := &fleet.PaginationMetadata{HasPreviousResults: opt.Page > 0, TotalResults: uint(filteredCount)} //nolint:gosec // dismiss G115
@@ -160,3 +144,26 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI
return avail, meta, nil
}
func (ds *Datastore) ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error {
stmt := `DELETE FROM fleet_maintained_apps WHERE slug NOT IN (?)`
var err error
var args []any
switch len(slugsToKeep) {
case 0:
stmt = `DELETE FROM fleet_maintained_apps`
default:
stmt, args, err = sqlx.In(stmt, slugsToKeep)
if err != nil {
return ctxerr.Wrap(ctx, err, "building sqlx.In statement for clearing removed maintained apps")
}
}
_, err = ds.writer(ctx).ExecContext(ctx, stmt, args...)
if err != nil {
return ctxerr.Wrap(ctx, err, "clearing removed maintained apps")
}
return nil
}
server/datastore/mysql/maintained_apps_test.go
+5
View File
@@ -22,6 +22,7 @@ func TestMaintainedApps(t *testing.T) {
{"UpsertMaintainedApps", testUpsertMaintainedApps},
{"Sync", testSync},
{"ListAndGetAvailableApps", testListAndGetAvailableApps},
{"SyncAndRemoveApps", testSyncAndRemoveApps},
}
for _, c := range cases {
@@ -425,3 +426,7 @@ func testListAndGetAvailableApps(t *testing.T, ds *Datastore) {
maintained3.TitleID = nil
require.Equal(t, maintained3, gotApp)
}
func testSyncAndRemoveApps(t *testing.T, ds *Datastore) {
maintained_apps.SyncAndRemoveApps(t, ds)
}
server/fleet/datastore.go
+4
View File
@@ -1961,6 +1961,10 @@ type Datastore interface {
// if a team is specified.
ListAvailableFleetMaintainedApps(ctx context.Context, teamID *uint, opt ListOptions) ([]MaintainedApp, *PaginationMetadata, error)
// ClearRemovedFleetMaintainedApps deletes all Fleet-maintained apps that are not in the given
// set of slugs.
ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error
// GetMaintainedAppByID gets a Fleet-maintained app by its ID, including software title ID if
// either the maintained app or a custom package/VPP app for the same app is installed on the specified team,
// if a team is specified.
server/fleet/maintained_apps.go
+21
View File
@@ -1,5 +1,7 @@
package fleet
import "net/http"
// MaintainedApp represents an app in the Fleet library of maintained apps,
// as stored in the fleet_library_apps table.
type MaintainedApp struct {
@@ -37,3 +39,22 @@ func (s *MaintainedApp) BundleIdentifier() string {
func (s *MaintainedApp) AuthzType() string {
return "maintained_app"
}
// NoMaintainedAppsInDatabaseError is the error type for no Fleet Maintained Apps in the database
type NoMaintainedAppsInDatabaseError struct {
ErrorWithUUID
}
// Error implements the error interface.
func (e *NoMaintainedAppsInDatabaseError) Error() string {
return `Fleet was unable to ingest the maintained apps list. Run fleetctl trigger name=maintained_apps to try repopulating the apps list.`
}
// StatusCode implements the go-kit http StatusCoder interface.
func (e *NoMaintainedAppsInDatabaseError) StatusCode() int {
return http.StatusNotFound
}
func (e *NoMaintainedAppsInDatabaseError) Is(target error) bool {
return target.Error() == e.Error()
}
server/mdm/maintainedapps/sync.go
+10 -1
View File
@@ -73,10 +73,14 @@ func Refresh(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) erro
return ctxerr.Wrap(ctx, err, "unmarshal apps list")
}
if appsList.Version != 2 {
return ctxerr.Errorf(ctx, "apps list is an incompatible version")
return ctxerr.New(ctx, "apps list is an incompatible version")
}
var gotApps []string
for _, app := range appsList.Apps {
gotApps = append(gotApps, app.Slug)
if app.UniqueIdentifier == "" {
app.UniqueIdentifier = app.Name
}
@@ -91,6 +95,11 @@ func Refresh(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) erro
}
}
// remove apps that were removed upstream
if err := ds.ClearRemovedFleetMaintainedApps(ctx, gotApps); err != nil {
return ctxerr.Wrap(ctx, err, "clear removed maintained apps during refresh")
}
return nil
}
server/mdm/maintainedapps/testing_utils.go
+60
View File
@@ -72,3 +72,63 @@ func ExpectedAppSlugs(t *testing.T) []string {
}
return slugs
}
func SyncAndRemoveApps(t *testing.T, ds fleet.Datastore) {
_, filename, _, _ := runtime.Caller(0)
base := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(filename))))
outputsDir := filepath.Join(base, "ee/maintained-apps/outputs")
b, err := os.ReadFile(filepath.Join(outputsDir, "apps.json"))
require.NoError(t, err)
var appsFile AppsList
require.NoError(t, json.Unmarshal(b, &appsFile))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
data, err := json.Marshal(&appsFile)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte(err.Error()))
return
}
_, _ = w.Write(data)
}))
defer srv.Close()
// not using t.Setenv because we want the env var to be unset on return of
// this call
os.Setenv("FLEET_DEV_MAINTAINED_APPS_BASE_URL", srv.URL)
defer os.Unsetenv("FLEET_DEV_MAINTAINED_APPS_BASE_URL")
err = Refresh(context.Background(), ds, log.NewNopLogger())
require.NoError(t, err)
originalApps, _, err := ds.ListAvailableFleetMaintainedApps(context.Background(), nil, fleet.ListOptions{})
require.NoError(t, err)
require.Equal(t, len(appsFile.Apps), len(originalApps))
// Modify the apps list to simulate removing an app from upstream
removedApp := appsFile.Apps[0]
appsFile.Apps = appsFile.Apps[1:]
err = Refresh(context.Background(), ds, log.NewNopLogger())
require.NoError(t, err)
modifiedApps, _, err := ds.ListAvailableFleetMaintainedApps(context.Background(), nil, fleet.ListOptions{})
require.NoError(t, err)
require.Equal(t, len(appsFile.Apps), len(modifiedApps))
require.Equal(t, len(originalApps)-1, len(modifiedApps))
for _, a := range modifiedApps {
require.NotEqual(t, removedApp.Slug, a.Slug)
}
// remove all apps from upstream.
appsFile.Apps = []appListing{}
err = Refresh(context.Background(), ds, log.NewNopLogger())
require.NoError(t, err)
modifiedApps, _, err = ds.ListAvailableFleetMaintainedApps(context.Background(), nil, fleet.ListOptions{})
require.ErrorIs(t, err, &fleet.NoMaintainedAppsInDatabaseError{})
require.Empty(t, modifiedApps)
}
server/mock/datastore_mock.go
+12
View File
@@ -1234,6 +1234,8 @@ type MaybeUpdateSetupExperienceVPPStatusFunc func(ctx context.Context, hostUUID
type ListAvailableFleetMaintainedAppsFunc func(ctx context.Context, teamID *uint, opt fleet.ListOptions) ([]fleet.MaintainedApp, *fleet.PaginationMetadata, error)
type ClearRemovedFleetMaintainedAppsFunc func(ctx context.Context, slugsToKeep []string) error
type GetMaintainedAppByIDFunc func(ctx context.Context, appID uint, teamID *uint) (*fleet.MaintainedApp, error)
type UpsertMaintainedAppFunc func(ctx context.Context, app *fleet.MaintainedApp) (*fleet.MaintainedApp, error)
@@ -3101,6 +3103,9 @@ type DataStore struct {
ListAvailableFleetMaintainedAppsFunc ListAvailableFleetMaintainedAppsFunc
ListAvailableFleetMaintainedAppsFuncInvoked bool
ClearRemovedFleetMaintainedAppsFunc ClearRemovedFleetMaintainedAppsFunc
ClearRemovedFleetMaintainedAppsFuncInvoked bool
GetMaintainedAppByIDFunc GetMaintainedAppByIDFunc
GetMaintainedAppByIDFuncInvoked bool
@@ -7418,6 +7423,13 @@ func (s *DataStore) ListAvailableFleetMaintainedApps(ctx context.Context, teamID
return s.ListAvailableFleetMaintainedAppsFunc(ctx, teamID, opt)
}
func (s *DataStore) ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error {
s.mu.Lock()
s.ClearRemovedFleetMaintainedAppsFuncInvoked = true
s.mu.Unlock()
return s.ClearRemovedFleetMaintainedAppsFunc(ctx, slugsToKeep)
}
func (s *DataStore) GetMaintainedAppByID(ctx context.Context, appID uint, teamID *uint) (*fleet.MaintainedApp, error) {
s.mu.Lock()
s.GetMaintainedAppByIDFuncInvoked = true