diff --git a/changes/33562-4.73.2-migration-fix b/changes/33562-4.73.2-migration-fix new file mode 100644 index 0000000000..07d11becd5 --- /dev/null +++ b/changes/33562-4.73.2-migration-fix @@ -0,0 +1 @@ +* Added logic to detect and fix migration issues caused by improperly published Fleet v4.73.2 Linux binary diff --git a/cmd/fleet/prepare.go b/cmd/fleet/prepare.go index 6a487dafaa..8af9aa63b2 100644 --- a/cmd/fleet/prepare.go +++ b/cmd/fleet/prepare.go @@ -51,6 +51,22 @@ To setup Fleet infrastructure, use one of the available commands. if err != nil { initFatal(err, "retrieving migration status") } + if status.StatusCode == fleet.NeedsFleetv4732Fix { + if !noPrompt { + printFleetv4732FixMessage() + bufio.NewScanner(os.Stdin).Scan() + } else { + fmt.Println("Applying automatic fix for mis-numbered v4.73.2 migrations") + } + if err := ds.FixFleetv4732Migrations(cmd.Context()); err != nil { + initFatal(err, "fixing v4.73.2 migrations") + } + // re-check status after fix + status, err = ds.MigrationStatus(cmd.Context()) + if err != nil { + initFatal(err, "retrieving migration status") + } + } switch status.StatusCode { case fleet.NoMigrationsCompleted: @@ -63,6 +79,8 @@ To setup Fleet infrastructure, use one of the available commands. printMissingMigrationsPrompt(status.MissingTable, status.MissingData) bufio.NewScanner(os.Stdin).Scan() } + case fleet.NeedsFleetv4732Fix, fleet.UnknownFleetv4732State: + printFleetv4732UnknownStateMessage(status.StatusCode) case fleet.UnknownMigrations: printUnknownMigrationsMessage(status.UnknownTable, status.UnknownData) if dev { @@ -113,6 +131,29 @@ func printMissingMigrationsPrompt(tables []int64, data []int64) { tablesAndDataToString(tables, data)) } +func printFleetv4732FixMessage() { + fmt.Printf("################################################################################\n" + + "# WARNING:\n" + + "# Your Fleet database has misnumbered migrations introduced in some released\n" + + "# v4.73.2 artifacts. Fleet will automatically perform this fix prior to database\n" + + "# migrations. Please back up your data before continuing.\n" + + "################################################################################\n") +} + +func printFleetv4732UnknownStateMessage(statusCode fleet.MigrationStatusCode) { + extra := "your Fleet database is in an unknown state." + if statusCode == fleet.NeedsFleetv4732Fix { + extra = "the automatic fix did not result in the expected state." + } + fmt.Print("################################################################################\n" + + "# WARNING:\n" + + "# Your Fleet database has misnumbered migrations introduced in some released\n" + + "# v4.73.2 artifacts. Fleet attempts to fix this problem automatically, however\n" + + "# " + extra + "\n" + + "# Please contact Fleet support for assistance in resolving this.\n" + + "################################################################################\n") +} + func tablesAndDataToString(tables, data []int64) string { switch { case len(tables) > 0 && len(data) == 0: diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index d6918fa473..566ce5ec95 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -269,6 +269,16 @@ the way that the Fleet server works. if dev { os.Exit(1) } + case fleet.NeedsFleetv4732Fix: + printFleetv4732FixNeededMessage() + if !config.Upgrades.AllowMissingMigrations { + os.Exit(1) + } + case fleet.UnknownFleetv4732State: + printFleetv4732UnknownStateMessage(migrationStatus.StatusCode) + if !config.Upgrades.AllowMissingMigrations { + os.Exit(1) + } case fleet.SomeMigrationsCompleted: tables, data := migrationStatus.MissingTable, migrationStatus.MissingData printMissingMigrationsWarning(tables, data) @@ -1592,6 +1602,22 @@ func printMissingMigrationsWarning(tables []int64, data []int64) { tablesAndDataToString(tables, data), os.Args[0]) } +func printFleetv4732FixNeededMessage() { + fmt.Printf("################################################################################\n"+ + "# WARNING:\n"+ + "# Your Fleet database has misnumbered migrations introduced in some released\n"+ + "# v4.73.2 artifacts. Fleet will automatically perform this fix prior to database\n"+ + "# migrations. Please back up your data before continuing.\n"+ + "#\n"+ + "# Run `%s prepare db` to perform migrations.\n"+ + "#\n"+ + "# To run the server without performing migrations:\n"+ + "# - Set environment variable FLEET_UPGRADES_ALLOW_MISSING_MIGRATIONS=1, or,\n"+ + "# - Set config updates.allow_missing_migrations to true, or,\n"+ + "# - Use command line argument --upgrades_allow_missing_migrations=true\n"+ + "################################################################################\n", os.Args[0]) +} + func initLicense(config configpkg.FleetConfig, devLicense, devExpiredLicense bool) (*fleet.LicenseInfo, error) { if devLicense { // This license key is valid for development only diff --git a/cmd/fleet/vuln_process.go b/cmd/fleet/vuln_process.go index f0a2147be1..22fcdbfa71 100644 --- a/cmd/fleet/vuln_process.go +++ b/cmd/fleet/vuln_process.go @@ -67,6 +67,8 @@ by an exit code of zero.`, switch status.StatusCode { case fleet.AllMigrationsCompleted: // only continue if db is considered up-to-date + case fleet.NeedsFleetv4732Fix, fleet.UnknownFleetv4732State: + migrationError = errors.New("database has misnumbered migrations from v4.73.2") case fleet.NoMigrationsCompleted: migrationError = errors.New("no migrations completed") case fleet.SomeMigrationsCompleted: diff --git a/server/datastore/mysql/migrations_test.go b/server/datastore/mysql/migrations_test.go index 2f8f683f52..f2cf15b7b8 100644 --- a/server/datastore/mysql/migrations_test.go +++ b/server/datastore/mysql/migrations_test.go @@ -7,6 +7,7 @@ import ( "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils" + "github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/data" "github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/tables" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/stretchr/testify/assert" @@ -56,6 +57,95 @@ func TestMigrationStatus(t *testing.T) { assert.Empty(t, status.MissingData) } +func TestV4732MigrationFix(t *testing.T) { + ds := createMySQLDSForMigrationTests(t, t.Name()) + t.Cleanup(func() { + ds.Close() + }) + status, err := ds.MigrationStatus(context.Background()) + require.NoError(t, err) + require.NotNil(t, status) + assert.EqualValues(t, fleet.NoMigrationsCompleted, status.StatusCode) + + recreate4732BadState(t, ds) + + status, err = ds.MigrationStatus(context.Background()) + require.NoError(t, err) + require.NotNil(t, status) + assert.EqualValues(t, fleet.NeedsFleetv4732Fix, status.StatusCode) + + err = ds.FixFleetv4732Migrations(context.Background()) + require.NoError(t, err) + + err = ds.MigrateTables(context.Background()) + require.NoError(t, err) + + status, err = ds.MigrationStatus(context.Background()) + require.NoError(t, err) + require.NotNil(t, status) + assert.EqualValues(t, fleet.AllMigrationsCompleted, status.StatusCode) + + // Insert a bad migration again which should trigger the unknown state + _, err = ds.writer(context.Background()).Exec(`INSERT INTO `+tables.MigrationClient.TableName+` (version_id, is_applied) VALUES (?, 1)`, fleet4732BadMigrationID1) + require.NoError(t, err) + status, err = ds.MigrationStatus(context.Background()) + require.NoError(t, err) + require.NotNil(t, status) + assert.EqualValues(t, fleet.UnknownFleetv4732State, status.StatusCode) +} + +// Apply the proper 4.73.2 migrations +func recreate4732GoodState(t *testing.T, ds *Datastore) { + var version int64 + var err error + + const maxDataMigration = 20230525175650 + + // Migrate up to 4.73.1 + for version < fleet4731GoodMigrationID { + err = tables.MigrationClient.UpByOne(ds.writer(context.Background()).DB, "") + require.NoError(t, err) + version, err = tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB) + require.NoError(t, err) + } + require.Equal(t, int64(fleet4731GoodMigrationID), version) + + // Apply data migrations which were deprecated before 4.73.2 and should never change so no need for + // upbyone, etc. but we'll verify below that we're at expected version + err = data.MigrationClient.Up(ds.writer(context.Background()).DB, "") + require.NoError(t, err) + version, err = data.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB) + require.NoError(t, err) + require.EqualValues(t, int64(maxDataMigration), version) + + // Apply the migrations from fleet v4.73.2 + err = tables.MigrationClient.UpByOne(ds.writer(context.Background()).DB, "") + require.NoError(t, err) + version, err = tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB) + require.NoError(t, err) + require.EqualValues(t, fleet4732GoodMigrationID2, version) + + err = tables.MigrationClient.UpByOne(ds.writer(context.Background()).DB, "") + require.NoError(t, err) + version, err = tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB) + require.NoError(t, err) + require.EqualValues(t, fleet4732GoodMigrationID1, version) +} + +// Recreate the bad state that some customers ended up with after running fleet v4.73.2 migrations +func recreate4732BadState(t *testing.T, ds *Datastore) { + recreate4732GoodState(t, ds) + + _, err := ds.writer(context.Background()).Exec(`UPDATE `+tables.MigrationClient.TableName+` SET version_id = ? WHERE version_id = ?`, fleet4732BadMigrationID1, fleet4732GoodMigrationID1) + require.NoError(t, err) + _, err = ds.writer(context.Background()).Exec(`UPDATE `+tables.MigrationClient.TableName+` SET version_id = ? WHERE version_id = ?`, fleet4732BadMigrationID2, fleet4732GoodMigrationID2) + require.NoError(t, err) + + version, err := tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB) + require.NoError(t, err) + require.EqualValues(t, fleet4732BadMigrationID1, version) +} + func TestMigrations(t *testing.T) { // Create the database (must use raw MySQL client to do this) ds := createMySQLDSForMigrationTests(t, t.Name()) diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index ab1a29b033..539b898b28 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -40,6 +40,16 @@ import ( const ( defaultSelectLimit = 1000000 mySQLTimestampFormat = "2006-01-02 15:04:05" // %Y/%m/%d %H:%M:%S + + // Migration IDs needed for fixing broken migrations that some customers encountered with fleet v4.73.2 + // See https://github.com/fleetdm/fleet/issues/33562 + fleet4732BadMigrationID1 = 20250918154557 // was 20250918154557_AddKernelHostCountsIndexForVulnQueries.go + fleet4732GoodMigrationID1 = 20250817154557 // 20250817154557_AddKernelHostCountsIndexForVulnQueries.go + + fleet4732BadMigrationID2 = 20250904115553 // was 20250904115553_OptimizeHostScriptResultsIndex.go + fleet4732GoodMigrationID2 = 20250816115553 // 20250816115553_OptimizeHostScriptResultsIndex.go + + fleet4731GoodMigrationID = 20250815130115 ) // Matches all non-word and '-' characters for replacement @@ -412,6 +422,11 @@ func (ds *Datastore) MigrationStatus(ctx context.Context) (*fleet.MigrationStatu if err != nil { return nil, fmt.Errorf("cannot load migrations: %w", err) } + // This will only return a non-nil status if we detect the specific broken state from v4.73.2 + status := ds.CheckFleetv4732BadMigrations(appliedTable) + if status != nil { + return status, nil + } return compareMigrations( tables.MigrationClient.Migrations, data.MigrationClient.Migrations, @@ -420,6 +435,62 @@ func (ds *Datastore) MigrationStatus(ctx context.Context) (*fleet.MigrationStatu ), nil } +// Checks for misnumbered migrations introduced in some released fleet v4.73.2 versions +func (ds *Datastore) CheckFleetv4732BadMigrations(appliedTable []int64) *fleet.MigrationStatus { + if len(appliedTable) == 0 { + return nil + } + // If the last 3 migrations are the "bad" 4.73.2 migrations and then the good 4.73.1 migration, in that order, + // we are in the known-bad 4.73.2 state and should apply the fix + if len(appliedTable) > 2 && + appliedTable[len(appliedTable)-1] == fleet4732BadMigrationID1 && + appliedTable[len(appliedTable)-2] == fleet4732BadMigrationID2 && + appliedTable[len(appliedTable)-3] == fleet4731GoodMigrationID { + return &fleet.MigrationStatus{ + StatusCode: fleet.NeedsFleetv4732Fix, + } + } + for _, v := range appliedTable { + if v == fleet4732BadMigrationID1 || v == fleet4732BadMigrationID2 { + return &fleet.MigrationStatus{ + StatusCode: fleet.UnknownFleetv4732State, + } + } + } + return nil +} + +func (ds *Datastore) FixFleetv4732Migrations(ctx context.Context) error { + // Update version ID of the bad migrations to the renumbered version IDs. Exactly 1 row should be affected + // by each query + stmt := `UPDATE ` + tables.MigrationClient.TableName + ` SET version_id = ? WHERE version_id = ?` + return ds.withTx(ctx, func(tx sqlx.ExtContext) error { + result, err := tx.ExecContext(ctx, stmt, fleet4732GoodMigrationID1, fleet4732BadMigrationID1) + if err != nil { + return err + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return ctxerr.Errorf(ctx, "expected to affect 1 row for migration %d, affected %d", fleet4732BadMigrationID1, affected) + } + result, err = tx.ExecContext(ctx, stmt, fleet4732GoodMigrationID2, fleet4732BadMigrationID2) + if err != nil { + return err + } + affected, err = result.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return ctxerr.Errorf(ctx, "expected to affect 1 row for migration %d, affected %d", fleet4732BadMigrationID2, affected) + } + return nil + }) +} + // It assumes some deployments may have performed migrations out of order. func compareMigrations(knownTable goose.Migrations, knownData goose.Migrations, appliedTable, appliedData []int64) *fleet.MigrationStatus { if len(appliedTable) == 0 && len(appliedData) == 0 { diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index cba1bcbd58..d73fc46c66 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -2555,6 +2555,11 @@ const ( AllMigrationsCompleted // UnknownMigrations means some unidentified migrations were detected on the database. UnknownMigrations + // NeedsFleetv4732Fix means the database needs the special fix migration for fleet v4.73.2 + NeedsFleetv4732Fix + // UnknownFleetv4732State means the database has the broken migrations from fleet v4.73.2 however + // it is not in the expected state and needs manual intervention. + UnknownFleetv4732State ) // TODO: we have a similar but different interface in the service package,