Fix for 4.73.2 misnumbered migrations (#33655)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #33562 Detects and if possible fixes migrations which were misnumbered in the released 4.73.2 Linux binary(it was based on the commit before the renumbering commit was added). This does not affect the released 4.73.2 docker images and this code does nothing on these since the migrations will not be detected We specifically look for the 3 most recent migrations being the mis-numbered 4.73.2 and 4.73.1 migrations in the expected order. If neither of the mis-numbered migrations are found, nothing is done. Likewise if the order is not right or the order is not exactly right(e.g. if intervening migrations, for instance from 4.74.0 have been applied) we do not apply the fix. Finally, the fix is only ever applied in the existing migration path and fleet will never try to apply the fleet automatically by just running the fleet server(though it will detect the condition and complain) # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] If paths of existing endpoints are modified without backwards compatibility, checked the frontend/CLI for any necessary changes ## Testing - [x] Added/updated automated tests - [x] Where appropriate, [automated tests simulate multiple hosts and test for host isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing) (updates to one hosts's records do not affect another) - [x] QA'd all new/changed functionality manually For unreleased bug fixes in a release candidate, one of: - [x] Confirmed that the fix is not expected to adversely impact load test results - [x] Alerted the release DRI if additional load testing is needed ## Database migrations - [x] Checked table schema to confirm autoupdate - [x] Checked schema for all modified table for columns that will auto-update timestamps during migration. - [x] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects. - [x] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`).
This commit is contained in:
Jordan Montgomery
@@ -0,0 +1 @@
|
||||
* Added logic to detect and fix migration issues caused by improperly published Fleet v4.73.2 Linux binary
|
||||
@@ -51,6 +51,22 @@ To setup Fleet infrastructure, use one of the available commands.
|
||||
if err != nil {
|
||||
initFatal(err, "retrieving migration status")
|
||||
}
|
||||
if status.StatusCode == fleet.NeedsFleetv4732Fix {
|
||||
if !noPrompt {
|
||||
printFleetv4732FixMessage()
|
||||
bufio.NewScanner(os.Stdin).Scan()
|
||||
} else {
|
||||
fmt.Println("Applying automatic fix for mis-numbered v4.73.2 migrations")
|
||||
}
|
||||
if err := ds.FixFleetv4732Migrations(cmd.Context()); err != nil {
|
||||
initFatal(err, "fixing v4.73.2 migrations")
|
||||
}
|
||||
// re-check status after fix
|
||||
status, err = ds.MigrationStatus(cmd.Context())
|
||||
if err != nil {
|
||||
initFatal(err, "retrieving migration status")
|
||||
}
|
||||
}
|
||||
|
||||
switch status.StatusCode {
|
||||
case fleet.NoMigrationsCompleted:
|
||||
@@ -63,6 +79,8 @@ To setup Fleet infrastructure, use one of the available commands.
|
||||
printMissingMigrationsPrompt(status.MissingTable, status.MissingData)
|
||||
bufio.NewScanner(os.Stdin).Scan()
|
||||
}
|
||||
case fleet.NeedsFleetv4732Fix, fleet.UnknownFleetv4732State:
|
||||
printFleetv4732UnknownStateMessage(status.StatusCode)
|
||||
case fleet.UnknownMigrations:
|
||||
printUnknownMigrationsMessage(status.UnknownTable, status.UnknownData)
|
||||
if dev {
|
||||
@@ -113,6 +131,29 @@ func printMissingMigrationsPrompt(tables []int64, data []int64) {
|
||||
tablesAndDataToString(tables, data))
|
||||
}
|
||||
|
||||
func printFleetv4732FixMessage() {
|
||||
fmt.Printf("################################################################################\n" +
|
||||
"# WARNING:\n" +
|
||||
"# Your Fleet database has misnumbered migrations introduced in some released\n" +
|
||||
"# v4.73.2 artifacts. Fleet will automatically perform this fix prior to database\n" +
|
||||
"# migrations. Please back up your data before continuing.\n" +
|
||||
"################################################################################\n")
|
||||
}
|
||||
|
||||
func printFleetv4732UnknownStateMessage(statusCode fleet.MigrationStatusCode) {
|
||||
extra := "your Fleet database is in an unknown state."
|
||||
if statusCode == fleet.NeedsFleetv4732Fix {
|
||||
extra = "the automatic fix did not result in the expected state."
|
||||
}
|
||||
fmt.Print("################################################################################\n" +
|
||||
"# WARNING:\n" +
|
||||
"# Your Fleet database has misnumbered migrations introduced in some released\n" +
|
||||
"# v4.73.2 artifacts. Fleet attempts to fix this problem automatically, however\n" +
|
||||
"# " + extra + "\n" +
|
||||
"# Please contact Fleet support for assistance in resolving this.\n" +
|
||||
"################################################################################\n")
|
||||
}
|
||||
|
||||
func tablesAndDataToString(tables, data []int64) string {
|
||||
switch {
|
||||
case len(tables) > 0 && len(data) == 0:
|
||||
|
||||
@@ -269,6 +269,16 @@ the way that the Fleet server works.
|
||||
if dev {
|
||||
os.Exit(1)
|
||||
}
|
||||
case fleet.NeedsFleetv4732Fix:
|
||||
printFleetv4732FixNeededMessage()
|
||||
if !config.Upgrades.AllowMissingMigrations {
|
||||
os.Exit(1)
|
||||
}
|
||||
case fleet.UnknownFleetv4732State:
|
||||
printFleetv4732UnknownStateMessage(migrationStatus.StatusCode)
|
||||
if !config.Upgrades.AllowMissingMigrations {
|
||||
os.Exit(1)
|
||||
}
|
||||
case fleet.SomeMigrationsCompleted:
|
||||
tables, data := migrationStatus.MissingTable, migrationStatus.MissingData
|
||||
printMissingMigrationsWarning(tables, data)
|
||||
@@ -1592,6 +1602,22 @@ func printMissingMigrationsWarning(tables []int64, data []int64) {
|
||||
tablesAndDataToString(tables, data), os.Args[0])
|
||||
}
|
||||
|
||||
func printFleetv4732FixNeededMessage() {
|
||||
fmt.Printf("################################################################################\n"+
|
||||
"# WARNING:\n"+
|
||||
"# Your Fleet database has misnumbered migrations introduced in some released\n"+
|
||||
"# v4.73.2 artifacts. Fleet will automatically perform this fix prior to database\n"+
|
||||
"# migrations. Please back up your data before continuing.\n"+
|
||||
"#\n"+
|
||||
"# Run `%s prepare db` to perform migrations.\n"+
|
||||
"#\n"+
|
||||
"# To run the server without performing migrations:\n"+
|
||||
"# - Set environment variable FLEET_UPGRADES_ALLOW_MISSING_MIGRATIONS=1, or,\n"+
|
||||
"# - Set config updates.allow_missing_migrations to true, or,\n"+
|
||||
"# - Use command line argument --upgrades_allow_missing_migrations=true\n"+
|
||||
"################################################################################\n", os.Args[0])
|
||||
}
|
||||
|
||||
func initLicense(config configpkg.FleetConfig, devLicense, devExpiredLicense bool) (*fleet.LicenseInfo, error) {
|
||||
if devLicense {
|
||||
// This license key is valid for development only
|
||||
|
||||
@@ -67,6 +67,8 @@ by an exit code of zero.`,
|
||||
switch status.StatusCode {
|
||||
case fleet.AllMigrationsCompleted:
|
||||
// only continue if db is considered up-to-date
|
||||
case fleet.NeedsFleetv4732Fix, fleet.UnknownFleetv4732State:
|
||||
migrationError = errors.New("database has misnumbered migrations from v4.73.2")
|
||||
case fleet.NoMigrationsCompleted:
|
||||
migrationError = errors.New("no migrations completed")
|
||||
case fleet.SomeMigrationsCompleted:
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
"github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils"
|
||||
"github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/data"
|
||||
"github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/tables"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -56,6 +57,95 @@ func TestMigrationStatus(t *testing.T) {
|
||||
assert.Empty(t, status.MissingData)
|
||||
}
|
||||
|
||||
func TestV4732MigrationFix(t *testing.T) {
|
||||
ds := createMySQLDSForMigrationTests(t, t.Name())
|
||||
t.Cleanup(func() {
|
||||
ds.Close()
|
||||
})
|
||||
status, err := ds.MigrationStatus(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, status)
|
||||
assert.EqualValues(t, fleet.NoMigrationsCompleted, status.StatusCode)
|
||||
|
||||
recreate4732BadState(t, ds)
|
||||
|
||||
status, err = ds.MigrationStatus(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, status)
|
||||
assert.EqualValues(t, fleet.NeedsFleetv4732Fix, status.StatusCode)
|
||||
|
||||
err = ds.FixFleetv4732Migrations(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ds.MigrateTables(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
status, err = ds.MigrationStatus(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, status)
|
||||
assert.EqualValues(t, fleet.AllMigrationsCompleted, status.StatusCode)
|
||||
|
||||
// Insert a bad migration again which should trigger the unknown state
|
||||
_, err = ds.writer(context.Background()).Exec(`INSERT INTO `+tables.MigrationClient.TableName+` (version_id, is_applied) VALUES (?, 1)`, fleet4732BadMigrationID1)
|
||||
require.NoError(t, err)
|
||||
status, err = ds.MigrationStatus(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, status)
|
||||
assert.EqualValues(t, fleet.UnknownFleetv4732State, status.StatusCode)
|
||||
}
|
||||
|
||||
// Apply the proper 4.73.2 migrations
|
||||
func recreate4732GoodState(t *testing.T, ds *Datastore) {
|
||||
var version int64
|
||||
var err error
|
||||
|
||||
const maxDataMigration = 20230525175650
|
||||
|
||||
// Migrate up to 4.73.1
|
||||
for version < fleet4731GoodMigrationID {
|
||||
err = tables.MigrationClient.UpByOne(ds.writer(context.Background()).DB, "")
|
||||
require.NoError(t, err)
|
||||
version, err = tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Equal(t, int64(fleet4731GoodMigrationID), version)
|
||||
|
||||
// Apply data migrations which were deprecated before 4.73.2 and should never change so no need for
|
||||
// upbyone, etc. but we'll verify below that we're at expected version
|
||||
err = data.MigrationClient.Up(ds.writer(context.Background()).DB, "")
|
||||
require.NoError(t, err)
|
||||
version, err = data.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, int64(maxDataMigration), version)
|
||||
|
||||
// Apply the migrations from fleet v4.73.2
|
||||
err = tables.MigrationClient.UpByOne(ds.writer(context.Background()).DB, "")
|
||||
require.NoError(t, err)
|
||||
version, err = tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, fleet4732GoodMigrationID2, version)
|
||||
|
||||
err = tables.MigrationClient.UpByOne(ds.writer(context.Background()).DB, "")
|
||||
require.NoError(t, err)
|
||||
version, err = tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, fleet4732GoodMigrationID1, version)
|
||||
}
|
||||
|
||||
// Recreate the bad state that some customers ended up with after running fleet v4.73.2 migrations
|
||||
func recreate4732BadState(t *testing.T, ds *Datastore) {
|
||||
recreate4732GoodState(t, ds)
|
||||
|
||||
_, err := ds.writer(context.Background()).Exec(`UPDATE `+tables.MigrationClient.TableName+` SET version_id = ? WHERE version_id = ?`, fleet4732BadMigrationID1, fleet4732GoodMigrationID1)
|
||||
require.NoError(t, err)
|
||||
_, err = ds.writer(context.Background()).Exec(`UPDATE `+tables.MigrationClient.TableName+` SET version_id = ? WHERE version_id = ?`, fleet4732BadMigrationID2, fleet4732GoodMigrationID2)
|
||||
require.NoError(t, err)
|
||||
|
||||
version, err := tables.MigrationClient.GetDBVersion(ds.writer(context.Background()).DB)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, fleet4732BadMigrationID1, version)
|
||||
}
|
||||
|
||||
func TestMigrations(t *testing.T) {
|
||||
// Create the database (must use raw MySQL client to do this)
|
||||
ds := createMySQLDSForMigrationTests(t, t.Name())
|
||||
|
||||
@@ -40,6 +40,16 @@ import (
|
||||
const (
|
||||
defaultSelectLimit = 1000000
|
||||
mySQLTimestampFormat = "2006-01-02 15:04:05" // %Y/%m/%d %H:%M:%S
|
||||
|
||||
// Migration IDs needed for fixing broken migrations that some customers encountered with fleet v4.73.2
|
||||
// See https://github.com/fleetdm/fleet/issues/33562
|
||||
fleet4732BadMigrationID1 = 20250918154557 // was 20250918154557_AddKernelHostCountsIndexForVulnQueries.go
|
||||
fleet4732GoodMigrationID1 = 20250817154557 // 20250817154557_AddKernelHostCountsIndexForVulnQueries.go
|
||||
|
||||
fleet4732BadMigrationID2 = 20250904115553 // was 20250904115553_OptimizeHostScriptResultsIndex.go
|
||||
fleet4732GoodMigrationID2 = 20250816115553 // 20250816115553_OptimizeHostScriptResultsIndex.go
|
||||
|
||||
fleet4731GoodMigrationID = 20250815130115
|
||||
)
|
||||
|
||||
// Matches all non-word and '-' characters for replacement
|
||||
@@ -412,6 +422,11 @@ func (ds *Datastore) MigrationStatus(ctx context.Context) (*fleet.MigrationStatu
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load migrations: %w", err)
|
||||
}
|
||||
// This will only return a non-nil status if we detect the specific broken state from v4.73.2
|
||||
status := ds.CheckFleetv4732BadMigrations(appliedTable)
|
||||
if status != nil {
|
||||
return status, nil
|
||||
}
|
||||
return compareMigrations(
|
||||
tables.MigrationClient.Migrations,
|
||||
data.MigrationClient.Migrations,
|
||||
@@ -420,6 +435,62 @@ func (ds *Datastore) MigrationStatus(ctx context.Context) (*fleet.MigrationStatu
|
||||
), nil
|
||||
}
|
||||
|
||||
// Checks for misnumbered migrations introduced in some released fleet v4.73.2 versions
|
||||
func (ds *Datastore) CheckFleetv4732BadMigrations(appliedTable []int64) *fleet.MigrationStatus {
|
||||
if len(appliedTable) == 0 {
|
||||
return nil
|
||||
}
|
||||
// If the last 3 migrations are the "bad" 4.73.2 migrations and then the good 4.73.1 migration, in that order,
|
||||
// we are in the known-bad 4.73.2 state and should apply the fix
|
||||
if len(appliedTable) > 2 &&
|
||||
appliedTable[len(appliedTable)-1] == fleet4732BadMigrationID1 &&
|
||||
appliedTable[len(appliedTable)-2] == fleet4732BadMigrationID2 &&
|
||||
appliedTable[len(appliedTable)-3] == fleet4731GoodMigrationID {
|
||||
return &fleet.MigrationStatus{
|
||||
StatusCode: fleet.NeedsFleetv4732Fix,
|
||||
}
|
||||
}
|
||||
for _, v := range appliedTable {
|
||||
if v == fleet4732BadMigrationID1 || v == fleet4732BadMigrationID2 {
|
||||
return &fleet.MigrationStatus{
|
||||
StatusCode: fleet.UnknownFleetv4732State,
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) FixFleetv4732Migrations(ctx context.Context) error {
|
||||
// Update version ID of the bad migrations to the renumbered version IDs. Exactly 1 row should be affected
|
||||
// by each query
|
||||
stmt := `UPDATE ` + tables.MigrationClient.TableName + ` SET version_id = ? WHERE version_id = ?`
|
||||
return ds.withTx(ctx, func(tx sqlx.ExtContext) error {
|
||||
result, err := tx.ExecContext(ctx, stmt, fleet4732GoodMigrationID1, fleet4732BadMigrationID1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return ctxerr.Errorf(ctx, "expected to affect 1 row for migration %d, affected %d", fleet4732BadMigrationID1, affected)
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, stmt, fleet4732GoodMigrationID2, fleet4732BadMigrationID2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err = result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return ctxerr.Errorf(ctx, "expected to affect 1 row for migration %d, affected %d", fleet4732BadMigrationID2, affected)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// It assumes some deployments may have performed migrations out of order.
|
||||
func compareMigrations(knownTable goose.Migrations, knownData goose.Migrations, appliedTable, appliedData []int64) *fleet.MigrationStatus {
|
||||
if len(appliedTable) == 0 && len(appliedData) == 0 {
|
||||
|
||||
@@ -2555,6 +2555,11 @@ const (
|
||||
AllMigrationsCompleted
|
||||
// UnknownMigrations means some unidentified migrations were detected on the database.
|
||||
UnknownMigrations
|
||||
// NeedsFleetv4732Fix means the database needs the special fix migration for fleet v4.73.2
|
||||
NeedsFleetv4732Fix
|
||||
// UnknownFleetv4732State means the database has the broken migrations from fleet v4.73.2 however
|
||||
// it is not in the expected state and needs manual intervention.
|
||||
UnknownFleetv4732State
|
||||
)
|
||||
|
||||
// TODO: we have a similar but different interface in the service package,
|
||||
|
||||
Reference in New Issue
Block a user