diff --git a/changes/46145-android-command-reconcile.md b/changes/46145-android-command-reconcile.md new file mode 100644 index 0000000000..8fa1678015 --- /dev/null +++ b/changes/46145-android-command-reconcile.md @@ -0,0 +1 @@ +- Fixed Android hosts staying stuck on a pending Lock, Wipe, or Clear passcode when Google never delivered the command's result to Fleet. Fleet now checks the command's outcome directly with Google once a day and updates the host, so the command can be re-issued. diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 833c851155..4468cb99e0 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -2567,6 +2567,36 @@ func newAndroidMDMDeviceReconcilerSchedule( return s, nil } +// newAndroidMDMCommandReconcilerSchedule periodically polls AMAPI for the outcome of Android MDM +// commands (Lock, Wipe, Clear passcode) that are still pending because their Pub/Sub COMMAND +// notification never arrived, so hosts don't stay stuck in a pending state. +func newAndroidMDMCommandReconcilerSchedule( + ctx context.Context, + instanceID string, + ds fleet.Datastore, + logger *slog.Logger, + licenseKey string, + newActivityFn fleet.NewActivityFunc, +) (*schedule.Schedule, error) { + const ( + name = string(fleet.CronMDMAndroidCommandReconciler) + // Daily is enough: a dropped notification is rare, and a day of reconciliation lag is invisible + // next to the indefinite wait an affected host has otherwise. + defaultInterval = 24 * time.Hour + ) + + logger = logger.With("cron", name) + s := schedule.New( + ctx, name, instanceID, defaultInterval, ds, ds, + schedule.WithLogger(logger), + schedule.WithJob("reconcile_android_commands", func(ctx context.Context) error { + return android_svc.ReconcileAndroidCommands(ctx, ds, logger, licenseKey, newActivityFn) + }), + ) + + return s, nil +} + func cronEnableAndroidAppReportsOnDefaultPolicy( ctx context.Context, instanceID string, diff --git a/cmd/fleet/cron_registration.go b/cmd/fleet/cron_registration.go index caf41648e2..01ec4ad118 100644 --- a/cmd/fleet/cron_registration.go +++ b/cmd/fleet/cron_registration.go @@ -270,6 +270,18 @@ func registerMDMCrons(ctx context.Context, deps cronSchedulesDeps) { ) }) + // Register Android MDM Command Reconciler schedule (recovers commands whose Pub/Sub notification was lost) + deps.register("failed to register mdm_android_command_reconciler schedule", func() (fleet.CronSchedule, error) { + return newAndroidMDMCommandReconcilerSchedule( + ctx, + deps.instanceID, + deps.ds, + deps.logger, + deps.config.License.Key, + deps.svc.NewActivity, + ) + }) + deps.register("failed to register enable_android_app_reports_on_default_policy cron", func() (fleet.CronSchedule, error) { return cronEnableAndroidAppReportsOnDefaultPolicy(ctx, deps.instanceID, deps.ds, deps.logger, deps.androidSvc) }) diff --git a/server/datastore/mysql/android.go b/server/datastore/mysql/android.go index 92514d11ac..9421fa4dbd 100644 --- a/server/datastore/mysql/android.go +++ b/server/datastore/mysql/android.go @@ -1296,6 +1296,30 @@ func (ds *Datastore) UpdateMDMAndroidCommandStatus(ctx context.Context, commandU return nil } +// ListPendingMDMAndroidCommands returns pending commands created before createdBefore, oldest first, capped at limit +// rows. The reconciler cron uses the age cutoff to skip commands that Pub/Sub is still likely to deliver, and the limit +// to bound how many AMAPI calls a single run makes. +func (ds *Datastore) ListPendingMDMAndroidCommands(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) { + const stmt = ` + SELECT + command_uuid, host_uuid, operation_name, command_type, status, + error_code, error_message, created_at, updated_at + FROM mdm_android_commands + WHERE status = ? AND created_at < ? + -- command_uuid breaks ties so rows with identical created_at keep a stable order between runs, + -- otherwise a full batch could return the same subset every time and starve the rest. + ORDER BY created_at, command_uuid + LIMIT ? + ` + var cmds []*android.MDMAndroidCommand + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &cmds, stmt, + string(android.MDMAndroidCommandStatusPending), createdBefore, limit, + ); err != nil { + return nil, ctxerr.Wrap(ctx, err, "listing pending mdm android commands") + } + return cmds, nil +} + // androidApplicableProfilesQuery computes, per host, the set of applicable profiles based on team and label scoping. Label // semantics must match the in-code Apple/Windows evaluator in server/mdm/reconcile: a dynamic label created after the host's // last label scan (h.label_updated_at < lbl.created_at) has unknown membership and preserves the host's current profile state — diff --git a/server/datastore/mysql/android_test.go b/server/datastore/mysql/android_test.go index 2e9a79a9d7..e31989239a 100644 --- a/server/datastore/mysql/android_test.go +++ b/server/datastore/mysql/android_test.go @@ -54,6 +54,7 @@ func TestAndroid(t *testing.T) { {"GetHostMDMAndroidProfiles", testGetHostMDMAndroidProfiles}, {"GetAndroidPolicyRequestByUUID", testGetAndroidPolicyRequestByUUID}, {"MDMAndroidCommandCRUD", testMDMAndroidCommandCRUD}, + {"ListPendingMDMAndroidCommands", testListPendingMDMAndroidCommands}, {"LockWipeHostViaAndroidMDM", testLockWipeHostViaAndroidMDM}, {"ListHostMDMAndroidProfilesPendingInstallWithVersion", testListHostMDMAndroidProfilesPendingInstallWithVersion}, {"BulkDeleteMDMAndroidHostProfiles", testBulkDeleteMDMAndroidHostProfiles}, @@ -2806,6 +2807,82 @@ func testMDMAndroidCommandCRUD(t *testing.T, ds *Datastore) { }) } +func testListPendingMDMAndroidCommands(t *testing.T, ds *Datastore) { + ctx := t.Context() + + // insertCommand creates a command row and backdates created_at so the age cutoff can be exercised + // without waiting. Returns the command_uuid. + insertCommand := func(t *testing.T, status string, age time.Duration) string { + cmdUUID := uuid.NewString() + require.NoError(t, ds.NewMDMAndroidCommand(ctx, &android.MDMAndroidCommand{ + CommandUUID: cmdUUID, + HostUUID: "host-" + cmdUUID, + OperationName: "enterprises/E1/devices/D1/operations/" + cmdUUID, + CommandType: string(android.MDMAndroidCommandTypeLock), + Status: status, + })) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, + `UPDATE mdm_android_commands SET created_at = NOW(6) - INTERVAL ? SECOND WHERE command_uuid = ?`, + int(age.Seconds()), cmdUUID) + return err + }) + return cmdUUID + } + + uuidsOf := func(cmds []*android.MDMAndroidCommand) []string { + got := make([]string, 0, len(cmds)) + for _, cmd := range cmds { + got = append(got, cmd.CommandUUID) + } + return got + } + + oldest := insertCommand(t, string(android.MDMAndroidCommandStatusPending), 72*time.Hour) + middle := insertCommand(t, string(android.MDMAndroidCommandStatusPending), 48*time.Hour) + newest := insertCommand(t, string(android.MDMAndroidCommandStatusPending), 25*time.Hour) + tooRecent := insertCommand(t, string(android.MDMAndroidCommandStatusPending), time.Hour) + acknowledged := insertCommand(t, string(android.MDMAndroidCommandStatusAcknowledged), 48*time.Hour) + errored := insertCommand(t, string(android.MDMAndroidCommandStatusError), 48*time.Hour) + + t.Run("returns only pending rows older than the cutoff, oldest first", func(t *testing.T) { + cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-24*time.Hour), 100) + require.NoError(t, err) + require.Equal(t, []string{oldest, middle, newest}, uuidsOf(cmds)) + require.NotContains(t, uuidsOf(cmds), tooRecent) + require.NotContains(t, uuidsOf(cmds), acknowledged) + require.NotContains(t, uuidsOf(cmds), errored) + }) + + t.Run("limit caps the batch to the oldest rows", func(t *testing.T) { + cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-24*time.Hour), 2) + require.NoError(t, err) + require.Equal(t, []string{oldest, middle}, uuidsOf(cmds)) + }) + + t.Run("returns all fields needed to reconcile", func(t *testing.T) { + cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-24*time.Hour), 1) + require.NoError(t, err) + require.Len(t, cmds, 1) + assert.Equal(t, oldest, cmds[0].CommandUUID) + assert.Equal(t, "host-"+oldest, cmds[0].HostUUID) + assert.Equal(t, "enterprises/E1/devices/D1/operations/"+oldest, cmds[0].OperationName) + assert.Equal(t, string(android.MDMAndroidCommandTypeLock), cmds[0].CommandType) + assert.Equal(t, string(android.MDMAndroidCommandStatusPending), cmds[0].Status) + // created_at drives the not-found grace period in the reconciler, so it has to come back + // populated. Only assert it predates the cutoff -- an exact age would be at the mercy of clock + // skew between the app and the database. + assert.False(t, cmds[0].CreatedAt.IsZero()) + assert.True(t, cmds[0].CreatedAt.Before(time.Now().Add(-24*time.Hour))) + }) + + t.Run("no matching rows returns an empty slice", func(t *testing.T) { + cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-365*24*time.Hour), 100) + require.NoError(t, err) + require.Empty(t, cmds) + }) +} + // newBareAndroidHostForTest inserts a minimal android-platform host row. Use this for tests // that exercise the host_mdm_actions layer and don't need a populated android_devices row // (use createAndroidHost + ds.NewAndroidHost for that). diff --git a/server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex.go b/server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex.go new file mode 100644 index 0000000000..cb33a65c96 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex.go @@ -0,0 +1,39 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20260807151355, Down_20260807151355) +} + +// Up_20260805182836 adds an index supporting the Android command reconciler's batch query +// (ListPendingMDMAndroidCommands), which reads +// +// WHERE status = 'pending' AND created_at < ? ORDER BY created_at, command_uuid LIMIT ? +// +// mdm_android_commands only had the primary key, the operation_name unique key, and a host_uuid +// key, none of which lead with status, so that query was a full table scan. The table grows with +// every Lock/Wipe/Clear-passcode ever issued while the pending rows the cron wants are a small +// slice of it, so the scan gets steadily more expensive as the table grows. +// +// status (equality) leads, created_at (range) follows -- the order MySQL needs to use both +// predicates from one index. InnoDB appends the primary key (command_uuid) to every secondary +// index, so this also satisfies the ORDER BY and the LIMIT can stop early instead of sorting. +// +// ALGORITHM=INPLACE, LOCK=NONE so the index builds without blocking command inserts. +func Up_20260807151355(tx *sql.Tx) error { + stmt := `ALTER TABLE mdm_android_commands + ADD INDEX idx_mdm_android_commands_status_created_at (status, created_at), + ALGORITHM=INPLACE, LOCK=NONE` + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("failed to add idx_mdm_android_commands_status_created_at: %w", err) + } + return nil +} + +func Down_20260807151355(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex_test.go b/server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex_test.go new file mode 100644 index 0000000000..1d6ec08115 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex_test.go @@ -0,0 +1,44 @@ +package tables + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20260807151355(t *testing.T) { + db := applyUpToPrev(t) + + // Seed a command so the migration is exercised against a non-empty table. + execNoErr(t, db, ` + INSERT INTO mdm_android_commands (command_uuid, host_uuid, operation_name, command_type, status) + VALUES ('cmd-uuid-1', 'host-uuid-1', 'enterprises/e1/devices/d1/operations/op1', 'LOCK', 'pending') + `) + + applyNext(t, db) + + rows, err := db.Query( + `SELECT column_name FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = 'mdm_android_commands' + AND index_name = 'idx_mdm_android_commands_status_created_at' + ORDER BY seq_in_index`, + ) + require.NoError(t, err) + defer rows.Close() + + var columns []string + for rows.Next() { + var columnName string + require.NoError(t, rows.Scan(&columnName)) + columns = append(columns, columnName) + } + require.NoError(t, rows.Err()) + require.Equal(t, []string{"status", "created_at"}, columns) + + // The seeded row survives the ALTER and is still readable through the new index's predicate. + var count int + require.NoError(t, db.QueryRow( + `SELECT COUNT(*) FROM mdm_android_commands WHERE status = 'pending' AND created_at < NOW(6)`, + ).Scan(&count)) + require.Equal(t, 1, count) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a705ba47c9..a5de05f97d 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1844,8 +1844,9 @@ CREATE TABLE `mdm_android_commands` ( `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`command_uuid`), UNIQUE KEY `idx_mdm_android_commands_operation_name` (`operation_name`), - KEY `idx_mdm_android_commands_host_uuid` (`host_uuid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + KEY `idx_mdm_android_commands_host_uuid` (`host_uuid`), + KEY `idx_mdm_android_commands_status_created_at` (`status`,`created_at`) +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; @@ -2318,9 +2319,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=587 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=588 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260317120000,1,'2020-01-01 01:01:01'),(496,20260318184559,1,'2020-01-01 01:01:01'),(497,20260319120000,1,'2020-01-01 01:01:01'),(498,20260323144117,1,'2020-01-01 01:01:01'),(499,20260324161944,1,'2020-01-01 01:01:01'),(500,20260324223334,1,'2020-01-01 01:01:01'),(501,20260326131501,1,'2020-01-01 01:01:01'),(502,20260326210603,1,'2020-01-01 01:01:01'),(503,20260331000000,1,'2020-01-01 01:01:01'),(504,20260401153000,1,'2020-01-01 01:01:01'),(505,20260401153001,1,'2020-01-01 01:01:01'),(506,20260401153503,1,'2020-01-01 01:01:01'),(507,20260403120000,1,'2020-01-01 01:01:01'),(508,20260409153713,1,'2020-01-01 01:01:01'),(509,20260409153714,1,'2020-01-01 01:01:01'),(510,20260409153715,1,'2020-01-01 01:01:01'),(511,20260409153716,1,'2020-01-01 01:01:01'),(512,20260409153717,1,'2020-01-01 01:01:01'),(513,20260409183610,1,'2020-01-01 01:01:01'),(514,20260410173222,1,'2020-01-01 01:01:01'),(515,20260422181702,1,'2020-01-01 01:01:01'),(516,20260423161823,1,'2020-01-01 01:01:01'),(517,20260423161824,1,'2020-01-01 01:01:01'),(518,20260518194422,1,'2020-01-01 01:01:01'),(519,20260522195224,1,'2020-01-01 01:01:01'),(520,20260522195225,1,'2020-01-01 01:01:01'),(521,20260522195226,1,'2020-01-01 01:01:01'),(522,20260522195227,1,'2020-01-01 01:01:01'),(523,20260522195229,1,'2020-01-01 01:01:01'),(524,20260522195230,1,'2020-01-01 01:01:01'),(525,20260522195231,1,'2020-01-01 01:01:01'),(526,20260522195232,1,'2020-01-01 01:01:01'),(527,20260522195233,1,'2020-01-01 01:01:01'),(528,20260522195234,1,'2020-01-01 01:01:01'),(529,20260522195235,1,'2020-01-01 01:01:01'),(530,20260527215817,1,'2020-01-01 01:01:01'),(531,20260527215818,1,'2020-01-01 01:01:01'),(532,20260528201143,1,'2020-01-01 01:01:01'),(533,20260528201150,1,'2020-01-01 01:01:01'),(534,20260528211626,1,'2020-01-01 01:01:01'),(535,20260528213326,1,'2020-01-01 01:01:01'),(536,20260529091823,1,'2020-01-01 01:01:01'),(537,20260529120000,1,'2020-01-01 01:01:01'),(538,20260601200727,1,'2020-01-01 01:01:01'),(539,20260603101320,1,'2020-01-01 01:01:01'),(540,20260603120000,1,'2020-01-01 01:01:01'),(541,20260604221206,1,'2020-01-01 01:01:01'),(542,20260605195941,1,'2020-01-01 01:01:01'),(543,20260606051849,1,'2020-01-01 01:01:01'),(544,20260608160653,1,'2020-01-01 01:01:01'),(545,20260608202705,1,'2020-01-01 01:01:01'),(546,20260608210432,1,'2020-01-01 01:01:01'),(547,20260610172952,1,'2020-01-01 01:01:01'),(548,20260624210253,1,'2020-01-01 01:01:01'),(549,20260624210311,1,'2020-01-01 01:01:01'),(550,20260626120000,1,'2020-01-01 01:01:01'),(551,20260702013055,1,'2020-01-01 01:01:01'),(552,20260702013056,1,'2020-01-01 01:01:01'),(553,20260702013057,1,'2020-01-01 01:01:01'),(554,20260702013058,1,'2020-01-01 01:01:01'),(555,20260702013059,1,'2020-01-01 01:01:01'),(556,20260702013100,1,'2020-01-01 01:01:01'),(557,20260702013101,1,'2020-01-01 01:01:01'),(558,20260702013102,1,'2020-01-01 01:01:01'),(559,20260702164518,1,'2020-01-01 01:01:01'),(560,20260717152653,1,'2020-01-01 01:01:01'),(561,20260723181401,1,'2020-01-01 01:01:01'),(562,20260723181402,1,'2020-01-01 01:01:01'),(563,20260723181403,1,'2020-01-01 01:01:01'),(564,20260723181404,1,'2020-01-01 01:01:01'),(565,20260723181405,1,'2020-01-01 01:01:01'),(566,20260723181406,1,'2020-01-01 01:01:01'),(567,20260723181407,1,'2020-01-01 01:01:01'),(568,20260723181408,1,'2020-01-01 01:01:01'),(569,20260723181409,1,'2020-01-01 01:01:01'),(570,20260723181410,1,'2020-01-01 01:01:01'),(571,20260723181411,1,'2020-01-01 01:01:01'),(572,20260724134801,1,'2020-01-01 01:01:01'),(573,20260727083533,1,'2020-01-01 01:01:01'),(574,20260727084359,1,'2020-01-01 01:01:01'),(575,20260729110229,1,'2020-01-01 01:01:01'),(576,20260729115013,1,'2020-01-01 01:01:01'),(577,20260731100711,1,'2020-01-01 01:01:01'),(578,20260731213352,1,'2020-01-01 01:01:01'),(579,20260803135530,1,'2020-01-01 01:01:01'),(580,20260803182251,1,'2020-01-01 01:01:01'),(581,20260805161502,1,'2020-01-01 01:01:01'),(582,20260806154139,1,'2020-01-01 01:01:01'),(583,20260806154150,1,'2020-01-01 01:01:01'),(584,20260806210232,1,'2020-01-01 01:01:01'),(585,20260807120050,1,'2020-01-01 01:01:01'),(586,20260807140831,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260317120000,1,'2020-01-01 01:01:01'),(496,20260318184559,1,'2020-01-01 01:01:01'),(497,20260319120000,1,'2020-01-01 01:01:01'),(498,20260323144117,1,'2020-01-01 01:01:01'),(499,20260324161944,1,'2020-01-01 01:01:01'),(500,20260324223334,1,'2020-01-01 01:01:01'),(501,20260326131501,1,'2020-01-01 01:01:01'),(502,20260326210603,1,'2020-01-01 01:01:01'),(503,20260331000000,1,'2020-01-01 01:01:01'),(504,20260401153000,1,'2020-01-01 01:01:01'),(505,20260401153001,1,'2020-01-01 01:01:01'),(506,20260401153503,1,'2020-01-01 01:01:01'),(507,20260403120000,1,'2020-01-01 01:01:01'),(508,20260409153713,1,'2020-01-01 01:01:01'),(509,20260409153714,1,'2020-01-01 01:01:01'),(510,20260409153715,1,'2020-01-01 01:01:01'),(511,20260409153716,1,'2020-01-01 01:01:01'),(512,20260409153717,1,'2020-01-01 01:01:01'),(513,20260409183610,1,'2020-01-01 01:01:01'),(514,20260410173222,1,'2020-01-01 01:01:01'),(515,20260422181702,1,'2020-01-01 01:01:01'),(516,20260423161823,1,'2020-01-01 01:01:01'),(517,20260423161824,1,'2020-01-01 01:01:01'),(518,20260518194422,1,'2020-01-01 01:01:01'),(519,20260522195224,1,'2020-01-01 01:01:01'),(520,20260522195225,1,'2020-01-01 01:01:01'),(521,20260522195226,1,'2020-01-01 01:01:01'),(522,20260522195227,1,'2020-01-01 01:01:01'),(523,20260522195229,1,'2020-01-01 01:01:01'),(524,20260522195230,1,'2020-01-01 01:01:01'),(525,20260522195231,1,'2020-01-01 01:01:01'),(526,20260522195232,1,'2020-01-01 01:01:01'),(527,20260522195233,1,'2020-01-01 01:01:01'),(528,20260522195234,1,'2020-01-01 01:01:01'),(529,20260522195235,1,'2020-01-01 01:01:01'),(530,20260527215817,1,'2020-01-01 01:01:01'),(531,20260527215818,1,'2020-01-01 01:01:01'),(532,20260528201143,1,'2020-01-01 01:01:01'),(533,20260528201150,1,'2020-01-01 01:01:01'),(534,20260528211626,1,'2020-01-01 01:01:01'),(535,20260528213326,1,'2020-01-01 01:01:01'),(536,20260529091823,1,'2020-01-01 01:01:01'),(537,20260529120000,1,'2020-01-01 01:01:01'),(538,20260601200727,1,'2020-01-01 01:01:01'),(539,20260603101320,1,'2020-01-01 01:01:01'),(540,20260603120000,1,'2020-01-01 01:01:01'),(541,20260604221206,1,'2020-01-01 01:01:01'),(542,20260605195941,1,'2020-01-01 01:01:01'),(543,20260606051849,1,'2020-01-01 01:01:01'),(544,20260608160653,1,'2020-01-01 01:01:01'),(545,20260608202705,1,'2020-01-01 01:01:01'),(546,20260608210432,1,'2020-01-01 01:01:01'),(547,20260610172952,1,'2020-01-01 01:01:01'),(548,20260624210253,1,'2020-01-01 01:01:01'),(549,20260624210311,1,'2020-01-01 01:01:01'),(550,20260626120000,1,'2020-01-01 01:01:01'),(551,20260702013055,1,'2020-01-01 01:01:01'),(552,20260702013056,1,'2020-01-01 01:01:01'),(553,20260702013057,1,'2020-01-01 01:01:01'),(554,20260702013058,1,'2020-01-01 01:01:01'),(555,20260702013059,1,'2020-01-01 01:01:01'),(556,20260702013100,1,'2020-01-01 01:01:01'),(557,20260702013101,1,'2020-01-01 01:01:01'),(558,20260702013102,1,'2020-01-01 01:01:01'),(559,20260702164518,1,'2020-01-01 01:01:01'),(560,20260717152653,1,'2020-01-01 01:01:01'),(561,20260723181401,1,'2020-01-01 01:01:01'),(562,20260723181402,1,'2020-01-01 01:01:01'),(563,20260723181403,1,'2020-01-01 01:01:01'),(564,20260723181404,1,'2020-01-01 01:01:01'),(565,20260723181405,1,'2020-01-01 01:01:01'),(566,20260723181406,1,'2020-01-01 01:01:01'),(567,20260723181407,1,'2020-01-01 01:01:01'),(568,20260723181408,1,'2020-01-01 01:01:01'),(569,20260723181409,1,'2020-01-01 01:01:01'),(570,20260723181410,1,'2020-01-01 01:01:01'),(571,20260723181411,1,'2020-01-01 01:01:01'),(572,20260724134801,1,'2020-01-01 01:01:01'),(573,20260727083533,1,'2020-01-01 01:01:01'),(574,20260727084359,1,'2020-01-01 01:01:01'),(575,20260729110229,1,'2020-01-01 01:01:01'),(576,20260729115013,1,'2020-01-01 01:01:01'),(577,20260731100711,1,'2020-01-01 01:01:01'),(578,20260731213352,1,'2020-01-01 01:01:01'),(579,20260803135530,1,'2020-01-01 01:01:01'),(580,20260803182251,1,'2020-01-01 01:01:01'),(581,20260805161502,1,'2020-01-01 01:01:01'),(582,20260806154139,1,'2020-01-01 01:01:01'),(583,20260806154150,1,'2020-01-01 01:01:01'),(584,20260806210232,1,'2020-01-01 01:01:01'),(585,20260807120050,1,'2020-01-01 01:01:01'),(586,20260807140831,1,'2020-01-01 01:01:01'),(587,20260807151355,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/fleet/cron_schedules.go b/server/fleet/cron_schedules.go index 1b68f2680f..68fb3d8242 100644 --- a/server/fleet/cron_schedules.go +++ b/server/fleet/cron_schedules.go @@ -73,6 +73,9 @@ const ( CronChartDataCollection CronScheduleName = "chart_data_collection" // Used by chart bounded context CronCleanupExpiredADUEChallenges CronScheduleName = "cleanup_expired_adue_challenges" CronAppleMDMOSUpdatesSchedule CronScheduleName = "apple_mdm_os_updates" + // CronMDMAndroidCommandReconciler polls AMAPI for the outcome of Android MDM commands whose Pub/Sub + // COMMAND notification never arrived, so they don't stay pending forever. Runs every 24h. + CronMDMAndroidCommandReconciler CronScheduleName = "mdm_android_command_reconciler" ) type CronSchedulesService interface { diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 57fb218410..e7f181f591 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -3964,6 +3964,11 @@ type AndroidDatastore interface { // a previously-issued command. Called by the Pub/Sub COMMAND handler on ack/error. UpdateMDMAndroidCommandStatus(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error + // ListPendingMDMAndroidCommands returns commands still in the pending status that were created + // before createdBefore, oldest first, capped at limit rows. Used by the command reconciler cron to + // find commands whose Pub/Sub COMMAND notification never arrived. + ListPendingMDMAndroidCommands(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) + // LockHostViaAndroidMDM inserts the LOCK row into mdm_android_commands and writes the lock_ref on host_mdm_actions in a // single transaction, mirroring WipeHostViaWindowsMDM. The caller must populate cmd.CommandUUID and cmd.OperationName // (returned by EnterprisesDevicesIssueCommand) before invoking. diff --git a/server/mdm/android/mock/client.go b/server/mdm/android/mock/client.go index 8130387731..2955f084d6 100644 --- a/server/mdm/android/mock/client.go +++ b/server/mdm/android/mock/client.go @@ -27,6 +27,8 @@ type EnterprisesDevicesDeleteFunc func(ctx context.Context, deviceName string) e type EnterprisesDevicesIssueCommandFunc func(ctx context.Context, deviceName string, command *androidmanagement.Command) (*androidmanagement.Operation, error) +type EnterprisesDevicesOperationsGetFunc func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) + type EnterprisesDevicesListPartialFunc func(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) type EnterprisesEnrollmentTokensCreateFunc func(ctx context.Context, enterpriseName string, token *androidmanagement.EnrollmentToken) (*androidmanagement.EnrollmentToken, error) @@ -67,6 +69,9 @@ type Client struct { EnterprisesDevicesIssueCommandFunc EnterprisesDevicesIssueCommandFunc EnterprisesDevicesIssueCommandFuncInvoked bool + EnterprisesDevicesOperationsGetFunc EnterprisesDevicesOperationsGetFunc + EnterprisesDevicesOperationsGetFuncInvoked bool + EnterprisesDevicesListPartialFunc EnterprisesDevicesListPartialFunc EnterprisesDevicesListPartialFuncInvoked bool @@ -146,6 +151,13 @@ func (p *Client) EnterprisesDevicesIssueCommand(ctx context.Context, deviceName return p.EnterprisesDevicesIssueCommandFunc(ctx, deviceName, command) } +func (p *Client) EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + p.mu.Lock() + p.EnterprisesDevicesOperationsGetFuncInvoked = true + p.mu.Unlock() + return p.EnterprisesDevicesOperationsGetFunc(ctx, operationName) +} + func (p *Client) EnterprisesDevicesListPartial(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) { p.mu.Lock() p.EnterprisesDevicesListPartialFuncInvoked = true diff --git a/server/mdm/android/service/androidmgmt/client.go b/server/mdm/android/service/androidmgmt/client.go index 5c814adf55..3e5dfaade8 100644 --- a/server/mdm/android/service/androidmgmt/client.go +++ b/server/mdm/android/service/androidmgmt/client.go @@ -46,6 +46,13 @@ type Client interface { // https://developers.google.com/android/management/reference/rest/v1/enterprises.devices/issueCommand EnterprisesDevicesIssueCommand(ctx context.Context, deviceName string, command *androidmanagement.Command) (*androidmanagement.Operation, error) + // EnterprisesDevicesOperationsGet fetches the current state of an Operation returned by + // EnterprisesDevicesIssueCommand. It is the authoritative source for a command's outcome and lets + // Fleet reconcile commands whose Pub/Sub COMMAND notification never arrived. operationName is the + // full AMAPI resource name (enterprises/X/devices/Y/operations/Z). See: + // https://developers.google.com/android/management/reference/rest/v1/enterprises.devices.operations/get + EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) + // EnterprisesDevicesListPartial lists devices for the given enterprise with partial fields. // Page size of 100 devices // See: https://developers.google.com/android/management/reference/rest/v1/enterprises.devices/list @@ -116,9 +123,36 @@ func IsNotModifiedError(err error) bool { // IsBadRequestError reports whether the AMAPI error indicates that the // request was invalid due to a client error. func IsBadRequestError(err error) bool { - var ae *googleapi.Error - if errors.As(err, &ae) { + if ae, ok := errors.AsType[*googleapi.Error](err); ok { return ae.Code == http.StatusBadRequest } return false } + +// IsNotFoundError reports whether the AMAPI error indicates that the requested +// resource does not exist. +func IsNotFoundError(err error) bool { + if ae, ok := errors.AsType[*googleapi.Error](err); ok { + return ae.Code == http.StatusNotFound + } + return false +} + +// IsAuthenticationError reports whether the AMAPI error indicates that the +// request was rejected over credentials or access, rather than anything about +// the resource that was requested. +func IsAuthenticationError(err error) bool { + if ae, ok := errors.AsType[*googleapi.Error](err); ok { + return ae.Code == http.StatusUnauthorized || ae.Code == http.StatusForbidden + } + return false +} + +// IsTooManyRequestsError reports whether the AMAPI error indicates that we +// exceeded the project's request quota. +func IsTooManyRequestsError(err error) bool { + if ae, ok := errors.AsType[*googleapi.Error](err); ok { + return ae.Code == http.StatusTooManyRequests + } + return false +} diff --git a/server/mdm/android/service/androidmgmt/google_client.go b/server/mdm/android/service/androidmgmt/google_client.go index f379ef63df..b5b5d026ea 100644 --- a/server/mdm/android/service/androidmgmt/google_client.go +++ b/server/mdm/android/service/androidmgmt/google_client.go @@ -246,6 +246,15 @@ func (g *GoogleClient) EnterprisesDevicesIssueCommand(ctx context.Context, devic return op, nil } +func (g *GoogleClient) EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + op, err := g.mgmt.Enterprises.Devices.Operations.Get(operationName).Context(ctx).Do() + if err != nil { + // Wrapped with %w so callers can classify the googleapi.Error (not found, quota exceeded). + return nil, fmt.Errorf("getting operation %s: %w", operationName, err) + } + return op, nil +} + func (g *GoogleClient) EnterprisesDevicesListPartial(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) { ret, err := g.mgmt.Enterprises.Devices.List(enterpriseName).Context(ctx).PageToken(pageToken).PageSize(100).Fields("nextPageToken", "devices/name").Do() if err != nil { diff --git a/server/mdm/android/service/androidmgmt/proxy_client.go b/server/mdm/android/service/androidmgmt/proxy_client.go index c5a500f624..0e93ed5dda 100644 --- a/server/mdm/android/service/androidmgmt/proxy_client.go +++ b/server/mdm/android/service/androidmgmt/proxy_client.go @@ -234,6 +234,17 @@ func (p *ProxyClient) EnterprisesDevicesIssueCommand(ctx context.Context, device return op, nil } +func (p *ProxyClient) EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + call := p.mgmt.Enterprises.Devices.Operations.Get(operationName).Context(ctx) + call.Header().Set("Authorization", "Bearer "+p.fleetServerSecret) + op, err := call.Do() + if err != nil { + // Wrapped with %w so callers can classify the googleapi.Error (not found, quota exceeded). + return nil, fmt.Errorf("getting operation %s: %w", operationName, err) + } + return op, nil +} + func (p *ProxyClient) EnterprisesDevicesListPartial(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) { call := p.mgmt.Enterprises.Devices.List(enterpriseName).Context(ctx).PageToken(pageToken).PageSize(100).Fields("nextPageToken", "devices/name") call.Header().Set("Authorization", "Bearer "+p.fleetServerSecret) diff --git a/server/mdm/android/service/pubsub.go b/server/mdm/android/service/pubsub.go index 68290a119a..a2718d290e 100644 --- a/server/mdm/android/service/pubsub.go +++ b/server/mdm/android/service/pubsub.go @@ -188,11 +188,12 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa } // Already-terminal rows. AMAPI may redeliver a notification at-least-once. - // For WIPE+acknowledged specifically, still re-run handleAndroidWipeAckUnenroll so transient DB + // For WIPE+acknowledged specifically, still re-run androidWipeAckUnenroll so transient DB // failures on the original delivery recover on this retry. if cmd.Status != string(android.MDMAndroidCommandStatusPending) { if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && cmd.Status == string(android.MDMAndroidCommandStatusAcknowledged) { - if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd, messageID, publishTime); err != nil { + if err := androidWipeAckUnenroll(ctx, svc.fleetDS, svc.newActivity, cmd, + svc.pubSubDedupRecorder(ctx, messageID, publishTime)); err != nil { return err } } @@ -201,28 +202,10 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa return nil } - newStatus := string(android.MDMAndroidCommandStatusAcknowledged) - var errCode, errMsg *string - if op.Error != nil { - newStatus = string(android.MDMAndroidCommandStatusError) - code := googleStatusCode(op.Error.Code) - message := op.Error.Message - errCode = &code - errMsg = &message - } - - if err := svc.fleetDS.UpdateMDMAndroidCommandStatus(ctx, cmd.CommandUUID, newStatus, errCode, errMsg); err != nil { - return ctxerr.Wrap(ctx, err, "update android command status from pub/sub") - } - - // WIPE ack is the authoritative signal that the device has been wiped (BYO: work profile removed; COBO: full factory reset). Flip - // host_mdm.enrolled to 0 here rather than waiting on a separate STATUS_REPORT / ENROLLMENT with state=DELETED, which AMAPI does - // not reliably send for a factory-reset COBO device (the agent is gone, nothing left to phone home). For BYO the DELETED - // notification typically arrives and is now a no-op because we already flipped state. - if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && newStatus == string(android.MDMAndroidCommandStatusAcknowledged) { - if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd, messageID, publishTime); err != nil { - return err - } + newStatus, errCode, errMsg := androidOperationTerminalState(&op) + if err := setAndroidCommandTerminalState(ctx, svc.fleetDS, svc.newActivity, cmd, newStatus, errCode, errMsg, + svc.pubSubDedupRecorder(ctx, messageID, publishTime)); err != nil { + return err } svc.logger.InfoContext(ctx, "android pub/sub COMMAND processed", @@ -234,11 +217,57 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa return nil } -// handleAndroidWipeAckUnenroll runs after a successful WIPE ack: flips host_mdm.enrolled, clears host_mdm_actions for BYO (so the +// androidOperationTerminalState maps a done AMAPI Operation to the terminal status to write on the +// mdm_android_commands row, plus the error code/message to record. A nil Operation.Error means the +// device executed the command successfully; a populated one means AMAPI or the device rejected it. +func androidOperationTerminalState(op *androidmanagement.Operation) (status string, errCode, errMsg *string) { + if op.Error == nil { + return string(android.MDMAndroidCommandStatusAcknowledged), nil, nil + } + code := googleStatusCode(op.Error.Code) + message := op.Error.Message + return string(android.MDMAndroidCommandStatusError), &code, &message +} + +// setAndroidCommandTerminalState moves a pending mdm_android_commands row to a terminal status and runs +// the post-WIPE-ack side effects. Shared by the Pub/Sub COMMAND handler and the command reconciler cron +// so the two paths cannot drift. onUnenrolled is passed through to androidWipeAckUnenroll; see its doc +// comment. +func setAndroidCommandTerminalState(ctx context.Context, ds fleet.Datastore, newActivityFn fleet.NewActivityFunc, + cmd *android.MDMAndroidCommand, status string, errCode, errMsg *string, onUnenrolled func(hostID uint), +) error { + // WIPE ack is the authoritative signal that the device has been wiped (BYO: work profile removed; COBO: full factory reset). Flip + // host_mdm.enrolled to 0 here rather than waiting on a separate STATUS_REPORT / ENROLLMENT with state=DELETED, which AMAPI does + // not reliably send for a factory-reset COBO device (the agent is gone, nothing left to phone home). For BYO the DELETED + // notification typically arrives and is now a no-op because we already flipped state. + // + // This runs before the status write, not after: androidWipeAckUnenroll is idempotent, so a failure + // here leaving the row pending is recoverable (Pub/Sub redelivers, and the reconciler cron only + // selects pending rows). Writing the status first would strand a row as acknowledged with its side + // effects never applied, which the reconciler could never pick up again. + if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && status == string(android.MDMAndroidCommandStatusAcknowledged) { + if err := androidWipeAckUnenroll(ctx, ds, newActivityFn, cmd, onUnenrolled); err != nil { + return err + } + } + + if err := ds.UpdateMDMAndroidCommandStatus(ctx, cmd.CommandUUID, status, errCode, errMsg); err != nil { + return ctxerr.Wrap(ctx, err, "update android command status") + } + return nil +} + +// androidWipeAckUnenroll runs after a successful WIPE ack: flips host_mdm.enrolled, clears host_mdm_actions for BYO (so the // "Wiped" badge does not stick on a host whose only the work profile was removed), and emits mdm_unenrolled if state actually // changed. Returns errors so Pub/Sub retries on transient DB failures. -func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *android.MDMAndroidCommand, messageID, publishTime string) error { - ah, err := svc.ds.AndroidHostLiteByHostUUID(ctx, cmd.HostUUID) +// +// onUnenrolled, when non-nil, runs only if this call actually flipped the host to unenrolled. The Pub/Sub +// path uses it to record dedup state for the notification that drove the wipe; the reconciler cron passes +// nil because it has no Pub/Sub message to dedup against. +func androidWipeAckUnenroll(ctx context.Context, ds fleet.Datastore, newActivityFn fleet.NewActivityFunc, + cmd *android.MDMAndroidCommand, onUnenrolled func(hostID uint), +) error { + ah, err := ds.AndroidHostLiteByHostUUID(ctx, cmd.HostUUID) if err != nil { return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: lookup host by uuid") } @@ -248,11 +277,11 @@ func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *andro // BYO needs host_mdm_actions cleared so IsWiped() returns false post-ack -- only the work // profile was removed, not the device. COBO leaves wipe_ref intact so the "Wiped" badge sticks. - if err := clearAndroidBYOWipeRef(ctx, svc.fleetDS, ah.Host.ID); err != nil { + if err := clearAndroidBYOWipeRef(ctx, ds, ah.Host.ID); err != nil { return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: clear byo wipe-ref") } - didUnenroll, err := svc.fleetDS.SetAndroidHostUnenrolled(ctx, ah.Host.ID) + didUnenroll, err := ds.SetAndroidHostUnenrolled(ctx, ah.Host.ID) if err != nil { return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: set host_mdm unenrolled") } @@ -277,13 +306,15 @@ func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *andro // it here means a STATUS_REPORT published before the wipe but delivered afterwards (Pub/Sub // is unordered) is dropped as stale by handlePubSubStatusReport, so it cannot re-enroll a // device that was just wiped. Only done when this delivery actually flipped state. - svc.recordPubSubProcessed(ctx, ah.Host.ID, messageID, pubSubEventTime("", publishTime)) + if onUnenrolled != nil { + onUnenrolled(ah.Host.ID) + } displayName := "" - if hosts, herr := svc.fleetDS.ListHostsLiteByIDs(ctx, []uint{ah.Host.ID}); herr == nil && len(hosts) == 1 && hosts[0] != nil { + if hosts, herr := ds.ListHostsLiteByIDs(ctx, []uint{ah.Host.ID}); herr == nil && len(hosts) == 1 && hosts[0] != nil { displayName = hosts[0].DisplayName() } - if err := svc.newActivity(ctx, nil, fleet.ActivityTypeMDMUnenrolled{ + if err := newActivityFn(ctx, nil, fleet.ActivityTypeMDMUnenrolled{ HostID: ah.Host.ID, HostDisplayName: displayName, InstalledFromDEP: false, @@ -416,6 +447,15 @@ func (svc *Service) recordPubSubProcessed(ctx context.Context, hostID uint, mess } } +// pubSubDedupRecorder builds the onUnenrolled callback for androidWipeAckUnenroll from a COMMAND +// notification's envelope. The COMMAND payload carries no device timestamp, so publishTime is the +// only available event time. +func (svc *Service) pubSubDedupRecorder(ctx context.Context, messageID, publishTime string) func(hostID uint) { + return func(hostID uint) { + svc.recordPubSubProcessed(ctx, hostID, messageID, pubSubEventTime("", publishTime)) + } +} + func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string, rawData []byte, messageID, publishTime string) error { err := svc.authenticatePubSub(ctx, token) if err != nil { diff --git a/server/mdm/android/service/reconcile_commands.go b/server/mdm/android/service/reconcile_commands.go new file mode 100644 index 0000000000..c6b34a64bd --- /dev/null +++ b/server/mdm/android/service/reconcile_commands.go @@ -0,0 +1,175 @@ +package service + +import ( + "context" + "log/slog" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mdm/android" + "github.com/fleetdm/fleet/v4/server/mdm/android/service/androidmgmt" +) + +const ( + // androidCommandReconcileMinAge is how long a command must sit in the pending status before we poll + // AMAPI for it. Pub/Sub delivers within seconds in normal operation, so anything younger than this is + // still expected to resolve on its own and polling it would only burn AMAPI quota. + androidCommandReconcileMinAge = 24 * time.Hour + + // androidCommandReconcileNotFoundGrace is how long we keep waiting on a command whose Operation AMAPI + // no longer knows about before declaring it failed. AMAPI drops Operation resources it has finished + // with, and it also 404s for a device that was deleted, so a NotFound on its own does not tell us the + // command is dead -- AMAPI may still be holding it (e.g. a WIPE waiting for the device to come back + // online). Once the row is older than GCP Pub/Sub's maximum retention no notification can arrive + // anymore, so at that point the row can only be stuck and marking it failed is what unsticks the host. + androidCommandReconcileNotFoundGrace = 7 * 24 * time.Hour + + // androidCommandReconcileBatchSize bounds how many commands (and therefore AMAPI calls) a single run + // makes. Combined with the rate limit below this caps a run at ~10 minutes of polling. Rows that don't + // fit are picked up by the next run: they are ordered oldest-first, so the most stuck ones go first. + androidCommandReconcileBatchSize = 500 + + // androidCommandReconcileCallsPerMinute is the AMAPI request rate the reconciler paces itself to, to + // stay well under the per-project request budget shared with the rest of Fleet's AMAPI traffic. + androidCommandReconcileCallsPerMinute = 50 + + // googleStatusCodeNotFound is google.rpc.Code NOT_FOUND, recorded on rows we fail because AMAPI no + // longer has the Operation. + googleStatusCodeNotFound = 5 +) + +// ReconcileAndroidCommands recovers Android MDM commands whose Pub/Sub COMMAND notification never +// arrived (Fleet's push endpoint down longer than GCP's retention, a subscription misconfiguration, a +// Google Cloud incident). Without this, such a command sits in mdm_android_commands.status='pending' +// forever, the host reads as perpetually pending lock/wipe/clear-passcode, and the admin cannot +// re-issue it. AMAPI's operations.get is the authoritative source for a command's outcome and, unlike +// Apple's and Windows' equivalents, needs neither the device to come back online nor AMAPI to re-send +// anything. +func ReconcileAndroidCommands(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, newActivityFn fleet.NewActivityFunc) error { + appConfig, err := ds.AppConfig(ctx) + if err != nil { + return ctxerr.Wrap(ctx, err, "get app config") + } + if !appConfig.MDM.AndroidEnabledAndConfigured { + return nil + } + + client := newAMAPIClient(ctx, logger, licenseKey) + + // Set the authentication secret for proxy client usage (a no-op for the Google client, which + // authenticates from its own env var and has no such asset). Without it every AMAPI call on the proxy + // path is rejected, so say so loudly rather than letting the run burn through the batch on 401s. + assets, err := ds.GetAllMDMConfigAssetsByName(ctx, []fleet.MDMAssetName{fleet.MDMAssetAndroidFleetServerSecret}, nil) + switch { + case err != nil: + logger.WarnContext(ctx, "could not read the android fleet server secret; AMAPI calls will fail if this Fleet uses the proxy client", "err", err) + default: + asset, ok := assets[fleet.MDMAssetAndroidFleetServerSecret] + if !ok || len(asset.Value) == 0 { + logger.WarnContext(ctx, "no android fleet server secret stored; AMAPI calls will fail if this Fleet uses the proxy client") + } else if err := client.SetAuthenticationSecret(string(asset.Value)); err != nil { + return ctxerr.Wrap(ctx, err, "set android fleet server secret") + } + } + + return reconcileAndroidCommands(ctx, ds, client, logger, newActivityFn, time.Now().UTC(), + time.Minute/androidCommandReconcileCallsPerMinute) +} + +// reconcileAndroidCommands is the testable core of ReconcileAndroidCommands. now anchors both the +// pending-age cutoff and the NotFound grace period; callInterval is the delay between AMAPI calls. +func reconcileAndroidCommands(ctx context.Context, ds fleet.Datastore, client androidmgmt.Client, logger *slog.Logger, + newActivityFn fleet.NewActivityFunc, now time.Time, callInterval time.Duration, +) error { + cmds, err := ds.ListPendingMDMAndroidCommands(ctx, now.Add(-androidCommandReconcileMinAge), androidCommandReconcileBatchSize) + if err != nil { + return ctxerr.Wrap(ctx, err, "list pending android commands for reconcile") + } + if len(cmds) == 0 { + return nil + } + + ticker := time.NewTicker(callInterval) + defer ticker.Stop() + + var resolved, stillRunning int + for i, cmd := range cmds { + // Pace ourselves between AMAPI calls, but don't pay the delay before the first one. + if i > 0 { + select { + case <-ticker.C: + case <-ctx.Done(): + return ctxerr.Wrap(ctx, ctx.Err(), "android command reconcile interrupted") + } + } + + op, err := client.EnterprisesDevicesOperationsGet(ctx, cmd.OperationName) + switch { + case androidmgmt.IsTooManyRequestsError(err): + // Out of AMAPI quota. Stop the run rather than hammering a rate-limited API; the remaining rows + // stay pending and the next run resumes with them (oldest first). + logger.WarnContext(ctx, "android command reconcile hit AMAPI quota, stopping run", + "command_uuid", cmd.CommandUUID, "resolved", resolved, "remaining", len(cmds)-i) + return ctxerr.Wrap(ctx, err, "android command reconcile exceeded AMAPI quota") + + case androidmgmt.IsAuthenticationError(err): + // Bad or missing credentials, or Fleet lost access to the enterprise. Every remaining call + // would be rejected the same way, so stop instead of working through the batch on errors that + // say nothing about the individual commands. + logger.ErrorContext(ctx, "android command reconcile rejected by AMAPI, stopping run", + "command_uuid", cmd.CommandUUID, "resolved", resolved, "remaining", len(cmds)-i, "err", err) + return ctxerr.Wrap(ctx, err, "android command reconcile rejected by AMAPI") + + case androidmgmt.IsNotFoundError(err): + age := now.Sub(cmd.CreatedAt) + if age < androidCommandReconcileNotFoundGrace { + logger.DebugContext(ctx, "android command operation not found in AMAPI, still within grace period", + "command_uuid", cmd.CommandUUID, "operation_name", cmd.OperationName, "age", age) + stillRunning++ + continue + } + errCode := googleStatusCode(googleStatusCodeNotFound) + errMsg := "Fleet did not receive a result for this command and Google no longer has a record of it." + // nil dedup recorder: this path is driven by the cron, not a Pub/Sub notification, so there is + // no messageId or publish time to record. + if err := setAndroidCommandTerminalState(ctx, ds, newActivityFn, cmd, + string(android.MDMAndroidCommandStatusError), &errCode, &errMsg, nil); err != nil { + logger.ErrorContext(ctx, "failed to fail android command with unknown operation", + "command_uuid", cmd.CommandUUID, "err", err) + ctxerr.Handle(ctx, err) + continue + } + resolved++ + logger.InfoContext(ctx, "android command operation unknown to AMAPI past grace period, marked error", + "command_uuid", cmd.CommandUUID, "operation_name", cmd.OperationName, "age", age) + + case err != nil: + // Transient AMAPI/network failure for this one command. Keep going: the rest of the batch is + // independent, and this row is retried on the next run. + logger.ErrorContext(ctx, "failed to get android command operation from AMAPI", + "command_uuid", cmd.CommandUUID, "operation_name", cmd.OperationName, "err", err) + ctxerr.Handle(ctx, ctxerr.Wrap(ctx, err, "get android command operation from AMAPI")) + + case !op.Done: + // Still queued at AMAPI (e.g. the device has not come online yet). Leave it pending. + stillRunning++ + + default: + status, errCode, errMsg := androidOperationTerminalState(op) + if err := setAndroidCommandTerminalState(ctx, ds, newActivityFn, cmd, status, errCode, errMsg, nil); err != nil { + logger.ErrorContext(ctx, "failed to apply reconciled android command status", + "command_uuid", cmd.CommandUUID, "status", status, "err", err) + ctxerr.Handle(ctx, err) + continue + } + resolved++ + logger.InfoContext(ctx, "android command reconciled from AMAPI", + "command_uuid", cmd.CommandUUID, "command_type", cmd.CommandType, "new_status", status) + } + } + + logger.DebugContext(ctx, "android command reconcile complete", + "checked", len(cmds), "resolved", resolved, "still_running", stillRunning) + return nil +} diff --git a/server/mdm/android/service/reconcile_commands_test.go b/server/mdm/android/service/reconcile_commands_test.go new file mode 100644 index 0000000000..5aa677db02 --- /dev/null +++ b/server/mdm/android/service/reconcile_commands_test.go @@ -0,0 +1,351 @@ +package service + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mdm/android" + android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/api/androidmanagement/v1" + "google.golang.org/api/googleapi" +) + +// reconcileNow is the fixed "current time" the reconcile tests run at, so command ages are exact. +var reconcileNow = time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + +// reconcileTestCallInterval keeps the reconciler's AMAPI pacing out of the tests' wall-clock time. +const reconcileTestCallInterval = time.Nanosecond + +// pendingCommandForReconcile builds a pending command row of the given type, created age ago. +func pendingCommandForReconcile(cmdUUID, cmdType string, age time.Duration) *android.MDMAndroidCommand { + return &android.MDMAndroidCommand{ + CommandUUID: cmdUUID, + HostUUID: "host-uuid-" + cmdUUID, + OperationName: "enterprises/E/devices/D/operations/" + cmdUUID, + CommandType: cmdType, + Status: string(android.MDMAndroidCommandStatusPending), + CreatedAt: reconcileNow.Add(-age), + } +} + +// newReconcileFixture wires a mock datastore and AMAPI client for the reconciler. cmds is what +// ListPendingMDMAndroidCommands returns; the caller shapes the client's operations.get behavior. +func newReconcileFixture(t *testing.T, cmds ...*android.MDMAndroidCommand) (*AndroidMockDS, *android_mock.Client, *slog.Logger) { + t.Helper() + mockDS := InitCommonDSMocks() + mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: true}}, nil + } + mockDS.ListPendingMDMAndroidCommandsFunc = func(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) { + require.Equal(t, reconcileNow.Add(-androidCommandReconcileMinAge), createdBefore) + require.Equal(t, androidCommandReconcileBatchSize, limit) + return cmds, nil + } + client := &android_mock.Client{} + client.InitCommonMocks() + // Discard log output: these tests assert on datastore effects, not on log lines. + return mockDS, client, slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// googleAPIError builds the *googleapi.Error shape the AMAPI clients return, so the reconciler's +// status-code classification is exercised the way it is in production. +func googleAPIError(code int, message string) error { + return &googleapi.Error{Code: code, Message: message} +} + +func TestReconcileAndroidCommands(t *testing.T) { + t.Run("done operation transitions the row to its terminal status", func(t *testing.T) { + for _, tc := range []struct { + name string + opError *androidmanagement.Status + expectedStatus string + expectedCode string + expectedMsg string + }{ + { + name: "no error means the device executed the command", + opError: nil, + expectedStatus: string(android.MDMAndroidCommandStatusAcknowledged), + }, + { + name: "populated error records the google.rpc code and message", + opError: &androidmanagement.Status{Code: 13, Message: "device does not support LOCK"}, + expectedStatus: string(android.MDMAndroidCommandStatusError), + expectedCode: "13", + expectedMsg: "device does not support LOCK", + }, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := pendingCommandForReconcile("cmd-done", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + require.Equal(t, cmd.OperationName, operationName) + return &androidmanagement.Operation{Name: operationName, Done: true, Error: tc.opError}, nil + } + var gotStatus string + var gotCode, gotMsg *string + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + require.Equal(t, cmd.CommandUUID, commandUUID) + gotStatus, gotCode, gotMsg = status, errorCode, errorMessage + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + + require.True(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked) + assert.Equal(t, tc.expectedStatus, gotStatus) + if tc.expectedCode == "" { + assert.Nil(t, gotCode) + assert.Nil(t, gotMsg) + } else { + require.NotNil(t, gotCode) + require.NotNil(t, gotMsg) + assert.Equal(t, tc.expectedCode, *gotCode) + assert.Equal(t, tc.expectedMsg, *gotMsg) + } + }) + } + }) + + t.Run("operation still running is left pending", func(t *testing.T) { + cmd := pendingCommandForReconcile("cmd-running", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + return &androidmanagement.Operation{Name: operationName, Done: false}, nil + } + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + t.Fatalf("a command AMAPI is still working on must not be transitioned") + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked) + }) + + t.Run("unknown operation inside the grace period is left pending", func(t *testing.T) { + // AMAPI 404s for an operation it has already discarded, but a notification can still arrive while + // the row is younger than Pub/Sub's retention, so we keep waiting. + cmd := pendingCommandForReconcile("cmd-404-young", string(android.MDMAndroidCommandTypeLock), + androidCommandReconcileNotFoundGrace-time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + return nil, googleAPIError(http.StatusNotFound, "Requested entity was not found.") + } + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + t.Fatalf("a command still inside the not-found grace period must not be transitioned") + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked) + }) + + t.Run("unknown operation past the grace period is marked error", func(t *testing.T) { + // Past Pub/Sub's retention no notification can arrive anymore, so the row can only be stuck. + cmd := pendingCommandForReconcile("cmd-404-old", string(android.MDMAndroidCommandTypeLock), + androidCommandReconcileNotFoundGrace+time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + return nil, googleAPIError(http.StatusNotFound, "Requested entity was not found.") + } + var gotStatus string + var gotCode, gotMsg *string + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + gotStatus, gotCode, gotMsg = status, errorCode, errorMessage + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + + require.True(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked) + assert.Equal(t, string(android.MDMAndroidCommandStatusError), gotStatus) + require.NotNil(t, gotCode) + assert.Equal(t, "5", *gotCode, "google.rpc.Code NOT_FOUND") + require.NotNil(t, gotMsg) + assert.NotEmpty(t, *gotMsg) + }) + + t.Run("acknowledged WIPE runs the unenroll side effects", func(t *testing.T) { + const hostID uint = 42 + cmd := pendingCommandForReconcile("cmd-wipe", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + return &androidmanagement.Operation{Name: operationName, Done: true}, nil + } + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + return nil + } + mockDS.AndroidHostLiteByHostUUIDFunc = func(ctx context.Context, hostUUID string) (*fleet.AndroidHost, error) { + require.Equal(t, cmd.HostUUID, hostUUID) + return &fleet.AndroidHost{Host: &fleet.Host{ID: hostID, UUID: hostUUID}}, nil + } + // BYO: the work profile was removed, so host_mdm_actions must be cleared for the "Wiped" badge to drop. + mockDS.GetHostMDMFunc = func(ctx context.Context, id uint) (*fleet.HostMDM, error) { + return &fleet.HostMDM{IsPersonalEnrollment: true}, nil + } + mockDS.ClearHostMDMActionsFunc = func(ctx context.Context, id uint) error { return nil } + mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) { + require.Equal(t, hostID, id) + return true, nil + } + mockDS.ListHostsLiteByIDsFunc = func(ctx context.Context, ids []uint) ([]*fleet.Host, error) { + return []*fleet.Host{{ID: hostID, Hostname: "wiped-host"}}, nil + } + var activities []fleet.ActivityDetails + newActivity := func(_ context.Context, _ *fleet.User, details fleet.ActivityDetails) error { + activities = append(activities, details) + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, newActivity, reconcileNow, reconcileTestCallInterval)) + + require.True(t, mockDS.ClearHostMDMActionsFuncInvoked, "BYO wipe must clear host_mdm_actions") + require.True(t, mockDS.SetAndroidHostUnenrolledFuncInvoked, "a wiped host must be flipped to unenrolled") + require.Len(t, activities, 1) + require.IsType(t, fleet.ActivityTypeMDMUnenrolled{}, activities[0]) + }) + + t.Run("a failed WIPE side effect leaves the command pending so the next run retries it", func(t *testing.T) { + // The reconciler only ever selects pending rows, so writing the terminal status before the + // unenroll side effect succeeds would strand the host: acknowledged, still enrolled, and never + // looked at again. + cmd := pendingCommandForReconcile("cmd-wipe-transient", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + return &androidmanagement.Operation{Name: operationName, Done: true}, nil + } + mockDS.AndroidHostLiteByHostUUIDFunc = func(ctx context.Context, hostUUID string) (*fleet.AndroidHost, error) { + return &fleet.AndroidHost{Host: &fleet.Host{ID: 55, UUID: hostUUID}}, nil + } + mockDS.GetHostMDMFunc = func(ctx context.Context, id uint) (*fleet.HostMDM, error) { + return &fleet.HostMDM{IsPersonalEnrollment: false}, nil + } + mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) { + return false, errors.New("simulated transient DB connection drop") + } + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + t.Fatalf("the command must stay pending when its wipe side effect fails") + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked) + }) + + t.Run("errored WIPE does not unenroll the host", func(t *testing.T) { + cmd := pendingCommandForReconcile("cmd-wipe-failed", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, cmd) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + return &androidmanagement.Operation{ + Name: operationName, + Done: true, + Error: &androidmanagement.Status{Code: 13, Message: "wipe failed"}, + }, nil + } + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + require.Equal(t, string(android.MDMAndroidCommandStatusError), status) + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + require.False(t, mockDS.SetAndroidHostUnenrolledFuncInvoked, "a failed wipe must leave the host enrolled") + }) + + t.Run("a failure on one command does not stop the rest of the batch", func(t *testing.T) { + failing := pendingCommandForReconcile("cmd-transient", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + updateFailing := pendingCommandForReconcile("cmd-update-fails", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + succeeding := pendingCommandForReconcile("cmd-ok", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, failing, updateFailing, succeeding) + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + if operationName == failing.OperationName { + return nil, errors.New("simulated transient network failure") + } + return &androidmanagement.Operation{Name: operationName, Done: true}, nil + } + var updated []string + mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error { + if commandUUID == updateFailing.CommandUUID { + return errors.New("simulated transient DB failure") + } + updated = append(updated, commandUUID) + return nil + } + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + require.Equal(t, []string{succeeding.CommandUUID}, updated, + "the reconciler must keep going past both an AMAPI failure and a DB failure") + }) + + t.Run("AMAPI quota error stops the run and surfaces an error", func(t *testing.T) { + first := pendingCommandForReconcile("cmd-quota", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + second := pendingCommandForReconcile("cmd-after-quota", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, first, second) + var calls int + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + calls++ + return nil, googleAPIError(http.StatusTooManyRequests, "Quota exceeded") + } + + err := reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval) + require.Error(t, err) + require.Equal(t, 1, calls, "the run must stop at the first quota error instead of hammering AMAPI") + require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked) + }) + + t.Run("AMAPI rejecting our credentials stops the run and surfaces an error", func(t *testing.T) { + // A missing or stale Fleet server secret, lost access to the enterprise, or (on the proxy path) + // fleetdm.com having no record of the enterprise, rejects every call identically -- working + // through the batch would only produce noise. + for _, statusCode := range []int{http.StatusUnauthorized, http.StatusForbidden} { + first := pendingCommandForReconcile("cmd-rejected", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + second := pendingCommandForReconcile("cmd-after-rejected", string(android.MDMAndroidCommandTypeLock), 48*time.Hour) + mockDS, client, logger := newReconcileFixture(t, first, second) + var calls int + client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) { + calls++ + return nil, googleAPIError(statusCode, "rejected") + } + + err := reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval) + require.Error(t, err, "status code %d", statusCode) + require.Equal(t, 1, calls, "status code %d must stop the run at the first rejection", statusCode) + require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked, + "a rejected call says nothing about the command, so nothing may be marked failed") + } + }) + + t.Run("nothing pending makes no AMAPI calls", func(t *testing.T) { + mockDS, client, logger := newReconcileFixture(t) + + require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)) + require.False(t, client.EnterprisesDevicesOperationsGetFuncInvoked) + }) + + t.Run("a datastore failure surfaces so the cron run is marked failed", func(t *testing.T) { + mockDS, client, logger := newReconcileFixture(t) + mockDS.ListPendingMDMAndroidCommandsFunc = func(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) { + return nil, errors.New("simulated DB outage") + } + + err := reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval) + require.ErrorContains(t, err, "simulated DB outage") + }) + + t.Run("android MDM turned off skips the run entirely", func(t *testing.T) { + mockDS, _, logger := newReconcileFixture(t) + mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: false}}, nil + } + + require.NoError(t, ReconcileAndroidCommands(t.Context(), &mockDS.DataStore, logger, "", noopNewActivity)) + require.False(t, mockDS.ListPendingMDMAndroidCommandsFuncInvoked) + }) +} diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 15dd928b98..8c4d2ae63a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -1988,6 +1988,8 @@ type GetMDMAndroidCommandByOperationNameFunc func(ctx context.Context, operation type UpdateMDMAndroidCommandStatusFunc func(ctx context.Context, commandUUID string, status string, errorCode *string, errorMessage *string) error +type ListPendingMDMAndroidCommandsFunc func(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) + type LockHostViaAndroidMDMFunc func(ctx context.Context, host *fleet.Host, cmd *android.MDMAndroidCommand) error type WipeHostViaAndroidMDMFunc func(ctx context.Context, host *fleet.Host, cmd *android.MDMAndroidCommand) error @@ -5251,6 +5253,9 @@ type DataStore struct { UpdateMDMAndroidCommandStatusFunc UpdateMDMAndroidCommandStatusFunc UpdateMDMAndroidCommandStatusFuncInvoked bool + ListPendingMDMAndroidCommandsFunc ListPendingMDMAndroidCommandsFunc + ListPendingMDMAndroidCommandsFuncInvoked bool + LockHostViaAndroidMDMFunc LockHostViaAndroidMDMFunc LockHostViaAndroidMDMFuncInvoked bool @@ -12602,6 +12607,13 @@ func (s *DataStore) UpdateMDMAndroidCommandStatus(ctx context.Context, commandUU return s.UpdateMDMAndroidCommandStatusFunc(ctx, commandUUID, status, errorCode, errorMessage) } +func (s *DataStore) ListPendingMDMAndroidCommands(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) { + s.mu.Lock() + s.ListPendingMDMAndroidCommandsFuncInvoked = true + s.mu.Unlock() + return s.ListPendingMDMAndroidCommandsFunc(ctx, createdBefore, limit) +} + func (s *DataStore) LockHostViaAndroidMDM(ctx context.Context, host *fleet.Host, cmd *android.MDMAndroidCommand) error { s.mu.Lock() s.LockHostViaAndroidMDMFuncInvoked = true diff --git a/tools/hangar/frontend/src/lib/fleetctlCrons.ts b/tools/hangar/frontend/src/lib/fleetctlCrons.ts index c3d51643a6..e5b4eb47c9 100644 --- a/tools/hangar/frontend/src/lib/fleetctlCrons.ts +++ b/tools/hangar/frontend/src/lib/fleetctlCrons.ts @@ -105,6 +105,12 @@ export const CRONS: CronInfo[] = [ interval: "1h", note: "Reconciles Android device existence with Google AMAPI.", }, + { + name: "mdm_android_command_reconciler", + group: "mdm", + interval: "24h", + note: "Resolves stuck Android MDM commands via AMAPI operations.get.", + }, // ---------- activity / maintenance ---------- { diff --git a/website/api/controllers/android-proxy/get-android-device-operation.js b/website/api/controllers/android-proxy/get-android-device-operation.js new file mode 100644 index 0000000000..a4efb51e59 --- /dev/null +++ b/website/api/controllers/android-proxy/get-android-device-operation.js @@ -0,0 +1,111 @@ +module.exports = { + + + friendlyName: 'Get android device operation', + + + description: 'Gets a long-running operation for a device of an Android enterprise. Fleet servers poll this to recover the outcome of an Android MDM command (Lock, Wipe, Clear passcode) whose Pub/Sub COMMAND notification never arrived.', + + + inputs: { + androidEnterpriseId: { + type: 'string', + required: true, + }, + deviceId: { + type: 'string', + required: true, + }, + operationId: { + type: 'string', + required: true, + }, + }, + + + exits: { + success: { description: 'The operation for a device of an Android enterprise was successfully retrieved.' }, + missingAuthHeader: { description: 'This request was missing an authorization header.', responseType: 'unauthorized'}, + unauthorized: { description: 'Invalid authentication token.', responseType: 'unauthorized'}, + // Unlike the other android-proxy actions, this one reserves 404 for a single meaning: the Android + // management API has no record of this operation. The Fleet server treats that as evidence the + // command can never complete and eventually marks it failed, so nothing about *this website's* + // records may return a 404. A missing AndroidEnterprise row and a loss of access to the enterprise + // both mean "we cannot answer for this enterprise" -- a 403, which the Fleet server classifies as + // an authorization failure and stops its whole reconciler run on. + enterpriseNotAccessible: { description: 'No Android enterprise found for this Fleet server, or Fleet is not authorized to manage it.', statusCode: 403 }, + operationNotFound: { description: 'The specified operation does not exist in this Android enterprise', responseType: 'notFound' }, + // The Fleet server classifies this status code to know it should stop polling and wait for the next + // reconciler run, so the Android management API's 429 has to survive the trip through this proxy. + tooManyRequests: { description: 'The Android management API rate limit was exceeded.', statusCode: 429 }, + }, + + + fn: async function ({ androidEnterpriseId, deviceId, operationId }) { + + // Extract fleetServerSecret from the Authorization header + let authHeader = this.req.get('authorization'); + let fleetServerSecret; + + if (authHeader && authHeader.startsWith('Bearer')) { + fleetServerSecret = authHeader.replace('Bearer', '').trim(); + } else { + throw 'missingAuthHeader'; + } + + // Authenticate this request + let thisAndroidEnterprise = await AndroidEnterprise.findOne({ + androidEnterpriseId: androidEnterpriseId + }); + + // Return a 403 (not a 404) if no records are found -- see the note on the exits above. + if (!thisAndroidEnterprise) { + throw 'enterpriseNotAccessible'; + } + // Return an unauthorized response if the provided secret does not match. + if (thisAndroidEnterprise.fleetServerSecret !== fleetServerSecret) { + throw 'unauthorized'; + } + + + // Get the shared Google API auth client with the getAndroidManagementAuthorizationClient helper. + // Note: we are doing this outside of the sails.helpers.flow.build() so any errors related to the website's credentials returned by the helper are not intercepted. + let androidManagementAuthClient = await sails.helpers.androidProxy.getAndroidManagementAuthorizationClient(); + + // Get the operation for this device. + // Note: We're using sails.helpers.flow.build here to handle any errors that occur using google's node library. + let getOperationResponse = await sails.helpers.flow.build(async () => { + let { google } = require('googleapis'); + let androidManagementConnection = google.androidmanagement({version: 'v1', auth: androidManagementAuthClient}); + // [?]: https://googleapis.dev/nodejs/googleapis/latest/androidmanagement/classes/Resource$Enterprises$Devices$Operations.html#get + let getOperationResult = await androidManagementConnection.enterprises.devices.operations.get({ + name: `enterprises/${androidEnterpriseId}/devices/${deviceId}/operations/${operationId}`, + }); + return getOperationResult.data; + }).intercept({status: 429}, ()=>{ + // If the Android management API returns a 429 response, log an additional warning that will trigger a help-p1 alert. + // Note: the error object is deliberately left out of this log -- gaxios errors carry the request + // config, including the Authorization header used to call Google. + sails.log.warn(`p1: Android management API rate limit exceeded! (When getting a device operation for Android enterprise ${androidEnterpriseId}.)`); + // Pass the 429 through to the Fleet server rather than collapsing it into a 500, so its reconciler + // can tell rate limiting apart from a generic failure. + return 'tooManyRequests'; + }).intercept({status: 403}, ()=>{ + // If the Android management API returns a 403 response, return an enterpriseNotAccessible (403) response to the Fleet server. + return 'enterpriseNotAccessible'; + }).intercept({status: 404}, ()=>{ + // If the Android management API returns a 404 response, return an operationNotFound (notFound) response to the Fleet server. + // The Fleet server treats this as "Google no longer has a record of this command". + return 'operationNotFound'; + }).intercept((err)=>{ + return new Error(`When attempting to get a device operation for an Android enterprise (${androidEnterpriseId}), an error occurred. Error: ${require('util').inspect(err)}`); + }); + + + // Return the operation data back to the Fleet server. + return getOperationResponse; + + } + + +}; diff --git a/website/config/routes.js b/website/config/routes.js index 5ad5dd54f4..004cee505a 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -1438,6 +1438,7 @@ module.exports.routes = { 'PATCH /api/android/v1/enterprises/:androidEnterpriseId/policies/:policyId': { action: 'android-proxy/modify-android-policies', csrf: false }, 'DELETE /api/android/v1/enterprises/:androidEnterpriseId': { action: 'android-proxy/delete-one-android-enterprise', csrf: false }, 'GET /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId': { action: 'android-proxy/get-android-device' }, + 'GET /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId/operations/:operationId': { action: 'android-proxy/get-android-device-operation' }, 'GET /api/android/v1/enterprises/:androidEnterpriseId/devices': { action: 'android-proxy/get-android-devices' }, 'DELETE /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId': { action: 'android-proxy/delete-android-device', csrf: false }, 'PATCH /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId': { action: 'android-proxy/modify-android-device', csrf: false },