From 9236a19342d33da8af88d39dc411f173ed7279c7 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 11 Dec 2023 11:29:17 -0600 Subject: [PATCH] Changed query performance statistics to uint64 to match osquery reports. (#15505) #15472 # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- changes/15472-performance-stat-types | 1 + server/datastore/mysql/hosts_test.go | 4 +- .../tables/20231207133731_FixStatsTypes.go | 40 +++++++++++ .../20231207133731_FixStatsTypes_test.go | 68 +++++++++++++++++++ server/datastore/mysql/packs_test.go | 12 ++-- .../datastore/mysql/scheduled_queries_test.go | 2 +- server/datastore/mysql/schema.sql | 16 ++--- server/fleet/queries.go | 14 ++-- server/fleet/scheduled_queries.go | 14 ++-- .../async/async_scheduled_query_stats_test.go | 2 +- server/service/osquery_utils/queries.go | 12 ++-- 11 files changed, 147 insertions(+), 38 deletions(-) create mode 100644 changes/15472-performance-stat-types create mode 100644 server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes.go create mode 100644 server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes_test.go diff --git a/changes/15472-performance-stat-types b/changes/15472-performance-stat-types new file mode 100644 index 0000000000..8550697ce5 --- /dev/null +++ b/changes/15472-performance-stat-types @@ -0,0 +1 @@ +Changed query performance statistics to uint64 to match osquery reports. \ No newline at end of file diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 19ba4044fb..260b8efa36 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -3598,7 +3598,7 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { PackID: pack1.ID, AverageMemory: 8000, Denylisted: false, - Executions: rand.Intn(1000), + Executions: uint64(rand.Intn(1000)), Interval: 30, LastExecuted: time.Now().UTC(), OutputSize: 1337, @@ -3619,7 +3619,7 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { PackID: pack2.ID, AverageMemory: 8000, Denylisted: false, - Executions: rand.Intn(1000), + Executions: uint64(rand.Intn(1000)), Interval: 30, LastExecuted: time.Now().UTC(), OutputSize: 1337, diff --git a/server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes.go b/server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes.go new file mode 100644 index 0000000000..02bda0d936 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes.go @@ -0,0 +1,40 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20231207133731, Down_20231207133731) +} + +func Up_20231207133731(tx *sql.Tx) error { + // Updating some bad data from osquery (which will be ignored in a later PR). + // Seems safer to update rather than delete. + stmt := ` + UPDATE scheduled_query_stats SET last_executed = '1970-01-01 00:00:01' WHERE YEAR(last_executed) = '0000'; + ` + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("fixing last_executed in scheduled_query_stats: %w", err) + } + + stmt = ` + ALTER TABLE scheduled_query_stats + MODIFY COLUMN average_memory BIGINT UNSIGNED NOT NULL, + MODIFY COLUMN executions BIGINT UNSIGNED NOT NULL, + MODIFY COLUMN output_size BIGINT UNSIGNED NOT NULL, + MODIFY COLUMN system_time BIGINT UNSIGNED NOT NULL, + MODIFY COLUMN user_time BIGINT UNSIGNED NOT NULL, + MODIFY COLUMN wall_time BIGINT UNSIGNED NOT NULL; + ` + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("changing data types for scheduled_query_stats: %w", err) + } + + return nil +} + +func Down_20231207133731(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes_test.go b/server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes_test.go new file mode 100644 index 0000000000..acf9816d82 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20231207133731_FixStatsTypes_test.go @@ -0,0 +1,68 @@ +package tables + +import ( + "github.com/stretchr/testify/require" + "math" + "testing" +) + +func TestUp_20231207133731(t *testing.T) { + db := applyUpToPrev(t) + + setupStmt := ` + INSERT INTO scheduled_query_stats (host_id, scheduled_query_id, average_memory, denylisted, executions, schedule_interval, output_size, system_time, user_time, wall_time, last_executed) VALUES + (?,?,?,?,?,?,?,?,?,?,?); + ` + + _, err := db.Exec(setupStmt, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, "2023-12-07 13:17:17") + require.NoError(t, err) + // Apply current migration. + applyNext(t, db) + + stmt := ` + SELECT host_id, average_memory FROM scheduled_query_stats WHERE host_id = 1; + ` + rows, err := db.Query(stmt) + require.NoError(t, rows.Err()) + require.NoError(t, err) + defer rows.Close() + count := 0 + for rows.Next() { + count += 1 + var hostId int + var avgMem uint64 + err := rows.Scan(&hostId, &avgMem) + require.NoError(t, err) + require.Equal(t, 1, hostId) + require.Equal(t, uint64(3), avgMem) + } + require.Equal(t, 1, count) + + _, err = db.Exec(setupStmt, 2, 2, uint64(math.MaxUint64), 4, uint64(math.MaxUint64-1), 6, uint64(math.MaxUint64-2), uint64(math.MaxUint64-3), uint64(math.MaxUint64-4), uint64(math.MaxUint64-5), "2023-12-07 13:17:17") + require.NoError(t, err) + + stmt = ` + SELECT host_id, average_memory, executions, output_size, system_time, user_time, wall_time FROM scheduled_query_stats WHERE host_id = 2; + ` + rows, err = db.Query(stmt) + require.NoError(t, rows.Err()) + require.NoError(t, err) + defer rows.Close() + count = 0 + for rows.Next() { + count += 1 + var hostId int + var avgMem, executions, outputSize, systemTime, userTime, wallTime uint64 + err := rows.Scan(&hostId, &avgMem, &executions, &outputSize, &systemTime, &userTime, &wallTime) + require.NoError(t, err) + require.Equal(t, 2, hostId) + require.Equal(t, uint64(math.MaxUint64), avgMem) + require.Equal(t, uint64(math.MaxUint64-1), executions) + require.Equal(t, uint64(math.MaxUint64-2), outputSize) + require.Equal(t, uint64(math.MaxUint64-3), systemTime) + require.Equal(t, uint64(math.MaxUint64-4), userTime) + require.Equal(t, uint64(math.MaxUint64-5), wallTime) + } + require.Equal(t, 1, count) + +} diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 750beaf969..864d4d5a10 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -444,15 +444,15 @@ func randomPackStatsForHost(packID uint, packName string, packType string, sched QueryName: sq.QueryName, Description: sq.Description, PackID: packID, - AverageMemory: rand.Intn(100), + AverageMemory: uint64(rand.Intn(100)), Denylisted: false, - Executions: rand.Intn(100), + Executions: uint64(rand.Intn(100)), Interval: rand.Intn(100), LastExecuted: time.Now(), - OutputSize: rand.Intn(1000), - SystemTime: rand.Intn(1000), - UserTime: rand.Intn(1000), - WallTime: rand.Intn(1000), + OutputSize: uint64(rand.Intn(1000)), + SystemTime: uint64(rand.Intn(1000)), + UserTime: uint64(rand.Intn(1000)), + WallTime: uint64(rand.Intn(1000)), }) } return []fleet.PackStats{ diff --git a/server/datastore/mysql/scheduled_queries_test.go b/server/datastore/mysql/scheduled_queries_test.go index 8b09a13311..74a441228d 100644 --- a/server/datastore/mysql/scheduled_queries_test.go +++ b/server/datastore/mysql/scheduled_queries_test.go @@ -508,7 +508,7 @@ func testScheduledQueriesAsyncBatchSaveStats(t *testing.T, ds *Datastore) { for hid, stats := range m { for _, st := range stats { ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { - var got int + var got uint64 err := sqlx.GetContext(ctx, tx, &got, `SELECT executions FROM scheduled_query_stats WHERE host_id = ? AND scheduled_query_id = ?`, hid, st.ScheduledQueryID) if err != nil { return err diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a6ec73c80a..1df4b03044 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -725,9 +725,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=230 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=231 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1184,15 +1184,15 @@ CREATE TABLE `scheduled_queries` ( CREATE TABLE `scheduled_query_stats` ( `host_id` int(10) unsigned NOT NULL, `scheduled_query_id` int(10) unsigned NOT NULL, - `average_memory` int(11) DEFAULT NULL, + `average_memory` bigint(20) unsigned NOT NULL, `denylisted` tinyint(1) DEFAULT NULL, - `executions` int(11) DEFAULT NULL, + `executions` bigint(20) unsigned NOT NULL, `schedule_interval` int(11) DEFAULT NULL, `last_executed` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `output_size` int(11) DEFAULT NULL, - `system_time` int(11) DEFAULT NULL, - `user_time` int(11) DEFAULT NULL, - `wall_time` int(11) DEFAULT NULL, + `output_size` bigint(20) unsigned NOT NULL, + `system_time` bigint(20) unsigned NOT NULL, + `user_time` bigint(20) unsigned NOT NULL, + `wall_time` bigint(20) unsigned NOT NULL, PRIMARY KEY (`host_id`,`scheduled_query_id`), KEY `scheduled_query_id` (`scheduled_query_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/server/fleet/queries.go b/server/fleet/queries.go index 0c59c3ab64..cfc655bbc4 100644 --- a/server/fleet/queries.go +++ b/server/fleet/queries.go @@ -410,16 +410,16 @@ type QueryStats struct { TeamID *uint `json:"team_id" db:"team_id"` // From osquery directly - AverageMemory int `json:"average_memory" db:"average_memory"` - Denylisted bool `json:"denylisted" db:"denylisted"` - Executions int `json:"executions" db:"executions"` + AverageMemory uint64 `json:"average_memory" db:"average_memory"` + Denylisted bool `json:"denylisted" db:"denylisted"` + Executions uint64 `json:"executions" db:"executions"` // Note schedule_interval is used for DB since "interval" is a reserved word in MySQL Interval int `json:"interval" db:"schedule_interval"` LastExecuted time.Time `json:"last_executed" db:"last_executed"` - OutputSize int `json:"output_size" db:"output_size"` - SystemTime int `json:"system_time" db:"system_time"` - UserTime int `json:"user_time" db:"user_time"` - WallTime int `json:"wall_time" db:"wall_time"` + OutputSize uint64 `json:"output_size" db:"output_size"` + SystemTime uint64 `json:"system_time" db:"system_time"` + UserTime uint64 `json:"user_time" db:"user_time"` + WallTime uint64 `json:"wall_time" db:"wall_time"` } // MapQueryReportsResultsToRows converts the scheduled query results as stored in Fleet's database diff --git a/server/fleet/scheduled_queries.go b/server/fleet/scheduled_queries.go index 4f3fca18b7..0eaac54cb5 100644 --- a/server/fleet/scheduled_queries.go +++ b/server/fleet/scheduled_queries.go @@ -151,16 +151,16 @@ type ScheduledQueryStats struct { PackID uint `json:"pack_id,omitempty" db:"pack_id"` // From osquery directly - AverageMemory int `json:"average_memory" db:"average_memory"` - Denylisted bool `json:"denylisted" db:"denylisted"` - Executions int `json:"executions" db:"executions"` + AverageMemory uint64 `json:"average_memory" db:"average_memory"` + Denylisted bool `json:"denylisted" db:"denylisted"` + Executions uint64 `json:"executions" db:"executions"` // Note schedule_interval is used for DB since "interval" is a reserved word in MySQL Interval int `json:"interval" db:"schedule_interval"` LastExecuted time.Time `json:"last_executed" db:"last_executed"` - OutputSize int `json:"output_size" db:"output_size"` - SystemTime int `json:"system_time" db:"system_time"` - UserTime int `json:"user_time" db:"user_time"` - WallTime int `json:"wall_time" db:"wall_time"` + OutputSize uint64 `json:"output_size" db:"output_size"` + SystemTime uint64 `json:"system_time" db:"system_time"` + UserTime uint64 `json:"user_time" db:"user_time"` + WallTime uint64 `json:"wall_time" db:"wall_time"` } // TeamID returns the team id if the stat is for a team query stat result diff --git a/server/service/async/async_scheduled_query_stats_test.go b/server/service/async/async_scheduled_query_stats_test.go index 699867c340..ddcd6632dd 100644 --- a/server/service/async/async_scheduled_query_stats_test.go +++ b/server/service/async/async_scheduled_query_stats_test.go @@ -242,7 +242,7 @@ func testRecordScheduledQueryStatsAsync(t *testing.T, ds *mock.Store, pool fleet var sqStat fleet.ScheduledQueryStats err = json.Unmarshal([]byte(res["p1\x00sq1"]), &sqStat) require.NoError(t, err) - require.Equal(t, 1, sqStat.Executions, res["p1\x00sq1"]) + require.Equal(t, uint64(1), sqStat.Executions, res["p1\x00sq1"]) count, err := redigo.Int(conn.Do("ZCARD", scheduledQueryStatsHostIDsKey)) require.NoError(t, err) diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index 91e5ae5e53..7c6a47b33b 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -1177,16 +1177,16 @@ func directIngestScheduledQueryStats(ctx context.Context, logger log.Logger, hos stats := fleet.ScheduledQueryStats{ ScheduledQueryName: scheduledName, PackName: packName, - AverageMemory: cast.ToInt(row["average_memory"]), + AverageMemory: cast.ToUint64(row["average_memory"]), Denylisted: cast.ToBool(row["denylisted"]), - Executions: cast.ToInt(row["executions"]), + Executions: cast.ToUint64(row["executions"]), Interval: cast.ToInt(row["interval"]), // Cast to int first to allow cast.ToTime to interpret the unix timestamp. LastExecuted: time.Unix(cast.ToInt64(row["last_executed"]), 0).UTC(), - OutputSize: cast.ToInt(row["output_size"]), - SystemTime: cast.ToInt(row["system_time"]), - UserTime: cast.ToInt(row["user_time"]), - WallTime: cast.ToInt(row["wall_time"]), + OutputSize: cast.ToUint64(row["output_size"]), + SystemTime: cast.ToUint64(row["system_time"]), + UserTime: cast.ToUint64(row["user_time"]), + WallTime: cast.ToUint64(row["wall_time"]), } packs[packName] = append(packs[packName], stats) }