From 4930ca2d0e12f68225cbffb5f33323bfc57b71ef Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 28 Feb 2022 13:55:14 -0500 Subject: [PATCH] Support listing software hosts count filtered by team (#4388) --- ...ue-4268-list-software-hosts-count-per-team | 1 + ...223113157_UpdateSoftwareHostCountsTable.go | 28 +++ ...3157_UpdateSoftwareHostCountsTable_test.go | 24 +++ server/datastore/mysql/schema.sql | 11 +- server/datastore/mysql/software.go | 166 +++++++++++------- server/datastore/mysql/software_test.go | 138 ++++++++++++++- server/datastore/mysql/testing_utils.go | 39 ++++ server/service/integration_core_test.go | 23 +++ 8 files changed, 354 insertions(+), 76 deletions(-) create mode 100644 changes/issue-4268-list-software-hosts-count-per-team create mode 100644 server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable.go create mode 100644 server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable_test.go diff --git a/changes/issue-4268-list-software-hosts-count-per-team b/changes/issue-4268-list-software-hosts-count-per-team new file mode 100644 index 0000000000..9ac6fd5015 --- /dev/null +++ b/changes/issue-4268-list-software-hosts-count-per-team @@ -0,0 +1 @@ +* Support filtering software hosts count per team. diff --git a/server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable.go b/server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable.go new file mode 100644 index 0000000000..e9d0da486a --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable.go @@ -0,0 +1,28 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20220223113157, Down_20220223113157) +} + +func Up_20220223113157(tx *sql.Tx) error { + alterStmt := `ALTER TABLE software_host_counts + ADD COLUMN team_id INT(10) UNSIGNED NOT NULL DEFAULT 0, + DROP PRIMARY KEY, + ADD PRIMARY KEY (software_id, team_id), + ADD INDEX idx_software_host_counts_team_id_hosts_count_software_id (team_id,hosts_count,software_id), + DROP INDEX idx_software_host_counts_host_count_software_id` + if _, err := tx.Exec(alterStmt); err != nil { + return errors.Wrap(err, "alter software_host_counts table") + } + return nil +} + +func Down_20220223113157(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable_test.go b/server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable_test.go new file mode 100644 index 0000000000..e92bee9e7d --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220223113157_UpdateSoftwareHostCountsTable_test.go @@ -0,0 +1,24 @@ +package tables + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUp_20220223113157(t *testing.T) { + db := applyUpToPrev(t) + + execNoErr(t, db, `INSERT INTO software_host_counts (software_id, hosts_count) VALUES (1, 1)`) + execNoErr(t, db, `INSERT INTO software_host_counts (software_id, hosts_count) VALUES (2, 10)`) + + // Apply current migration. + applyNext(t, db) + + var count int + require.NoError(t, db.Get(&count, `SELECT count(*) FROM software_host_counts WHERE team_id = 0`)) + assert.Equal(t, 2, count) + require.NoError(t, db.Get(&count, `SELECT SUM(hosts_count) FROM software_host_counts WHERE team_id = 0`)) + assert.Equal(t, 11, count) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index f3a7559cb8..5ef1afd621 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -318,9 +318,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=124 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -563,9 +563,10 @@ CREATE TABLE `software_host_counts` ( `hosts_count` int(10) unsigned NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`software_id`), - KEY `idx_software_host_counts_host_count_software_id` (`hosts_count`,`software_id`), - KEY `idx_software_host_counts_updated_at_software_id` (`updated_at`,`software_id`) + `team_id` int(10) unsigned NOT NULL DEFAULT '0', + PRIMARY KEY (`software_id`,`team_id`), + KEY `idx_software_host_counts_updated_at_software_id` (`updated_at`,`software_id`), + KEY `idx_software_host_counts_team_id_hosts_count_software_id` (`team_id`,`hosts_count`,`software_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 72d06bfcc2..1d2749dc85 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -347,6 +347,12 @@ func selectSoftwareSQL(hostID *uint, opts fleet.SoftwareListOptions) (string, [] goqu.I("shc.hosts_count"), goqu.I("shc.updated_at").As("counts_updated_at"), ) + + if opts.TeamID != nil { + ds = ds.Where(goqu.I("shc.team_id").Eq(opts.TeamID)) + } else { + ds = ds.Where(goqu.I("shc.team_id").Eq(0)) + } } ds = appendListOptionsToSelect(ds, opts.ListOptions) @@ -640,88 +646,122 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software // After aggregation, it cleans up unused software (e.g. software installed // on removed hosts, software uninstalled on hosts, etc.) func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error { - resetStmt := ` - UPDATE software_host_counts - SET hosts_count = 0, updated_at = ?` + const ( + resetStmt = ` + UPDATE software_host_counts + SET hosts_count = 0, updated_at = ?` - queryStmt := ` - SELECT count(*), software_id - FROM host_software - WHERE software_id > 0 - GROUP BY software_id` + // team_id is added to the select list to have the same structure as + // the teamCountsStmt, making it easier to use a common implementation + globalCountsStmt = ` + SELECT count(*), 0 as team_id, software_id + FROM host_software + WHERE software_id > 0 + GROUP BY software_id` - insertStmt := ` - INSERT INTO software_host_counts - (software_id, hosts_count, updated_at) - VALUES - %s - ON DUPLICATE KEY UPDATE - hosts_count = VALUES(hosts_count), - updated_at = VALUES(updated_at)` - valuesPart := `(?, ?, ?),` + teamCountsStmt = ` + SELECT count(*), h.team_id, hs.software_id + FROM host_software hs + INNER JOIN hosts h + ON hs.host_id = h.id + WHERE h.team_id IS NOT NULL AND hs.software_id > 0 + GROUP BY hs.software_id, h.team_id` + + insertStmt = ` + INSERT INTO software_host_counts + (software_id, hosts_count, team_id, updated_at) + VALUES + %s + ON DUPLICATE KEY UPDATE + hosts_count = VALUES(hosts_count), + updated_at = VALUES(updated_at)` + + valuesPart = `(?, ?, ?, ?),` + + cleanupSoftwareStmt = ` + DELETE s + FROM software s + LEFT JOIN software_host_counts shc + ON s.id = shc.software_id + WHERE + shc.software_id IS NULL OR + (shc.team_id = 0 AND shc.hosts_count = 0)` + + cleanupTeamStmt = ` + DELETE shc + FROM software_host_counts shc + LEFT JOIN teams t + ON t.id = shc.team_id + WHERE + shc.team_id > 0 AND + t.id IS NULL` + ) // first, reset all counts to 0 if _, err := ds.writer.ExecContext(ctx, resetStmt, updatedAt); err != nil { return ctxerr.Wrap(ctx, err, "reset all software_host_counts to 0") } - // next get a cursor for the counts for each software - rows, err := ds.reader.QueryContext(ctx, queryStmt) - if err != nil { - return ctxerr.Wrap(ctx, err, "read counts from host_software") - } - defer rows.Close() - - // use a loop to iterate to prevent loading all in one go in memory, as it - // could get pretty big at >100K hosts with 1000+ software each. - const batchSize = 100 - var batchCount int - args := make([]interface{}, 0, batchSize*3) - for rows.Next() { - var count int - var sid uint - - if err := rows.Scan(&count, &sid); err != nil { - return ctxerr.Wrap(ctx, err, "scan row into variables") + // next get a cursor for the global and team counts for each software + stmtLabel := []string{"global", "team"} + for i, countStmt := range []string{globalCountsStmt, teamCountsStmt} { + rows, err := ds.reader.QueryContext(ctx, countStmt) + if err != nil { + return ctxerr.Wrapf(ctx, err, "read %s counts from host_software", stmtLabel[i]) } + defer rows.Close() - args = append(args, sid, count, updatedAt) - batchCount++ + // use a loop to iterate to prevent loading all in one go in memory, as it + // could get pretty big at >100K hosts with 1000+ software each. Use a write + // batch to prevent making too many single-row inserts. + const batchSize = 100 + var batchCount int + args := make([]interface{}, 0, batchSize*4) + for rows.Next() { + var ( + count int + teamID uint + sid uint + ) - if batchCount == batchSize { - values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") - if _, err := ds.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { - return ctxerr.Wrap(ctx, err, "insert batch into software_host_counts") + if err := rows.Scan(&count, &teamID, &sid); err != nil { + return ctxerr.Wrapf(ctx, err, "scan %s row into variables", stmtLabel[i]) } - args = args[:0] - batchCount = 0 + args = append(args, sid, count, teamID, updatedAt) + batchCount++ + + if batchCount == batchSize { + values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") + if _, err := ds.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { + return ctxerr.Wrapf(ctx, err, "insert %s batch into software_host_counts", stmtLabel[i]) + } + + args = args[:0] + batchCount = 0 + } } - } - if batchCount > 0 { - values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") - if _, err := ds.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { - return ctxerr.Wrap(ctx, err, "insert last batch into software_host_counts") + if batchCount > 0 { + values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") + if _, err := ds.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { + return ctxerr.Wrapf(ctx, err, "insert last %s batch into software_host_counts", stmtLabel[i]) + } } - } - if err := rows.Err(); err != nil { - return ctxerr.Wrap(ctx, err, "iterate over host_software counts") + if err := rows.Err(); err != nil { + return ctxerr.Wrapf(ctx, err, "iterate over %s host_software counts", stmtLabel[i]) + } + rows.Close() } - cleanupStmt := ` - DELETE FROM - software - WHERE - NOT EXISTS ( - SELECT 1 - FROM - software_host_counts shc - WHERE - software.id = shc.software_id AND - shc.hosts_count > 0)` - if _, err := ds.writer.ExecContext(ctx, cleanupStmt); err != nil { + // remove any unused software (global counts = 0) + if _, err := ds.writer.ExecContext(ctx, cleanupSoftwareStmt); err != nil { return ctxerr.Wrap(ctx, err, "delete unused software") } + + // remove any software count row for teams that don't exist anymore + if _, err := ds.writer.ExecContext(ctx, cleanupTeamStmt); err != nil { + return ctxerr.Wrap(ctx, err, "delete software_host_counts for non-existing teams") + } return nil } diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 3028a8dd6f..be326d05ab 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" @@ -578,14 +579,14 @@ func testSoftwareCalculateHostsPerSoftware(t *testing.T, ds *Datastore) { {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, } - require.NoError(t, ds.UpdateHostSoftware(context.Background(), host1.ID, software1)) - require.NoError(t, ds.UpdateHostSoftware(context.Background(), host2.ID, software2)) + require.NoError(t, ds.UpdateHostSoftware(ctx, host1.ID, software1)) + require.NoError(t, ds.UpdateHostSoftware(ctx, host2.ID, software2)) err := ds.CalculateHostsPerSoftware(ctx, time.Now()) require.NoError(t, err) - swOpts := fleet.SoftwareListOptions{WithHostCounts: true, ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} - swCounts := listSoftwareCheckCount(t, ds, 4, 4, swOpts, false) + globalOpts := fleet.SoftwareListOptions{WithHostCounts: true, ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} + globalCounts := listSoftwareCheckCount(t, ds, 4, 4, globalOpts, false) want := []fleet.Software{ {Name: "foo", Version: "0.0.3", HostsCount: 2}, @@ -593,25 +594,25 @@ func testSoftwareCalculateHostsPerSoftware(t *testing.T, ds *Datastore) { {Name: "foo", Version: "v0.0.2", HostsCount: 1}, {Name: "bar", Version: "0.0.3", HostsCount: 1}, } - cmpNameVersionCount(want, swCounts) + cmpNameVersionCount(want, globalCounts) // update host2, remove "bar" software software2 = []fleet.Software{ {Name: "foo", Version: "v0.0.2", Source: "chrome_extensions"}, {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, } - require.NoError(t, ds.UpdateHostSoftware(context.Background(), host2.ID, software2)) + require.NoError(t, ds.UpdateHostSoftware(ctx, host2.ID, software2)) err = ds.CalculateHostsPerSoftware(ctx, time.Now()) require.NoError(t, err) - swCounts = listSoftwareCheckCount(t, ds, 3, 3, swOpts, false) + globalCounts = listSoftwareCheckCount(t, ds, 3, 3, globalOpts, false) want = []fleet.Software{ {Name: "foo", Version: "0.0.3", HostsCount: 2}, {Name: "foo", Version: "0.0.1", HostsCount: 1}, {Name: "foo", Version: "v0.0.2", HostsCount: 1}, } - cmpNameVersionCount(want, swCounts) + cmpNameVersionCount(want, globalCounts) // create a software entry without any host and any counts _, err = ds.writer.ExecContext(ctx, `INSERT INTO software (name, version, source) VALUES ('baz', '0.0.1', 'testing')`) @@ -638,6 +639,127 @@ func testSoftwareCalculateHostsPerSoftware(t *testing.T, ds *Datastore) { {Name: "foo", Version: "v0.0.2", HostsCount: 0}, } cmpNameVersionCount(want, allSw) + + // create 2 teams and assign a new host to each + team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1"}) + require.NoError(t, err) + team2, err := ds.NewTeam(ctx, &fleet.Team{Name: "team2"}) + require.NoError(t, err) + host3 := test.NewHost(t, ds, "host3", "", "host3key", "host3uuid", time.Now()) + require.NoError(t, ds.AddHostsToTeam(ctx, &team1.ID, []uint{host3.ID})) + host4 := test.NewHost(t, ds, "host4", "", "host4key", "host4uuid", time.Now()) + require.NoError(t, ds.AddHostsToTeam(ctx, &team2.ID, []uint{host4.ID})) + + // assign existing host1 to team1 too, so we have a team with multiple hosts + require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID})) + // use some software for host3 and host4 + software3 := []fleet.Software{ + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + } + software4 := []fleet.Software{ + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, + } + require.NoError(t, ds.UpdateHostSoftware(ctx, host3.ID, software3)) + require.NoError(t, ds.UpdateHostSoftware(ctx, host4.ID, software4)) + + // at this point, there's no counts per team, only global counts + globalCounts = listSoftwareCheckCount(t, ds, 3, 3, globalOpts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 2}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + {Name: "foo", Version: "v0.0.2", HostsCount: 1}, + } + cmpNameVersionCount(want, globalCounts) + + team1Opts := fleet.SoftwareListOptions{WithHostCounts: true, TeamID: ptr.Uint(team1.ID), ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} + team1Counts := listSoftwareCheckCount(t, ds, 0, 0, team1Opts, false) + want = []fleet.Software{} + cmpNameVersionCount(want, team1Counts) + + // after a call to Calculate, the global counts are updated and the team counts appear + err = ds.CalculateHostsPerSoftware(ctx, time.Now()) + require.NoError(t, err) + + globalCounts = listSoftwareCheckCount(t, ds, 4, 4, globalOpts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 4}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + {Name: "foo", Version: "v0.0.2", HostsCount: 1}, + {Name: "bar", Version: "0.0.3", HostsCount: 1}, + } + cmpNameVersionCount(want, globalCounts) + + team1Counts = listSoftwareCheckCount(t, ds, 2, 2, team1Opts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 2}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + } + cmpNameVersionCount(want, team1Counts) + + team2Opts := fleet.SoftwareListOptions{WithHostCounts: true, TeamID: ptr.Uint(team2.ID), ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} + team2Counts := listSoftwareCheckCount(t, ds, 2, 2, team2Opts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 1}, + {Name: "bar", Version: "0.0.3", HostsCount: 1}, + } + cmpNameVersionCount(want, team2Counts) + + // update host4 (team2), remove "bar" software + software4 = []fleet.Software{ + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + } + require.NoError(t, ds.UpdateHostSoftware(ctx, host4.ID, software4)) + + err = ds.CalculateHostsPerSoftware(ctx, time.Now()) + require.NoError(t, err) + + globalCounts = listSoftwareCheckCount(t, ds, 3, 3, globalOpts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 4}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + {Name: "foo", Version: "v0.0.2", HostsCount: 1}, + } + cmpNameVersionCount(want, globalCounts) + + team1Counts = listSoftwareCheckCount(t, ds, 2, 2, team1Opts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 2}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + } + cmpNameVersionCount(want, team1Counts) + + team2Counts = listSoftwareCheckCount(t, ds, 1, 1, team2Opts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 1}, + } + cmpNameVersionCount(want, team2Counts) + + // update host4 (team2), remove all software and delete team + software4 = []fleet.Software{} + require.NoError(t, ds.UpdateHostSoftware(ctx, host4.ID, software4)) + require.NoError(t, ds.DeleteTeam(ctx, team2.ID)) + + // this call will remove team2 from the software host counts table + err = ds.CalculateHostsPerSoftware(ctx, time.Now()) + require.NoError(t, err) + + globalCounts = listSoftwareCheckCount(t, ds, 3, 3, globalOpts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 3}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + {Name: "foo", Version: "v0.0.2", HostsCount: 1}, + } + cmpNameVersionCount(want, globalCounts) + + team1Counts = listSoftwareCheckCount(t, ds, 2, 2, team1Opts, false) + want = []fleet.Software{ + {Name: "foo", Version: "0.0.3", HostsCount: 2}, + {Name: "foo", Version: "0.0.1", HostsCount: 1}, + } + cmpNameVersionCount(want, team1Counts) + + listSoftwareCheckCount(t, ds, 0, 0, team2Opts, false) } func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 88c931a67e..cd7d26f89f 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -2,7 +2,9 @@ package mysql import ( "context" + "database/sql" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -10,6 +12,7 @@ import ( "runtime" "strings" "testing" + "text/tabwriter" "time" "github.com/WatchBeam/clock" @@ -337,3 +340,39 @@ func TruncateTables(t testing.TB, ds *Datastore, tables ...string) { return nil })) } + +// this is meant to be used for debugging/testing that statement uses an efficient +// plan (e.g. makes use of an index, avoids full scans, etc.) using the data already +// created for tests. Calls to this function should be temporary and removed when +// done investigating the plan, so it is expected that this function will be detected +// as unused. +func explainSQLStatement(w io.Writer, db sqlx.QueryerContext, stmt string, args ...interface{}) { //nolint:deadcode,unused + var rows []struct { + ID int `db:"id"` + SelectType string `db:"select_type"` + Table sql.NullString `db:"table"` + Partitions sql.NullString `db:"partitions"` + Type sql.NullString `db:"type"` + PossibleKeys sql.NullString `db:"possible_keys"` + Key sql.NullString `db:"key"` + KeyLen sql.NullInt64 `db:"key_len"` + Ref sql.NullString `db:"ref"` + Rows sql.NullInt64 `db:"rows"` + Filtered sql.NullFloat64 `db:"filtered"` + Extra sql.NullString `db:"Extra"` + } + if err := sqlx.SelectContext(context.Background(), db, &rows, "EXPLAIN "+stmt, args...); err != nil { + panic(err) + } + fmt.Fprint(w, "\n\n", strings.Repeat("-", 60), "\n", stmt, "\n", strings.Repeat("-", 60), "\n") + tw := tabwriter.NewWriter(w, 0, 1, 1, ' ', tabwriter.Debug) + + fmt.Fprintln(tw, "id\tselect_type\ttable\tpartitions\ttype\tpossible_keys\tkey\tkey_len\tref\trows\tfiltered\textra") + for _, row := range rows { + fmt.Fprintf(tw, "%d\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\t%d\t%f\t%s\n", row.ID, row.SelectType, row.Table.String, row.Partitions.String, + row.Type.String, row.PossibleKeys.String, row.Key.String, row.KeyLen.Int64, row.Ref.String, row.Rows.Int64, row.Filtered.Float64, row.Extra.String) + } + if err := tw.Flush(); err != nil { + panic(err) + } +} diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 35c9088591..693c1249a3 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -2894,6 +2894,14 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { } } + // create a team and make the last 3 hosts part of it (meaning 3 that use + // sws[19], 2 for sws[18], and 1 for sws[17]) + tm, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + Name: t.Name(), + }) + require.NoError(t, err) + require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &tm.ID, []uint{hosts[19].ID, hosts[18].ID, hosts[17].ID})) + assertResp := func(resp listSoftwareResponse, want []fleet.Software, ts time.Time, counts ...int) { require.Len(t, resp.Software, len(want)) for i := range resp.Software { @@ -2915,6 +2923,11 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, nil, time.Time{}) + // same with a team filter + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) + assertResp(lsResp, nil, time.Time{}) + // calculate hosts counts hostsCountTs := time.Now().UTC() require.NoError(t, s.ds.CalculateHostsPerSoftware(context.Background(), hostsCountTs)) @@ -2968,6 +2981,16 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, nil, time.Time{}) + + // filter by the team, 2 by page + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "0", "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) + assertResp(lsResp, []fleet.Software{sws[19], sws[18]}, hostsCountTs, 3, 2) + + // filter by the team, 2 by page, next page + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "1", "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) + assertResp(lsResp, []fleet.Software{sws[17]}, hostsCountTs, 1) } func (s *integrationTestSuite) TestChangeUserEmail() {