diff --git a/changes/13574-cache-policy-results b/changes/13574-cache-policy-results new file mode 100644 index 0000000000..b1baa11cc5 --- /dev/null +++ b/changes/13574-cache-policy-results @@ -0,0 +1,2 @@ +- policy results are now cached in mysql for faster sort operations on policy counts. counts are +updated by the cleanups_then_aggregation cron job 1X per hour by default. \ No newline at end of file diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 4c4dda15f7..65eb427f9d 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -780,6 +780,12 @@ func newCleanupsAndAggregationSchedule( return ds.UpdateQueryAggregatedStats(ctx) }, ), + schedule.WithJob( + "policy_aggregated_stats", + func(ctx context.Context) error { + return ds.UpdateHostPolicyCounts(ctx) + }, + ), schedule.WithJob( "aggregated_munki_and_mdm", func(ctx context.Context) error { diff --git a/server/datastore/mysql/migrations/tables/20231121054530_CreatePolicyStatsTable.go b/server/datastore/mysql/migrations/tables/20231121054530_CreatePolicyStatsTable.go new file mode 100644 index 0000000000..bdb97731ac --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20231121054530_CreatePolicyStatsTable.go @@ -0,0 +1,45 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20231121054530, Down_20231121054530) +} + +func Up_20231121054530(tx *sql.Tx) error { + stmt := ` + CREATE TABLE policy_stats ( + id int(10) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + policy_id int(10) unsigned NOT NULL, + -- inherited_team_id is used to indicate the row contains inherited + -- global policies counts for a team, otherwise it will be 0. This allows us + -- to use the UNIQUE KEY constraint with this column to avoid duplicate rows + -- when policies.team_id is null. + -- A foreign key constraint is not used here because team 0 is not a valid team + inherited_team_id int(10) unsigned NOT NULL DEFAULT 0, + -- cached counts for the policy / team + passing_host_count MEDIUMINT UNSIGNED NOT NULL DEFAULT 0, + failing_host_count MEDIUMINT UNSIGNED NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + FOREIGN KEY (policy_id) REFERENCES policies(id) ON DELETE CASCADE, + UNIQUE KEY policy_team_unique (policy_id, inherited_team_id) +); +` + + _, err := tx.Exec(stmt) + if err != nil { + return fmt.Errorf("failed to create policy_stats table: %w", err) + } + + return nil +} + +func Down_20231121054530(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20231121054530_CreatePolicyStatsTable_test.go b/server/datastore/mysql/migrations/tables/20231121054530_CreatePolicyStatsTable_test.go new file mode 100644 index 0000000000..d284521c3b --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20231121054530_CreatePolicyStatsTable_test.go @@ -0,0 +1,83 @@ +package tables + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20231121054530(t *testing.T) { + db := applyUpToPrev(t) + + const ( + insertUsersStmt = `INSERT INTO users (id, name, password, salt, email) VALUES (?, ?, ?, ?, ?)` + + insertPolicyStmt = `INSERT INTO policies ( + team_id, name, query, description, author_id, platforms, critical + ) VALUES (?, ?, ?, ?, ?, ?, ?)` + + insertTeamStmt = `INSERT INTO teams (name) VALUES (?)` + + deletePolicyStmt = `DELETE FROM policies WHERE id = ?` + + loadPolicyStatsStmt = `SELECT + id, policy_id, inherited_team_id, passing_host_count, failing_host_count + FROM policy_stats WHERE id = ?` + ) + + // Apply current migration. + applyNext(t, db) + + // Create a user + _, err := db.Exec(insertUsersStmt, 1, "user1", "password999", "salt999", "foo") + require.NoError(t, err) + + // Create a team + res, err := db.Exec(insertTeamStmt, "team1") + require.NoError(t, err) + teamID, _ := res.LastInsertId() + + // Create a global policy + res, err = db.Exec(insertPolicyStmt, nil, "global-policy", "SELECT 1;", "Global policy description", 1, "all", false) + require.NoError(t, err) + globalPolicyStatID, _ := res.LastInsertId() + + // Insert a policy_stats entry for the global policy (globally) + _, err = db.Exec(`INSERT INTO policy_stats (policy_id, inherited_team_id, passing_host_count, failing_host_count) VALUES (?, 0, ?, ?)`, globalPolicyStatID, 100, 10) + require.NoError(t, err) + + // Insert a policy_stats entry for the team inheriting the global policy + _, err = db.Exec(`INSERT INTO policy_stats (policy_id, inherited_team_id, passing_host_count, failing_host_count) VALUES (?, ?, ?, ?)`, globalPolicyStatID, teamID, 50, 5) + require.NoError(t, err) + + // Verify the entries in the policy_stats table + var id int + var policyID int64 + var inheritedTeamID int64 + var passingCount, failingCount int + + // Verify global policy stats (global level) + err = db.QueryRow(loadPolicyStatsStmt, 1).Scan(&id, &policyID, &inheritedTeamID, &passingCount, &failingCount) + require.NoError(t, err) + require.Equal(t, globalPolicyStatID, policyID) + require.Equal(t, int64(0), inheritedTeamID) + + // Verify global policy stats (team level) + err = db.QueryRow(loadPolicyStatsStmt, 2).Scan(&id, &policyID, &inheritedTeamID, &passingCount, &failingCount) + require.NoError(t, err) + require.Equal(t, globalPolicyStatID, policyID) + require.Equal(t, teamID, inheritedTeamID) + + // Verify global policy stats still exist (global level) + err = db.QueryRow(loadPolicyStatsStmt, globalPolicyStatID).Scan(&id, &policyID, &inheritedTeamID, &passingCount, &failingCount) + require.NoError(t, err) + + // Delete the global policy and check that its policy_stats entry is also deleted + _, err = db.Exec(deletePolicyStmt, globalPolicyStatID) + require.NoError(t, err) + + err = db.QueryRow(loadPolicyStatsStmt, globalPolicyStatID).Scan(&id, &policyID, &inheritedTeamID, &passingCount, &failingCount) + require.Error(t, err) + require.ErrorIs(t, err, sql.ErrNoRows) +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 4c1870d5a5..8a74e45498 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -62,10 +62,13 @@ func (ds *Datastore) PolicyByName(ctx context.Context, name string) (*fleet.Poli fmt.Sprint(`SELECT `+policyCols+`, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, - (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, - (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count + COALESCE(ps.passing_host_count, 0) as passing_host_count, + COALESCE(ps.failing_host_count, 0) as failing_host_count FROM policies p LEFT JOIN users u ON p.author_id = u.id + LEFT JOIN policy_stats ps ON p.id = ps.policy_id + AND ((p.team_id IS NULL AND ps.inherited_team_id = 0) + OR (p.team_id IS NOT NULL AND ps.inherited_team_id = p.team_id)) WHERE p.name=?`), name) if err != nil { if err == sql.ErrNoRows { @@ -87,14 +90,17 @@ func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint, teamID *uint) var policy fleet.Policy err := sqlx.GetContext(ctx, q, &policy, fmt.Sprintf(` - SELECT `+policyCols+`, + SELECT %s, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, - (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, - (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count + COALESCE(ps.passing_host_count, 0) as passing_host_count, + COALESCE(ps.failing_host_count, 0) as failing_host_count FROM policies p LEFT JOIN users u ON p.author_id = u.id - WHERE p.id=? AND %s`, teamWhere), + LEFT JOIN policy_stats ps ON p.id = ps.policy_id + AND ((p.team_id IS NULL AND ps.inherited_team_id = 0) + OR (p.team_id IS NOT NULL AND ps.inherited_team_id = p.team_id)) + WHERE p.id=? AND %s`, policyCols, teamWhere), args...) if err != nil { if err == sql.ErrNoRows { @@ -292,112 +298,75 @@ func (ds *Datastore) RecordPolicyQueryExecutions(ctx context.Context, host *flee } func (ds *Datastore) ListGlobalPolicies(ctx context.Context, opts fleet.ListOptions) ([]*fleet.Policy, error) { - return listPoliciesDB(ctx, ds.reader(ctx), nil, nil, opts) + return listPoliciesDB(ctx, ds.reader(ctx), nil, opts) } // returns the list of policies associated with the provided teamID, or the // global policies if teamID is nil. The pass/fail host counts are the totals // regardless of hosts' team if countsForTeamID is nil, or the totals just for // hosts that belong to the provided countsForTeamID if it is not nil. -func listPoliciesDB(ctx context.Context, q sqlx.QueryerContext, teamID, countsForTeamID *uint, opts fleet.ListOptions) ([]*fleet.Policy, error) { +func listPoliciesDB(ctx context.Context, q sqlx.QueryerContext, teamID *uint, opts fleet.ListOptions) ([]*fleet.Policy, error) { var args []interface{} - var initialQuery string - - // Sorting by failing host counts requires an expensive join with the - // policy membership table and may result in long response times - if opts.OrderKey == "failing_host_count" { - if countsForTeamID != nil { - initialQuery = ` - SELECT p.id - FROM policies p - LEFT JOIN ( - SELECT pm.policy_id, - COUNT(*) AS failing_host_count - FROM policy_membership pm - INNER JOIN hosts h ON pm.host_id = h.id AND pm.passes = false AND h.team_id = ? - GROUP BY pm.policy_id - ) AS subq ON p.id = subq.policy_id - ` - args = append(args, *countsForTeamID) - } else { - initialQuery = ` - SELECT p.id - FROM policies p - LEFT JOIN ( - SELECT pm.policy_id, - COUNT(*) AS failing_host_count - FROM policy_membership pm - WHERE pm.passes = false - GROUP BY pm.policy_id - ) AS subq ON p.id = subq.policy_id - ` - } - } else { - initialQuery = "SELECT id FROM policies" - } - - if teamID != nil { - initialQuery += " WHERE team_id = ?" - args = append(args, *teamID) - } else { - initialQuery += " WHERE team_id IS NULL" - } - - initialQuery, args = searchLike(initialQuery, args, opts.MatchQuery, policySearchColumns...) - initialQuery, args = appendListOptionsWithCursorToSQL(initialQuery, args, &opts) - - var ids []uint - err := sqlx.SelectContext(ctx, q, &ids, initialQuery, args...) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "retrieving policy ids") - } - - if len(ids) == 0 { - return []*fleet.Policy{}, nil - } - - args = []interface{}{} // reset args - - counts := ` - (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, - (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count -` - if countsForTeamID != nil { - counts = ` - (select count(*) from policy_membership pm inner join hosts h on pm.host_id = h.id where pm.policy_id=p.id and pm.passes=true and h.team_id = ?) as passing_host_count, - (select count(*) from policy_membership pm inner join hosts h on pm.host_id = h.id where pm.policy_id=p.id and pm.passes=false and h.team_id = ?) as failing_host_count -` - args = append(args, *countsForTeamID, *countsForTeamID) - } - - query := fmt.Sprintf(` - SELECT `+policyCols+`, + query := ` + SELECT ` + policyCols + `, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, - %s + COALESCE(ps.passing_host_count, 0) AS passing_host_count, + COALESCE(ps.failing_host_count, 0) AS failing_host_count FROM policies p LEFT JOIN users u ON p.author_id = u.id - WHERE p.id IN (?)`, counts) + LEFT JOIN policy_stats ps ON p.id = ps.policy_id AND ps.inherited_team_id = 0 + ` - args = append(args, ids) - - query, args, err = sqlx.In(query, args...) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building query to get policies by ID") + if teamID != nil { + query += " WHERE team_id = ?" + args = append(args, *teamID) + } else { + query += " WHERE team_id IS NULL" } - // removing pagination options to avoid double pagination - opts.Page = 0 - opts.PerPage = 0 - + query, args = searchLike(query, args, opts.MatchQuery, policySearchColumns...) query, args = appendListOptionsWithCursorToSQL(query, args, &opts) var policies []*fleet.Policy - err = sqlx.SelectContext(ctx, q, &policies, query, args...) + err := sqlx.SelectContext(ctx, q, &policies, query, args...) if err != nil { return nil, ctxerr.Wrap(ctx, err, "listing policies") } + + return policies, nil +} + +// getInheritedPoliciesForTeam returns the list of global policies with the +// passing and failing host counts for the provided teamID +func getInheritedPoliciesForTeam(ctx context.Context, q sqlx.QueryerContext, TeamID uint, opts fleet.ListOptions) ([]*fleet.Policy, error) { + var args []interface{} + + query := ` + SELECT + ` + policyCols + `, + COALESCE(u.name, '') AS author_name, + COALESCE(u.email, '') AS author_email, + COALESCE(ps.passing_host_count, 0) as passing_host_count, + COALESCE(ps.failing_host_count, 0) as failing_host_count + FROM policies p + LEFT JOIN users u ON p.author_id = u.id + LEFT JOIN policy_stats ps ON p.id = ps.policy_id AND ps.inherited_team_id = ? + WHERE p.team_id IS NULL + ` + + args = append(args, TeamID) + + query, args = searchLike(query, args, opts.MatchQuery, policySearchColumns...) + query, _ = appendListOptionsToSQL(query, &opts) + + var policies []*fleet.Policy + err := sqlx.SelectContext(ctx, q, &policies, query, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "listing inherited policies") + } + return policies, nil } @@ -431,10 +400,13 @@ func (ds *Datastore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fl sql := `SELECT ` + policyCols + `, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, - (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, - (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count + COALESCE(ps.passing_host_count, 0) as passing_host_count, + COALESCE(ps.failing_host_count, 0) as failing_host_count FROM policies p LEFT JOIN users u ON p.author_id = u.id + LEFT JOIN policy_stats ps ON p.id = ps.policy_id + AND ((p.team_id IS NULL AND ps.inherited_team_id = 0) + OR (p.team_id IS NOT NULL AND ps.inherited_team_id = p.team_id)) WHERE p.id IN (?)` query, args, err := sqlx.In(sql, ids) if err != nil { @@ -561,12 +533,12 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u } func (ds *Datastore) ListTeamPolicies(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies, inheritedPolicies []*fleet.Policy, err error) { - teamPolicies, err = listPoliciesDB(ctx, ds.reader(ctx), &teamID, nil, opts) + teamPolicies, err = listPoliciesDB(ctx, ds.reader(ctx), &teamID, opts) if err != nil { return nil, nil, err } // get inherited (global) policies with counts of hosts for that team - inheritedPolicies, err = listPoliciesDB(ctx, ds.reader(ctx), nil, &teamID, iopts) + inheritedPolicies, err = getInheritedPoliciesForTeam(ctx, ds.reader(ctx), teamID, iopts) if err != nil { return nil, nil, err } @@ -1133,3 +1105,56 @@ func amountPolicyViolationDaysDB(ctx context.Context, tx sqlx.QueryerContext) (i return int(counts.FailingHostCount), int(counts.TotalHostCount), nil } + +func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { + // Update Counts for Inherited Global Policies for each Team + _, err := ds.writer(ctx).ExecContext(ctx, ` + INSERT INTO policy_stats (policy_id, inherited_team_id, passing_host_count, failing_host_count) + SELECT + p.id, + t.id AS inherited_team_id, + ( + SELECT COUNT(*) + FROM policy_membership pm + INNER JOIN hosts h ON pm.host_id = h.id + WHERE pm.policy_id = p.id AND pm.passes = true AND h.team_id = t.id + ) AS passing_host_count, + ( + SELECT COUNT(*) + FROM policy_membership pm + INNER JOIN hosts h ON pm.host_id = h.id + WHERE pm.policy_id = p.id AND pm.passes = false AND h.team_id = t.id + ) AS failing_host_count + FROM policies p + CROSS JOIN teams t + WHERE p.team_id IS NULL + GROUP BY p.id, t.id + ON DUPLICATE KEY UPDATE + passing_host_count = VALUES(passing_host_count), + failing_host_count = VALUES(failing_host_count); + `) + if err != nil { + return ctxerr.Wrap(ctx, err, "update host policy counts for inherited global policies") + } + + // Update Counts for Global and Team Policies + _, err = ds.writer(ctx).ExecContext(ctx, ` + INSERT INTO policy_stats (policy_id, inherited_team_id, passing_host_count, failing_host_count) + SELECT + p.id, + 0 AS inherited_team_id, -- using 0 to represent global scope + COALESCE(SUM(IF(pm.passes IS NULL, 0, pm.passes = 1)), 0), + COALESCE(SUM(IF(pm.passes IS NULL, 0, pm.passes = 0)), 0) + FROM policies p + LEFT JOIN policy_membership pm ON p.id = pm.policy_id + GROUP BY p.id + ON DUPLICATE KEY UPDATE + passing_host_count = VALUES(passing_host_count), + failing_host_count = VALUES(failing_host_count); + `) + if err != nil { + return ctxerr.Wrap(ctx, err, "update host policy counts for global and team policies") + } + + return nil +} diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 76506e4381..fe311fdbaf 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -53,6 +53,7 @@ func TestPolicies(t *testing.T) { {"TestListGlobalPoliciesCanPaginate", testListGlobalPoliciesCanPaginate}, {"TestListTeamPoliciesCanPaginate", testListTeamPoliciesCanPaginate}, {"TestCountPolicies", testCountPolicies}, + {"TestUpdatePolicyHostCounts", testUpdatePolicyHostCounts}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -277,6 +278,8 @@ func testPoliciesMembershipView(deferred bool, t *testing.T, ds *Datastore) { require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host2, map[uint]*bool{p2.ID: nil}, time.Now(), deferred)) + require.NoError(t, ds.UpdateHostPolicyCounts(ctx)) + policies, err := ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) require.NoError(t, err) require.Len(t, policies, 2) @@ -292,6 +295,8 @@ func testPoliciesMembershipView(deferred bool, t *testing.T, ds *Datastore) { require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host1, map[uint]*bool{p.ID: ptr.Bool(false)}, time.Now(), deferred)) require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host2, map[uint]*bool{p2.ID: ptr.Bool(false)}, time.Now(), deferred)) + require.NoError(t, ds.UpdateHostPolicyCounts(ctx)) + policies, err = ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) require.NoError(t, err) require.Len(t, policies, 2) @@ -349,6 +354,8 @@ func testPoliciesMembershipView(deferred bool, t *testing.T, ds *Datastore) { require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host4, map[uint]*bool{t2pol.ID: ptr.Bool(false), t2pol2.ID: ptr.Bool(true), p.ID: ptr.Bool(false)}, time.Now(), deferred)) require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host5, map[uint]*bool{t2pol.ID: ptr.Bool(true), t2pol2.ID: ptr.Bool(true), p2.ID: ptr.Bool(true)}, time.Now(), deferred)) + require.NoError(t, ds.UpdateHostPolicyCounts(ctx)) + t1Pols, t1Inherited, err := ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) require.NoError(t, err) require.Len(t, t1Pols, 1) @@ -1028,6 +1035,11 @@ func testPoliciesByID(t *testing.T, ds *Datastore) { user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) policy1 := newTestPolicy(t, ds, user1, "policy1", "darwin", nil) _ = newTestPolicy(t, ds, user1, "policy2", "darwin", nil) + host1 := newTestHostWithPlatform(t, ds, "host1", "darwin", nil) + + require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), host1, map[uint]*bool{policy1.ID: ptr.Bool(true)}, time.Now(), false)) + require.NoError(t, ds.UpdateHostPolicyCounts(context.Background())) + policiesByID, err := ds.PoliciesByID(context.Background(), []uint{1, 2}) require.NoError(t, err) assert.Equal(t, len(policiesByID), 2) @@ -1035,6 +1047,7 @@ func testPoliciesByID(t *testing.T, ds *Datastore) { assert.Equal(t, policiesByID[1].Name, policy1.Name) assert.Equal(t, policiesByID[2].ID, uint(2)) assert.Equal(t, policiesByID[2].Name, "policy2") + assert.Equal(t, uint(1), policiesByID[1].PassingHostCount) _, err = ds.PoliciesByID(context.Background(), []uint{1, 2, 3}) require.Error(t, err) @@ -1100,7 +1113,11 @@ func testTeamPolicyTransfer(t *testing.T, ds *Datastore) { require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host2, map[uint]*bool{team1Policy.ID: ptr.Bool(false), globalPolicy.ID: ptr.Bool(true)}, time.Now(), false)) require.NoError(t, ds.RecordPolicyQueryExecutions(ctx, host2, map[uint]*bool{team1Policy.ID: ptr.Bool(true), globalPolicy.ID: ptr.Bool(true)}, time.Now(), false)) + require.NoError(t, ds.UpdateHostPolicyCounts(ctx)) + checkPassingCount := func(tm1, tm1Inherited, tm2Inherited, global uint) { + t.Helper() + require.NoError(t, ds.UpdateHostPolicyCounts(ctx)) policies, inherited, err := ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) require.NoError(t, err) require.Len(t, policies, 1) @@ -2429,3 +2446,61 @@ func testCountPolicies(t *testing.T, ds *Datastore) { require.NoError(t, err) assert.Equal(t, 10, globalCount) } + +func testUpdatePolicyHostCounts(t *testing.T, ds *Datastore) { + // new policy + policy, err := ds.NewGlobalPolicy(context.Background(), nil, fleet.PolicyPayload{Name: "global policy 1"}) + require.NoError(t, err) + + team, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + // create 4 team hosts + var teamHosts []*fleet.Host + for i := 0; i < 4; i++ { + h, err := ds.NewHost(context.Background(), &fleet.Host{OsqueryHostID: ptr.String(fmt.Sprintf("host%d", i)), NodeKey: ptr.String(fmt.Sprintf("host%d", i)), TeamID: &team.ID}) + require.NoError(t, err) + teamHosts = append(teamHosts, h) + } + + // create 4 global hosts + var globalHosts []*fleet.Host + for i := 4; i < 8; i++ { + h, err := ds.NewHost(context.Background(), &fleet.Host{OsqueryHostID: ptr.String(fmt.Sprintf("host%d", i)), NodeKey: ptr.String(fmt.Sprintf("host%d", i)), TeamID: nil}) + require.NoError(t, err) + globalHosts = append(globalHosts, h) + } + + // add policy responses + for _, h := range teamHosts { + res := map[uint]*bool{ + policy.ID: ptr.Bool(true), + } + err = ds.RecordPolicyQueryExecutions(context.Background(), h, res, time.Now(), false) + require.NoError(t, err) + } + + for _, h := range globalHosts { + res := map[uint]*bool{ + policy.ID: ptr.Bool(true), + } + err = ds.RecordPolicyQueryExecutions(context.Background(), h, res, time.Now(), false) + require.NoError(t, err) + } + + // check policy host counts before update + policy, err = ds.Policy(context.Background(), policy.ID) + require.NoError(t, err) + require.Equal(t, uint(0), policy.FailingHostCount) + require.Equal(t, uint(0), policy.PassingHostCount) + + // update policy host counts + err = ds.UpdateHostPolicyCounts(context.Background()) + require.NoError(t, err) + + // check policy host counts + policy, err = ds.Policy(context.Background(), policy.ID) + require.NoError(t, err) + require.Equal(t, uint(0), policy.FailingHostCount) + require.Equal(t, uint(8), policy.PassingHostCount) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index c3d2472a06..27ff6bbab2 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -722,9 +722,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=222 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=223 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1069,6 +1069,21 @@ CREATE TABLE `policy_membership` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `policy_stats` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `policy_id` int(10) unsigned NOT NULL, + `inherited_team_id` int(10) unsigned NOT NULL DEFAULT '0', + `passing_host_count` mediumint(8) unsigned NOT NULL DEFAULT '0', + `failing_host_count` mediumint(8) unsigned NOT NULL DEFAULT '0', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `policy_team_unique` (`policy_id`,`inherited_team_id`), + CONSTRAINT `policy_stats_ibfk_1` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `queries` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index f086026f60..e8b3788935 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -529,6 +529,7 @@ type Datastore interface { PoliciesByID(ctx context.Context, ids []uint) (map[uint]*Policy, error) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) CountPolicies(ctx context.Context, teamID *uint, matchQuery string) (int, error) + UpdateHostPolicyCounts(ctx context.Context) error PolicyQueriesForHost(ctx context.Context, host *Host) (map[string]string, error) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 21224f0d80..31e1f24a6a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -392,6 +392,8 @@ type DeleteGlobalPoliciesFunc func(ctx context.Context, ids []uint) ([]uint, err type CountPoliciesFunc func(ctx context.Context, teamID *uint, matchQuery string) (int, error) +type UpdateHostPolicyCountsFunc func(ctx context.Context) error + type PolicyQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error) type AsyncBatchInsertPolicyMembershipFunc func(ctx context.Context, batch []fleet.PolicyMembershipResult) error @@ -1314,6 +1316,9 @@ type DataStore struct { CountPoliciesFunc CountPoliciesFunc CountPoliciesFuncInvoked bool + UpdateHostPolicyCountsFunc UpdateHostPolicyCountsFunc + UpdateHostPolicyCountsFuncInvoked bool + PolicyQueriesForHostFunc PolicyQueriesForHostFunc PolicyQueriesForHostFuncInvoked bool @@ -3166,6 +3171,13 @@ func (s *DataStore) CountPolicies(ctx context.Context, teamID *uint, matchQuery return s.CountPoliciesFunc(ctx, teamID, matchQuery) } +func (s *DataStore) UpdateHostPolicyCounts(ctx context.Context) error { + s.mu.Lock() + s.UpdateHostPolicyCountsFuncInvoked = true + s.mu.Unlock() + return s.UpdateHostPolicyCountsFunc(ctx) +} + func (s *DataStore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { s.mu.Lock() s.PolicyQueriesForHostFuncInvoked = true