From 592a7450e3b9a5d9650d19bffbdaf7eb5b6fd599 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 4 Mar 2024 12:35:27 -0600 Subject: [PATCH] Enabling setting host status webhook at the team level via REST API and fleetctl apply/gitops. (#17186) Enabling setting host status webhook at the team level via REST API and fleetctl apply/gitops. #14916 Example payload: ```json { "data": { "days_unseen": 3, "host_ids": [ 10724, 10726, 10738, 10739, 10740, 10741, 10742, 10744, 10745, 10746, 10747, 10748, 10749 ], "team_id": 3, "total_hosts": 15, "unseen_hosts": 13 }, "text": "More than 86.67% of your hosts have not checked into Fleet for more than 3 days. You've been sent this message because the Host status webhook is enabled in your Fleet instance." } ``` # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- changes/17094-per-team-host-status-webhook | 1 + cmd/fleet/serve_test.go | 4 +- cmd/fleetctl/apply_test.go | 54 ++++++++++ cmd/fleetctl/gitops_test.go | 35 ++++++ .../testdata/expectedGetTeamsJson.json | 12 +++ .../testdata/expectedGetTeamsYaml.yml | 4 + .../testdata/gitops/team_config_no_paths.yml | 9 +- .../macosSetupExpectedTeam1And2Empty.yml | 4 + .../macosSetupExpectedTeam1And2Set.yml | 4 + .../testdata/macosSetupExpectedTeam1Empty.yml | 2 + ee/server/service/teams.go | 43 +++++--- server/datastore/mysql/hosts.go | 47 ++++---- server/datastore/mysql/hosts_test.go | 48 ++++++--- server/fleet/datastore.go | 2 +- server/fleet/teams.go | 18 ++-- server/mock/datastore_mock.go | 6 +- server/service/client.go | 12 +++ server/service/integration_enterprise_test.go | 6 ++ server/webhooks/host_status.go | 62 +++++++++-- server/webhooks/host_status_test.go | 100 ++++++++++++++++-- 20 files changed, 398 insertions(+), 75 deletions(-) create mode 100644 changes/17094-per-team-host-status-webhook diff --git a/changes/17094-per-team-host-status-webhook b/changes/17094-per-team-host-status-webhook new file mode 100644 index 0000000000..56eaf2dd73 --- /dev/null +++ b/changes/17094-per-team-host-status-webhook @@ -0,0 +1 @@ +Enabling setting host status webhook at the team level via REST API and fleetctl apply/gitops. diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index 7dd690966b..0da6a384ed 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -255,7 +255,7 @@ func TestAutomationsSchedule(t *testing.T) { calledOnce := make(chan struct{}) calledTwice := make(chan struct{}) - ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, daysCount int) (int, int, error) { + ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, teamID *uint, daysCount int) (int, []uint, error) { defer func() { select { case <-calledOnce: @@ -268,7 +268,7 @@ func TestAutomationsSchedule(t *testing.T) { close(calledOnce) } }() - return 10, 6, nil + return 10, []uint{1, 2, 3, 4, 5, 6}, nil } ctx, cancelFunc := context.WithCancel(context.Background()) diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 3de289565a..b2584ae524 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -385,6 +385,60 @@ spec: assert.Equal(t, newMDMSettings, teamsByName["team1"].Config.MDM) // enroll secret not cleared since not provided assert.Equal(t, []*fleet.EnrollSecret{{Secret: "BBB"}}, enrolledSecretsCalled[uint(42)]) + + // Apply team host_status_webhook + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + webhook_settings: + host_status_webhook: + days_count: 14 + destination_url: https://example.com + enable_host_status_webhook: true + host_percentage: 25 +`, + ) + + require.Equal(t, "[+] applied 1 teams\n", runAppForTest(t, []string{"apply", "-f", filename})) + // Ensure the webhook settings are applied + assert.Equal( + t, fleet.HostStatusWebhookSettings{ + DaysCount: 14, + DestinationURL: "https://example.com", + Enable: true, + HostPercentage: 25, + }, teamsByName["team1"].Config.WebhookSettings.HostStatusWebhook, + ) + assert.Equal(t, fleet.FailingPoliciesWebhookSettings{}, teamsByName["team1"].Config.WebhookSettings.FailingPoliciesWebhook) + // enroll secret not cleared since not provided + assert.Equal(t, []*fleet.EnrollSecret{{Secret: "BBB"}}, enrolledSecretsCalled[uint(42)]) + + // Apply empty webhook settings + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + webhook_settings: +`, + ) + + require.Equal(t, "[+] applied 1 teams\n", runAppForTest(t, []string{"apply", "-f", filename})) + // Ensure the webhook settings have not changed + assert.Equal( + t, fleet.HostStatusWebhookSettings{ + DaysCount: 14, + DestinationURL: "https://example.com", + Enable: true, + HostPercentage: 25, + }, teamsByName["team1"].Config.WebhookSettings.HostStatusWebhook, + ) } func writeTmpYml(t *testing.T, contents string) string { diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 6c2f033cf4..4f77c2b112 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -514,4 +514,39 @@ func TestFullTeamGitOps(t *testing.T) { assert.Len(t, appliedScripts, 1) assert.Len(t, appliedMacProfiles, 1) assert.Len(t, appliedWinProfiles, 1) + assert.True(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) + assert.Equal(t, "https://example.com/host_status_webhook", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) + + // Now clear the settings + tmpFile, err := os.CreateTemp(t.TempDir(), "*.yml") + require.NoError(t, err) + secret := "TestSecret" + t.Setenv("TEST_SECRET", secret) + + _, err = tmpFile.WriteString( + ` +controls: +queries: +policies: +agent_options: +name: ${TEST_TEAM_NAME} +team_settings: + secrets: [{"secret":"${TEST_SECRET}"}] +`, + ) + require.NoError(t, err) + + // Dry run + savedTeam = nil + _ = runAppForTest(t, []string{"gitops", "-f", tmpFile.Name(), "--dry-run"}) + assert.Nil(t, savedTeam) + + // Real run + _ = runAppForTest(t, []string{"gitops", "-f", tmpFile.Name()}) + require.NotNil(t, savedTeam) + assert.Equal(t, teamName, savedTeam.Name) + require.Len(t, enrolledSecrets, 1) + assert.Equal(t, secret, enrolledSecrets[0].Secret) + assert.False(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) + assert.Equal(t, "", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) } diff --git a/cmd/fleetctl/testdata/expectedGetTeamsJson.json b/cmd/fleetctl/testdata/expectedGetTeamsJson.json index 1ef05fc823..3fabfffecb 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsJson.json +++ b/cmd/fleetctl/testdata/expectedGetTeamsJson.json @@ -12,6 +12,12 @@ "host_expiry_window": 0 }, "webhook_settings": { + "host_status_webhook": { + "enable_host_status_webhook": false, + "destination_url": "", + "host_percentage": 0, + "days_count": 0 + }, "failing_policies_webhook": { "enable_failing_policies_webhook": false, "destination_url": "", @@ -81,6 +87,12 @@ "host_expiry_window": 15 }, "webhook_settings": { + "host_status_webhook": { + "enable_host_status_webhook": false, + "destination_url": "", + "host_percentage": 0, + "days_count": 0 + }, "failing_policies_webhook": { "enable_failing_policies_webhook": false, "destination_url": "", diff --git a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml index c81c5ebfa3..1249b3e5fd 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml @@ -26,6 +26,8 @@ spec: enable_end_user_authentication: false macos_setup_assistant: scripts: null + webhook_settings: + host_status_webhook: null name: team1 --- apiVersion: v1 @@ -64,4 +66,6 @@ spec: enable_end_user_authentication: false macos_setup_assistant: scripts: null + webhook_settings: + host_status_webhook: null name: team2 diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 6a4fbdf2de..3295e75bbb 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -4,10 +4,11 @@ team_settings: - secret: "SampleSecret123" - secret: "ABC" webhook_settings: - failing_policies_webhook: - enable_failing_policies_webhook: true - destination_url: https://example.tines.com/webhook - policy_ids: [1, 2, 3, 4, 5, 6 ,7, 8, 9] + host_status_webhook: + days_count: 14 + destination_url: https://example.com/host_status_webhook + enable_host_status_webhook: true + host_percentage: 25 features: enable_host_users: true enable_software_inventory: true diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml index 2eefecaf5c..7315325b48 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml @@ -26,6 +26,8 @@ spec: deadline_days: null grace_period_days: null scripts: null + webhook_settings: + host_status_webhook: null name: tm1 --- apiVersion: v1 @@ -54,4 +56,6 @@ spec: deadline_days: null grace_period_days: null scripts: null + webhook_settings: + host_status_webhook: null name: tm2 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml index 50069b989d..1cce56630c 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml @@ -26,6 +26,8 @@ spec: deadline_days: null grace_period_days: null scripts: null + webhook_settings: + host_status_webhook: null name: tm1 --- apiVersion: v1 @@ -54,4 +56,6 @@ spec: deadline_days: null grace_period_days: null scripts: null + webhook_settings: + host_status_webhook: null name: tm2 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml index fdb20cddfe..c6e8b1653b 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml @@ -26,5 +26,7 @@ spec: windows_settings: custom_settings: null scripts: null + webhook_settings: + host_status_webhook: null name: tm1 diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index a194e78b6d..a615012568 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -869,24 +869,33 @@ func (svc *Service) createTeamFromSpec( } } + invalid := &fleet.InvalidArgumentError{} if enableDiskEncryption && !appCfg.MDM.AtLeastOnePlatformEnabledAndConfigured() { - return nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError("mdm", - `Couldn't edit enable_disk_encryption. Neither macOS MDM nor Windows is turned on. Visit https://fleetdm.com/docs/using-fleet to learn how to turn on MDM.`)) + invalid.Append( + "mdm", + `Couldn't edit enable_disk_encryption. Neither macOS MDM nor Windows is turned on. Visit https://fleetdm.com/docs/using-fleet to learn how to turn on MDM.`, + ) } var hostExpirySettings fleet.HostExpirySettings if spec.HostExpirySettings != nil { if spec.HostExpirySettings.HostExpiryEnabled && spec.HostExpirySettings.HostExpiryWindow <= 0 { - return nil, ctxerr.Wrap( - ctx, fleet.NewInvalidArgumentError( - "host_expiry_settings.host_expiry_window", - `When enabling host expiry, host expiry window must be a positive number.`, - ), + invalid.Append( + "host_expiry_settings.host_expiry_window", "When enabling host expiry, host expiry window must be a positive number.", ) } hostExpirySettings = *spec.HostExpirySettings } + hostStatusWebhook := fleet.HostStatusWebhookSettings{} + if spec.WebhookSettings.HostStatusWebhook != nil { + fleet.ValidateEnabledHostStatusIntegrations(*spec.WebhookSettings.HostStatusWebhook, invalid) + hostStatusWebhook = *spec.WebhookSettings.HostStatusWebhook + } + if invalid.HasErrors() { + return nil, ctxerr.Wrap(ctx, invalid) + } + if dryRun { return &fleet.Team{Name: spec.Name}, nil } @@ -904,6 +913,9 @@ func (svc *Service) createTeamFromSpec( MacOSSetup: macOSSetup, }, HostExpirySettings: hostExpirySettings, + WebhookSettings: fleet.TeamWebhookSettings{ + HostStatusWebhook: hostStatusWebhook, + }, }, Secrets: secrets, }) @@ -1041,18 +1053,25 @@ func (svc *Service) editTeamFromSpec( } // if host_expiry_settings are not provided, do not change them + invalid := &fleet.InvalidArgumentError{} if spec.HostExpirySettings != nil { if spec.HostExpirySettings.HostExpiryEnabled && spec.HostExpirySettings.HostExpiryWindow <= 0 { - return ctxerr.Wrap( - ctx, fleet.NewInvalidArgumentError( - "host_expiry_settings.host_expiry_window", - `When enabling host expiry, host expiry window must be a positive number.`, - ), + invalid.Append( + "host_expiry_settings.host_expiry_window", "When enabling host expiry, host expiry window must be a positive number.", ) } team.Config.HostExpirySettings = *spec.HostExpirySettings } + // If host status webhook is not provided, do not change it + if spec.WebhookSettings.HostStatusWebhook != nil { + fleet.ValidateEnabledHostStatusIntegrations(*spec.WebhookSettings.HostStatusWebhook, invalid) + team.Config.WebhookSettings.HostStatusWebhook = *spec.WebhookSettings.HostStatusWebhook + } + if invalid.HasErrors() { + return ctxerr.Wrap(ctx, invalid) + } + if dryRun { return nil } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index fada41f8d9..cd41d5e607 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -2819,30 +2819,41 @@ func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, users return nil } -func (ds *Datastore) TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error) { - var counts struct { - Total int `db:"total"` - Unseen int `db:"unseen"` +func (ds *Datastore) TotalAndUnseenHostsSince(ctx context.Context, teamID *uint, daysCount int) (total int, unseen []uint, err error) { + // convert daysCount to integer number of seconds for more precision in sql query + var args []interface{} + totalQuery := `SELECT COUNT(*) FROM hosts` + if teamID != nil { + totalQuery += " WHERE team_id = ?" + args = append(args, *teamID) } - // convert daysCount to integer number of seconds for more precision in sql query - unseenSeconds := daysCount * 24 * 60 * 60 - - err = sqlx.GetContext(ctx, ds.reader(ctx), &counts, - `SELECT - COUNT(*) as total, - SUM(IF(TIMESTAMPDIFF(SECOND, COALESCE(hst.seen_time, h.created_at), CURRENT_TIMESTAMP) >= ?, 1, 0)) as unseen - FROM hosts h - LEFT JOIN host_seen_times hst - ON h.id = hst.host_id`, - unseenSeconds, - ) + err = sqlx.GetContext(ctx, ds.reader(ctx), &total, totalQuery, args...) if err != nil { - return 0, 0, ctxerr.Wrap(ctx, err, "getting total and unseen host counts") + return 0, nil, ctxerr.Wrap(ctx, err, "getting total host counts") } - return counts.Total, counts.Unseen, nil + unseenSeconds := daysCount * 24 * 60 * 60 + args = []interface{}{unseenSeconds} + unseenQuery := `SELECT id + FROM hosts h + LEFT JOIN host_seen_times hst + ON h.id = hst.host_id + WHERE TIMESTAMPDIFF(SECOND, COALESCE(hst.seen_time, h.created_at), CURRENT_TIMESTAMP) >= ?` + + if teamID != nil { + unseenQuery += " AND team_id = ?" + args = append(args, *teamID) + } + + err = sqlx.SelectContext(ctx, ds.reader(ctx), &unseen, unseenQuery, args...) + + if err != nil { + return total, nil, ctxerr.Wrap(ctx, err, "getting unseen host counts") + } + + return } func (ds *Datastore) DeleteHosts(ctx context.Context, ids []uint) error { diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 8790daf494..fa699ecdc6 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -2726,7 +2726,7 @@ func testHostsSaveUsersWithoutUid(t *testing.T, ds *Datastore) { assert.Equal(t, host.Users[0].Uid, u2.Uid) } -func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { +func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) *fleet.Host { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -2741,41 +2741,61 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { }) require.NoError(t, err) require.NotNil(t, host) + return host } func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { - addHostSeenLast(t, ds, 1, 0) + host1 := addHostSeenLast(t, ds, 1, 0) - total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1) + total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), nil, 1) require.NoError(t, err) assert.Equal(t, 1, total) - assert.Equal(t, 0, unseen) + assert.Len(t, unseen, 0) - addHostSeenLast(t, ds, 2, 2) - addHostSeenLast(t, ds, 3, 4) + host2 := addHostSeenLast(t, ds, 2, 2) + host3 := addHostSeenLast(t, ds, 3, 4) - total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), 1) + total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), nil, 1) require.NoError(t, err) assert.Equal(t, 3, total) - assert.Equal(t, 2, unseen) + assert.Len(t, unseen, 2) // host not counted as unseen if less than a full 24 hours has passed _, err = ds.writer(context.Background()).ExecContext(context.Background(), `UPDATE host_seen_times SET seen_time = ? WHERE host_id = 2`, time.Now().Add(-1*time.Duration(1)*86399*time.Second)) require.NoError(t, err) - total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), 1) + total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), nil, 1) require.NoError(t, err) assert.Equal(t, 3, total) - assert.Equal(t, 1, unseen) + assert.Len(t, unseen, 1) // host counted as unseen if more than 24 hours has passed _, err = ds.writer(context.Background()).ExecContext(context.Background(), `UPDATE host_seen_times SET seen_time = ? WHERE host_id = 2`, time.Now().Add(-1*time.Duration(1)*86401*time.Second)) require.NoError(t, err) - total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), 1) + total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), nil, 1) require.NoError(t, err) assert.Equal(t, 3, total) - assert.Equal(t, 2, unseen) + require.Len(t, unseen, 2) + assert.Equal(t, host2.ID, unseen[0]) + assert.Equal(t, host3.ID, unseen[1]) + + // Test team hosts + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), &team1.ID, 1) + require.NoError(t, err) + assert.Equal(t, 0, total) + assert.Len(t, unseen, 0) + + require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID, host3.ID})) + total, unseen, err = ds.TotalAndUnseenHostsSince(context.Background(), &team1.ID, 1) + require.NoError(t, err) + assert.Equal(t, 2, total) + require.Len(t, unseen, 1) + assert.Equal(t, host3.ID, unseen[0]) + } func testHostsListByPolicy(t *testing.T, ds *Datastore) { @@ -4920,10 +4940,10 @@ func testHostsNoSeenTime(t *testing.T, ds *Datastore) { require.Equal(t, h1.ID, foundHosts[1].ID) require.Equal(t, foundHosts[1].SeenTime, foundHosts[1].CreatedAt) - total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1) + total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), nil, 1) require.NoError(t, err) require.Equal(t, total, 2) - require.Equal(t, unseen, 0) + require.Len(t, unseen, 0) h3, err := ds.NewHost(context.Background(), &fleet.Host{ ID: 3, diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 30d9b0a42f..2105310605 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -272,7 +272,7 @@ type Datastore interface { // AddHostsToTeam adds hosts to an existing team, clearing their team settings if teamID is nil. AddHostsToTeam(ctx context.Context, teamID *uint, hostIDs []uint) error - TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error) + TotalAndUnseenHostsSince(ctx context.Context, teamID *uint, daysCount int) (total int, unseen []uint, err error) // DeleteHosts deletes associated tables for multiple hosts. // diff --git a/server/fleet/teams.go b/server/fleet/teams.go index f0e124ab14..110bddb718 100644 --- a/server/fleet/teams.go +++ b/server/fleet/teams.go @@ -148,6 +148,7 @@ type TeamConfig struct { } type TeamWebhookSettings struct { + HostStatusWebhook HostStatusWebhookSettings `json:"host_status_webhook"` FailingPoliciesWebhook FailingPoliciesWebhookSettings `json:"failing_policies_webhook"` } @@ -395,12 +396,17 @@ type TeamSpec struct { // If the agent_options key is present but empty in the YAML, will be set to // "null" (JSON null). Otherwise, if the key is present and set, it will be // set to the agent options JSON object. - AgentOptions json.RawMessage `json:"agent_options,omitempty"` // marshals as "null" if omitempty is not set - HostExpirySettings *HostExpirySettings `json:"host_expiry_settings,omitempty"` - Secrets []EnrollSecret `json:"secrets,omitempty"` - Features *json.RawMessage `json:"features"` - MDM TeamSpecMDM `json:"mdm"` - Scripts optjson.Slice[string] `json:"scripts"` + AgentOptions json.RawMessage `json:"agent_options,omitempty"` // marshals as "null" if omitempty is not set + HostExpirySettings *HostExpirySettings `json:"host_expiry_settings,omitempty"` + Secrets []EnrollSecret `json:"secrets,omitempty"` + Features *json.RawMessage `json:"features"` + MDM TeamSpecMDM `json:"mdm"` + Scripts optjson.Slice[string] `json:"scripts"` + WebhookSettings TeamSpecWebhookSettings `json:"webhook_settings"` +} + +type TeamSpecWebhookSettings struct { + HostStatusWebhook *HostStatusWebhookSettings `json:"host_status_webhook"` } // TeamSpecFromTeam returns a TeamSpec constructed from the given Team. diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 81ace3f297..621da9766d 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -200,7 +200,7 @@ type HostLiteByIDFunc func(ctx context.Context, id uint) (*fleet.HostLite, error type AddHostsToTeamFunc func(ctx context.Context, teamID *uint, hostIDs []uint) error -type TotalAndUnseenHostsSinceFunc func(ctx context.Context, daysCount int) (total int, unseen int, err error) +type TotalAndUnseenHostsSinceFunc func(ctx context.Context, teamID *uint, daysCount int) (total int, unseen []uint, err error) type DeleteHostsFunc func(ctx context.Context, ids []uint) error @@ -2744,11 +2744,11 @@ func (s *DataStore) AddHostsToTeam(ctx context.Context, teamID *uint, hostIDs [] return s.AddHostsToTeamFunc(ctx, teamID, hostIDs) } -func (s *DataStore) TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error) { +func (s *DataStore) TotalAndUnseenHostsSince(ctx context.Context, teamID *uint, daysCount int) (total int, unseen []uint, err error) { s.mu.Lock() s.TotalAndUnseenHostsSinceFuncInvoked = true s.mu.Unlock() - return s.TotalAndUnseenHostsSinceFunc(ctx, daysCount) + return s.TotalAndUnseenHostsSinceFunc(ctx, teamID, daysCount) } func (s *DataStore) DeleteHosts(ctx context.Context, ids []uint) error { diff --git a/server/service/client.go b/server/service/client.go index a5ea7e4dab..eeaa245c7b 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -913,6 +913,18 @@ func (c *Client) DoGitOps( } team["scripts"] = scripts team["secrets"] = config.TeamSettings["secrets"] + team["webhook_settings"] = map[string]interface{}{} + clearHostStatusWebhook := true + if webhookSettings, ok := config.TeamSettings["webhook_settings"]; ok { + if hostStatusWebhook, ok := webhookSettings.(map[string]interface{})["host_status_webhook"]; ok { + clearHostStatusWebhook = false + team["webhook_settings"].(map[string]interface{})["host_status_webhook"] = hostStatusWebhook + } + } + if clearHostStatusWebhook { + // Clear out any existing host_status_webhook settings + team["webhook_settings"].(map[string]interface{})["host_status_webhook"] = map[string]interface{}{} + } team["mdm"] = map[string]interface{}{} mdmAppConfig = team["mdm"].(map[string]interface{}) } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 8d6047cb41..142b216ae3 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -1315,9 +1315,15 @@ func (s *integrationEnterpriseTestSuite) TestExternalIntegrationsTeamConfig() { Enable: true, DestinationURL: "http://example.com", }, + HostStatusWebhook: fleet.HostStatusWebhookSettings{ + Enable: true, + DestinationURL: "http://example.com/host_status_webhook", + }, }}, http.StatusOK, &tmResp) require.True(t, tmResp.Team.Config.WebhookSettings.FailingPoliciesWebhook.Enable) require.Equal(t, "http://example.com", tmResp.Team.Config.WebhookSettings.FailingPoliciesWebhook.DestinationURL) + require.True(t, tmResp.Team.Config.WebhookSettings.HostStatusWebhook.Enable) + require.Equal(t, "http://example.com/host_status_webhook", tmResp.Team.Config.WebhookSettings.HostStatusWebhook.DestinationURL) // add an unknown automation - does not exist at the global level s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", team.ID), fleet.TeamPayload{Integrations: &fleet.TeamIntegrations{ diff --git a/server/webhooks/host_status.go b/server/webhooks/host_status.go index 37bb453040..5900dcddc7 100644 --- a/server/webhooks/host_status.go +++ b/server/webhooks/host_status.go @@ -3,12 +3,12 @@ package webhooks import ( "context" "fmt" - "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/hashicorp/go-multierror" ) func TriggerHostStatusWebhook( @@ -16,6 +16,13 @@ func TriggerHostStatusWebhook( ds fleet.Datastore, logger kitlog.Logger, ) error { + multiErr := &multierror.Error{} + multiErr = multierror.Append(multiErr, triggerGlobalHostStatusWebhook(ctx, ds, logger)) + multiErr = multierror.Append(multiErr, triggerTeamHostStatusWebhook(ctx, ds, logger)) + return multiErr.ErrorOrNil() +} + +func triggerGlobalHostStatusWebhook(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error { appConfig, err := ds.AppConfig(ctx) if err != nil { return ctxerr.Wrap(ctx, err, "getting app config") @@ -25,30 +32,39 @@ func TriggerHostStatusWebhook( return nil } - level.Debug(logger).Log("enabled", "true") + level.Debug(logger).Log("global", "true", "enable_host_status_webhook", "true") - total, unseen, err := ds.TotalAndUnseenHostsSince(ctx, appConfig.WebhookSettings.HostStatusWebhook.DaysCount) + return processWebhook(ctx, ds, nil, appConfig.WebhookSettings.HostStatusWebhook) +} + +func processWebhook(ctx context.Context, ds fleet.Datastore, teamID *uint, settings fleet.HostStatusWebhookSettings) error { + total, unseen, err := ds.TotalAndUnseenHostsSince(ctx, teamID, settings.DaysCount) if err != nil { return ctxerr.Wrap(ctx, err, "getting total and unseen hosts") } - percentUnseen := float64(unseen) * 100.0 / float64(total) - if percentUnseen >= appConfig.WebhookSettings.HostStatusWebhook.HostPercentage { - url := appConfig.WebhookSettings.HostStatusWebhook.DestinationURL + unseenCount := len(unseen) + percentUnseen := float64(unseenCount) * 100.0 / float64(total) + if percentUnseen >= settings.HostPercentage { + url := settings.DestinationURL message := fmt.Sprintf( "More than %.2f%% of your hosts have not checked into Fleet for more than %d days. "+ "You've been sent this message because the Host status webhook is enabled in your Fleet instance.", - percentUnseen, appConfig.WebhookSettings.HostStatusWebhook.DaysCount, + percentUnseen, settings.DaysCount, ) payload := map[string]interface{}{ "text": message, "data": map[string]interface{}{ - "unseen_hosts": unseen, + "unseen_hosts": unseenCount, "total_hosts": total, - "days_unseen": appConfig.WebhookSettings.HostStatusWebhook.DaysCount, + "days_unseen": settings.DaysCount, + "host_ids": unseen, }, } + if teamID != nil { + payload["data"].(map[string]interface{})["team_id"] = *teamID + } err = server.PostJSONWithTimeout(ctx, url, &payload) if err != nil { @@ -58,3 +74,31 @@ func TriggerHostStatusWebhook( return nil } + +func triggerTeamHostStatusWebhook(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error { + + teams, err := ds.TeamsSummary(ctx) + if err != nil { + return ctxerr.Wrap(ctx, err, "getting teams summary") + } + // We try to send a webhook for each team. If one team fails, we continue. + multiErr := &multierror.Error{} + for _, teamSummary := range teams { + id := teamSummary.ID + team, err := ds.Team(ctx, id) + if err != nil { + multiErr = multierror.Append(multiErr, ctxerr.Wrap(ctx, err, "getting team")) + continue + } + if !team.Config.WebhookSettings.HostStatusWebhook.Enable { + continue + } + level.Debug(logger).Log("team", id, "enable_host_status_webhook", "true") + err = processWebhook(ctx, ds, &id, team.Config.WebhookSettings.HostStatusWebhook) + if err != nil { + multiErr = multierror.Append(multiErr, ctxerr.Wrap(ctx, err, "processing webhook")) + } + } + + return multiErr.ErrorOrNil() +} diff --git a/server/webhooks/host_status_test.go b/server/webhooks/host_status_test.go index d1da424f36..2f15bfc405 100644 --- a/server/webhooks/host_status_test.go +++ b/server/webhooks/host_status_test.go @@ -18,11 +18,13 @@ func TestTriggerHostStatusWebhook(t *testing.T) { ds := new(mock.Store) requestBody := "" + count := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestBodyBytes, err := io.ReadAll(r.Body) require.NoError(t, err) requestBody = string(requestBodyBytes) + count++ })) defer ts.Close() @@ -41,24 +43,110 @@ func TestTriggerHostStatusWebhook(t *testing.T) { return ac, nil } - ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, daysCount int) (int, int, error) { + ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, teamID *uint, daysCount int) (int, []uint, error) { assert.Equal(t, 2, daysCount) - return 10, 6, nil + return 10, []uint{1, 2, 3, 4, 5, 6}, nil + } + + ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) { + return nil, nil } require.NoError(t, TriggerHostStatusWebhook(context.Background(), ds, kitlog.NewNopLogger())) assert.Equal( t, - `{"data":{"days_unseen":2,"total_hosts":10,"unseen_hosts":6},"text":"More than 60.00% of your hosts have not checked into Fleet for more than 2 days. You've been sent this message because the Host status webhook is enabled in your Fleet instance."}`, + `{"data":{"days_unseen":2,"host_ids":[1,2,3,4,5,6],"total_hosts":10,"unseen_hosts":6},"text":"More than 60.00% of your hosts have not checked into Fleet for more than 2 days. You've been sent this message because the Host status webhook is enabled in your Fleet instance."}`, requestBody, ) - requestBody = "" + assert.Equal(t, 1, count) - ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, daysCount int) (int, int, error) { + requestBody = "" + ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, teamID *uint, daysCount int) (int, []uint, error) { assert.Equal(t, 2, daysCount) - return 10, 1, nil + return 10, []uint{1}, nil } require.NoError(t, TriggerHostStatusWebhook(context.Background(), ds, kitlog.NewNopLogger())) assert.Equal(t, "", requestBody) + assert.Equal(t, 1, count) +} + +func TestTriggerHostStatusWebhookTeam(t *testing.T) { + ds := new(mock.Store) + + requestBody := "" + count := 0 + + ts := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + requestBodyBytes, err := io.ReadAll(r.Body) + require.NoError(t, err) + requestBody = string(requestBodyBytes) + count++ + }, + ), + ) + defer ts.Close() + + ac := &fleet.AppConfig{ + WebhookSettings: fleet.WebhookSettings{ + HostStatusWebhook: fleet.HostStatusWebhookSettings{ + Enable: false, + DestinationURL: ts.URL, + HostPercentage: 43, + DaysCount: 3, + }, + }, + } + teamSettings := fleet.HostStatusWebhookSettings{ + Enable: true, + DestinationURL: ts.URL, + HostPercentage: 43, + DaysCount: 2, + } + + ds.AppConfigFunc = func(context.Context) (*fleet.AppConfig, error) { + return ac, nil + } + + ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, teamID *uint, daysCount int) (int, []uint, error) { + assert.Equal(t, 2, daysCount) + assert.Equal(t, uint(1), *teamID) + return 10, []uint{1, 2, 3, 4, 5, 6}, nil + } + + ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) { + return []*fleet.TeamSummary{{ID: 1}}, nil + } + ds.TeamFunc = func(ctx context.Context, id uint) (*fleet.Team, error) { + assert.Equal(t, uint(1), id) + return &fleet.Team{ + ID: 1, + Config: fleet.TeamConfig{ + WebhookSettings: fleet.TeamWebhookSettings{ + HostStatusWebhook: teamSettings, + }, + }, + }, nil + } + + require.NoError(t, TriggerHostStatusWebhook(context.Background(), ds, kitlog.NewNopLogger())) + assert.Equal( + t, + `{"data":{"days_unseen":2,"host_ids":[1,2,3,4,5,6],"team_id":1,"total_hosts":10,"unseen_hosts":6},"text":"More than 60.00% of your hosts have not checked into Fleet for more than 2 days. You've been sent this message because the Host status webhook is enabled in your Fleet instance."}`, + requestBody, + ) + assert.Equal(t, 1, count) + + requestBody = "" + ds.TotalAndUnseenHostsSinceFunc = func(ctx context.Context, teamID *uint, daysCount int) (int, []uint, error) { + assert.Equal(t, 2, daysCount) + assert.Equal(t, uint(1), *teamID) + return 10, []uint{1}, nil + } + + require.NoError(t, TriggerHostStatusWebhook(context.Background(), ds, kitlog.NewNopLogger())) + assert.Equal(t, "", requestBody) + assert.Equal(t, 1, count) }