From 8f66a50f2bf0ecfe6cd6e718ad8919f57339f91f Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 22 Jun 2022 16:38:11 -0400 Subject: [PATCH] Add hosts enrolled by operating system to anonymous usage stats (#6329) --- changes/issue-5378-add-hosts-by-osversion | 1 + cmd/fleet/serve_test.go | 7 +- docs/Using-Fleet/Usage-statistics.md | 46 ++++++++++++- .../admin/AppSettingsPage/cards/constants.ts | 44 ++++++++++++ server/datastore/mysql/hosts.go | 32 +++++++-- server/datastore/mysql/statistics.go | 5 +- server/datastore/mysql/statistics_test.go | 68 ++++++++++++++++++- server/fleet/statistics.go | 32 +++++---- .../webhooks/receive-usage-analytics.js | 1 + website/api/models/HistoricalUsageSnapshot.js | 1 + website/assets/js/cloud.setup.js | 2 +- 11 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 changes/issue-5378-add-hosts-by-osversion diff --git a/changes/issue-5378-add-hosts-by-osversion b/changes/issue-5378-add-hosts-by-osversion new file mode 100644 index 0000000000..d4d263e78a --- /dev/null +++ b/changes/issue-5378-add-hosts-by-osversion @@ -0,0 +1 @@ +* Added the number of hosts enrolled by OS and OS version to anonymous usage statistics. diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index ec2c11f9a0..3d96b4d451 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -54,6 +54,11 @@ func TestMaybeSendStatistics(t *testing.T) { SystemUsersEnabled: true, HostsStatusWebHookEnabled: true, NumWeeklyActiveUsers: 111, + HostsEnrolledByOperatingSystem: map[string][]fleet.HostsCountByOSVersion{ + "linux": { + fleet.HostsCountByOSVersion{Version: "1.2.3", NumEnrolled: 22}, + }, + }, }, true, nil } recorded := false @@ -65,7 +70,7 @@ func TestMaybeSendStatistics(t *testing.T) { err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL, &fleet.LicenseInfo{Tier: "premium"}) require.NoError(t, err) assert.True(t, recorded) - assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","numHostsEnrolled":999,"numUsers":99,"numTeams":9,"numPolicies":0,"numLabels":3,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true,"numWeeklyActiveUsers":111}`, requestBody) + assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","numHostsEnrolled":999,"numUsers":99,"numTeams":9,"numPolicies":0,"numLabels":3,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true,"numWeeklyActiveUsers":111,"hostsEnrolledByOperatingSystem":{"linux":[{"version":"1.2.3","numEnrolled":22}]}}`, requestBody) } func TestMaybeSendStatisticsSkipsSendingIfNotNeeded(t *testing.T) { diff --git a/docs/Using-Fleet/Usage-statistics.md b/docs/Using-Fleet/Usage-statistics.md index 1ae91b0ad2..04e4c6fe60 100644 --- a/docs/Using-Fleet/Usage-statistics.md +++ b/docs/Using-Fleet/Usage-statistics.md @@ -24,7 +24,51 @@ Fleet Device Management Inc. periodically collects anonymous information about y "vulnDetectionEnabled": true, "systemUsersEnabled": true, "hostStatusWebhookEnabled": true, - "numWeeklyActiveUsers": 999 + "numWeeklyActiveUsers": 999, + "hostsEnrolledByOperatingSystem": { + "macos": [ + { + "version": "12.3.1", + "numEnrolled": 999 + } + ], + "windows": [ + { + "version": "10, version 21H2 (W)", + "numEnrolled": 999 + } + ], + "ubuntuLinux": [ + { + "version": "22.04 'Jammy Jellyfish' (LTS)", + "numEnrolled": 999 + } + ], + "centosLinux": [ + { + "version": "12.3.1", + "numEnrolled": 999 + } + ], + "debianLinux": [ + { + "version": "11 (Bullseye)", + "numEnrolled": 999 + } + ], + "redhatLinux": [ + { + "version": "9", + "numEnrolled": 999 + } + ], + "amazonLinux": [ + { + "version": "AMI", + "numEnrolled": 999 + } + ] + } } ``` diff --git a/frontend/pages/admin/AppSettingsPage/cards/constants.ts b/frontend/pages/admin/AppSettingsPage/cards/constants.ts index 308c573ace..fc3d1a733b 100644 --- a/frontend/pages/admin/AppSettingsPage/cards/constants.ts +++ b/frontend/pages/admin/AppSettingsPage/cards/constants.ts @@ -80,6 +80,50 @@ export const usageStatsPreview = { systemUsersEnabled: true, hostStatusWebhookEnabled: true, numWeeklyActiveUsers: 999, + hostsEnrolledByOperatingSystem: { + macos: [ + { + version: "12.3.1", + numEnrolled: 999, + }, + ], + windows: [ + { + version: "10, version 21H2 (W)", + numEnrolled: 999, + }, + ], + ubuntuLinux: [ + { + version: "22.04 'Jammy Jellyfish' (LTS)", + numEnrolled: 999, + }, + ], + centosLinux: [ + { + version: "12.3.1", + numEnrolled: 999, + }, + ], + debianLinux: [ + { + version: "11 (Bullseye)", + numEnrolled: 999, + }, + ], + redhatLinux: [ + { + version: "9", + numEnrolled: 999, + }, + ], + amazonLinux: [ + { + version: "AMI", + numEnrolled: 999, + }, + ], + }, }; export default { diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 73b1db2b93..6d824c689c 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -375,13 +375,33 @@ LIMIT return &host, nil } -func amountEnrolledHostsDB(ctx context.Context, db sqlx.QueryerContext) (int, error) { - var amount int - err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM hosts`) - if err != nil { - return 0, err +func amountEnrolledHostsByOSDB(ctx context.Context, db sqlx.QueryerContext) (byOS map[string][]fleet.HostsCountByOSVersion, totalCount int, err error) { + var hostsByOS []struct { + Platform string `db:"platform"` + OSVersion string `db:"os_version"` + NumHosts int `db:"num_hosts"` } - return amount, nil + + const stmt = ` + SELECT platform, os_version, count(*) as num_hosts + FROM hosts + GROUP BY platform, os_version + ` + if err := sqlx.SelectContext(ctx, db, &hostsByOS, stmt); err != nil { + return nil, 0, err + } + + byOS = make(map[string][]fleet.HostsCountByOSVersion) + for _, h := range hostsByOS { + totalCount += h.NumHosts + byVersion := byOS[h.Platform] + byVersion = append(byVersion, fleet.HostsCountByOSVersion{ + Version: h.OSVersion, + NumEnrolled: h.NumHosts, + }) + byOS[h.Platform] = byVersion + } + return byOS, totalCount, nil } func (ds *Datastore) ListHosts(ctx context.Context, filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) { diff --git a/server/datastore/mysql/statistics.go b/server/datastore/mysql/statistics.go index f4b7882b58..4965a6d989 100644 --- a/server/datastore/mysql/statistics.go +++ b/server/datastore/mysql/statistics.go @@ -19,9 +19,9 @@ type statistics struct { func (ds *Datastore) ShouldSendStatistics(ctx context.Context, frequency time.Duration, license *fleet.LicenseInfo) (fleet.StatisticsPayload, bool, error) { computeStats := func(stats *fleet.StatisticsPayload, since time.Time) error { - amountEnrolledHosts, err := amountEnrolledHostsDB(ctx, ds.writer) + enrolledHostsByOS, amountEnrolledHosts, err := amountEnrolledHostsByOSDB(ctx, ds.writer) if err != nil { - return ctxerr.Wrap(ctx, err, "amount enrolled hosts") + return ctxerr.Wrap(ctx, err, "amount enrolled hosts by os") } amountUsers, err := amountUsersDB(ctx, ds.writer) if err != nil { @@ -58,6 +58,7 @@ func (ds *Datastore) ShouldSendStatistics(ctx context.Context, frequency time.Du stats.SystemUsersEnabled = appConfig.HostSettings.EnableHostUsers stats.HostsStatusWebHookEnabled = appConfig.WebhookSettings.HostStatusWebhook.Enable stats.NumWeeklyActiveUsers = amountWeeklyUsers + stats.HostsEnrolledByOperatingSystem = enrolledHostsByOS return nil } diff --git a/server/datastore/mysql/statistics_test.go b/server/datastore/mysql/statistics_test.go index fd8ba7d814..8c7db4d929 100644 --- a/server/datastore/mysql/statistics_test.go +++ b/server/datastore/mysql/statistics_test.go @@ -140,6 +140,7 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { time.Sleep(1100 * time.Millisecond) // ensure the DB timestamp is not in the same second + // create a few more hosts, with platforms and os versions _, err = ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -151,6 +152,56 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { PrimaryIP: "192.168.1.2", PrimaryMac: "30-65-EC-6F-C4-59", OsqueryHostID: "S", + Platform: "rhel", + OSVersion: "Fedora 35", + }) + require.NoError(t, err) + + _, err = ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "3", + UUID: "3", + Hostname: "foo.local3", + PrimaryIP: "192.168.1.3", + PrimaryMac: "40-65-EC-6F-C4-59", + OsqueryHostID: "T", + Platform: "rhel", + OSVersion: "Fedora 35", + }) + require.NoError(t, err) + + _, err = ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "4", + UUID: "4", + Hostname: "foo.local4", + PrimaryIP: "192.168.1.4", + PrimaryMac: "50-65-EC-6F-C4-59", + OsqueryHostID: "U", + Platform: "macos", + OSVersion: "10.11.12", + }) + require.NoError(t, err) + + _, err = ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "5", + UUID: "5", + Hostname: "foo.local5", + PrimaryIP: "192.168.1.5", + PrimaryMac: "60-65-EC-6F-C4-59", + OsqueryHostID: "V", + Platform: "rhel", + OSVersion: "Fedora 36", }) require.NoError(t, err) @@ -159,9 +210,20 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { require.NoError(t, err) assert.True(t, shouldSend) assert.Equal(t, firstIdentifier, stats.AnonymousIdentifier) - assert.Equal(t, stats.NumHostsEnrolled, 2) + assert.Equal(t, stats.NumHostsEnrolled, 5) assert.Equal(t, stats.NumUsers, 2) - assert.Equal(t, stats.NumWeeklyActiveUsers, 0) // no active user since last stats were sent + assert.Equal(t, stats.NumWeeklyActiveUsers, 0) // no active user since last stats were sent + require.Len(t, stats.HostsEnrolledByOperatingSystem, 3) // empty platform, rhel and macos + require.ElementsMatch(t, []fleet.HostsCountByOSVersion{ + {Version: "Fedora 35", NumEnrolled: 2}, + {Version: "Fedora 36", NumEnrolled: 1}, + }, stats.HostsEnrolledByOperatingSystem["rhel"]) + require.ElementsMatch(t, []fleet.HostsCountByOSVersion{ + {Version: "10.11.12", NumEnrolled: 1}, + }, stats.HostsEnrolledByOperatingSystem["macos"]) + require.ElementsMatch(t, []fleet.HostsCountByOSVersion{ + {Version: "", NumEnrolled: 1}, + }, stats.HostsEnrolledByOperatingSystem[""]) // Create multiple new sessions for a single user _, err = ds.NewSession(context.Background(), u1.ID, "session_key2") @@ -178,7 +240,7 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { require.NoError(t, err) assert.True(t, shouldSend) assert.Equal(t, firstIdentifier, stats.AnonymousIdentifier) - assert.Equal(t, stats.NumHostsEnrolled, 2) + assert.Equal(t, stats.NumHostsEnrolled, 5) assert.Equal(t, stats.NumUsers, 2) assert.Equal(t, stats.NumWeeklyActiveUsers, 1) } diff --git a/server/fleet/statistics.go b/server/fleet/statistics.go index f4c9c44fcb..6a573a4b41 100644 --- a/server/fleet/statistics.go +++ b/server/fleet/statistics.go @@ -3,19 +3,25 @@ package fleet import "time" type StatisticsPayload struct { - AnonymousIdentifier string `json:"anonymousIdentifier"` - FleetVersion string `json:"fleetVersion"` - LicenseTier string `json:"licenseTier"` - NumHostsEnrolled int `json:"numHostsEnrolled"` - NumUsers int `json:"numUsers"` - NumTeams int `json:"numTeams"` - NumPolicies int `json:"numPolicies"` - NumLabels int `json:"numLabels"` - SoftwareInventoryEnabled bool `json:"softwareInventoryEnabled"` - VulnDetectionEnabled bool `json:"vulnDetectionEnabled"` - SystemUsersEnabled bool `json:"systemUsersEnabled"` - HostsStatusWebHookEnabled bool `json:"hostsStatusWebHookEnabled"` - NumWeeklyActiveUsers int `json:"numWeeklyActiveUsers"` + AnonymousIdentifier string `json:"anonymousIdentifier"` + FleetVersion string `json:"fleetVersion"` + LicenseTier string `json:"licenseTier"` + NumHostsEnrolled int `json:"numHostsEnrolled"` + NumUsers int `json:"numUsers"` + NumTeams int `json:"numTeams"` + NumPolicies int `json:"numPolicies"` + NumLabels int `json:"numLabels"` + SoftwareInventoryEnabled bool `json:"softwareInventoryEnabled"` + VulnDetectionEnabled bool `json:"vulnDetectionEnabled"` + SystemUsersEnabled bool `json:"systemUsersEnabled"` + HostsStatusWebHookEnabled bool `json:"hostsStatusWebHookEnabled"` + NumWeeklyActiveUsers int `json:"numWeeklyActiveUsers"` + HostsEnrolledByOperatingSystem map[string][]HostsCountByOSVersion `json:"hostsEnrolledByOperatingSystem"` +} + +type HostsCountByOSVersion struct { + Version string `json:"version"` + NumEnrolled int `json:"numEnrolled"` } const ( diff --git a/website/api/controllers/webhooks/receive-usage-analytics.js b/website/api/controllers/webhooks/receive-usage-analytics.js index a9222bdb04..c753231c04 100644 --- a/website/api/controllers/webhooks/receive-usage-analytics.js +++ b/website/api/controllers/webhooks/receive-usage-analytics.js @@ -21,6 +21,7 @@ module.exports = { systemUsersEnabled: { type: 'boolean', defaultsTo: false }, hostStatusWebhookEnabled: { type: 'boolean', defaultsTo: false }, numWeeklyActiveUsers: { type: 'number', defaultsTo: 0 }, + hostsEnrolledByOperatingSystem: { type: 'json', defaultsTo: {} }, }, diff --git a/website/api/models/HistoricalUsageSnapshot.js b/website/api/models/HistoricalUsageSnapshot.js index 70cb11172b..8b60d760d5 100644 --- a/website/api/models/HistoricalUsageSnapshot.js +++ b/website/api/models/HistoricalUsageSnapshot.js @@ -25,6 +25,7 @@ module.exports = { systemUsersEnabled: { required: true, type: 'boolean' }, hostStatusWebhookEnabled: { required: true, type: 'boolean' }, numWeeklyActiveUsers: { required: true, type: 'number' }, + hostsEnrolledByOperatingSystem: { required: true, type: 'json' }, // ╔═╗╔╦╗╔╗ ╔═╗╔╦╗╔═╗ // ║╣ ║║║╠╩╗║╣ ║║╚═╗ diff --git a/website/assets/js/cloud.setup.js b/website/assets/js/cloud.setup.js index bcb24c79ce..711015f9c8 100644 --- a/website/assets/js/cloud.setup.js +++ b/website/assets/js/cloud.setup.js @@ -13,7 +13,7 @@ Cloud.setup({ /* eslint-disable */ - methods: {"downloadSitemap":{"verb":"GET","url":"/sitemap.xml","args":[]},"receiveUsageAnalytics":{"verb":"POST","url":"/api/v1/webhooks/receive-usage-analytics","args":["anonymousIdentifier","fleetVersion","licenseTier","numHostsEnrolled","numUsers","numTeams","numPolicies","numLabels","softwareInventoryEnabled","vulnDetectionEnabled","systemUsersEnabled","hostStatusWebhookEnabled", "numWeeklyActiveUsers"]},"receiveFromGithub":{"verb":"GET","url":"/api/v1/webhooks/github","args":["botSignature","action","sender","repository","changes","issue","comment","pull_request","label"]},"deliverContactFormMessage":{"verb":"POST","url":"/api/v1/deliver-contact-form-message","args":["emailAddress","topic","firstName","lastName","message"]},"sendPasswordRecoveryEmail":{"verb":"POST","url":"/api/v1/entrance/send-password-recovery-email","args":["emailAddress"]},"signup":{"verb":"POST","url":"/api/v1/customers/signup","args":["emailAddress","password","organization","firstName","lastName"]},"updateProfile":{"verb":"POST","url":"/api/v1/account/update-profile","args":["firstName","lastName","organization","emailAddress"]},"updatePassword":{"verb":"POST","url":"/api/v1/account/update-password","args":["oldPassword","newPassword"]},"updateBillingCard":{"verb":"POST","url":"/api/v1/account/update-billing-card","args":["stripeToken","billingCardLast4","billingCardBrand","billingCardExpMonth","billingCardExpYear"]},"login":{"verb":"POST","url":"/api/v1/customers/login","args":["emailAddress","password","rememberMe"]},"logout":{"verb":"GET","url":"/api/v1/account/logout","args":[]},"createQuote":{"verb":"POST","url":"/api/v1/customers/create-quote","args":["numberOfHosts"]},"saveBillingInfoAndSubscribe":{"verb":"POST","url":"/api/v1/customers/save-billing-info-and-subscribe","args":["quoteId","paymentSource"]},"updatePasswordAndLogin":{"verb":"POST","url":"/api/v1/entrance/update-password-and-login","args":["password","token"]},"deliverDemoSignup":{"verb":"POST","url":"/api/v1/deliver-demo-signup","args":[]}} + methods: {"downloadSitemap":{"verb":"GET","url":"/sitemap.xml","args":[]},"receiveUsageAnalytics":{"verb":"POST","url":"/api/v1/webhooks/receive-usage-analytics","args":["anonymousIdentifier","fleetVersion","licenseTier","numHostsEnrolled","numUsers","numTeams","numPolicies","numLabels","softwareInventoryEnabled","vulnDetectionEnabled","systemUsersEnabled","hostStatusWebhookEnabled", "numWeeklyActiveUsers", "hostsEnrolledByOperatingSystem"]},"receiveFromGithub":{"verb":"GET","url":"/api/v1/webhooks/github","args":["botSignature","action","sender","repository","changes","issue","comment","pull_request","label"]},"deliverContactFormMessage":{"verb":"POST","url":"/api/v1/deliver-contact-form-message","args":["emailAddress","topic","firstName","lastName","message"]},"sendPasswordRecoveryEmail":{"verb":"POST","url":"/api/v1/entrance/send-password-recovery-email","args":["emailAddress"]},"signup":{"verb":"POST","url":"/api/v1/customers/signup","args":["emailAddress","password","organization","firstName","lastName"]},"updateProfile":{"verb":"POST","url":"/api/v1/account/update-profile","args":["firstName","lastName","organization","emailAddress"]},"updatePassword":{"verb":"POST","url":"/api/v1/account/update-password","args":["oldPassword","newPassword"]},"updateBillingCard":{"verb":"POST","url":"/api/v1/account/update-billing-card","args":["stripeToken","billingCardLast4","billingCardBrand","billingCardExpMonth","billingCardExpYear"]},"login":{"verb":"POST","url":"/api/v1/customers/login","args":["emailAddress","password","rememberMe"]},"logout":{"verb":"GET","url":"/api/v1/account/logout","args":[]},"createQuote":{"verb":"POST","url":"/api/v1/customers/create-quote","args":["numberOfHosts"]},"saveBillingInfoAndSubscribe":{"verb":"POST","url":"/api/v1/customers/save-billing-info-and-subscribe","args":["quoteId","paymentSource"]},"updatePasswordAndLogin":{"verb":"POST","url":"/api/v1/entrance/update-password-and-login","args":["password","token"]},"deliverDemoSignup":{"verb":"POST","url":"/api/v1/deliver-demo-signup","args":[]}} /* eslint-enable */ });