From 6ee6dba4a496dbba9f915f1ede85b717e0edf563 Mon Sep 17 00:00:00 2001 From: gillespi314 <73313222+gillespi314@users.noreply.github.com> Date: Mon, 6 Dec 2021 14:39:00 -0600 Subject: [PATCH] Update anonymous usage statistics to record Fleet feature adoption (#3056) * Collect additional anonymous usage statistics * Update fleetdm.com and docs for usage stats * Fix lint-go errors * Fix failing test * update webhook inputs * Update receive-usage-analytics.js * fix failing test * Update statistics_test.go Co-authored-by: eashaw --- cmd/fleet/serve.go | 14 ++--- cmd/fleet/serve_test.go | 26 +++++--- docs/01-Using-Fleet/11-Usage-statistics.md | 15 ++++- .../admin/AppConfigForm/AppConfigForm.jsx | 19 ++++-- server/datastore/mysql/policies.go | 9 +++ server/datastore/mysql/statistics.go | 46 +++++++++++--- server/datastore/mysql/statistics_test.go | 62 ++++++++++++++++++- server/datastore/mysql/teams.go | 9 +++ server/datastore/mysql/users.go | 10 ++- server/fleet/datastore.go | 2 +- server/fleet/statistics.go | 14 ++++- server/mock/datastore_mock.go | 6 +- .../webhooks/receive-usage-analytics.js | 19 +++--- website/api/models/HistoricalUsageSnapshot.js | 11 +++- 14 files changed, 212 insertions(+), 50 deletions(-) diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 6c1f313f50..d416170cc5 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -292,7 +292,7 @@ the way that the Fleet server works. } } - cancelBackground := runCrons(ds, task, kitlog.With(logger, "component", "crons"), config) + cancelBackground := runCrons(ds, task, kitlog.With(logger, "component", "crons"), config, license) // Flush seen hosts every second go func() { @@ -501,7 +501,7 @@ const ( lockKeyWebhooks = "webhooks" ) -func trySendStatistics(ctx context.Context, ds fleet.Datastore, frequency time.Duration, url string) error { +func trySendStatistics(ctx context.Context, ds fleet.Datastore, frequency time.Duration, url string, license *fleet.LicenseInfo) error { ac, err := ds.AppConfig(ctx) if err != nil { return err @@ -510,7 +510,7 @@ func trySendStatistics(ctx context.Context, ds fleet.Datastore, frequency time.D return nil } - stats, shouldSend, err := ds.ShouldSendStatistics(ctx, frequency) + stats, shouldSend, err := ds.ShouldSendStatistics(ctx, frequency, license) if err != nil { return err } @@ -525,7 +525,7 @@ func trySendStatistics(ctx context.Context, ds fleet.Datastore, frequency time.D return ds.RecordStatisticsSent(ctx) } -func runCrons(ds fleet.Datastore, task *async.Task, logger kitlog.Logger, config config.FleetConfig) context.CancelFunc { +func runCrons(ds fleet.Datastore, task *async.Task, logger kitlog.Logger, config config.FleetConfig, license *fleet.LicenseInfo) context.CancelFunc { ctx, cancelBackground := context.WithCancel(context.Background()) ourIdentifier, err := server.GenerateRandomText(64) @@ -537,7 +537,7 @@ func runCrons(ds fleet.Datastore, task *async.Task, logger kitlog.Logger, config task.StartCollectors(ctx, config.Osquery.AsyncHostCollectInterval, config.Osquery.AsyncHostCollectMaxJitterPercent, kitlog.With(logger, "cron", "async_task")) - go cronCleanups(ctx, ds, kitlog.With(logger, "cron", "cleanups"), ourIdentifier) + go cronCleanups(ctx, ds, kitlog.With(logger, "cron", "cleanups"), ourIdentifier, license) go cronVulnerabilities( ctx, ds, kitlog.With(logger, "cron", "vulnerabilities"), ourIdentifier, config) go cronWebhooks(ctx, ds, kitlog.With(logger, "cron", "webhooks"), ourIdentifier) @@ -545,7 +545,7 @@ func runCrons(ds fleet.Datastore, task *async.Task, logger kitlog.Logger, config return cancelBackground } -func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, identifier string) { +func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, identifier string, license *fleet.LicenseInfo) { ticker := time.NewTicker(10 * time.Second) for { level.Debug(logger).Log("waiting", "on ticker") @@ -594,7 +594,7 @@ func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, level.Error(logger).Log("err", "cleaning expired hosts", "details", err) } - err = trySendStatistics(ctx, ds, fleet.StatisticsFrequency, "https://fleetdm.com/api/v1/webhooks/receive-usage-analytics") + err = trySendStatistics(ctx, ds, fleet.StatisticsFrequency, "https://fleetdm.com/api/v1/webhooks/receive-usage-analytics", license) if err != nil { level.Error(logger).Log("err", "sending statistics", "details", err) } diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index 705a1756af..4391590e0d 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -37,11 +37,19 @@ func TestMaybeSendStatistics(t *testing.T) { return &fleet.AppConfig{ServerSettings: fleet.ServerSettings{EnableAnalytics: true}}, nil } - ds.ShouldSendStatisticsFunc = func(ctx context.Context, frequency time.Duration) (fleet.StatisticsPayload, bool, error) { + ds.ShouldSendStatisticsFunc = func(ctx context.Context, frequency time.Duration, license *fleet.LicenseInfo) (fleet.StatisticsPayload, bool, error) { return fleet.StatisticsPayload{ - AnonymousIdentifier: "ident", - FleetVersion: "1.2.3", - NumHostsEnrolled: 999, + AnonymousIdentifier: "ident", + FleetVersion: "1.2.3", + LicenseTier: "premium", + NumHostsEnrolled: 999, + NumUsers: 99, + NumTeams: 9, + NumPolicies: 0, + SoftwareInventoryEnabled: true, + VulnDetectionEnabled: true, + SystemUsersEnabled: true, + HostsStatusWebHookEnabled: true, }, true, nil } recorded := false @@ -50,10 +58,10 @@ func TestMaybeSendStatistics(t *testing.T) { return nil } - err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL) + err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL, &fleet.LicenseInfo{Tier: "premium"}) require.NoError(t, err) assert.True(t, recorded) - assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","numHostsEnrolled":999}`, requestBody) + assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","numHostsEnrolled":999,"numUsers":99,"numTeams":9,"numPolicies":0,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true}`, requestBody) } func TestMaybeSendStatisticsSkipsSendingIfNotNeeded(t *testing.T) { @@ -70,7 +78,7 @@ func TestMaybeSendStatisticsSkipsSendingIfNotNeeded(t *testing.T) { return &fleet.AppConfig{ServerSettings: fleet.ServerSettings{EnableAnalytics: true}}, nil } - ds.ShouldSendStatisticsFunc = func(ctx context.Context, frequency time.Duration) (fleet.StatisticsPayload, bool, error) { + ds.ShouldSendStatisticsFunc = func(ctx context.Context, frequency time.Duration, license *fleet.LicenseInfo) (fleet.StatisticsPayload, bool, error) { return fleet.StatisticsPayload{}, false, nil } recorded := false @@ -79,7 +87,7 @@ func TestMaybeSendStatisticsSkipsSendingIfNotNeeded(t *testing.T) { return nil } - err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL) + err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL, &fleet.LicenseInfo{Tier: "premium"}) require.NoError(t, err) assert.False(t, recorded) assert.False(t, called) @@ -99,7 +107,7 @@ func TestMaybeSendStatisticsSkipsIfNotConfigured(t *testing.T) { return &fleet.AppConfig{}, nil } - err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL) + err := trySendStatistics(context.Background(), ds, fleet.StatisticsFrequency, ts.URL, &fleet.LicenseInfo{Tier: "premium"}) require.NoError(t, err) assert.False(t, called) } diff --git a/docs/01-Using-Fleet/11-Usage-statistics.md b/docs/01-Using-Fleet/11-Usage-statistics.md index 816ea33775..0b528610f1 100644 --- a/docs/01-Using-Fleet/11-Usage-statistics.md +++ b/docs/01-Using-Fleet/11-Usage-statistics.md @@ -12,9 +12,18 @@ Fleet Device Management Inc. periodically collects anonymous information about y ```json { - "anonymous_identifier": 1, - "fleet_version": "x.x.x", - "hosts_enrolled_count": 12345 + "anonymousIdentifier": "9pnzNmrES3mQG66UQtd29cYTiX2+fZ4CYxDvh495720=", + "fleetVersion": "x.x.x", + "licenseTier": "free", + "numHostsEnrolled": 12345, + "numUsers": 12, + "numTeams": 3, + "numPolicies": 5, + "numLabels": 20, + "softwareInventoryEnabled": true, + "vulnDetectionEnabled": true, + "systemUsersEnabled": true, + "hostStatusWebhookEnabled": true, } ``` diff --git a/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx b/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx index e018082f1e..d36f8a899b 100644 --- a/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx +++ b/frontend/components/forms/admin/AppConfigForm/AppConfigForm.jsx @@ -288,10 +288,19 @@ class AppConfigForm extends Component { return null; } - const json = { - anonymous_identifier: "wmTH972f06USpahr41LHpgLKAhgZL", - fleet_version: "x.x.x", - hosts_enrolled_count: 12345, + const stats = { + anonymousIdentifier: "9pnzNmrES3mQG66UQtd29cYTiX2+fZ4CYxDvh495720=", + fleetVersion: "x.x.x", + licenseTier: "free", + numHostsEnrolled: 12345, + numUsers: 12, + numTeams: 3, + numPolicies: 5, + numLabels: 20, + softwareInventoryEnabled: true, + vulnDetectionEnabled: true, + systemUsersEnabled: true, + hostStatusWebhookEnabled: true, }; return ( @@ -301,7 +310,7 @@ class AppConfigForm extends Component { className={`${baseClass}__usage-stats-preview-modal`} >

An example JSON payload sent to Fleet Device Management Inc.

-
+