Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

additional stats (#19078)

Browse Source 
Additional telemetry stats to report to Heroku.
FYI - @eashaw, we could later add to datadog as well.

---------

Co-authored-by: Noah Talerman <47070608+noahtalerman@users.noreply.github.com>
This commit is contained in:
Sharon Katz
2024-05-24 15:06:10 -04:00
committed by GitHub
co-authored by Noah Talerman
parent 4545e960ae
commit f75f0776e6
6 changed files with 91 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changes/19072-additional-stats
+1
View File
@@ -0,0 +1 @@
- Added additional statistics items as part of our quality telemetry.
cmd/fleet/serve_test.go
+7 -1
View File
@@ -91,6 +91,12 @@ func TestMaybeSendStatistics(t *testing.T) {
LicenseTier: "premium",
NumHostsEnrolled: 999,
NumUsers: 99,
NumSoftwareVersions: 100,
NumHostSoftwares: 101,
NumSoftwareTitles: 102,
NumHostSoftwareInstalledPaths: 103,
NumSoftwareCPEs: 104,
NumSoftwareCVEs: 105,
NumTeams: 9,
NumPolicies: 0,
NumLabels: 3,
@@ -128,7 +134,7 @@ func TestMaybeSendStatistics(t *testing.T) {
require.NoError(t, err)
assert.True(t, recorded)
require.True(t, cleanedup)
assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","organization":"Fleet","numHostsEnrolled":999,"numUsers":99,"numTeams":9,"numPolicies":0,"numLabels":3,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true,"mdmMacOsEnabled":false,"hostExpiryEnabled":false,"mdmWindowsEnabled":false,"liveQueryDisabled":false,"numWeeklyActiveUsers":111,"numWeeklyPolicyViolationDaysActual":0,"numWeeklyPolicyViolationDaysPossible":0,"hostsEnrolledByOperatingSystem":{"linux":[{"version":"1.2.3","numEnrolled":22}]},"hostsEnrolledByOrbitVersion":[],"hostsEnrolledByOsqueryVersion":[],"storedErrors":[],"numHostsNotResponding":0}`, requestBody)
assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","organization":"Fleet","numHostsEnrolled":999,"numUsers":99,"numSoftwareVersions":100,"numHostSoftwares":101,"numSoftwareTitles":102,"numHostSoftwareInstalledPaths":103,"numSoftwareCPEs":104,"numSoftwareCVEs":105,"numTeams":9,"numPolicies":0,"numLabels":3,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true,"mdmMacOsEnabled":false,"hostExpiryEnabled":false,"mdmWindowsEnabled":false,"liveQueryDisabled":false,"numWeeklyActiveUsers":111,"numWeeklyPolicyViolationDaysActual":0,"numWeeklyPolicyViolationDaysPossible":0,"hostsEnrolledByOperatingSystem":{"linux":[{"version":"1.2.3","numEnrolled":22}]},"hostsEnrolledByOrbitVersion":[],"hostsEnrolledByOsqueryVersion":[],"storedErrors":[],"numHostsNotResponding":0}`, requestBody)
}
func TestMaybeSendStatisticsSkipsSendingIfNotNeeded(t *testing.T) {
server/datastore/mysql/statistics.go
+31 -1
View File
@@ -28,10 +28,34 @@ func (ds *Datastore) ShouldSendStatistics(ctx context.Context, frequency time.Du
if err != nil {
return ctxerr.Wrap(ctx, err, "amount enrolled hosts by os")
}
amountUsers, err := amountUsersDB(ctx, ds.writer(ctx))
amountUsers, err := tableRowsCount(ctx, ds.writer(ctx), "users")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount users")
}
amountSoftwaresVersions, err := tableRowsCount(ctx, ds.writer(ctx), "software")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount software")
}
amountHostSoftwares, err := tableRowsCount(ctx, ds.writer(ctx), "host_software")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount host_software")
}
amountSoftwareTitles, err := tableRowsCount(ctx, ds.writer(ctx), "software_titles")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount software_titles")
}
amountHostSoftwareInstalledPaths, err := tableRowsCount(ctx, ds.writer(ctx), "host_software_installed_paths")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount host_software_installed_paths")
}
amountSoftwareCpes, err := tableRowsCount(ctx, ds.writer(ctx), "software_cpe")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount software_cpe")
}
amountSoftwareCves, err := tableRowsCount(ctx, ds.writer(ctx), "software_cve")
if err != nil {
return ctxerr.Wrap(ctx, err, "amount software_cve")
}
amountTeams, err := amountTeamsDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount teams")
@@ -77,6 +101,12 @@ func (ds *Datastore) ShouldSendStatistics(ctx context.Context, frequency time.Du
stats.NumHostsEnrolled = amountEnrolledHosts
stats.NumUsers = amountUsers
stats.NumSoftwareVersions = amountSoftwaresVersions
stats.NumHostSoftwares = amountHostSoftwares
stats.NumSoftwareTitles = amountSoftwareTitles
stats.NumHostSoftwareInstalledPaths = amountHostSoftwareInstalledPaths
stats.NumSoftwareCPEs = amountSoftwareCpes
stats.NumSoftwareCVEs = amountSoftwareCves
stats.NumTeams = amountTeams
stats.NumPolicies = amountPolicies
stats.NumLabels = amountLabels
server/datastore/mysql/statistics_test.go
+24
View File
@@ -64,6 +64,12 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) {
assert.Equal(t, "Fleet", stats.Organization)
assert.Equal(t, 0, stats.NumHostsEnrolled)
assert.Equal(t, 0, stats.NumUsers)
assert.Equal(t, 0, stats.NumSoftwareVersions)
assert.Equal(t, 0, stats.NumHostSoftwares)
assert.Equal(t, 0, stats.NumSoftwareTitles)
assert.Equal(t, 0, stats.NumHostSoftwareInstalledPaths)
assert.Equal(t, 0, stats.NumSoftwareCPEs)
assert.Equal(t, 0, stats.NumSoftwareCVEs)
assert.Equal(t, 0, stats.NumTeams)
assert.Equal(t, 0, stats.NumPolicies)
assert.Equal(t, builtinLabels, stats.NumLabels)
@@ -202,6 +208,12 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) {
assert.Equal(t, "Fleet", stats.Organization)
assert.Equal(t, 1, stats.NumHostsEnrolled)
assert.Equal(t, 2, stats.NumUsers)
assert.Equal(t, 0, stats.NumSoftwareVersions)
assert.Equal(t, 0, stats.NumHostSoftwares)
assert.Equal(t, 0, stats.NumSoftwareTitles)
assert.Equal(t, 0, stats.NumHostSoftwareInstalledPaths)
assert.Equal(t, 0, stats.NumSoftwareCPEs)
assert.Equal(t, 0, stats.NumSoftwareCVEs)
assert.Equal(t, 1, stats.NumTeams)
assert.Equal(t, 1, stats.NumPolicies)
assert.Equal(t, builtinLabels+1, stats.NumLabels)
@@ -300,6 +312,12 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) {
assert.Equal(t, "Fleet", stats.Organization)
assert.Equal(t, 5, stats.NumHostsEnrolled)
assert.Equal(t, 2, stats.NumUsers)
assert.Equal(t, 0, stats.NumSoftwareVersions)
assert.Equal(t, 0, stats.NumHostSoftwares)
assert.Equal(t, 0, stats.NumSoftwareTitles)
assert.Equal(t, 0, stats.NumHostSoftwareInstalledPaths)
assert.Equal(t, 0, stats.NumSoftwareCPEs)
assert.Equal(t, 0, stats.NumSoftwareCVEs)
assert.Equal(t, 0, stats.NumWeeklyActiveUsers) // no active user since last stats were sent
require.Len(t, stats.HostsEnrolledByOperatingSystem, 3) // empty platform, rhel and macos
assert.Equal(t, 5, stats.NumWeeklyPolicyViolationDaysActual)
@@ -338,6 +356,12 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) {
assert.Equal(t, "Fleet", stats.Organization)
assert.Equal(t, 5, stats.NumHostsEnrolled)
assert.Equal(t, 2, stats.NumUsers)
assert.Equal(t, 0, stats.NumSoftwareVersions)
assert.Equal(t, 0, stats.NumHostSoftwares)
assert.Equal(t, 0, stats.NumSoftwareTitles)
assert.Equal(t, 0, stats.NumHostSoftwareInstalledPaths)
assert.Equal(t, 0, stats.NumSoftwareCPEs)
assert.Equal(t, 0, stats.NumSoftwareCVEs)
assert.Equal(t, 1, stats.NumWeeklyActiveUsers)
assert.Equal(t, 0, stats.NumWeeklyPolicyViolationDaysActual)
assert.Equal(t, 0, stats.NumWeeklyPolicyViolationDaysPossible)
server/datastore/mysql/users.go
+4 -4
View File
@@ -276,13 +276,13 @@ func (ds *Datastore) DeleteUser(ctx context.Context, id uint) error {
return ds.deleteEntity(ctx, usersTable, id)
}
func amountUsersDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM users`)
func tableRowsCount(ctx context.Context, db sqlx.QueryerContext, tableName string) (int, error) {
var count int
err := sqlx.GetContext(ctx, db, &count, fmt.Sprintf(`SELECT count(*) FROM %s`, tableName))
if err != nil {
return 0, err
}
return amount, nil
return count, nil
}
func amountActiveUsersSinceDB(ctx context.Context, db sqlx.QueryerContext, since time.Time) (int, error) {
server/fleet/statistics.go
+24 -18
View File
@@ -6,24 +6,30 @@ import (
)
type StatisticsPayload struct {
AnonymousIdentifier string `json:"anonymousIdentifier"`
FleetVersion string `json:"fleetVersion"`
LicenseTier string `json:"licenseTier"`
Organization string `json:"organization"`
NumHostsEnrolled int `json:"numHostsEnrolled"`
NumUsers int `json:"numUsers"`
NumTeams int `json:"numTeams"`
NumPolicies int `json:"numPolicies"`
NumLabels int `json:"numLabels"`
SoftwareInventoryEnabled bool `json:"softwareInventoryEnabled"`
VulnDetectionEnabled bool `json:"vulnDetectionEnabled"`
SystemUsersEnabled bool `json:"systemUsersEnabled"`
HostsStatusWebHookEnabled bool `json:"hostsStatusWebHookEnabled"`
MDMMacOsEnabled bool `json:"mdmMacOsEnabled"`
HostExpiryEnabled bool `json:"hostExpiryEnabled"`
MDMWindowsEnabled bool `json:"mdmWindowsEnabled"`
LiveQueryDisabled bool `json:"liveQueryDisabled"`
NumWeeklyActiveUsers int `json:"numWeeklyActiveUsers"`
AnonymousIdentifier string `json:"anonymousIdentifier"`
FleetVersion string `json:"fleetVersion"`
LicenseTier string `json:"licenseTier"`
Organization string `json:"organization"`
NumHostsEnrolled int `json:"numHostsEnrolled"`
NumUsers int `json:"numUsers"`
NumSoftwareVersions int `json:"numSoftwareVersions"`
NumHostSoftwares int `json:"numHostSoftwares"`
NumSoftwareTitles int `json:"numSoftwareTitles"`
NumHostSoftwareInstalledPaths int `json:"numHostSoftwareInstalledPaths"`
NumSoftwareCPEs int `json:"numSoftwareCPEs"`
NumSoftwareCVEs int `json:"numSoftwareCVEs"`
NumTeams int `json:"numTeams"`
NumPolicies int `json:"numPolicies"`
NumLabels int `json:"numLabels"`
SoftwareInventoryEnabled bool `json:"softwareInventoryEnabled"`
VulnDetectionEnabled bool `json:"vulnDetectionEnabled"`
SystemUsersEnabled bool `json:"systemUsersEnabled"`
HostsStatusWebHookEnabled bool `json:"hostsStatusWebHookEnabled"`
MDMMacOsEnabled bool `json:"mdmMacOsEnabled"`
HostExpiryEnabled bool `json:"hostExpiryEnabled"`
MDMWindowsEnabled bool `json:"mdmWindowsEnabled"`
LiveQueryDisabled bool `json:"liveQueryDisabled"`
NumWeeklyActiveUsers int `json:"numWeeklyActiveUsers"`
// NumWeeklyPolicyViolationDaysActual is an aggregate count of actual policy violation days. One
// policy violation day is added for each policy that a host is failing as of the time the count
// is incremented. The count increments once per 24-hour interval and resets each week.