diff --git a/changes/43910-implement-chart-module b/changes/43910-implement-chart-module new file mode 100644 index 0000000000..3420492d39 --- /dev/null +++ b/changes/43910-implement-chart-module @@ -0,0 +1 @@ +- Implemented the chart bounded context and schema to support charting capabilities in Fleet diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index df6d972817..61b47b38d3 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -15,6 +15,7 @@ import ( eewebhooks "github.com/fleetdm/fleet/v4/ee/server/webhooks" "github.com/fleetdm/fleet/v4/server" activity_api "github.com/fleetdm/fleet/v4/server/activity/api" + chart_api "github.com/fleetdm/fleet/v4/server/chart/api" "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/contexts/license" @@ -1222,6 +1223,7 @@ func newCleanupsAndAggregationSchedule( androidSvc android.Service, activitySvc activity_api.Service, acmeSvc acme_api.Service, + chartSvc chart_api.Service, ) (*schedule.Schedule, error) { const ( name = string(fleet.CronCleanupsThenAggregation) @@ -1473,6 +1475,31 @@ func newCleanupsAndAggregationSchedule( schedule.WithJob("cleanup_orphaned_nano_refetch_commands", func(ctx context.Context) error { return ds.CleanupOrphanedNanoRefetchCommands(ctx) }), + schedule.WithJob("cleanup_chart_data", func(ctx context.Context) error { + return chartSvc.CleanupData(ctx, 30) + }), + ) + + return s, nil +} + +func newChartDataCollectionSchedule( + ctx context.Context, + instanceID string, + ds fleet.Datastore, + chartSvc chart_api.Service, + logger *slog.Logger, +) (*schedule.Schedule, error) { + const ( + name = string(fleet.CronChartDataCollection) + defaultInterval = 10 * time.Minute + ) + s := schedule.New( + ctx, name, instanceID, defaultInterval, ds, ds, + schedule.WithLogger(logger.With("cron", name)), + schedule.WithJob("collect_chart_datasets", func(ctx context.Context) error { + return chartSvc.CollectDatasets(ctx, time.Now()) + }), ) return s, nil diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 864e1ead34..4f2ff00c36 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -39,10 +39,14 @@ import ( "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/acl/acmeacl" "github.com/fleetdm/fleet/v4/server/acl/activityacl" + "github.com/fleetdm/fleet/v4/server/acl/chartacl" activity_api "github.com/fleetdm/fleet/v4/server/activity/api" activity_bootstrap "github.com/fleetdm/fleet/v4/server/activity/bootstrap" apiendpoints "github.com/fleetdm/fleet/v4/server/api_endpoints" "github.com/fleetdm/fleet/v4/server/authz" + "github.com/fleetdm/fleet/v4/server/chart" + chart_api "github.com/fleetdm/fleet/v4/server/chart/api" + chart_bootstrap "github.com/fleetdm/fleet/v4/server/chart/bootstrap" configpkg "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/contexts/installersize" @@ -1112,6 +1116,21 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev // Inject the ACME service module into the main service svc.SetACMEService(acmeSvc) + // Bootstrap chart bounded context + chartSvc, chartRoutes := createChartBoundedContext(dbConns, svc, logger) + + if os.Getenv("FLEET_SKIP_CHART_DATA_COLLECTION") == "" { + if err := cronSchedules.StartCronSchedule( + func() (fleet.CronSchedule, error) { + return newChartDataCollectionSchedule(ctx, instanceID, ds, chartSvc, logger) + }, + ); err != nil { + initFatal(err, "failed to register chart_data_collection schedule") + } + } else { + logger.InfoContext(ctx, "skipping chart data collection cron (FLEET_SKIP_CHART_DATA_COLLECTION is set)") + } + // Perform a cleanup of cron_stats outside of the cronSchedules because the // schedule package uses cron_stats entries to decide whether a schedule will // run or not (see https://github.com/fleetdm/fleet/issues/9486). @@ -1171,7 +1190,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev func() (fleet.CronSchedule, error) { commander := apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPushService) return newCleanupsAndAggregationSchedule( - ctx, instanceID, ds, svc, logger, redisWrapperDS, &config, commander, softwareInstallStore, bootstrapPackageStore, softwareTitleIconStore, androidSvc, activitySvc, acmeSvc, + ctx, instanceID, ds, svc, logger, redisWrapperDS, &config, commander, softwareInstallStore, bootstrapPackageStore, softwareTitleIconStore, androidSvc, activitySvc, acmeSvc, chartSvc, ) }, ); err != nil { @@ -1482,7 +1501,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev extra = append(extra, service.WithHTTPSigVerifier(httpSigVerifier)) apiHandler = service.MakeHandler(svc, config, httpLogger, limiterStore, redisPool, carveStore, - []endpointer.HandlerRoutesFunc{android_service.GetRoutes(svc, androidSvc), activityRoutes, acmeRoutes}, extra...) + []endpointer.HandlerRoutesFunc{android_service.GetRoutes(svc, androidSvc), activityRoutes, acmeRoutes, chartRoutes}, extra...) if err := apiendpoints.Init(apiHandler); err != nil { panic(fmt.Sprintf("error initializing API endpoints: %v", err)) @@ -1922,6 +1941,25 @@ func createACMEServiceModule(ds fleet.Datastore, dbConns *common_mysql.DBConnect return acmeSvc, acmeRoutes } +func createChartBoundedContext(dbConns *common_mysql.DBConnections, svc fleet.Service, logger *slog.Logger) (chart_api.Service, endpointer.HandlerRoutesFunc) { + legacyAuthorizer, err := authz.NewAuthorizer() + if err != nil { + initFatal(err, "initializing chart authorizer") + } + chartAuthorizer := authz.NewAuthorizerAdapter(legacyAuthorizer) + chartViewer := chartacl.NewFleetViewerAdapter() + chartSvc, chartRoutesFn := chart_bootstrap.New(dbConns, chartAuthorizer, chartViewer, logger) + // Register all chart types here. The registry is used to validate chart types in the API + // and to iterate over all chart types when generating chart data. + chartSvc.RegisterDataset(&chart.UptimeDataset{}) + // Create auth middleware for chart bounded context + chartAuthMiddleware := func(next endpoint.Endpoint) endpoint.Endpoint { + return auth.AuthenticatedUser(svc, next) + } + chartRoutes := chartRoutesFn(chartAuthMiddleware) + return chartSvc, chartRoutes +} + func createActivityBoundedContext(svc fleet.Service, dbConns *common_mysql.DBConnections, logger *slog.Logger) (activity_api.Service, endpointer.HandlerRoutesFunc) { legacyAuthorizer, err := authz.NewAuthorizer() if err != nil { diff --git a/pkg/str/str.go b/pkg/str/str.go index 5fa9b27506..070784e9c1 100644 --- a/pkg/str/str.go +++ b/pkg/str/str.go @@ -1,6 +1,9 @@ package str -import "strings" +import ( + "strconv" + "strings" +) func SplitAndTrim(s string, delimiter string, removeEmpty bool) []string { parts := strings.Split(s, delimiter) @@ -13,3 +16,31 @@ func SplitAndTrim(s string, delimiter string, removeEmpty bool) []string { } return cleaned } + +// ParseUintList parses a comma-separated string of unsigned integers, trimming +// whitespace around each value and silently skipping values that cannot be +// parsed. Returns nil for an empty input. +func ParseUintList(s string) []uint { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + result := make([]uint, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if v, err := strconv.ParseUint(p, 10, 0); err == nil { + result = append(result, uint(v)) + } + } + return result +} + +// ParseStringList parses a comma-separated string into a slice of strings, +// trimming whitespace and dropping empty values. Returns nil for an empty +// input. +func ParseStringList(s string) []string { + if s == "" { + return nil + } + return SplitAndTrim(s, ",", true) +} diff --git a/pkg/str/str_test.go b/pkg/str/str_test.go index b2df000ba9..b91320e0cd 100644 --- a/pkg/str/str_test.go +++ b/pkg/str/str_test.go @@ -86,3 +86,100 @@ func TestSplitAndTrim(t *testing.T) { }) } } + +func TestParseUintList(t *testing.T) { + tests := []struct { + name string + input string + expected []uint + }{ + { + name: "empty string returns nil", + input: "", + expected: nil, + }, + { + name: "single value", + input: "42", + expected: []uint{42}, + }, + { + name: "multiple values", + input: "1,2,3", + expected: []uint{1, 2, 3}, + }, + { + name: "trims whitespace", + input: " 1 , 2 , 3 ", + expected: []uint{1, 2, 3}, + }, + { + name: "skips non-numeric values", + input: "1,abc,2,,3", + expected: []uint{1, 2, 3}, + }, + { + name: "skips negative values", + input: "1,-2,3", + expected: []uint{1, 3}, + }, + { + name: "all invalid returns empty slice", + input: "a,b,c", + expected: []uint{}, + }, + { + name: "zero is valid", + input: "0,1", + expected: []uint{0, 1}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ParseUintList(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestParseStringList(t *testing.T) { + tests := []struct { + name string + input string + expected []string + }{ + { + name: "empty string returns nil", + input: "", + expected: nil, + }, + { + name: "single value", + input: "foo", + expected: []string{"foo"}, + }, + { + name: "multiple values", + input: "foo,bar,baz", + expected: []string{"foo", "bar", "baz"}, + }, + { + name: "trims whitespace", + input: " foo , bar , baz ", + expected: []string{"foo", "bar", "baz"}, + }, + { + name: "drops empty values", + input: "foo,,bar, ,baz", + expected: []string{"foo", "bar", "baz"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ParseStringList(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/server/acl/chartacl/fleet_adapter.go b/server/acl/chartacl/fleet_adapter.go new file mode 100644 index 0000000000..47e0e15c44 --- /dev/null +++ b/server/acl/chartacl/fleet_adapter.go @@ -0,0 +1,52 @@ +// Package chartacl provides the anti-corruption layer between the chart +// bounded context and legacy Fleet code. +// +// This package is the ONLY place that imports both chart types and fleet / +// viewer-context types. It translates between them, letting the chart +// context stay free of direct server/fleet or server/contexts/viewer imports +// (which the arch_test enforces). +package chartacl + +import ( + "context" + "errors" + + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" +) + +// FleetViewerAdapter resolves the current authenticated viewer into the +// minimal information the chart service needs to decide team scope. It has +// no state and no Fleet-service dependency — the viewer context is the only +// input it needs. +type FleetViewerAdapter struct{} + +// NewFleetViewerAdapter returns an adapter suitable for passing to +// chart/bootstrap.New. +func NewFleetViewerAdapter() *FleetViewerAdapter { + return &FleetViewerAdapter{} +} + +// Ensure FleetViewerAdapter implements api.ViewerProvider. +var _ api.ViewerProvider = (*FleetViewerAdapter)(nil) + +// ViewerScope reads the user from the viewer context and reports whether they +// have global access (isGlobal) or, otherwise, the list of team IDs they +// belong to. Returns an error when no viewer is in context — chart endpoints +// sit behind authenticated middleware, so the absence of a viewer means the +// request never went through auth and we refuse to serve data. +func (a *FleetViewerAdapter) ViewerScope(ctx context.Context) (bool, []uint, error) { + vc, ok := viewer.FromContext(ctx) + if !ok || vc.User == nil { + return false, nil, errors.New("chart: no authenticated viewer in context") + } + u := vc.User + if u.GlobalRole != nil && *u.GlobalRole != "" { + return true, nil, nil + } + ids := make([]uint, 0, len(u.Teams)) + for _, t := range u.Teams { + ids = append(ids, t.ID) + } + return false, ids, nil +} diff --git a/server/acl/chartacl/fleet_adapter_test.go b/server/acl/chartacl/fleet_adapter_test.go new file mode 100644 index 0000000000..d9bb0fbb4b --- /dev/null +++ b/server/acl/chartacl/fleet_adapter_test.go @@ -0,0 +1,80 @@ +package chartacl + +import ( + "testing" + + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestViewerScopeNoViewerInContextErrors(t *testing.T) { + a := NewFleetViewerAdapter() + _, _, err := a.ViewerScope(t.Context()) + require.Error(t, err, "missing viewer should fail closed, not silently return global=false") +} + +func TestViewerScopeGlobalUser(t *testing.T) { + a := NewFleetViewerAdapter() + + role := fleet.RoleAdmin + ctx := viewer.NewContext(t.Context(), viewer.Viewer{ + User: &fleet.User{GlobalRole: &role}, + }) + + isGlobal, teamIDs, err := a.ViewerScope(ctx) + require.NoError(t, err) + assert.True(t, isGlobal) + assert.Nil(t, teamIDs, "global user's team list is not used by the caller") +} + +func TestViewerScopeGlobalUserEmptyRoleStringIsNotGlobal(t *testing.T) { + // Defensive: a User with a non-nil but empty GlobalRole string should not + // be treated as global. This matches the User.HasAnyGlobalRole semantics. + a := NewFleetViewerAdapter() + + empty := "" + ctx := viewer.NewContext(t.Context(), viewer.Viewer{ + User: &fleet.User{GlobalRole: &empty}, + }) + + isGlobal, _, err := a.ViewerScope(ctx) + require.NoError(t, err) + assert.False(t, isGlobal) +} + +func TestViewerScopeTeamUser(t *testing.T) { + a := NewFleetViewerAdapter() + + ctx := viewer.NewContext(t.Context(), viewer.Viewer{ + User: &fleet.User{ + Teams: []fleet.UserTeam{ + {Team: fleet.Team{ID: 3}, Role: fleet.RoleObserver}, + {Team: fleet.Team{ID: 7}, Role: fleet.RoleMaintainer}, + }, + }, + }) + + isGlobal, teamIDs, err := a.ViewerScope(ctx) + require.NoError(t, err) + assert.False(t, isGlobal) + assert.Equal(t, []uint{3, 7}, teamIDs) +} + +func TestViewerScopeTeamUserNoTeams(t *testing.T) { + // Authenticated user with neither a global role nor any team memberships. + // The chart service treats this as "team-scoped with zero teams" and + // returns empty data — the adapter's job is just to report the scope + // faithfully. + a := NewFleetViewerAdapter() + + ctx := viewer.NewContext(t.Context(), viewer.Viewer{ + User: &fleet.User{}, + }) + + isGlobal, teamIDs, err := a.ViewerScope(ctx) + require.NoError(t, err) + assert.False(t, isGlobal) + assert.Empty(t, teamIDs) +} diff --git a/server/chart/api/chart.go b/server/chart/api/chart.go new file mode 100644 index 0000000000..0ffda3d057 --- /dev/null +++ b/server/chart/api/chart.go @@ -0,0 +1,136 @@ +package api + +import ( + "context" + "time" +) + +// SampleStrategy describes how a dataset's samples combine within a bucket and +// whether rows can collapse across buckets when the bitmap is unchanged. +type SampleStrategy string + +const ( + // SampleStrategyAccumulate means each sample is a partial observation. + // Writes: every row is born closed (valid_to set at insert time to bucketEnd). + // Within-bucket samples OR-merge into the existing row via ODKU; a sample in + // a new bucket just creates a new row with a new valid_from. No explicit + // close step, no cross-bucket collapse. + // Reads: bucket value = OR of every row whose interval overlaps the bucket + // ("hosts observed at any point during the bucket"). + // Used for datasets like uptime and software usage. + // @todo: implement job to collapse identical consecutive rows + // to optimize storage and query performance. + SampleStrategyAccumulate SampleStrategy = "accumulate" + + // SampleStrategySnapshot means each sample is the full state of a single moment. + // Writes: rows are always keyed to 1h boundaries (so row transitions align + // to hour marks regardless of tz). Within a 1h write-bucket, the latest + // sample's bitmap overwrites via ODKU — last sample wins. Across buckets, + // unchanged state keeps the row open (valid_to = sentinel); a changed sample + // closes the prior row at the new hour boundary and opens a new one. + // Reads: bucket value = OR across entities of each entity's row active at + // bucketEnd ("state as of the end of the bucket"). An entity whose row was + // closed mid-bucket with no replacement is absent at bucketEnd. + // Used for datasets like CVE and software inventory. + SampleStrategySnapshot SampleStrategy = "snapshot" +) + +// Dataset defines the interface for a chartable dataset. +type Dataset interface { + // Name returns the dataset identifier used in the DB and API path. + Name() string + + // DefaultResolutionHours returns the default display granularity in hours + // (1 for uptime, 24 for CVE). Used when the caller doesn't specify + // RequestOpts.Resolution. Unrelated to write-side granularity — all + // collectors write at 1h regardless of display resolution; see + // SampleStrategy for details. + DefaultResolutionHours() int + + // SampleStrategy returns how samples combine within and across buckets. + SampleStrategy() SampleStrategy + + // Collect is called by the cron job to populate data in bulk. + Collect(ctx context.Context, store DatasetStore, now time.Time) error + + // DefaultVisualization returns the default visualization type (e.g. "line", "heatmap"). + DefaultVisualization() string +} + +// DatasetStore is the narrow interface that datasets need for their Collect +// method. It is satisfied by the chart internal Datastore, keeping dataset +// implementations decoupled from internals. +type DatasetStore interface { + // FindRecentlySeenHostIDs returns host IDs that have reported since the + // given cutoff. Used by datasets like uptime that derive their sample from + // recent host activity. + FindRecentlySeenHostIDs(ctx context.Context, since time.Time) ([]uint, error) + + // RecordBucketData writes one or more entity bitmaps for the given bucket + // using the specified sample strategy. See SampleStrategy for semantics. + RecordBucketData( + ctx context.Context, + dataset string, + bucketStart time.Time, + bucketSize time.Duration, + strategy SampleStrategy, + entityBitmaps map[string][]byte, + ) error +} + +// Host is a minimal host type for authorization checks within the chart bounded context. +// The JSON tags matter: the OPA rego policy reads object.team_id via the JSON-encoded +// input, so renaming or dropping the tag silently breaks team-scoped authorization. +type Host struct { + ID uint `json:"id"` + TeamID *uint `json:"team_id"` +} + +// AuthzType implements platform_authz.AuthzTyper. +func (h *Host) AuthzType() string { return "host" } + +// DataPoint represents a single data point in the chart response. +type DataPoint struct { + Timestamp time.Time `json:"timestamp"` + Value int `json:"value"` +} + +// Response is the API response for chart data. +type Response struct { + Metric string `json:"metric"` + Visualization string `json:"visualization"` + TotalHosts int `json:"total_hosts"` + Resolution string `json:"resolution"` + Days int `json:"days"` + Filters Filters `json:"filters"` + Data []DataPoint `json:"data"` +} + +// RequestOpts captures the parsed query parameters for a chart request. +type RequestOpts struct { + Days int + // Resolution is the display granularity in hours. Must be 0 or a positive + // divisor of 24. 0 means "use the dataset's default resolution." + Resolution int + // TZOffsetMinutes is the client's UTC offset as reported by JavaScript's + // Date.getTimezoneOffset() (positive = west of UTC, e.g. CDT = 300). + // Used to align hourly bucket boundaries to local time. + TZOffsetMinutes int + // TeamID scopes the request to a single team. nil = global (authz + data + // both fall back to the user's accessible scope). *TeamID == 0 means + // hosts with no team assignment, matching Fleet's convention elsewhere. + TeamID *uint + LabelIDs []uint + Platforms []string + IncludeHostIDs []uint + ExcludeHostIDs []uint +} + +// Filters captures the applied filters for a chart request. +type Filters struct { + TeamID *uint `json:"fleet_id,omitempty"` + LabelIDs []uint `json:"label_ids,omitempty"` + Platforms []string `json:"platforms,omitempty"` + IncludeHostIDs []uint `json:"include_host_ids,omitempty"` + ExcludeHostIDs []uint `json:"exclude_host_ids,omitempty"` +} diff --git a/server/chart/api/http/types.go b/server/chart/api/http/types.go new file mode 100644 index 0000000000..40d63074ff --- /dev/null +++ b/server/chart/api/http/types.go @@ -0,0 +1,29 @@ +// Package http provides HTTP request/response types for the chart bounded context. +package http + +import "github.com/fleetdm/fleet/v4/server/chart/api" + +// GetChartDataRequest is the HTTP request for the chart data endpoint. +type GetChartDataRequest struct { + Metric string `url:"metric"` + Days int `query:"days,optional"` + Resolution int `query:"resolution,optional"` + TZOffset int `query:"tz_offset,optional"` + // TeamID is a pointer so we can distinguish "absent" (auto-scope to the + // viewer) from fleet_id=0 (no-team hosts, a valid Fleet filter). + // Exposed as fleet_id on the wire per the teams→fleets rename; the Go + // field name stays TeamID to match the rest of the codebase. + TeamID *uint `query:"fleet_id,optional"` + LabelIDs string `query:"label_ids,optional"` + Platforms string `query:"platforms,optional"` + IncludeHostIDs string `query:"include_host_ids,optional"` + ExcludeHostIDs string `query:"exclude_host_ids,optional"` +} + +// GetChartDataResponse is the HTTP response for the chart data endpoint. +type GetChartDataResponse struct { + *api.Response + Err error `json:"error,omitempty"` +} + +func (r GetChartDataResponse) Error() error { return r.Err } diff --git a/server/chart/api/service.go b/server/chart/api/service.go new file mode 100644 index 0000000000..18ea1cc4f3 --- /dev/null +++ b/server/chart/api/service.go @@ -0,0 +1,45 @@ +// Package api provides the public API for the chart bounded context. +// External code should use this package to interact with the chart service. +package api + +import ( + "context" + "time" +) + +// Service is the composite interface for the chart service module. +// Bootstrap returns this type. +type Service interface { + // GetChartData returns time-series chart data for the given metric. + GetChartData(ctx context.Context, metric string, opts RequestOpts) (*Response, error) + + // RegisterDataset registers a chart dataset. + RegisterDataset(ds Dataset) + + // CollectDatasets runs Collect on all registered datasets for the given timestamp. + CollectDatasets(ctx context.Context, now time.Time) error + + // CleanupData deletes chart data rows older than the specified number of days. + CleanupData(ctx context.Context, days int) error +} + +// ViewerProvider exposes authorization-relevant information about the current +// authenticated viewer. Implementations typically read the viewer context, so +// this is the seam that keeps the chart bounded context free of direct +// server/fleet imports. +type ViewerProvider interface { + // ViewerScope reports what the authenticated viewer is allowed to see + // across teams. + // + // - isGlobal == true: the viewer has a global role and can see every + // team plus no-team hosts. teamIDs is ignored in that case. + // - isGlobal == false: the viewer is team-scoped; teamIDs lists the IDs + // of every team the viewer has any role on. An empty slice means the + // viewer authenticated but has no team memberships (unusual — they + // should see no hosts). + // + // Returns an error if no viewer is in context (should never happen behind + // the authenticated middleware, but bounded contexts fail closed rather + // than leak data on misconfiguration). + ViewerScope(ctx context.Context) (isGlobal bool, teamIDs []uint, err error) +} diff --git a/server/chart/arch_test.go b/server/chart/arch_test.go new file mode 100644 index 0000000000..17d5d8f980 --- /dev/null +++ b/server/chart/arch_test.go @@ -0,0 +1,112 @@ +package chart_test + +import ( + "regexp" + "slices" + "testing" + + "github.com/fleetdm/fleet/v4/server/archtest" +) + +const m = archtest.ModuleName + +var ( + fleetDeps = regexp.MustCompile(`^github\.com/fleetdm/`) + + // Common allowed dependencies across chart packages. + chartPkgs = []string{ + m + "/server/chart", + m + "/server/chart/api", + m + "/server/chart/api/http", + m + "/server/chart/internal/types", + } + + platformPkgs = []string{ + m + "/server/platform/...", + m + "/server/contexts/...", + m + "/pkg/fleethttp", + m + "/pkg/str", + } +) + +// TestChartPackageDependencies runs architecture tests for all chart packages. +// Each package has specific rules about what dependencies are allowed. +func TestChartPackageDependencies(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + pkg string + shouldNotDepend []string // defaults to m + "/..." if empty + ignoreDeps []string + }{ + { + // Root package only depends on api (for dataset implementations). + name: "root package only depends on api", + pkg: m + "/server/chart", + ignoreDeps: []string{m + "/server/chart/api"}, + }, + { + name: "api package has no Fleet dependencies", + pkg: m + "/server/chart/api", + }, + { + name: "api/http only depends on api", + pkg: m + "/server/chart/api/http", + ignoreDeps: []string{m + "/server/chart/api"}, + }, + { + name: "internal/types only depends on api", + pkg: m + "/server/chart/internal/types", + ignoreDeps: []string{m + "/server/chart/api"}, + }, + { + name: "internal/mysql depends on chart, types, and platform", + pkg: m + "/server/chart/internal/mysql", + ignoreDeps: slices.Concat(chartPkgs, platformPkgs), + }, + { + name: "internal/service depends on chart and platform packages", + pkg: m + "/server/chart/internal/service", + ignoreDeps: slices.Concat(chartPkgs, platformPkgs), + }, + { + name: "bootstrap depends on chart and platform packages", + pkg: m + "/server/chart/bootstrap", + ignoreDeps: slices.Concat([]string{ + m + "/server/chart/internal/mysql", + m + "/server/chart/internal/service", + }, chartPkgs, platformPkgs), + }, + { + name: "all packages only depend on chart and platform", + pkg: m + "/server/chart/...", + ignoreDeps: slices.Concat([]string{ + m + "/server/chart/internal/mysql", + m + "/server/chart/internal/service", + }, chartPkgs, platformPkgs), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + shouldNotDepend := tc.shouldNotDepend + if len(shouldNotDepend) == 0 { + shouldNotDepend = []string{m + "/..."} + } + + test := archtest.NewPackageTest(t, tc.pkg). + OnlyInclude(fleetDeps). + ShouldNotDependOn(shouldNotDepend...). + WithTests() + + if len(tc.ignoreDeps) > 0 { + test.IgnoreDeps(tc.ignoreDeps...) + } + + test.Check() + }) + } +} diff --git a/server/chart/blob.go b/server/chart/blob.go new file mode 100644 index 0000000000..462df9b07c --- /dev/null +++ b/server/chart/blob.go @@ -0,0 +1,84 @@ +// Package chart provides blob utility helpers, dataset implementations, and +// shared constants for the chart bounded context. Public API types live in +// server/chart/api; internal types (HostFilter, Datastore) live in +// server/chart/internal/types. +package chart + +import ( + "encoding/binary" + "math/bits" +) + +// HostIDsToBlob builds a byte slice with bits set at positions corresponding to +// the given host IDs. Bit N of the blob = host ID N. +func HostIDsToBlob(ids []uint) []byte { + if len(ids) == 0 { + return nil + } + + // Find the max ID to size the blob. + var maxID uint + for _, id := range ids { + if id > maxID { + maxID = id + } + } + + blob := make([]byte, maxID/8+1) + for _, id := range ids { + blob[id/8] |= 1 << (id % 8) + } + return blob +} + +// BlobPopcount returns the number of set bits in the blob. +func BlobPopcount(blob []byte) int { + count := 0 + // Process 8 bytes at a time for performance. + i := 0 + for ; i+8 <= len(blob); i += 8 { + v := binary.LittleEndian.Uint64(blob[i : i+8]) + count += bits.OnesCount64(v) + } + for ; i < len(blob); i++ { + count += bits.OnesCount8(blob[i]) + } + return count +} + +// BlobAND returns a new blob that is the bitwise AND of a and b. +// The result length is min(len(a), len(b)) — bits beyond the shorter blob are implicitly zero. +func BlobAND(a, b []byte) []byte { + if a == nil || b == nil { + return nil + } + n := min(len(a), len(b)) + if n == 0 { + return nil + } + result := make([]byte, n) + a = a[:n] + b = b[:n] + for i := range n { + result[i] = a[i] & b[i] //nolint:gosec // a and b are bounded to n via slicing above + } + return result +} + +// BlobOR returns a new blob that is the bitwise OR of a and b. +// The result length is max(len(a), len(b)) — the shorter blob is zero-extended. +func BlobOR(a, b []byte) []byte { + long, short := a, b + if len(b) > len(a) { + long, short = b, a + } + if len(long) == 0 { + return nil + } + result := make([]byte, len(long)) + copy(result, long) + for i := range short { + result[i] |= short[i] + } + return result +} diff --git a/server/chart/blob_test.go b/server/chart/blob_test.go new file mode 100644 index 0000000000..fe85271927 --- /dev/null +++ b/server/chart/blob_test.go @@ -0,0 +1,105 @@ +package chart + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHostIDsToBlob(t *testing.T) { + t.Run("nil for empty input", func(t *testing.T) { + assert.Nil(t, HostIDsToBlob(nil)) + assert.Nil(t, HostIDsToBlob([]uint{})) + }) + + t.Run("single host", func(t *testing.T) { + blob := HostIDsToBlob([]uint{0}) + require.Len(t, blob, 1) + assert.Equal(t, byte(0x01), blob[0]) + }) + + t.Run("host ID 7", func(t *testing.T) { + blob := HostIDsToBlob([]uint{7}) + require.Len(t, blob, 1) + assert.Equal(t, byte(0x80), blob[0]) + }) + + t.Run("host ID 8 starts second byte", func(t *testing.T) { + blob := HostIDsToBlob([]uint{8}) + require.Len(t, blob, 2) + assert.Equal(t, byte(0x00), blob[0]) + assert.Equal(t, byte(0x01), blob[1]) + }) + + t.Run("multiple hosts", func(t *testing.T) { + blob := HostIDsToBlob([]uint{0, 1, 8, 16}) + require.Len(t, blob, 3) + assert.Equal(t, byte(0x03), blob[0]) // bits 0,1 + assert.Equal(t, byte(0x01), blob[1]) // bit 8 + assert.Equal(t, byte(0x01), blob[2]) // bit 16 + }) + + t.Run("large host ID", func(t *testing.T) { + blob := HostIDsToBlob([]uint{1000}) + require.Len(t, blob, 126) // 1000/8+1 + assert.Equal(t, byte(0x01), blob[125]) + }) +} + +func TestBlobPopcount(t *testing.T) { + assert.Equal(t, 0, BlobPopcount(nil)) + assert.Equal(t, 0, BlobPopcount([]byte{})) + assert.Equal(t, 1, BlobPopcount([]byte{0x01})) + assert.Equal(t, 8, BlobPopcount([]byte{0xFF})) + assert.Equal(t, 3, BlobPopcount([]byte{0x07})) + + // Multi-byte + assert.Equal(t, 4, BlobPopcount([]byte{0x0F, 0x00})) + assert.Equal(t, 16, BlobPopcount([]byte{0xFF, 0xFF})) + + // Exercises the uint64 fast path (>= 8 bytes) + blob := make([]byte, 16) + blob[0] = 0xFF // 8 bits + blob[15] = 0x01 // 1 bit + assert.Equal(t, 9, BlobPopcount(blob)) +} + +func TestBlobAND(t *testing.T) { + assert.Nil(t, BlobAND([]byte{}, []byte{})) + assert.Nil(t, BlobAND([]byte{0xFF}, []byte{})) + + result := BlobAND([]byte{0xFF, 0x0F}, []byte{0x0F, 0xFF}) + assert.Equal(t, []byte{0x0F, 0x0F}, result) + + // Different lengths: result is min length + result = BlobAND([]byte{0xFF, 0xFF, 0xFF}, []byte{0x0F}) + assert.Equal(t, []byte{0x0F}, result) +} + +func TestBlobOR(t *testing.T) { + assert.Nil(t, BlobOR(nil, nil)) + + // One nil + result := BlobOR([]byte{0x0F}, nil) + assert.Equal(t, []byte{0x0F}, result) + + result = BlobOR([]byte{0xF0, 0x00}, []byte{0x0F, 0xFF}) + assert.Equal(t, []byte{0xFF, 0xFF}, result) + + // Different lengths: result is max length + result = BlobOR([]byte{0x01}, []byte{0x02, 0xFF}) + assert.Equal(t, []byte{0x03, 0xFF}, result) +} + +func TestRoundTrip(t *testing.T) { + ids := []uint{1, 5, 10, 42, 100, 255} + blob := HostIDsToBlob(ids) + assert.Equal(t, len(ids), BlobPopcount(blob)) + + // Filter to only even IDs + filterIDs := []uint{10, 42, 100} + filterBlob := HostIDsToBlob(filterIDs) + filtered := BlobAND(blob, filterBlob) + assert.Equal(t, 3, BlobPopcount(filtered)) +} diff --git a/server/chart/bootstrap/bootstrap.go b/server/chart/bootstrap/bootstrap.go new file mode 100644 index 0000000000..b815c1efb5 --- /dev/null +++ b/server/chart/bootstrap/bootstrap.go @@ -0,0 +1,32 @@ +// Package bootstrap provides the public entry point for the chart bounded context. +// It wires together internal components and exposes them for use in serve.go. +package bootstrap + +import ( + "log/slog" + + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/fleetdm/fleet/v4/server/chart/internal/mysql" + "github.com/fleetdm/fleet/v4/server/chart/internal/service" + platform_authz "github.com/fleetdm/fleet/v4/server/platform/authz" + eu "github.com/fleetdm/fleet/v4/server/platform/endpointer" + platform_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + "github.com/go-kit/kit/endpoint" +) + +// New creates a new chart service module and returns its service and route handler. +func New( + dbConns *platform_mysql.DBConnections, + authorizer platform_authz.Authorizer, + viewerProvider api.ViewerProvider, + logger *slog.Logger, +) (api.Service, func(authMiddleware endpoint.Middleware) eu.HandlerRoutesFunc) { + ds := mysql.NewDatastore(dbConns, logger) + svc := service.NewService(authorizer, ds, viewerProvider, logger) + + routesFn := func(authMiddleware endpoint.Middleware) eu.HandlerRoutesFunc { + return service.GetRoutes(svc, authMiddleware) + } + + return svc, routesFn +} diff --git a/server/chart/datasets.go b/server/chart/datasets.go new file mode 100644 index 0000000000..b6e4cd3bb7 --- /dev/null +++ b/server/chart/datasets.go @@ -0,0 +1,48 @@ +package chart + +import ( + "context" + "time" + + "github.com/fleetdm/fleet/v4/server/chart/api" +) + +// uptimeRecentlySeenWindow must match the cron schedule cadence so each sample +// reflects activity since the last run. +const uptimeRecentlySeenWindow = 10 * time.Minute + +// UptimeDataset implements api.Dataset for host uptime tracking. +type UptimeDataset struct{} + +func (u *UptimeDataset) Name() string { return "uptime" } +func (u *UptimeDataset) DefaultResolutionHours() int { return 3 } +func (u *UptimeDataset) SampleStrategy() api.SampleStrategy { return api.SampleStrategyAccumulate } +func (u *UptimeDataset) DefaultVisualization() string { return "checkerboard" } + +func (u *UptimeDataset) Collect(ctx context.Context, store api.DatasetStore, now time.Time) error { + hostIDs, err := store.FindRecentlySeenHostIDs(ctx, now.Add(-uptimeRecentlySeenWindow)) + if err != nil { + return err + } + if len(hostIDs) == 0 { + return nil + } + bucketStart := now.UTC().Truncate(time.Hour) + return store.RecordBucketData(ctx, u.Name(), bucketStart, time.Hour, u.SampleStrategy(), + // The empty string key means "all entities" since uptime isn't tracked per host. + // The value is a bitmap of host IDs that were active in this bucket. + map[string][]byte{"": HostIDsToBlob(hostIDs)}) +} + +// CVEDataset implements api.Dataset for host CVE tracking. +type CVEDataset struct{} + +func (c *CVEDataset) Name() string { return "cve" } +func (c *CVEDataset) DefaultResolutionHours() int { return 24 } +func (c *CVEDataset) SampleStrategy() api.SampleStrategy { return api.SampleStrategySnapshot } +func (c *CVEDataset) DefaultVisualization() string { return "line" } + +// @todo implement CVE dataset collection. +func (c *CVEDataset) Collect(_ context.Context, _ api.DatasetStore, _ time.Time) error { + return nil +} diff --git a/server/chart/internal/mysql/charts.go b/server/chart/internal/mysql/charts.go new file mode 100644 index 0000000000..c5108eb128 --- /dev/null +++ b/server/chart/internal/mysql/charts.go @@ -0,0 +1,161 @@ +// Package mysql provides the MySQL datastore implementation for the chart bounded context. +package mysql + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/server/chart/internal/types" + "github.com/fleetdm/fleet/v4/server/contexts/ctxdb" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + platform_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + "github.com/jmoiron/sqlx" +) + +// Datastore is the MySQL implementation of the chart datastore. +type Datastore struct { + primary *sqlx.DB + replica *sqlx.DB + logger *slog.Logger +} + +// NewDatastore creates a new MySQL datastore for the chart bounded context. +func NewDatastore(conns *platform_mysql.DBConnections, logger *slog.Logger) *Datastore { + return &Datastore{primary: conns.Primary, replica: conns.Replica, logger: logger} +} + +// Ensure Datastore implements types.Datastore at compile time. +var _ types.Datastore = (*Datastore)(nil) + +func (ds *Datastore) reader(ctx context.Context) sqlx.QueryerContext { + if ctxdb.IsPrimaryRequired(ctx) { + return ds.primary + } + return ds.replica +} + +func (ds *Datastore) writer(_ context.Context) *sqlx.DB { + return ds.primary +} + +// rebind rewrites a query from ? placeholders to the driver-specific format. +func (ds *Datastore) rebind(query string) string { + return ds.primary.Rebind(query) +} + +func (ds *Datastore) GetHostIDsForFilter(ctx context.Context, hostFilter *types.HostFilter) ([]uint, error) { + subquery, args := buildHostFilterClauses(hostFilter) + + query := fmt.Sprintf(`SELECT h.id FROM hosts h WHERE 1=1 %s`, subquery) + + query, args, err := sqlx.In(query, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "expand host IDs filter query args") + } + query = ds.rebind(query) + + var ids []uint + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &ids, query, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get host IDs for filter") + } + return ids, nil +} + +// FindRecentlySeenHostIDs returns host IDs with any activity signal at or after `since`. +// "Activity signal" is the most recent of host_seen_times.seen_time, nano_enrollments.last_seen_at, +// or host details/creation timestamps. +func (ds *Datastore) FindRecentlySeenHostIDs(ctx context.Context, since time.Time) ([]uint, error) { + const query = ` + SELECT h.id + FROM hosts h + LEFT JOIN host_seen_times hst ON h.id = hst.host_id + LEFT JOIN nano_enrollments ne ON ne.id = h.uuid + AND ne.type IN ('Device', 'User Enrollment (Device)') + WHERE COALESCE( + GREATEST( + COALESCE(hst.seen_time, ne.last_seen_at), + COALESCE(ne.last_seen_at, hst.seen_time) + ), + NULLIF(h.detail_updated_at, '2000-01-01 00:00:00'), + h.created_at + ) >= ?` + + var ids []uint + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &ids, query, since.UTC()); err != nil { + return nil, ctxerr.Wrap(ctx, err, "find recently seen host IDs") + } + return ids, nil +} + +// buildHostFilterClauses translates a HostFilter into SQL WHERE clauses for +// the hosts table. Uses "h" as the table alias. Args may contain slices — +// caller must use sqlx.In to expand them. +func buildHostFilterClauses(filter *types.HostFilter) (string, []any) { + if filter == nil { + return "", nil + } + + var clauses []string + var args []any + + if filter.TeamIDs != nil { + // Empty non-nil: caller is team-scoped with zero accessible teams; + // emit a guaranteed-empty clause so we never run IN () and never return + // hosts the caller can't see. + if len(filter.TeamIDs) == 0 { + clauses = append(clauses, "1=0") + } else { + // Split "no team" (id 0) from real team ids — the two map to + // different SQL (IS NULL vs = ?), so they're OR-ed together when + // both are present. + var positive []uint + includesNoTeam := false + for _, tid := range filter.TeamIDs { + if tid == 0 { + includesNoTeam = true + } else { + positive = append(positive, tid) + } + } + switch { + case includesNoTeam && len(positive) > 0: + clauses = append(clauses, "(h.team_id IS NULL OR h.team_id IN (?))") + args = append(args, positive) + case includesNoTeam: + clauses = append(clauses, "h.team_id IS NULL") + default: + clauses = append(clauses, "h.team_id IN (?)") + args = append(args, positive) + } + } + } + + if len(filter.LabelIDs) > 0 { + clauses = append(clauses, "h.id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id IN (?))") + args = append(args, filter.LabelIDs) + } + + if len(filter.Platforms) > 0 { + clauses = append(clauses, "h.platform IN (?)") + args = append(args, filter.Platforms) + } + + if len(filter.IncludeHostIDs) > 0 { + clauses = append(clauses, "h.id IN (?)") + args = append(args, filter.IncludeHostIDs) + } + + if len(filter.ExcludeHostIDs) > 0 { + clauses = append(clauses, "h.id NOT IN (?)") + args = append(args, filter.ExcludeHostIDs) + } + + if len(clauses) == 0 { + return "", nil + } + + return " AND " + strings.Join(clauses, " AND "), args +} diff --git a/server/chart/internal/mysql/data.go b/server/chart/internal/mysql/data.go new file mode 100644 index 0000000000..a972e303d9 --- /dev/null +++ b/server/chart/internal/mysql/data.go @@ -0,0 +1,363 @@ +package mysql + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/server/chart" + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/jmoiron/sqlx" +) + +// scdOpenSentinel is the end-of-time marker used for valid_to on currently-open +// snapshot rows. Also used as a filter to distinguish open rows from closed ones. +var scdOpenSentinel = time.Date(9999, 12, 31, 0, 0, 0, 0, time.UTC) + +// scdUpsertBatch caps how many entity rows are written per INSERT statement. +const scdUpsertBatch = 200 + +// scdRow is a single row of host_scd_data as fetched by GetSCDData. +type scdRow struct { + EntityID string `db:"entity_id"` + HostBitmap []byte `db:"host_bitmap"` + ValidFrom time.Time `db:"valid_from"` + ValidTo time.Time `db:"valid_to"` +} + +func (ds *Datastore) RecordBucketData( + ctx context.Context, + dataset string, + bucketStart time.Time, + bucketSize time.Duration, + strategy api.SampleStrategy, + entityBitmaps map[string][]byte, +) error { + if len(entityBitmaps) == 0 { + return nil + } + bucketStart = bucketStart.UTC() + + switch strategy { + case api.SampleStrategyAccumulate: + return ds.recordAccumulate(ctx, dataset, bucketStart, bucketSize, entityBitmaps) + case api.SampleStrategySnapshot: + return ds.recordSnapshot(ctx, dataset, bucketStart, entityBitmaps) + default: + return ctxerr.Errorf(ctx, "unknown sample strategy: %s", strategy) + } +} + +// recordAccumulate OR-merges each entity's new bitmap into the row keyed by +// (dataset, entity_id, bucketStart). Rows are always explicitly closed at +// bucketStart+bucketSize; there is no cross-bucket collapse. A new bucket +// always starts a fresh row (different valid_from, different unique key), so +// the first sample in a new bucket never inherits the prior bucket's bitmap. +func (ds *Datastore) recordAccumulate( + ctx context.Context, + dataset string, + bucketStart time.Time, + bucketSize time.Duration, + entityBitmaps map[string][]byte, +) error { + validTo := bucketStart.Add(bucketSize) + + entityIDs := make([]string, 0, len(entityBitmaps)) + for id := range entityBitmaps { + entityIDs = append(entityIDs, id) + } + + // Fetch the current in-bucket bitmaps so we can OR-merge before writing. + existing := make(map[string][]byte, len(entityIDs)) + if len(entityIDs) > 0 { + query, args, err := sqlx.In( + `SELECT entity_id, host_bitmap FROM host_scd_data + WHERE dataset = ? AND valid_from = ? AND entity_id IN (?)`, + dataset, bucketStart, entityIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "expand accumulate select args") + } + query = ds.rebind(query) + + type row struct { + EntityID string `db:"entity_id"` + HostBitmap []byte `db:"host_bitmap"` + } + var rows []row + if err := sqlx.SelectContext(ctx, ds.writer(ctx), &rows, query, args...); err != nil { + return ctxerr.Wrap(ctx, err, "fetch in-bucket bitmaps") + } + for _, r := range rows { + existing[r.EntityID] = r.HostBitmap + } + } + + type upsertRow struct { + entityID string + bitmap []byte + } + toUpsert := make([]upsertRow, 0, len(entityBitmaps)) + for entityID, newBitmap := range entityBitmaps { + merged := chart.BlobOR(existing[entityID], newBitmap) + toUpsert = append(toUpsert, upsertRow{entityID: entityID, bitmap: merged}) + } + + for i := 0; i < len(toUpsert); i += scdUpsertBatch { + end := min(i+scdUpsertBatch, len(toUpsert)) + batch := toUpsert[i:end] + + placeholders := make([]string, 0, len(batch)) + args := make([]any, 0, len(batch)*5) + for _, r := range batch { + placeholders = append(placeholders, "(?, ?, ?, ?, ?)") + args = append(args, dataset, r.entityID, r.bitmap, bucketStart, validTo) + } + // Concatenating hardcoded "(?,?,?,?,?)" placeholder strings, not user input. + stmt := `INSERT INTO host_scd_data (dataset, entity_id, host_bitmap, valid_from, valid_to) VALUES ` + //nolint:gosec // G202 + strings.Join(placeholders, ", ") + + ` ON DUPLICATE KEY UPDATE host_bitmap = VALUES(host_bitmap)` + if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "upsert accumulate rows") + } + } + return nil +} + +// recordSnapshot reconciles the current per-entity bitmaps against open rows. +// Unchanged entities keep their open row (valid_to = sentinel extends naturally). +// Changed entities get their open row closed at bucketStart (if it opened in a +// prior bucket) and a new open row inserted for this bucket. Entities absent +// from the input whose open rows still exist are closed. +func (ds *Datastore) recordSnapshot( + ctx context.Context, + dataset string, + bucketStart time.Time, + entityBitmaps map[string][]byte, +) error { + type openRow struct { + EntityID string `db:"entity_id"` + HostBitmap []byte `db:"host_bitmap"` + ValidFrom time.Time `db:"valid_from"` + } + var openRows []openRow + if err := sqlx.SelectContext(ctx, ds.writer(ctx), &openRows, + `SELECT entity_id, host_bitmap, valid_from + FROM host_scd_data + WHERE dataset = ? AND valid_to = ?`, + dataset, scdOpenSentinel); err != nil { + return ctxerr.Wrap(ctx, err, "fetch open SCD rows") + } + + openByEntity := make(map[string]openRow, len(openRows)) + for _, r := range openRows { + openByEntity[r.EntityID] = r + } + + var toClose []string + type upsertRow struct { + entityID string + bitmap []byte + } + var toUpsert []upsertRow + + for entityID, bitmap := range entityBitmaps { + existing, hasOpen := openByEntity[entityID] + if hasOpen && bytes.Equal(existing.HostBitmap, bitmap) { + continue // unchanged state — leave the row alone + } + if hasOpen && existing.ValidFrom.Before(bucketStart) { + toClose = append(toClose, entityID) + } + toUpsert = append(toUpsert, upsertRow{entityID: entityID, bitmap: bitmap}) + } + + // Entities that disappeared entirely — close their open rows. If the row + // opened this bucket the close leaves a zero-length historical record; that's + // fine and callers can filter valid_from < valid_to if they care. + for entityID := range openByEntity { + if _, ok := entityBitmaps[entityID]; !ok { + toClose = append(toClose, entityID) + } + } + + if len(toClose) > 0 { + closeQuery, closeArgs, err := sqlx.In( + `UPDATE host_scd_data SET valid_to = ? + WHERE dataset = ? AND valid_to = ? AND entity_id IN (?)`, + bucketStart, dataset, scdOpenSentinel, toClose) + if err != nil { + return ctxerr.Wrap(ctx, err, "expand close SCD query args") + } + closeQuery = ds.rebind(closeQuery) + if _, err := ds.writer(ctx).ExecContext(ctx, closeQuery, closeArgs...); err != nil { + return ctxerr.Wrap(ctx, err, "close stale SCD rows") + } + } + + // Snapshot inserts leave valid_to at its DEFAULT (the sentinel). ODKU on + // uniq_entity_bucket means same-bucket overwrites collapse onto this bucket's + // row; new-bucket writes create a fresh row whose predecessor (if any) was + // just closed above. + for i := 0; i < len(toUpsert); i += scdUpsertBatch { + end := min(i+scdUpsertBatch, len(toUpsert)) + batch := toUpsert[i:end] + + placeholders := make([]string, 0, len(batch)) + args := make([]any, 0, len(batch)*4) + for _, r := range batch { + placeholders = append(placeholders, "(?, ?, ?, ?)") + args = append(args, dataset, r.entityID, r.bitmap, bucketStart) + } + // Concatenating hardcoded "(?,?,?,?)" placeholder strings, not user input. + stmt := `INSERT INTO host_scd_data (dataset, entity_id, host_bitmap, valid_from) VALUES ` + //nolint:gosec // G202 + strings.Join(placeholders, ", ") + + ` ON DUPLICATE KEY UPDATE host_bitmap = VALUES(host_bitmap)` + if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "upsert snapshot rows") + } + } + + return nil +} + +// GetSCDData walks buckets of bucketSize across [startDate, endDate] and, for +// each bucket, aggregates the rows whose interval touches the bucket according +// to the sample strategy: +// - Accumulate: OR every overlapping row (across all entities, unless the +// caller restricted entityIDs). For single-entity datasets like uptime this +// is "hosts observed at any point in bucket." For multi-entity datasets +// like (future) software usage it's "distinct hosts seen doing anything +// tracked during the bucket" — the entity dimension collapses into the +// union of hosts touching any entity. +// - Snapshot: per entity, take the row active at bucketEnd; OR across +// entities. "State as of the end of the bucket" — for multi-entity datasets +// like CVE, this is the union of hosts affected by any tracked entity at +// bucketEnd. +// +// filterMask is AND-ed into every bucket's merged bitmap so results reflect +// only hosts visible to the caller. Returns numBuckets = +// (endDate - startDate) / bucketSize data points, labeled by bucket *start* +// (the first label is startDate + bucketSize; the last label is endDate). +// Zero-valued buckets are included with value 0, not omitted. +// +// The caller is responsible for passing bucket-aligned startDate/endDate (e.g. +// local-midnight-aligned for tz-sensitive rendering); the walker does not +// truncate. +func (ds *Datastore) GetSCDData( + ctx context.Context, + dataset string, + startDate, endDate time.Time, + bucketSize time.Duration, + strategy api.SampleStrategy, + filterMask []byte, + entityIDs []string, +) ([]api.DataPoint, error) { + startDate = startDate.UTC() + endDate = endDate.UTC() + + numBuckets := int(endDate.Sub(startDate) / bucketSize) + if numBuckets <= 0 { + return nil, nil + } + + // Fetch every row whose validity interval overlaps any of the buckets. The + // walker filters precisely per bucket; this just narrows the scan. + firstBucketStart := startDate.Add(bucketSize) + lastBucketEnd := endDate.Add(bucketSize) + args := []any{dataset, lastBucketEnd, firstBucketStart} + var entityClause string + if len(entityIDs) > 0 { + entityClause = " AND entity_id IN (?)" + args = append(args, entityIDs) + } + + query := fmt.Sprintf(` + SELECT entity_id, host_bitmap, valid_from, valid_to + FROM host_scd_data + WHERE dataset = ? + AND valid_from < ? + AND valid_to > ?%s`, entityClause) + + expanded, expandedArgs, err := sqlx.In(query, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "expand SCD query args") + } + expanded = ds.rebind(expanded) + + var rows []scdRow + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &rows, expanded, expandedArgs...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get SCD data") + } + + results := make([]api.DataPoint, numBuckets) + for i := range numBuckets { + bucketStart := startDate.Add(time.Duration(i+1) * bucketSize) + bucketEnd := bucketStart.Add(bucketSize) + merged := aggregateBucket(rows, bucketStart, bucketEnd, strategy) + if merged != nil { + merged = chart.BlobAND(merged, filterMask) + } + results[i] = api.DataPoint{ + Timestamp: bucketStart, + Value: chart.BlobPopcount(merged), + } + } + return results, nil +} + +// aggregateBucket returns the merged bitmap for a single bucket given the +// sample strategy. For Accumulate, ORs every overlapping row (entity dimension +// collapses into the union — correct for "distinct hosts seen doing anything +// tracked"). For Snapshot, picks the row active at bucketEnd per entity and +// ORs across entities. +func aggregateBucket(rows []scdRow, bucketStart, bucketEnd time.Time, strategy api.SampleStrategy) []byte { + if strategy == api.SampleStrategySnapshot { + // Per entity, the row "active at bucketEnd" is the one whose + // [valid_from, valid_to) covers the instant bucketEnd-ε. For interval + // boundaries, that's valid_from < bucketEnd AND valid_to >= bucketEnd. + // Write semantics ensure at most one such row per (entity, moment). + var merged []byte + seen := make(map[string]struct{}) + for _, r := range rows { + if !r.ValidFrom.Before(bucketEnd) || r.ValidTo.Before(bucketEnd) { + continue + } + if _, dup := seen[r.EntityID]; dup { + continue + } + seen[r.EntityID] = struct{}{} + merged = chart.BlobOR(merged, r.HostBitmap) + } + return merged + } + + // Accumulate: OR every row that overlaps the bucket. + var merged []byte + for _, r := range rows { + if !r.ValidFrom.Before(bucketEnd) || !r.ValidTo.After(bucketStart) { + continue + } + merged = chart.BlobOR(merged, r.HostBitmap) + } + return merged +} + +// CleanupSCDData deletes closed SCD rows whose valid_to is older than the +// retention cutoff. Open rows (valid_to = sentinel) are always preserved. +func (ds *Datastore) CleanupSCDData(ctx context.Context, days int) error { + // Compute the cutoff in Go (UTC) so the retention boundary doesn't depend + // on the MySQL session time zone — all valid_to writes are UTC. + cutoff := time.Now().UTC().AddDate(0, 0, -days) + _, err := ds.writer(ctx).ExecContext(ctx, + `DELETE FROM host_scd_data + WHERE valid_to < ? + AND valid_to <> ?`, + cutoff, scdOpenSentinel) + if err != nil { + return ctxerr.Wrap(ctx, err, "cleanup SCD data") + } + return nil +} diff --git a/server/chart/internal/mysql/data_test.go b/server/chart/internal/mysql/data_test.go new file mode 100644 index 0000000000..6328b92202 --- /dev/null +++ b/server/chart/internal/mysql/data_test.go @@ -0,0 +1,109 @@ +package mysql + +import ( + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/chart" + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/stretchr/testify/assert" +) + +func TestAggregateBucketAccumulate(t *testing.T) { + bucketStart := time.Date(2026, 4, 21, 0, 0, 0, 0, time.UTC) + bucketEnd := bucketStart.Add(24 * time.Hour) + + // Three accumulate rows within the bucket, each observed during a different + // hour. Accumulate semantics = union of all overlapping rows. + rows := []scdRow{ + {EntityID: "", HostBitmap: chart.HostIDsToBlob([]uint{1, 2}), ValidFrom: bucketStart.Add(2 * time.Hour), ValidTo: bucketStart.Add(3 * time.Hour)}, + {EntityID: "", HostBitmap: chart.HostIDsToBlob([]uint{3}), ValidFrom: bucketStart.Add(10 * time.Hour), ValidTo: bucketStart.Add(11 * time.Hour)}, + {EntityID: "", HostBitmap: chart.HostIDsToBlob([]uint{2, 4}), ValidFrom: bucketStart.Add(15 * time.Hour), ValidTo: bucketStart.Add(16 * time.Hour)}, + } + + got := aggregateBucket(rows, bucketStart, bucketEnd, api.SampleStrategyAccumulate) + assert.Equal(t, 4, chart.BlobPopcount(got), "union of {1,2}, {3}, {2,4} = {1,2,3,4}") +} + +func TestAggregateBucketAccumulateMultiEntity(t *testing.T) { + bucketStart := time.Date(2026, 4, 21, 14, 0, 0, 0, time.UTC) + bucketEnd := bucketStart.Add(time.Hour) + + // Future-style multi-entity accumulate dataset (e.g. software usage): + // entity = software name; bitmap = hosts that used that software this hour. + // Bucket value = distinct hosts using any tracked software during the hour. + rows := []scdRow{ + {EntityID: "slack", HostBitmap: chart.HostIDsToBlob([]uint{1, 2}), ValidFrom: bucketStart, ValidTo: bucketEnd}, + {EntityID: "zoom", HostBitmap: chart.HostIDsToBlob([]uint{2, 3}), ValidFrom: bucketStart, ValidTo: bucketEnd}, + {EntityID: "chrome", HostBitmap: chart.HostIDsToBlob([]uint{4}), ValidFrom: bucketStart, ValidTo: bucketEnd}, + } + + got := aggregateBucket(rows, bucketStart, bucketEnd, api.SampleStrategyAccumulate) + assert.Equal(t, 4, chart.BlobPopcount(got), "union across entities = {1,2,3,4}") +} + +func TestAggregateBucketSnapshotEndOfBucket(t *testing.T) { + bucketStart := time.Date(2026, 4, 21, 0, 0, 0, 0, time.UTC) + bucketEnd := bucketStart.Add(24 * time.Hour) + + // One entity "cve-A" changed state mid-bucket: affected hosts were {1,2,3} + // from hr 0 to hr 14, then {1,2} from hr 14 onward (H3 patched). + // End-of-bucket semantics should return only the *latest* state, not the OR. + rows := []scdRow{ + {EntityID: "cve-A", HostBitmap: chart.HostIDsToBlob([]uint{1, 2, 3}), ValidFrom: bucketStart, ValidTo: bucketStart.Add(14 * time.Hour)}, + {EntityID: "cve-A", HostBitmap: chart.HostIDsToBlob([]uint{1, 2}), ValidFrom: bucketStart.Add(14 * time.Hour), ValidTo: time.Date(9999, 12, 31, 0, 0, 0, 0, time.UTC)}, + } + + got := aggregateBucket(rows, bucketStart, bucketEnd, api.SampleStrategySnapshot) + assert.Equal(t, 2, chart.BlobPopcount(got), "end-of-bucket state is {1,2}, not union {1,2,3}") +} + +func TestAggregateBucketSnapshotMultipleEntities(t *testing.T) { + bucketStart := time.Date(2026, 4, 21, 0, 0, 0, 0, time.UTC) + bucketEnd := bucketStart.Add(24 * time.Hour) + + sentinel := time.Date(9999, 12, 31, 0, 0, 0, 0, time.UTC) + + // Two entities, each with an end-of-bucket state; snapshot returns OR across + // entities of each's latest row. + rows := []scdRow{ + // cve-A: latest state {1,2} + {EntityID: "cve-A", HostBitmap: chart.HostIDsToBlob([]uint{1, 2, 3}), ValidFrom: bucketStart, ValidTo: bucketStart.Add(14 * time.Hour)}, + {EntityID: "cve-A", HostBitmap: chart.HostIDsToBlob([]uint{1, 2}), ValidFrom: bucketStart.Add(14 * time.Hour), ValidTo: sentinel}, + // cve-B: latest state {3,4} + {EntityID: "cve-B", HostBitmap: chart.HostIDsToBlob([]uint{3, 4}), ValidFrom: bucketStart.Add(5 * time.Hour), ValidTo: sentinel}, + } + + got := aggregateBucket(rows, bucketStart, bucketEnd, api.SampleStrategySnapshot) + assert.Equal(t, 4, chart.BlobPopcount(got), "union of cve-A end-state {1,2} and cve-B end-state {3,4}") +} + +func TestAggregateBucketSnapshotEntityDisappears(t *testing.T) { + bucketStart := time.Date(2026, 4, 21, 0, 0, 0, 0, time.UTC) + bucketEnd := bucketStart.Add(24 * time.Hour) + + // Entity was active early in bucket but its row was closed mid-bucket with + // no replacement (entity disappeared — e.g., last affected host patched). + // End-of-bucket semantics exclude it: no row is active at bucketEnd. + rows := []scdRow{ + {EntityID: "cve-A", HostBitmap: chart.HostIDsToBlob([]uint{1, 2, 3}), ValidFrom: bucketStart, ValidTo: bucketStart.Add(14 * time.Hour)}, + } + + got := aggregateBucket(rows, bucketStart, bucketEnd, api.SampleStrategySnapshot) + assert.Equal(t, 0, chart.BlobPopcount(got), "entity closed mid-bucket is absent at bucketEnd") +} + +func TestAggregateBucketSnapshotRowClosedExactlyAtBucketEnd(t *testing.T) { + bucketStart := time.Date(2026, 4, 21, 0, 0, 0, 0, time.UTC) + bucketEnd := bucketStart.Add(24 * time.Hour) + + // Row's valid_to == bucketEnd. The row represents state up to (but not + // including) bucketEnd — i.e., the state just before the bucket ends. + // That's exactly what end-of-bucket semantics should pick. + rows := []scdRow{ + {EntityID: "cve-A", HostBitmap: chart.HostIDsToBlob([]uint{1, 2}), ValidFrom: bucketStart, ValidTo: bucketEnd}, + } + + got := aggregateBucket(rows, bucketStart, bucketEnd, api.SampleStrategySnapshot) + assert.Equal(t, 2, chart.BlobPopcount(got), "row whose valid_to equals bucketEnd covers bucketEnd-ε") +} diff --git a/server/chart/internal/service/endpoint_utils.go b/server/chart/internal/service/endpoint_utils.go new file mode 100644 index 0000000000..cff0598c5b --- /dev/null +++ b/server/chart/internal/service/endpoint_utils.go @@ -0,0 +1,68 @@ +package service + +import ( + "context" + "encoding/json" + "io" + "net/http" + + "github.com/fleetdm/fleet/v4/server/chart/api" + eu "github.com/fleetdm/fleet/v4/server/platform/endpointer" + platform_http "github.com/fleetdm/fleet/v4/server/platform/http" + "github.com/go-kit/kit/endpoint" + kithttp "github.com/go-kit/kit/transport/http" + "github.com/gorilla/mux" +) + +// encodeResponse encodes the response as JSON using the common Fleet encoding pattern. +func encodeResponse(ctx context.Context, w http.ResponseWriter, response any) error { + return eu.EncodeCommonResponse(ctx, w, response, + func(w http.ResponseWriter, response any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(response) + }, + nil, // no domain-specific error encoder; standard fleet errors are handled by common encoder + ) +} + +// makeDecoder creates a decoder for the given request type. +func makeDecoder(iface any, requestBodySizeLimit int64) kithttp.DecodeRequestFunc { + return eu.MakeDecoder(iface, func(body io.Reader, req any) error { + return json.NewDecoder(body).Decode(req) + }, nil, nil, nil, nil, requestBodySizeLimit) +} + +// handlerFunc is the handler function type for chart service endpoints. +type handlerFunc func(ctx context.Context, request any, svc api.Service) (platform_http.Errorer, error) + +type chartEndpointer struct { + svc api.Service +} + +func (e *chartEndpointer) CallHandlerFunc(f handlerFunc, ctx context.Context, request any, svc any) (platform_http.Errorer, error) { + return f(ctx, request, svc.(api.Service)) +} + +func (e *chartEndpointer) Service() any { + return e.svc +} + +// Compile-time check to ensure chartEndpointer implements Endpointer. +var _ eu.Endpointer[handlerFunc] = &chartEndpointer{} + +func newChartEndpointer(svc api.Service, authMiddleware endpoint.Middleware, opts []kithttp.ServerOption, r *mux.Router, + versions ...string, +) *eu.CommonEndpointer[handlerFunc] { + return &eu.CommonEndpointer[handlerFunc]{ + EP: &chartEndpointer{ + svc: svc, + }, + MakeDecoderFn: makeDecoder, + EncodeFn: encodeResponse, + Opts: opts, + AuthMiddleware: authMiddleware, + Router: r, + Versions: versions, + } +} diff --git a/server/chart/internal/service/handler.go b/server/chart/internal/service/handler.go new file mode 100644 index 0000000000..b1de99b84d --- /dev/null +++ b/server/chart/internal/service/handler.go @@ -0,0 +1,54 @@ +package service + +import ( + "context" + + "github.com/fleetdm/fleet/v4/pkg/str" + "github.com/fleetdm/fleet/v4/server/chart/api" + api_http "github.com/fleetdm/fleet/v4/server/chart/api/http" + eu "github.com/fleetdm/fleet/v4/server/platform/endpointer" + platform_http "github.com/fleetdm/fleet/v4/server/platform/http" + "github.com/go-kit/kit/endpoint" + kithttp "github.com/go-kit/kit/transport/http" + "github.com/gorilla/mux" +) + +// GetRoutes returns a function that registers chart routes on the router using the provided +// authMiddleware. +func GetRoutes(svc api.Service, authMiddleware endpoint.Middleware) eu.HandlerRoutesFunc { + return func(r *mux.Router, opts []kithttp.ServerOption) { + attachFleetAPIRoutes(r, svc, authMiddleware, opts) + } +} + +func attachFleetAPIRoutes(r *mux.Router, svc api.Service, authMiddleware endpoint.Middleware, opts []kithttp.ServerOption) { + apiVersions := []string{"v1", "2022-04"} + ue := newChartEndpointer(svc, authMiddleware, opts, r, apiVersions...) + ue.GET("/api/_version_/fleet/charts/{metric}", getChartDataEndpoint, api_http.GetChartDataRequest{}) +} + +func getChartDataEndpoint(ctx context.Context, request any, svc api.Service) (platform_http.Errorer, error) { + req := request.(*api_http.GetChartDataRequest) + + days := req.Days + if days == 0 { + days = 7 + } + + opts := api.RequestOpts{ + Days: days, + Resolution: req.Resolution, + TZOffsetMinutes: req.TZOffset, + TeamID: req.TeamID, + LabelIDs: str.ParseUintList(req.LabelIDs), + Platforms: str.ParseStringList(req.Platforms), + IncludeHostIDs: str.ParseUintList(req.IncludeHostIDs), + ExcludeHostIDs: str.ParseUintList(req.ExcludeHostIDs), + } + + resp, err := svc.GetChartData(ctx, req.Metric, opts) + if err != nil { + return api_http.GetChartDataResponse{Err: err}, nil + } + return api_http.GetChartDataResponse{Response: resp}, nil +} diff --git a/server/chart/internal/service/host_cache.go b/server/chart/internal/service/host_cache.go new file mode 100644 index 0000000000..bb5c9047bc --- /dev/null +++ b/server/chart/internal/service/host_cache.go @@ -0,0 +1,130 @@ +package service + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "time" + + "github.com/fleetdm/fleet/v4/server/chart/internal/types" + "golang.org/x/sync/singleflight" +) + +// hostFilterCacheTTL is how long a host-ID bitmap is served from cache before +// being recomputed. Kept well under the chart collection cadence (10m) so data +// and mask staleness stay roughly aligned. +const hostFilterCacheTTL = 60 * time.Second + +// hostBitmapFetcher is the signature used by cache callers to compute a fresh +// bitmap on a miss. Returning an error bypasses caching for that call. +type hostBitmapFetcher func(ctx context.Context) ([]byte, error) + +// hostFilterCache maps a canonicalized HostFilter to the bitmap of host IDs +// that match it. Entries are considered valid for ttl; concurrent misses for +// the same key are collapsed via singleflight. Expired entries are swept from +// the map opportunistically on each write, so stale keys (e.g. one-off +// IncludeHostIDs filters) don't accumulate indefinitely. +type hostFilterCache struct { + ttl time.Duration + clock func() time.Time + sf singleflight.Group + mu sync.RWMutex + entries map[string]hostFilterCacheEntry +} + +type hostFilterCacheEntry struct { + bitmap []byte + expiresAt time.Time +} + +func newHostFilterCache(ttl time.Duration) *hostFilterCache { + return &hostFilterCache{ + ttl: ttl, + clock: time.Now, + entries: make(map[string]hostFilterCacheEntry), + } +} + +// Get returns the cached bitmap for the filter or computes a fresh one via +// fetch on miss/expiry. Concurrent misses for the same filter share one fetch. +func (c *hostFilterCache) Get(ctx context.Context, filter *types.HostFilter, fetch hostBitmapFetcher) ([]byte, error) { + key := hashHostFilter(filter) + + c.mu.RLock() + entry, ok := c.entries[key] + c.mu.RUnlock() + if ok && c.clock().Before(entry.expiresAt) { + return entry.bitmap, nil + } + + val, err, _ := c.sf.Do(key, func() (any, error) { + // Re-check after acquiring the singleflight slot: another goroutine + // may have populated the cache while we were waiting. + c.mu.RLock() + entry, ok := c.entries[key] + c.mu.RUnlock() + if ok && c.clock().Before(entry.expiresAt) { + return entry.bitmap, nil + } + + bitmap, err := fetch(ctx) + if err != nil { + return nil, err + } + now := c.clock() + c.mu.Lock() + // Sweep expired entries so keys we never see again don't leak. Cheap + // because this only runs on misses (already the slow path). + for k, e := range c.entries { + if !now.Before(e.expiresAt) { + delete(c.entries, k) + } + } + c.entries[key] = hostFilterCacheEntry{ + bitmap: bitmap, + expiresAt: now.Add(c.ttl), + } + c.mu.Unlock() + return bitmap, nil + }) + if err != nil { + return nil, err + } + return val.([]byte), nil +} + +// hashHostFilter produces a deterministic string key for a HostFilter. Slice +// fields are sorted and copied so caller mutations can't affect keying; a +// shared separator that can't appear in the encoded values keeps distinct +// filters from collapsing to the same key. +// +// TeamIDs specifically distinguishes nil from empty-non-nil — the two have +// different semantics (no filter vs match nothing) and must never share a +// cache entry. +func hashHostFilter(f *types.HostFilter) string { + if f == nil { + return "nil" + } + teams := slices.Clone(f.TeamIDs) + slices.Sort(teams) + labels := slices.Clone(f.LabelIDs) + slices.Sort(labels) + platforms := slices.Clone(f.Platforms) + slices.Sort(platforms) + include := slices.Clone(f.IncludeHostIDs) + slices.Sort(include) + exclude := slices.Clone(f.ExcludeHostIDs) + slices.Sort(exclude) + + var b strings.Builder + if f.TeamIDs == nil { + b.WriteString("teams=nil") + } else { + fmt.Fprintf(&b, "teams=%v", teams) + } + fmt.Fprintf(&b, "|labels=%v|platforms=%s|include=%v|exclude=%v", + labels, strings.Join(platforms, ","), include, exclude) + return b.String() +} diff --git a/server/chart/internal/service/host_cache_test.go b/server/chart/internal/service/host_cache_test.go new file mode 100644 index 0000000000..40a4426ecf --- /dev/null +++ b/server/chart/internal/service/host_cache_test.go @@ -0,0 +1,184 @@ +package service + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/chart/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHashHostFilterDeterministic(t *testing.T) { + t.Run("slice order and duplicates don't change the key", func(t *testing.T) { + a := &types.HostFilter{ + LabelIDs: []uint{3, 1, 2}, + Platforms: []string{"windows", "darwin"}, + IncludeHostIDs: []uint{9, 7}, + ExcludeHostIDs: []uint{5, 4}, + } + b := &types.HostFilter{ + LabelIDs: []uint{1, 2, 3}, + Platforms: []string{"darwin", "windows"}, + IncludeHostIDs: []uint{7, 9}, + ExcludeHostIDs: []uint{4, 5}, + } + assert.Equal(t, hashHostFilter(a), hashHostFilter(b)) + }) + + t.Run("teams distinguishes nil, empty, zero-team, and specific teams", func(t *testing.T) { + // Four semantically distinct values that must not share a cache key: + // nil — no team filter (global user, no team_id) + // empty slice — match nothing (team user with zero accessible teams) + // [0] — no-team hosts (team_id=0 query) + // [5] — specific team + keys := map[string]struct{}{ + hashHostFilter(&types.HostFilter{TeamIDs: nil}): {}, + hashHostFilter(&types.HostFilter{TeamIDs: []uint{}}): {}, + hashHostFilter(&types.HostFilter{TeamIDs: []uint{0}}): {}, + hashHostFilter(&types.HostFilter{TeamIDs: []uint{5}}): {}, + hashHostFilter(&types.HostFilter{TeamIDs: []uint{1, 2}}): {}, + } + assert.Len(t, keys, 5, "all five team-scope variants must produce distinct keys") + }) + + t.Run("label vs include collision guard", func(t *testing.T) { + // Without a separator, labels=[1,2] + include=[] could collide with + // labels=[] + include=[1,2] if the key was naively concatenated. + a := &types.HostFilter{LabelIDs: []uint{1, 2}} + b := &types.HostFilter{IncludeHostIDs: []uint{1, 2}} + assert.NotEqual(t, hashHostFilter(a), hashHostFilter(b)) + }) +} + +func TestHostFilterCacheServesFromCacheUntilTTL(t *testing.T) { + cache := newHostFilterCache(10 * time.Second) + + // Override the clock so TTL behavior is deterministic. + var now atomic.Int64 + now.Store(time.Now().UnixNano()) + cache.clock = func() time.Time { return time.Unix(0, now.Load()) } + + var calls atomic.Int32 + fetch := func(_ context.Context) ([]byte, error) { + calls.Add(1) + return []byte{0x0F}, nil + } + + filter := &types.HostFilter{LabelIDs: []uint{1}} + for range 5 { + b, err := cache.Get(t.Context(), filter, fetch) + require.NoError(t, err) + assert.Equal(t, []byte{0x0F}, b) + } + assert.Equal(t, int32(1), calls.Load(), "repeated gets within TTL should hit the cache") + + // Advance past TTL. + now.Add(int64(11 * time.Second)) + _, err := cache.Get(t.Context(), filter, fetch) + require.NoError(t, err) + assert.Equal(t, int32(2), calls.Load(), "expired entry should trigger a refetch") +} + +func TestHostFilterCacheDistinctFiltersMissSeparately(t *testing.T) { + cache := newHostFilterCache(time.Minute) + + var calls atomic.Int32 + fetch := func(_ context.Context) ([]byte, error) { + calls.Add(1) + return []byte{0xFF}, nil + } + + _, err := cache.Get(t.Context(), &types.HostFilter{TeamIDs: []uint{1}}, fetch) + require.NoError(t, err) + _, err = cache.Get(t.Context(), &types.HostFilter{TeamIDs: []uint{2}}, fetch) + require.NoError(t, err) + + assert.Equal(t, int32(2), calls.Load(), "different filter keys should each trigger a fetch") +} + +func TestHostFilterCacheSingleflightCoalescesConcurrentMisses(t *testing.T) { + cache := newHostFilterCache(time.Minute) + + var calls atomic.Int32 + unblock := make(chan struct{}) + fetch := func(_ context.Context) ([]byte, error) { + calls.Add(1) + <-unblock // hold the fetch until all goroutines are parked on singleflight + return []byte{0x01}, nil + } + + filter := &types.HostFilter{LabelIDs: []uint{42}} + + var wg sync.WaitGroup + const goroutines = 20 + wg.Add(goroutines) + for range goroutines { + go func() { + defer wg.Done() + b, err := cache.Get(t.Context(), filter, fetch) + assert.NoError(t, err) + assert.Equal(t, []byte{0x01}, b) + }() + } + + // Give the goroutines a moment to all reach Get; then release the fetch. + time.Sleep(20 * time.Millisecond) + close(unblock) + wg.Wait() + + assert.Equal(t, int32(1), calls.Load(), "singleflight should coalesce concurrent misses") +} + +func TestHostFilterCacheSweepsExpiredEntriesOnWrite(t *testing.T) { + cache := newHostFilterCache(10 * time.Second) + + var now atomic.Int64 + now.Store(time.Now().UnixNano()) + cache.clock = func() time.Time { return time.Unix(0, now.Load()) } + + fetch := func(_ context.Context) ([]byte, error) { return []byte{0x01}, nil } + + // Seed two entries that will later be expired. + _, err := cache.Get(t.Context(), &types.HostFilter{LabelIDs: []uint{1}}, fetch) + require.NoError(t, err) + _, err = cache.Get(t.Context(), &types.HostFilter{LabelIDs: []uint{2}}, fetch) + require.NoError(t, err) + + cache.mu.RLock() + assert.Len(t, cache.entries, 2) + cache.mu.RUnlock() + + // Advance past TTL and write a new entry — the two stale entries should + // be swept during the write path. + now.Add(int64(11 * time.Second)) + _, err = cache.Get(t.Context(), &types.HostFilter{LabelIDs: []uint{3}}, fetch) + require.NoError(t, err) + + cache.mu.RLock() + defer cache.mu.RUnlock() + assert.Len(t, cache.entries, 1, "expired entries should be swept on write") +} + +func TestHostFilterCacheDoesNotCacheErrors(t *testing.T) { + cache := newHostFilterCache(time.Minute) + + var calls atomic.Int32 + sentinel := errors.New("boom") + fetch := func(_ context.Context) ([]byte, error) { + calls.Add(1) + return nil, sentinel + } + + filter := &types.HostFilter{} + _, err := cache.Get(t.Context(), filter, fetch) + require.ErrorIs(t, err, sentinel) + _, err = cache.Get(t.Context(), filter, fetch) + require.ErrorIs(t, err, sentinel) + + assert.Equal(t, int32(2), calls.Load(), "failed fetches must not poison the cache") +} diff --git a/server/chart/internal/service/service.go b/server/chart/internal/service/service.go new file mode 100644 index 0000000000..767ff67747 --- /dev/null +++ b/server/chart/internal/service/service.go @@ -0,0 +1,217 @@ +// Package service provides the service implementation for the chart bounded context. +package service + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/fleetdm/fleet/v4/server/chart" + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/fleetdm/fleet/v4/server/chart/internal/types" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + platform_authz "github.com/fleetdm/fleet/v4/server/platform/authz" + platform_http "github.com/fleetdm/fleet/v4/server/platform/http" +) + +// Service is the chart bounded context service implementation. +type Service struct { + authz platform_authz.Authorizer + store types.Datastore + viewer api.ViewerProvider + datasets map[string]api.Dataset + hostCache *hostFilterCache + logger *slog.Logger +} + +// NewService creates a new chart service. +func NewService(authz platform_authz.Authorizer, store types.Datastore, viewerProvider api.ViewerProvider, logger *slog.Logger) *Service { + return &Service{ + authz: authz, + store: store, + viewer: viewerProvider, + datasets: make(map[string]api.Dataset), + hostCache: newHostFilterCache(hostFilterCacheTTL), + logger: logger, + } +} + +// Ensure Service implements api.Service at compile time. +var _ api.Service = (*Service)(nil) + +func (s *Service) RegisterDataset(ds api.Dataset) { + s.datasets[ds.Name()] = ds +} + +func (s *Service) CollectDatasets(ctx context.Context, now time.Time) error { + for name, dataset := range s.datasets { + if err := dataset.Collect(ctx, s.store, now); err != nil { + // Log and continue — don't let one dataset failure block others. + if s.logger != nil { + s.logger.ErrorContext(ctx, "collect chart dataset", "dataset", name, "err", ctxerr.Wrap(ctx, err, "collect chart dataset")) + } + } + } + return nil +} + +func (s *Service) GetChartData(ctx context.Context, metric string, opts api.RequestOpts) (*api.Response, error) { + // Resolve scope first: for authz we need the right action + subject, and + // for data we need the effective team set. Fail closed if there's no + // viewer — the authenticated middleware should have placed one in ctx. + isGlobal, viewerTeamIDs, err := s.viewer.ViewerScope(ctx) + if err != nil { + return nil, err + } + + // Build the authz subject + action. Two distinct cases: + // - Explicit team_id: Host{TeamID: opts.TeamID} + ActionRead. Rego's + // read rule for hosts requires team_role(subject, object.team_id) to + // match, so a team user asking for a team they don't have a role on + // is rejected by policy (not by us). Global users pass via the + // global-role rules, which don't care about team_id. + // - No team_id: Host{} + ActionList. Rego's list rules pass global + // users unconditionally and pass team users who have a list-capable + // role on any of their teams. The service then scopes data below. + authzSubject := &api.Host{TeamID: opts.TeamID} + authzAction := platform_authz.ActionRead + if opts.TeamID == nil { + authzAction = platform_authz.ActionList + } + if err := s.authz.Authorize(ctx, authzSubject, authzAction); err != nil { + return nil, err + } + + dataset, ok := s.datasets[metric] + if !ok { + return nil, &platform_http.BadRequestError{Message: fmt.Sprintf("unknown chart metric: %s", metric)} + } + + // Validate days preset. + validDays := map[int]struct{}{1: {}, 7: {}, 14: {}, 30: {}} + if _, ok := validDays[opts.Days]; !ok { + return nil, &platform_http.BadRequestError{Message: fmt.Sprintf("invalid days value: %d (must be 1, 7, 14, or 30)", opts.Days)} + } + + // Resolution must be 0 or a positive divisor of 24. + if opts.Resolution < 0 || (opts.Resolution != 0 && 24%opts.Resolution != 0) { + return nil, &platform_http.BadRequestError{Message: fmt.Sprintf("invalid resolution value: %d (must be 0 or a positive divisor of 24)", opts.Resolution)} + } + + hours := opts.Resolution + if hours <= 0 { + hours = dataset.DefaultResolutionHours() + } + bucketSize := time.Duration(hours) * time.Hour + + startDate, endDate := computeBucketRange(time.Now(), bucketSize, opts.Days, opts.TZOffsetMinutes) + + // Build the host filter. The bitmap mask always encodes "currently visible + // hosts" — team scoping, label/platform/include/exclude, and incidentally + // dropping hosts deleted since the SCD rows were written. + hostFilter := &types.HostFilter{ + TeamIDs: effectiveTeamIDs(opts.TeamID, isGlobal, viewerTeamIDs), + LabelIDs: opts.LabelIDs, + Platforms: opts.Platforms, + IncludeHostIDs: opts.IncludeHostIDs, + ExcludeHostIDs: opts.ExcludeHostIDs, + } + + filterMask, err := s.hostCache.Get(ctx, hostFilter, func(ctx context.Context) ([]byte, error) { + hostIDs, err := s.store.GetHostIDsForFilter(ctx, hostFilter) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "fetch host IDs for chart filter") + } + return chart.HostIDsToBlob(hostIDs), nil + }) + if err != nil { + return nil, err + } + + data, err := s.store.GetSCDData(ctx, metric, startDate, endDate, bucketSize, dataset.SampleStrategy(), filterMask, nil) + if err != nil { + return nil, err + } + + return &api.Response{ + Metric: metric, + Visualization: dataset.DefaultVisualization(), + TotalHosts: chart.BlobPopcount(filterMask), + Resolution: formatResolution(bucketSize), + Days: opts.Days, + Filters: api.Filters{ + TeamID: opts.TeamID, + LabelIDs: opts.LabelIDs, + Platforms: opts.Platforms, + IncludeHostIDs: opts.IncludeHostIDs, + ExcludeHostIDs: opts.ExcludeHostIDs, + }, + Data: data, + }, nil +} + +// effectiveTeamIDs decides the team scope applied at SQL time. +// +// explicit team_id? → just that team (authz rule above already ensured +// the caller has access to it, or is global) +// global user, no team_id → nil, meaning "no team filter" +// team user, no team_id → the viewer's accessible teams. Empty-but-non-nil +// here means the user has no teams at all; SQL +// emits 1=0 so they see nothing. +func effectiveTeamIDs(requestedTeamID *uint, isGlobal bool, viewerTeamIDs []uint) []uint { + if requestedTeamID != nil { + return []uint{*requestedTeamID} + } + if isGlobal { + return nil + } + // Return a non-nil slice even when empty — the SQL builder treats non-nil + // as "scoped" and emits a no-match clause, which is what we want for a + // team user with zero team memberships. + if viewerTeamIDs == nil { + return []uint{} + } + return viewerTeamIDs +} + +func (s *Service) CleanupData(ctx context.Context, days int) error { + return s.store.CleanupSCDData(ctx, days) +} + +// computeBucketRange returns a (startDate, endDate) UTC pair such that the +// GetSCDData walker will emit (days*24h)/bucketSize data points labeled at +// bucket boundaries aligned to the client's local time. The last label is +// endDate — i.e., the current (possibly ongoing) bucket in the client's tz. +func computeBucketRange(now time.Time, bucketSize time.Duration, days, tzOffsetMinutes int) (time.Time, time.Time) { + loc := time.FixedZone("client", -tzOffsetMinutes*60) + localNow := now.In(loc) + + var alignedEnd time.Time + if bucketSize < 24*time.Hour { + // Align to the current local bucket within the day. + step := max(int(bucketSize/time.Hour), 1) + alignedHour := (localNow.Hour() / step) * step + alignedEnd = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), alignedHour, 0, 0, 0, loc) + } else { + // Daily (or coarser) — align to the start of today's local day. + alignedEnd = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc) + } + + endDate := alignedEnd.UTC() + startDate := endDate.Add(-time.Duration(days) * 24 * time.Hour) + return startDate, endDate +} + +func formatResolution(bucketSize time.Duration) string { + switch { + case bucketSize == time.Hour: + return "hourly" + case bucketSize == 24*time.Hour: + return "daily" + case bucketSize < 24*time.Hour: + return fmt.Sprintf("%d-hour", int(bucketSize/time.Hour)) + default: + return fmt.Sprintf("%d-day", int(bucketSize/(24*time.Hour))) + } +} diff --git a/server/chart/internal/service/service_test.go b/server/chart/internal/service/service_test.go new file mode 100644 index 0000000000..04146bcaf7 --- /dev/null +++ b/server/chart/internal/service/service_test.go @@ -0,0 +1,487 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/chart" + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/fleetdm/fleet/v4/server/chart/internal/types" + platform_authz "github.com/fleetdm/fleet/v4/server/platform/authz" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockAuthorizer always allows access. +type mockAuthorizer struct{} + +func (m *mockAuthorizer) Authorize(_ context.Context, _ platform_authz.AuthzTyper, _ platform_authz.Action) error { + return nil +} + +// recordingAuthorizer captures the subject and action handed to Authorize so +// tests can assert against the authz input. allow controls the return value. +type recordingAuthorizer struct { + gotSubject platform_authz.AuthzTyper + gotAction platform_authz.Action + allow bool +} + +func (r *recordingAuthorizer) Authorize(_ context.Context, subject platform_authz.AuthzTyper, action platform_authz.Action) error { + r.gotSubject = subject + r.gotAction = action + if r.allow { + return nil + } + return errors.New("forbidden") +} + +// mockViewerProvider returns pre-programmed viewer scope. Default (zero +// value) represents a global user — convenient for the many tests that don't +// care about team scoping. +type mockViewerProvider struct { + isGlobal bool + teamIDs []uint + err error +} + +func (m *mockViewerProvider) ViewerScope(_ context.Context) (bool, []uint, error) { + return m.isGlobal, m.teamIDs, m.err +} + +// globalViewer returns a viewer provider for a global user (sees everything). +func globalViewer() *mockViewerProvider { return &mockViewerProvider{isGlobal: true} } + +// mockDatastore implements types.Datastore for unit tests. +type mockDatastore struct { + getSCDDataFunc func(ctx context.Context, dataset string, startDate, endDate time.Time, bucketSize time.Duration, strategy api.SampleStrategy, filterMask []byte, entityIDs []string) ([]api.DataPoint, error) + getHostIDsForFilterFunc func(ctx context.Context, hostFilter *types.HostFilter) ([]uint, error) + findRecentlySeenHostIDsFn func(ctx context.Context, since time.Time) ([]uint, error) + recordBucketDataFn func(ctx context.Context, dataset string, bucketStart time.Time, bucketSize time.Duration, strategy api.SampleStrategy, entityBitmaps map[string][]byte) error + recordBucketDataInvoked bool +} + +func (m *mockDatastore) FindRecentlySeenHostIDs(ctx context.Context, since time.Time) ([]uint, error) { + if m.findRecentlySeenHostIDsFn != nil { + return m.findRecentlySeenHostIDsFn(ctx, since) + } + return nil, nil +} + +func (m *mockDatastore) RecordBucketData(ctx context.Context, dataset string, bucketStart time.Time, bucketSize time.Duration, strategy api.SampleStrategy, entityBitmaps map[string][]byte) error { + m.recordBucketDataInvoked = true + if m.recordBucketDataFn != nil { + return m.recordBucketDataFn(ctx, dataset, bucketStart, bucketSize, strategy, entityBitmaps) + } + return nil +} + +func (m *mockDatastore) GetSCDData(ctx context.Context, dataset string, startDate, endDate time.Time, bucketSize time.Duration, strategy api.SampleStrategy, filterMask []byte, entityIDs []string) ([]api.DataPoint, error) { + if m.getSCDDataFunc != nil { + return m.getSCDDataFunc(ctx, dataset, startDate, endDate, bucketSize, strategy, filterMask, entityIDs) + } + return nil, nil +} + +func (m *mockDatastore) GetHostIDsForFilter(ctx context.Context, hostFilter *types.HostFilter) ([]uint, error) { + if m.getHostIDsForFilterFunc != nil { + return m.getHostIDsForFilterFunc(ctx, hostFilter) + } + return nil, nil +} + +func (m *mockDatastore) CleanupSCDData(_ context.Context, _ int) error { + return nil +} + +func TestGetChartDataUnknownMetric(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + + _, err := svc.GetChartData(t.Context(), "nonexistent", api.RequestOpts{Days: 7}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown chart metric") +} + +func TestGetChartDataInvalidDays(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 5}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid days value") +} + +func TestGetChartDataInvalidResolution(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + cases := []struct { + name string + resolution int + }{ + {"not a divisor of 24", 5}, + {"negative divisor of 24", -2}, + {"negative non-divisor", -5}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7, Resolution: tc.resolution}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid resolution value") + }) + } +} + +func TestGetChartDataUptimeDefault(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + // Drive TotalHosts via the host-ID list: bitmap popcount = 200. + ds.getHostIDsForFilterFunc = func(_ context.Context, _ *types.HostFilter) ([]uint, error) { + ids := make([]uint, 200) + for i := range ids { + ids[i] = uint(i + 1) + } + return ids, nil + } + + var gotBucketSize time.Duration + var gotStart, gotEnd time.Time + var gotStrategy api.SampleStrategy + var gotMask []byte + ds.getSCDDataFunc = func(_ context.Context, dataset string, start, end time.Time, bucketSize time.Duration, strategy api.SampleStrategy, mask []byte, _ []string) ([]api.DataPoint, error) { + assert.Equal(t, "uptime", dataset) + gotBucketSize = bucketSize + gotStart = start + gotEnd = end + gotStrategy = strategy + gotMask = mask + return []api.DataPoint{{Timestamp: start, Value: 42}}, nil + } + + resp, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.NoError(t, err) + assert.Equal(t, "uptime", resp.Metric) + assert.Equal(t, "checkerboard", resp.Visualization) + assert.Equal(t, "3-hour", resp.Resolution) + assert.Equal(t, 200, resp.TotalHosts) + assert.Equal(t, 7, resp.Days) + assert.Equal(t, 3*time.Hour, gotBucketSize) + assert.Equal(t, api.SampleStrategyAccumulate, gotStrategy) + assert.Equal(t, 200, chart.BlobPopcount(gotMask), "filter mask should encode all 200 host IDs") + // Span must be exactly 7 days. + assert.Equal(t, 7*24*time.Hour, gotEnd.Sub(gotStart)) +} + +func TestGetChartDataUptimeResolution(t *testing.T) { + for _, tc := range []struct { + resolution int + resolutionStr string + bucketSize time.Duration + }{ + {0, "3-hour", 3 * time.Hour}, + {1, "hourly", time.Hour}, + {2, "2-hour", 2 * time.Hour}, + {4, "4-hour", 4 * time.Hour}, + {8, "8-hour", 8 * time.Hour}, + } { + t.Run(tc.resolutionStr, func(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + var gotBucketSize time.Duration + ds.getSCDDataFunc = func(_ context.Context, _ string, _, _ time.Time, bucketSize time.Duration, _ api.SampleStrategy, _ []byte, _ []string) ([]api.DataPoint, error) { + gotBucketSize = bucketSize + return nil, nil + } + + resp, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 30, Resolution: tc.resolution}) + require.NoError(t, err) + assert.Equal(t, tc.resolutionStr, resp.Resolution) + assert.Equal(t, tc.bucketSize, gotBucketSize) + }) + } +} + +func TestGetChartDataCVEResolution(t *testing.T) { + // Resolution applies uniformly regardless of the dataset's default: + // omitted → dataset default (24h for CVE), specified → that value in hours. + for _, tc := range []struct { + name string + resolution int + resolutionStr string + bucketSize time.Duration + }{ + {"default", 0, "daily", 24 * time.Hour}, + {"hourly override", 1, "hourly", time.Hour}, + {"4-hour override", 4, "4-hour", 4 * time.Hour}, + } { + t.Run(tc.name, func(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.CVEDataset{}) + + var gotBucketSize time.Duration + var gotStrategy api.SampleStrategy + ds.getSCDDataFunc = func(_ context.Context, _ string, _, _ time.Time, bucketSize time.Duration, strategy api.SampleStrategy, _ []byte, _ []string) ([]api.DataPoint, error) { + gotBucketSize = bucketSize + gotStrategy = strategy + return nil, nil + } + + resp, err := svc.GetChartData(t.Context(), "cve", api.RequestOpts{Days: 30, Resolution: tc.resolution}) + require.NoError(t, err) + assert.Equal(t, tc.resolutionStr, resp.Resolution) + assert.Equal(t, tc.bucketSize, gotBucketSize) + assert.Equal(t, api.SampleStrategySnapshot, gotStrategy) + }) + } +} + +func TestGetChartDataWithHostFilters(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + var gotFilter *types.HostFilter + ds.getHostIDsForFilterFunc = func(_ context.Context, hostFilter *types.HostFilter) ([]uint, error) { + gotFilter = hostFilter + return []uint{10, 20}, nil + } + ds.getSCDDataFunc = func(_ context.Context, _ string, _, _ time.Time, _ time.Duration, _ api.SampleStrategy, mask []byte, _ []string) ([]api.DataPoint, error) { + assert.Equal(t, 2, chart.BlobPopcount(mask), "mask should encode the 2 host IDs returned") + return []api.DataPoint{{Value: 2}}, nil + } + + teamID := uint(5) + resp, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{ + Days: 7, + TeamID: &teamID, + LabelIDs: []uint{1, 2}, + Platforms: []string{"darwin"}, + }) + require.NoError(t, err) + + require.NotNil(t, gotFilter) + assert.Equal(t, []uint{5}, gotFilter.TeamIDs, "explicit team_id becomes a single-element scope") + assert.Equal(t, []uint{1, 2}, gotFilter.LabelIDs) + assert.Equal(t, []string{"darwin"}, gotFilter.Platforms) + + assert.Equal(t, 2, resp.TotalHosts, "TotalHosts is now popcount of filter mask") + require.NotNil(t, resp.Filters.TeamID) + assert.Equal(t, uint(5), *resp.Filters.TeamID, "response echoes what the caller asked for") + assert.Equal(t, []uint{1, 2}, resp.Filters.LabelIDs) + assert.Equal(t, []string{"darwin"}, resp.Filters.Platforms) +} + +func TestGetChartDataAuthzScope(t *testing.T) { + t.Run("no fleet_id → ActionList with Host{} (rego allows team users)", func(t *testing.T) { + auth := &recordingAuthorizer{allow: true} + svc := NewService(auth, &mockDatastore{}, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.NoError(t, err) + + host, ok := auth.gotSubject.(*api.Host) + require.True(t, ok, "authz subject should be *api.Host") + assert.Nil(t, host.TeamID, "without an explicit fleet_id, the subject's TeamID stays nil") + assert.Equal(t, platform_authz.ActionList, auth.gotAction, + "no fleet_id uses ActionList so rego's team-list rule can pass team users") + }) + + t.Run("explicit fleet_id=5 → ActionRead with Host{TeamID:5} (rego enforces exact team)", func(t *testing.T) { + auth := &recordingAuthorizer{allow: true} + svc := NewService(auth, &mockDatastore{}, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + teamID := uint(5) + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7, TeamID: &teamID}) + require.NoError(t, err) + + host, ok := auth.gotSubject.(*api.Host) + require.True(t, ok) + require.NotNil(t, host.TeamID) + assert.Equal(t, uint(5), *host.TeamID) + assert.Equal(t, platform_authz.ActionRead, auth.gotAction, + "explicit fleet_id uses ActionRead so rego's team-read rule can enforce exact-team access") + }) + + t.Run("authz denial propagates", func(t *testing.T) { + auth := &recordingAuthorizer{allow: false} + svc := NewService(auth, &mockDatastore{}, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.Error(t, err) + assert.Contains(t, err.Error(), "forbidden") + }) + + t.Run("viewer provider error propagates before authz", func(t *testing.T) { + auth := &recordingAuthorizer{allow: true} + viewer := &mockViewerProvider{err: errors.New("no viewer in context")} + svc := NewService(auth, &mockDatastore{}, viewer, nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no viewer") + assert.Nil(t, auth.gotSubject, "authz must not run when viewer resolution failed") + }) +} + +func TestGetChartDataScopesDataByViewer(t *testing.T) { + t.Run("global user, no fleet_id → nil TeamIDs (no team filter)", func(t *testing.T) { + ds := &mockDatastore{} + var gotFilter *types.HostFilter + ds.getHostIDsForFilterFunc = func(_ context.Context, f *types.HostFilter) ([]uint, error) { + gotFilter = f + return []uint{1, 2, 3}, nil + } + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.NoError(t, err) + require.NotNil(t, gotFilter) + assert.Nil(t, gotFilter.TeamIDs, "global user with no fleet_id gets no team filter") + }) + + t.Run("team user, no fleet_id → their accessible teams", func(t *testing.T) { + ds := &mockDatastore{} + var gotFilter *types.HostFilter + ds.getHostIDsForFilterFunc = func(_ context.Context, f *types.HostFilter) ([]uint, error) { + gotFilter = f + return []uint{10, 11}, nil + } + viewer := &mockViewerProvider{isGlobal: false, teamIDs: []uint{3, 7}} + svc := NewService(&mockAuthorizer{}, ds, viewer, nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.NoError(t, err) + require.NotNil(t, gotFilter) + assert.Equal(t, []uint{3, 7}, gotFilter.TeamIDs, + "team user without explicit fleet_id is scoped to the union of their teams") + }) + + t.Run("team user with zero accessible teams → empty non-nil TeamIDs (SQL no-match)", func(t *testing.T) { + ds := &mockDatastore{} + var gotFilter *types.HostFilter + ds.getHostIDsForFilterFunc = func(_ context.Context, f *types.HostFilter) ([]uint, error) { + gotFilter = f + return nil, nil + } + viewer := &mockViewerProvider{isGlobal: false, teamIDs: nil} + svc := NewService(&mockAuthorizer{}, ds, viewer, nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + resp, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7}) + require.NoError(t, err) + require.NotNil(t, gotFilter) + require.NotNil(t, gotFilter.TeamIDs, "empty-not-nil signals 'team-scoped with no teams'") + assert.Empty(t, gotFilter.TeamIDs) + assert.Equal(t, 0, resp.TotalHosts, "no accessible teams means no hosts and no data") + }) + + t.Run("explicit fleet_id overrides viewer scope", func(t *testing.T) { + ds := &mockDatastore{} + var gotFilter *types.HostFilter + ds.getHostIDsForFilterFunc = func(_ context.Context, f *types.HostFilter) ([]uint, error) { + gotFilter = f + return []uint{1}, nil + } + // Viewer sees teams 3, 7 — but caller explicitly asks for team 3. + viewer := &mockViewerProvider{isGlobal: false, teamIDs: []uint{3, 7}} + svc := NewService(&mockAuthorizer{}, ds, viewer, nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + teamID := uint(3) + _, err := svc.GetChartData(t.Context(), "uptime", api.RequestOpts{Days: 7, TeamID: &teamID}) + require.NoError(t, err) + require.NotNil(t, gotFilter) + assert.Equal(t, []uint{3}, gotFilter.TeamIDs, + "explicit fleet_id narrows to that team; authz (not the filter) enforced access above") + }) +} + +func TestComputeBucketRange(t *testing.T) { + t.Run("hourly UTC", func(t *testing.T) { + now := time.Date(2026, 4, 8, 14, 37, 12, 0, time.UTC) + start, end := computeBucketRange(now, time.Hour, 1, 0) + assert.Equal(t, time.Date(2026, 4, 8, 14, 0, 0, 0, time.UTC), end) + assert.Equal(t, time.Date(2026, 4, 7, 14, 0, 0, 0, time.UTC), start) + }) + + t.Run("sub-daily resolution aligns to step", func(t *testing.T) { + now := time.Date(2026, 4, 8, 15, 30, 0, 0, time.UTC) + _, end := computeBucketRange(now, 4*time.Hour, 1, 0) + // 15 / 4 * 4 = 12 — aligned to nearest step hour within the day. + assert.Equal(t, time.Date(2026, 4, 8, 12, 0, 0, 0, time.UTC), end) + }) + + t.Run("hourly with tz offset aligns to local hour", func(t *testing.T) { + // 14:37 UTC = 07:37 PDT (offset +420 minutes). Local hour 07 → end at 07:00 PDT = 14:00 UTC. + now := time.Date(2026, 4, 8, 14, 37, 0, 0, time.UTC) + _, end := computeBucketRange(now, time.Hour, 1, 420) + assert.Equal(t, time.Date(2026, 4, 8, 14, 0, 0, 0, time.UTC), end) + }) + + t.Run("daily with tz offset aligns to local midnight", func(t *testing.T) { + // 14:37 UTC = 07:37 PDT. Local midnight = 00:00 PDT = 07:00 UTC. + now := time.Date(2026, 4, 8, 14, 37, 0, 0, time.UTC) + start, end := computeBucketRange(now, 24*time.Hour, 7, 420) + assert.Equal(t, time.Date(2026, 4, 8, 7, 0, 0, 0, time.UTC), end) + assert.Equal(t, time.Date(2026, 4, 1, 7, 0, 0, 0, time.UTC), start) + }) +} + +func TestCollectDatasetsUptime(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.UptimeDataset{}) + + now := time.Date(2026, 4, 8, 14, 37, 0, 0, time.UTC) + wantBucketStart := time.Date(2026, 4, 8, 14, 0, 0, 0, time.UTC) + + ds.findRecentlySeenHostIDsFn = func(_ context.Context, since time.Time) ([]uint, error) { + assert.Equal(t, now.Add(-10*time.Minute), since) + return []uint{1, 2, 3}, nil + } + ds.recordBucketDataFn = func(_ context.Context, dataset string, bucketStart time.Time, bucketSize time.Duration, strategy api.SampleStrategy, entityBitmaps map[string][]byte) error { + assert.Equal(t, "uptime", dataset) + assert.Equal(t, wantBucketStart, bucketStart) + assert.Equal(t, time.Hour, bucketSize) + assert.Equal(t, api.SampleStrategyAccumulate, strategy) + require.Len(t, entityBitmaps, 1) + assert.NotEmpty(t, entityBitmaps[""]) + return nil + } + + err := svc.CollectDatasets(t.Context(), now) + require.NoError(t, err) + assert.True(t, ds.recordBucketDataInvoked) +} + +func TestUptimeDatasetMetadata(t *testing.T) { + d := &chart.UptimeDataset{} + assert.Equal(t, "uptime", d.Name()) + assert.Equal(t, 3, d.DefaultResolutionHours()) + assert.Equal(t, api.SampleStrategyAccumulate, d.SampleStrategy()) + assert.Equal(t, "checkerboard", d.DefaultVisualization()) +} + +func TestCVEDatasetMetadata(t *testing.T) { + d := &chart.CVEDataset{} + assert.Equal(t, "cve", d.Name()) + assert.Equal(t, 24, d.DefaultResolutionHours()) + assert.Equal(t, api.SampleStrategySnapshot, d.SampleStrategy()) + assert.Equal(t, "line", d.DefaultVisualization()) +} diff --git a/server/chart/internal/types/chart.go b/server/chart/internal/types/chart.go new file mode 100644 index 0000000000..93bc22f1e9 --- /dev/null +++ b/server/chart/internal/types/chart.go @@ -0,0 +1,75 @@ +// Package types provides internal types and interfaces for the chart bounded context. +package types + +import ( + "context" + "time" + + "github.com/fleetdm/fleet/v4/server/chart/api" +) + +// HostFilter is the internal filter used by the service and datastore to narrow +// SCD queries to a specific set of hosts. +// +// TeamIDs semantics — the distinction between nil and empty matters: +// - nil: no team filter applied (all hosts across all teams, including no-team). +// This is the global-user-no-explicit-team-id case. +// - empty non-nil ([]uint{}): caller is team-scoped but has zero accessible +// teams. SQL falls through to a no-match clause so the user sees nothing. +// - single 0 ([]uint{0}): hosts with no team assignment (team_id IS NULL). +// - other values: team_id IN (list). Mixed with a 0 entry yields +// "(team_id IS NULL OR team_id IN (non-zero list))". +type HostFilter struct { + TeamIDs []uint + LabelIDs []uint + Platforms []string + IncludeHostIDs []uint + ExcludeHostIDs []uint +} + +// Datastore is the internal datastore interface for the chart bounded context. +type Datastore interface { + // FindRecentlySeenHostIDs returns host IDs that have reported since the + // given cutoff. Used by datasets like uptime that derive their sample from + // recent host activity. + FindRecentlySeenHostIDs(ctx context.Context, since time.Time) ([]uint, error) + + // RecordBucketData writes one or more entity bitmaps for the given bucket using + // the specified sample strategy. See api.SampleStrategy for the semantics of + // each strategy. + RecordBucketData( + ctx context.Context, + dataset string, + bucketStart time.Time, + bucketSize time.Duration, + strategy api.SampleStrategy, + entityBitmaps map[string][]byte, + ) error + + // GetSCDData returns per-bucket distinct-host counts for a dataset over the + // given range at the given bucket size. Aggregation within a bucket depends + // on the sample strategy: + // - Accumulate: OR every row that overlaps the bucket ("hosts observed at + // any point during the bucket"). + // - Snapshot: for each entity, pick the row active at bucketEnd, then OR + // across entities ("state as of the end of the bucket"). + // filterMask is always applied via bitmap AND — callers build it via + // GetHostIDsForFilter + chart.HostIDsToBlob, usually through a cache. + // The entity filter is applied via entity_id IN. + GetSCDData( + ctx context.Context, + dataset string, + startDate, endDate time.Time, + bucketSize time.Duration, + strategy api.SampleStrategy, + filterMask []byte, + entityIDs []string, + ) ([]api.DataPoint, error) + + // GetHostIDsForFilter returns the host IDs that match the given host filter. + GetHostIDsForFilter(ctx context.Context, hostFilter *HostFilter) ([]uint, error) + + // CleanupSCDData deletes closed SCD rows whose valid_to is older than the + // retention cutoff. Open rows (valid_to = sentinel) are never deleted. + CleanupSCDData(ctx context.Context, days int) error +} diff --git a/server/datastore/mysql/migrations/tables/20260423161823_AddHostSCDData.go b/server/datastore/mysql/migrations/tables/20260423161823_AddHostSCDData.go new file mode 100644 index 0000000000..7cae3b8b24 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20260423161823_AddHostSCDData.go @@ -0,0 +1,47 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20260423161823, Down_20260423161823) +} + +func Up_20260423161823(tx *sql.Tx) error { + // host_scd_data is the unified storage for all chart datasets. Rows are + // interval-based (valid_from, valid_to) bitmaps, written by one of two sample + // strategies: + // - Accumulate: rows are explicitly closed at bucket boundaries; same-bucket + // samples are OR-merged into the existing row via ON DUPLICATE KEY UPDATE. + // Used for datasets like uptime where each sample is a partial observation. + // - Snapshot: rows stay open (valid_to = sentinel) until the bitmap changes, + // at which point the prior row is closed and a new one inserted. Used for + // datasets like CVE where each sample is the full state. + // See server/chart/internal/mysql/data.go for the write and read paths. + _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS host_scd_data ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + dataset VARCHAR(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, + entity_id VARCHAR(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + host_bitmap MEDIUMBLOB NOT NULL, + valid_from DATETIME NOT NULL, + valid_to DATETIME NOT NULL DEFAULT '9999-12-31 00:00:00', + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + UNIQUE KEY uniq_entity_bucket (dataset, entity_id, valid_from), + KEY idx_dataset_range (dataset, valid_from, valid_to) + ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci + `) + if err != nil { + return fmt.Errorf("create host_scd_data table: %w", err) + } + + return nil +} + +func Down_20260423161823(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20260423161823_AddHostSCDData_test.go b/server/datastore/mysql/migrations/tables/20260423161823_AddHostSCDData_test.go new file mode 100644 index 0000000000..ce97d8c7c2 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20260423161823_AddHostSCDData_test.go @@ -0,0 +1,9 @@ +package tables + +import "testing" + +func TestUp_20260423161823(t *testing.T) { + db := applyUpToPrev(t) + + applyNext(t, db) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 651567d83d..a960e0f48e 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1096,6 +1096,22 @@ CREATE TABLE `host_recovery_key_passwords` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `host_scd_data` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `dataset` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, + `entity_id` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `host_bitmap` mediumblob NOT NULL, + `valid_from` datetime NOT NULL, + `valid_to` datetime NOT NULL DEFAULT '9999-12-31 00:00:00', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_entity_bucket` (`dataset`,`entity_id`,`valid_from`), + KEY `idx_dataset_range` (`dataset`,`valid_from`,`valid_to`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `host_scim_user` ( `host_id` int unsigned NOT NULL, `scim_user_id` int unsigned NOT NULL, @@ -1934,9 +1950,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=516 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=517 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260317120000,1,'2020-01-01 01:01:01'),(496,20260318184559,1,'2020-01-01 01:01:01'),(497,20260319120000,1,'2020-01-01 01:01:01'),(498,20260323144117,1,'2020-01-01 01:01:01'),(499,20260324161944,1,'2020-01-01 01:01:01'),(500,20260324223334,1,'2020-01-01 01:01:01'),(501,20260326131501,1,'2020-01-01 01:01:01'),(502,20260326210603,1,'2020-01-01 01:01:01'),(503,20260331000000,1,'2020-01-01 01:01:01'),(504,20260401153000,1,'2020-01-01 01:01:01'),(505,20260401153001,1,'2020-01-01 01:01:01'),(506,20260401153503,1,'2020-01-01 01:01:01'),(507,20260403120000,1,'2020-01-01 01:01:01'),(508,20260409153713,1,'2020-01-01 01:01:01'),(509,20260409153714,1,'2020-01-01 01:01:01'),(510,20260409153715,1,'2020-01-01 01:01:01'),(511,20260409153716,1,'2020-01-01 01:01:01'),(512,20260409153717,1,'2020-01-01 01:01:01'),(513,20260409183610,1,'2020-01-01 01:01:01'),(514,20260410173222,1,'2020-01-01 01:01:01'),(515,20260422181702,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260317120000,1,'2020-01-01 01:01:01'),(496,20260318184559,1,'2020-01-01 01:01:01'),(497,20260319120000,1,'2020-01-01 01:01:01'),(498,20260323144117,1,'2020-01-01 01:01:01'),(499,20260324161944,1,'2020-01-01 01:01:01'),(500,20260324223334,1,'2020-01-01 01:01:01'),(501,20260326131501,1,'2020-01-01 01:01:01'),(502,20260326210603,1,'2020-01-01 01:01:01'),(503,20260331000000,1,'2020-01-01 01:01:01'),(504,20260401153000,1,'2020-01-01 01:01:01'),(505,20260401153001,1,'2020-01-01 01:01:01'),(506,20260401153503,1,'2020-01-01 01:01:01'),(507,20260403120000,1,'2020-01-01 01:01:01'),(508,20260409153713,1,'2020-01-01 01:01:01'),(509,20260409153714,1,'2020-01-01 01:01:01'),(510,20260409153715,1,'2020-01-01 01:01:01'),(511,20260409153716,1,'2020-01-01 01:01:01'),(512,20260409153717,1,'2020-01-01 01:01:01'),(513,20260409183610,1,'2020-01-01 01:01:01'),(514,20260410173222,1,'2020-01-01 01:01:01'),(515,20260422181702,1,'2020-01-01 01:01:01'),(516,20260423161823,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/fleet/cron_schedules.go b/server/fleet/cron_schedules.go index dff478e449..72bc407c84 100644 --- a/server/fleet/cron_schedules.go +++ b/server/fleet/cron_schedules.go @@ -53,6 +53,7 @@ const ( // Runs every 5 minutes. CronSendRecoveryLockCommands CronScheduleName = "send_recovery_lock_commands" CronAppleMDMWorker CronScheduleName = "apple_mdm_worker" + CronChartDataCollection CronScheduleName = "chart_data_collection" // Used by chart bounded context ) type CronSchedulesService interface { diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 62191474c6..7348a431b4 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -567,7 +567,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { w.Header().Set("x-exit", "invalidEmailDomain") } w.WriteHeader(statusCode) - resp := []byte(fmt.Sprintf("status: %d", statusCode)) + resp := fmt.Appendf(nil, "status: %d", status) if statusCode == http.StatusOK && strings.Contains(r.URL.RawQuery, "deliveryMethod=json") { rawBody, err := io.ReadAll(r.Body) require.NoError(s.T(), err) @@ -23569,7 +23569,6 @@ func (s *integrationMDMTestSuite) TestManagedLocalAccount() { })) t.Run("Enrollment flow", func(t *testing.T) { - // DEP-enroll the first host and run the post-enrollment worker s.runDEPSchedule() depURLToken := loadEnrollmentProfileDEPToken(t, s.ds) diff --git a/tools/charts-backfill/README.md b/tools/charts-backfill/README.md new file mode 100644 index 0000000000..55530978aa --- /dev/null +++ b/tools/charts-backfill/README.md @@ -0,0 +1,33 @@ +# charts-backfill + +Generates synthetic chart data for development and testing. Writes rows to +`host_hourly_data_blobs` using `ON DUPLICATE KEY UPDATE`, so it is safe to +re-run. + +## Usage + +```bash +go run ./tools/charts-backfill --dataset uptime --days 30 +go run ./tools/charts-backfill --dataset uptime --days 7 --host-ids 1,2,3 +go run ./tools/charts-backfill --dataset cve --days 30 --entity-ids CVE-2024-1,CVE-2024-2 +go run ./tools/charts-backfill --mysql-dsn "fleet:fleet@tcp(localhost:3306)/fleet" +``` + +## Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--dataset` | `uptime` | Dataset name (`uptime`, `policy`, `cve`, ...) | +| `--days` | `30` | Number of days to backfill | +| `--start-date` | `now - days` | Start date (`YYYY-MM-DD`) | +| `--entity-ids` | `""` | Comma-separated entity IDs (e.g. CVE IDs); `""` for non-entity datasets | +| `--host-ids` | all hosts | Comma-separated host IDs to include | +| `--mysql-dsn` | local dev | MySQL connection string | + +## Datasets + +- **Hourly blob** (default): 24 rows/day per entity, one per hour. +- **Daily blob** (`cve`): one row/day with `hour = -1` (whole-day sentinel). + +Density (fraction of hosts marked active) varies by dataset — see +`densityRange` in `main.go`. diff --git a/tools/charts-backfill/main.go b/tools/charts-backfill/main.go new file mode 100644 index 0000000000..4de571d438 --- /dev/null +++ b/tools/charts-backfill/main.go @@ -0,0 +1,222 @@ +// charts-backfill generates realistic chart data for development and testing. +// Writes rows to host_scd_data in closed form (explicit valid_to); the live +// collector can then extend from these rows via its normal write path. +// Safe to re-run — uses ON DUPLICATE KEY UPDATE to merge new data. +// +// Usage: +// +// go run ./tools/charts-backfill --dataset uptime --days 30 +// go run ./tools/charts-backfill --dataset uptime --days 7 --host-ids 1,2,3 +// go run ./tools/charts-backfill --mysql-dsn "fleet:fleet@tcp(localhost:3306)/fleet" +package main + +import ( + "database/sql" + "flag" + "log" + "math/rand/v2" + "time" + + "github.com/fleetdm/fleet/v4/pkg/str" + "github.com/fleetdm/fleet/v4/server/chart" + _ "github.com/go-sql-driver/mysql" +) + +// dailyDatasets bucket at 24h granularity; all others are hourly. +var dailyDatasets = map[string]struct{}{ + "cve": {}, +} + +func main() { + dataset := flag.String("dataset", "uptime", "dataset name (e.g. uptime, policy, cve)") + days := flag.Int("days", 30, "number of days to backfill") + startDate := flag.String("start-date", "", "start date (YYYY-MM-DD), defaults to now - days") + entityIDsStr := flag.String("entity-ids", "", "comma-separated entity IDs (default: '' for non-entity datasets)") + hostIDsStr := flag.String("host-ids", "", "comma-separated host IDs (default: all from hosts table)") + dsn := flag.String("mysql-dsn", "fleet:fleet@tcp(localhost:3306)/fleet?parseTime=true", "MySQL connection string") + flag.Parse() + + var start time.Time + if *startDate != "" { + s, err := time.Parse("2006-01-02", *startDate) + if err != nil { + log.Fatalf("invalid start-date %q: %v", *startDate, err) + } + start = s + } else { + start = time.Now().UTC().AddDate(0, 0, -(*days - 1)) + } + start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, time.UTC) + + db, err := sql.Open("mysql", *dsn) + if err != nil { + log.Fatalf("failed to connect to mysql: %v", err) + } + + if err := db.Ping(); err != nil { + db.Close() + log.Fatalf("failed to ping mysql: %v", err) + } + defer db.Close() + + hostIDs := str.ParseUintList(*hostIDsStr) + if len(hostIDs) == 0 { + hostIDs, err = queryHostIDs(db) + if err != nil { + log.Fatalf("failed to query host IDs: %v", err) //nolint:gocritic // dev tool, OS reclaims db handle on exit + } + if len(hostIDs) == 0 { + log.Fatal("no hosts found in database") + } + } + + entityIDs := str.ParseStringList(*entityIDsStr) + if len(entityIDs) == 0 { + entityIDs = []string{""} + } + + log.Printf("backfilling dataset=%q, days=%d, start=%s, hosts=%d, entities=%d", + *dataset, *days, start.Format("2006-01-02"), len(hostIDs), len(entityIDs)) + + startTime := time.Now() + totalRows := backfill(db, *dataset, *days, start, hostIDs, entityIDs) + log.Printf("done: %d SCD rows inserted/updated in %.1fs", totalRows, time.Since(startTime).Seconds()) +} + +func backfill(db *sql.DB, dataset string, days int, start time.Time, hostIDs []uint, entityIDs []string) int { + if _, ok := dailyDatasets[dataset]; ok { + return backfillDaily(db, dataset, days, start, hostIDs, entityIDs) + } + return backfillHourly(db, dataset, days, start, hostIDs, entityIDs) +} + +func backfillHourly(db *sql.DB, dataset string, days int, start time.Time, hostIDs []uint, entityIDs []string) int { + totalRows := 0 + for day := range days { + date := start.AddDate(0, 0, day) + + for _, entityID := range entityIDs { + hourlyHosts := generateHourlyHosts(dataset, hostIDs) + + for hour, activeHosts := range hourlyHosts { + if len(activeHosts) == 0 { + continue + } + validFrom := date.Add(time.Duration(hour) * time.Hour) + validTo := validFrom.Add(time.Hour) + blob := chart.HostIDsToBlob(activeHosts) + + _, err := db.Exec( + `INSERT INTO host_scd_data (dataset, entity_id, host_bitmap, valid_from, valid_to) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE host_bitmap = VALUES(host_bitmap), valid_to = VALUES(valid_to)`, + dataset, entityID, blob, validFrom, validTo) + if err != nil { + log.Fatalf("insert hourly SCD row failed on %s hour %d: %v", validFrom, hour, err) + } + totalRows++ + } + } + + if (day+1)%5 == 0 || day == days-1 { + log.Printf(" day %d/%d (%s) — %d rows so far", + day+1, days, date.Format("2006-01-02"), totalRows) + } + } + return totalRows +} + +func backfillDaily(db *sql.DB, dataset string, days int, start time.Time, hostIDs []uint, entityIDs []string) int { + totalRows := 0 + minDensity, maxDensity := densityRange(dataset) + n := len(hostIDs) + + for day := range days { + date := start.AddDate(0, 0, day) + + for _, entityID := range entityIDs { + density := minDensity + rand.Float64()*(maxDensity-minDensity) //nolint:gosec // dev data generator, not crypto + count := int(float64(n) * density) + if count == 0 { + continue + } + active := make([]uint, count) + for i, idx := range rand.Perm(n)[:count] { + active[i] = hostIDs[idx] + } + blob := chart.HostIDsToBlob(active) + validFrom := date + validTo := date.AddDate(0, 0, 1) + + _, err := db.Exec( + `INSERT INTO host_scd_data (dataset, entity_id, host_bitmap, valid_from, valid_to) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE host_bitmap = VALUES(host_bitmap), valid_to = VALUES(valid_to)`, + dataset, entityID, blob, validFrom, validTo) + if err != nil { + log.Fatalf("insert daily SCD row failed on %s entity %q: %v", date, entityID, err) + } + totalRows++ + } + + if (day+1)%5 == 0 || day == days-1 { + log.Printf(" day %d/%d (%s) — %d rows so far", + day+1, days, date.Format("2006-01-02"), totalRows) + } + } + + return totalRows +} + +// generateHourlyHosts returns a map of hour -> active host IDs for a single day. +func generateHourlyHosts(dataset string, hostIDs []uint) map[int][]uint { + minDensity, maxDensity := densityRange(dataset) + n := len(hostIDs) + result := make(map[int][]uint, 24) + + for hour := range 24 { + density := minDensity + rand.Float64()*(maxDensity-minDensity) //nolint:gosec // dev data generator, not crypto + count := int(float64(n) * density) + if count == 0 { + continue + } + active := make([]uint, count) + for i, idx := range rand.Perm(n)[:count] { + active[i] = hostIDs[idx] + } + result[hour] = active + } + + return result +} + +func densityRange(dataset string) (float64, float64) { + switch dataset { + case "uptime": + return 0.0, 1.0 + case "policy": + return 0.05, 0.20 + case "cve": + return 0.10, 0.30 + default: + return 0.40, 0.80 + } +} + +func queryHostIDs(db *sql.DB) ([]uint, error) { + rows, err := db.Query("SELECT id FROM hosts ORDER BY id") + if err != nil { + return nil, err + } + defer rows.Close() + + var ids []uint + for rows.Next() { + var id uint + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} diff --git a/tools/charts-collect/README.md b/tools/charts-collect/README.md new file mode 100644 index 0000000000..ad6c46f62b --- /dev/null +++ b/tools/charts-collect/README.md @@ -0,0 +1,49 @@ +# charts-collect + +Fetches live data from a Fleet instance via the REST API and writes chart rows +into a local database. Designed to run hourly via cron. + +## What it collects + +- **Uptime** — fetches currently online hosts and ORs them into the current + hour's `host_hourly_data_blobs` row (`dataset='uptime'`). +- **CVE** — fetches per-host vulnerabilities, inverts into per-CVE host + bitmaps, and reconciles into `host_scd_data` (`dataset='cve'`). Unchanged + CVEs keep their open row; changed bitmaps close the prior-day row and open + a new one for today; intra-day changes overwrite today's row via ODKU. + +## Usage + +```bash +go run ./tools/charts-collect \ + --fleet-url https://dogfood.fleetdm.com \ + --fleet-token + +go run ./tools/charts-collect \ + --fleet-url https://dogfood.fleetdm.com \ + --fleet-token \ + --mysql-dsn "fleet:fleet@tcp(localhost:3306)/fleet" +``` + +## Flags and env vars + +| Flag | Env | Description | +|------|-----|-------------| +| `--fleet-url` | `FLEET_URL` | Fleet server URL (required) | +| `--fleet-token` | `FLEET_TOKEN` | Fleet API token (required) | +| `--mysql-dsn` | `MYSQL_DSN` | Full MySQL DSN | + +If `--mysql-dsn` / `MYSQL_DSN` is not set, the DSN is assembled from the same +env vars used by the fleet server (so the same values can be reused, e.g. via +Render `fromService`): + +- `FLEET_MYSQL_ADDRESS` +- `FLEET_MYSQL_USERNAME` +- `FLEET_MYSQL_PASSWORD` +- `FLEET_MYSQL_DATABASE` + +## Notes + +- SCD encoding constants (`9999-12-31` open sentinel, batch size) mirror + `server/chart/internal/mysql/scd.go`. Keep in sync when either side changes. +- Errors in one collector (uptime/cve) are logged but do not block the other. diff --git a/tools/charts-collect/main.go b/tools/charts-collect/main.go new file mode 100644 index 0000000000..d42e4e564a --- /dev/null +++ b/tools/charts-collect/main.go @@ -0,0 +1,433 @@ +// charts-collect fetches live data from a Fleet instance via API and writes +// chart data into a local database. Designed to run hourly via cron. +// +// Uptime: fetches currently online hosts and OR-merges them into the +// current-hour accumulate row (dataset='uptime'). Rows are closed at hour +// boundaries; no cross-bucket collapse. +// CVE: fetches per-host vulnerability data, builds per-CVE host bitmaps, and +// reconciles them into host_scd_data (dataset='cve') as snapshot rows. +// Unchanged CVEs keep their existing open row; changed bitmaps close the prior +// row at today's midnight (UTC) and open a new one; intra-day changes +// overwrite today's row via ODKU. +// +// Usage: +// +// go run ./tools/charts-collect --fleet-url https://dogfood.fleetdm.com --fleet-token +// go run ./tools/charts-collect --fleet-url https://dogfood.fleetdm.com --fleet-token --mysql-dsn "fleet:fleet@tcp(localhost:3306)/fleet" +// +// Env vars: +// - FLEET_URL / FLEET_TOKEN: API target (same as --fleet-url / --fleet-token). +// - MYSQL_DSN: full DSN (same as --mysql-dsn). +// - FLEET_MYSQL_ADDRESS, FLEET_MYSQL_DATABASE, FLEET_MYSQL_USERNAME, +// FLEET_MYSQL_PASSWORD: used to assemble a DSN when MYSQL_DSN/--mysql-dsn +// is not set. Matches the fleet server's env var names so the same values +// can be reused (e.g. via Render fromService). +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/pkg/fleethttp" + "github.com/fleetdm/fleet/v4/server/chart" + _ "github.com/go-sql-driver/mysql" +) + +const ( + perPage = 500 + // scdUpsertBatch mirrors the constant in server/chart/internal/mysql/data.go — + // the collector writes to the same table out-of-process and must keep the + // encoding in sync. + scdUpsertBatch = 200 +) + +// scdOpenSentinel is the end-of-time marker used for valid_to on open snapshot +// rows. Must match the DEFAULT in the host_scd_data table. +var scdOpenSentinel = time.Date(9999, 12, 31, 0, 0, 0, 0, time.UTC) + +func main() { + fleetURL := flag.String("fleet-url", os.Getenv("FLEET_URL"), "Fleet server URL (or FLEET_URL env var)") + fleetToken := flag.String("fleet-token", os.Getenv("FLEET_TOKEN"), "Fleet API token (or FLEET_TOKEN env var)") + dsn := flag.String("mysql-dsn", os.Getenv("MYSQL_DSN"), "MySQL connection string (or MYSQL_DSN env var; falls back to FLEET_MYSQL_* env vars)") + flag.Parse() + + if *fleetURL == "" || *fleetToken == "" { + log.Fatal("--fleet-url and --fleet-token are required (or set FLEET_URL and FLEET_TOKEN)") + } + + if *dsn == "" { + built, err := dsnFromEnv() + if err != nil { + log.Fatalf("build mysql dsn: %v", err) + } + *dsn = built + } + + db, err := sql.Open("mysql", *dsn) + if err != nil { + log.Fatalf("connect to mysql: %v", err) + } + + if err := db.Ping(); err != nil { + db.Close() + log.Fatalf("ping mysql: %v", err) + } + defer db.Close() + + api := &apiClient{ + baseURL: *fleetURL, + token: *fleetToken, + http: fleethttp.NewClient(fleethttp.WithTimeout(30 * time.Second)), + } + + if err := collectUptime(api, db); err != nil { + log.Printf("ERROR uptime collection: %v", err) + } + if err := collectCVE(api, db); err != nil { + log.Printf("ERROR cve collection: %v", err) + } +} + +// dsnFromEnv builds a MySQL DSN from the standard FLEET_MYSQL_* env vars used +// by the fleet server. Returns an error if any required piece is missing so we +// don't silently fall back to localhost defaults in a production cron. +func dsnFromEnv() (string, error) { + addr := os.Getenv("FLEET_MYSQL_ADDRESS") + user := os.Getenv("FLEET_MYSQL_USERNAME") + pass := os.Getenv("FLEET_MYSQL_PASSWORD") + db := os.Getenv("FLEET_MYSQL_DATABASE") + + var missing []string + if addr == "" { + missing = append(missing, "FLEET_MYSQL_ADDRESS") + } + if user == "" { + missing = append(missing, "FLEET_MYSQL_USERNAME") + } + if db == "" { + missing = append(missing, "FLEET_MYSQL_DATABASE") + } + if len(missing) > 0 { + return "", fmt.Errorf("missing env vars: %s (or set --mysql-dsn / MYSQL_DSN)", strings.Join(missing, ", ")) + } + + // Raw password — matches how the fleet server formats its DSN + // (server/platform/mysql/common.go). go-sql-driver does not URL-decode + // the password field, so encoding it here would corrupt the value. + return fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true", user, pass, addr, db), nil +} + +// apiClient wraps HTTP calls to the Fleet API. +type apiClient struct { + baseURL string + token string + http *http.Client +} + +func (a *apiClient) get(path string) (*http.Response, error) { + url := a.baseURL + path + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", a.token)) + + resp, err := a.http.Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("GET %s: status %d", path, resp.StatusCode) + } + return resp, nil +} + +// --- API response types (minimal) --- + +type hostsResponse struct { + Hosts []struct { + ID uint `json:"id"` + } `json:"hosts"` +} + +type hostSoftwareResponse struct { + Software []struct { + InstalledVersions []struct { + Vulnerabilities []string `json:"vulnerabilities"` + } `json:"installed_versions"` + } `json:"software"` + Meta *struct { + HasNextResults bool `json:"has_next_results"` + } `json:"meta"` +} + +// --- Uptime collection --- + +func collectUptime(api *apiClient, db *sql.DB) error { + log.Println("collecting uptime data...") + + hostIDs, err := fetchHostIDs(api, "status=online") + if err != nil { + return fmt.Errorf("fetch online hosts: %w", err) + } + log.Printf(" %d online hosts", len(hostIDs)) + + if len(hostIDs) == 0 { + return nil + } + + now := time.Now().UTC() + bucketStart := now.Truncate(time.Hour) + validTo := bucketStart.Add(time.Hour) + newBlob := chart.HostIDsToBlob(hostIDs) + + // OR with existing in-bucket bitmap (accumulate semantic). + var existing []byte + err = db.QueryRow( + `SELECT host_bitmap FROM host_scd_data + WHERE dataset = 'uptime' AND entity_id = '' AND valid_from = ?`, + bucketStart, + ).Scan(&existing) + if err == nil { + newBlob = chart.BlobOR(existing, newBlob) + } + + _, err = db.Exec( + `INSERT INTO host_scd_data (dataset, entity_id, host_bitmap, valid_from, valid_to) + VALUES ('uptime', '', ?, ?, ?) + ON DUPLICATE KEY UPDATE host_bitmap = VALUES(host_bitmap)`, + newBlob, bucketStart, validTo, + ) + if err != nil { + return fmt.Errorf("write uptime SCD row: %w", err) + } + + log.Printf(" wrote uptime row: %d hosts, valid_from %s", chart.BlobPopcount(newBlob), bucketStart) + return nil +} + +// --- CVE collection --- + +func collectCVE(api *apiClient, db *sql.DB) error { + log.Println("collecting CVE data...") + + hostIDs, err := fetchHostIDs(api, "") + if err != nil { + return fmt.Errorf("fetch all hosts: %w", err) + } + log.Printf(" %d total hosts", len(hostIDs)) + + // Invert per-host fetches into per-CVE host sets. + fetchStart := time.Now() + cveHosts := make(map[string][]uint) + for i, hostID := range hostIDs { + cves, err := fetchHostCVEs(api, hostID) + if err != nil { + log.Printf(" warning: host %d: %v", hostID, err) + continue + } + for _, cve := range cves { + cveHosts[cve] = append(cveHosts[cve], hostID) + } + if (i+1)%50 == 0 { + log.Printf(" fetched %d/%d hosts, %d unique CVEs so far (%.1fs)", + i+1, len(hostIDs), len(cveHosts), time.Since(fetchStart).Seconds()) + } + } + log.Printf(" %d unique CVEs found in %.1fs", len(cveHosts), time.Since(fetchStart).Seconds()) + + // Build the desired entity->bitmap map for today's 24h bucket. + entityBitmaps := make(map[string][]byte, len(cveHosts)) + for cve, hosts := range cveHosts { + entityBitmaps[cve] = chart.HostIDsToBlob(hosts) + } + + // Snapshot rows are keyed to 1h boundaries (not 24h) so that row transitions + // fall on hour marks. This lets tz-offset users' local-day queries resolve + // "state at end of my day" to a row boundary observed at or before that + // moment, rather than being pulled forward by the artificial UTC-midnight + // transition that 24h keying would impose. + writeStart := time.Now() + bucketStart := time.Now().UTC().Truncate(time.Hour) + if err := reconcileSnapshot(db, "cve", entityBitmaps, bucketStart); err != nil { + return fmt.Errorf("reconcile SCD: %w", err) + } + log.Printf(" reconciled %d entities in %.1fs", len(entityBitmaps), time.Since(writeStart).Seconds()) + return nil +} + +// reconcileSnapshot mirrors Datastore.recordSnapshot in +// server/chart/internal/mysql/data.go. +func reconcileSnapshot(db *sql.DB, dataset string, entityBitmaps map[string][]byte, bucketStart time.Time) error { + rows, err := db.Query( + `SELECT entity_id, host_bitmap, valid_from + FROM host_scd_data + WHERE dataset = ? AND valid_to = ?`, + dataset, scdOpenSentinel) + if err != nil { + return fmt.Errorf("fetch open SCD rows: %w", err) + } + defer rows.Close() + type openRow struct { + bitmap []byte + validFrom time.Time + } + openByEntity := make(map[string]openRow) + for rows.Next() { + var entityID string + var bitmap []byte + var validFrom time.Time + if err := rows.Scan(&entityID, &bitmap, &validFrom); err != nil { + return fmt.Errorf("scan open SCD row: %w", err) + } + openByEntity[entityID] = openRow{bitmap: bitmap, validFrom: validFrom} + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate open SCD rows: %w", err) + } + + var toClose []string + var toUpsert []struct { + entityID string + bitmap []byte + } + + for entityID, bitmap := range entityBitmaps { + existing, hasOpen := openByEntity[entityID] + if hasOpen && bytes.Equal(existing.bitmap, bitmap) { + continue + } + if hasOpen && existing.validFrom.Before(bucketStart) { + toClose = append(toClose, entityID) + } + toUpsert = append(toUpsert, struct { + entityID string + bitmap []byte + }{entityID, bitmap}) + } + + for entityID := range openByEntity { + if _, ok := entityBitmaps[entityID]; !ok { + toClose = append(toClose, entityID) + } + } + + if len(toClose) > 0 { + placeholders := make([]string, len(toClose)) + args := []any{bucketStart, dataset, scdOpenSentinel} + for i, e := range toClose { + placeholders[i] = "?" + args = append(args, e) + } + // Concatenating hardcoded "?" placeholder strings, not user input. + stmt := fmt.Sprintf( //nolint:gosec // G202 + `UPDATE host_scd_data SET valid_to = ? + WHERE dataset = ? AND valid_to = ? AND entity_id IN (%s)`, + strings.Join(placeholders, ",")) + if _, err := db.Exec(stmt, args...); err != nil { + return fmt.Errorf("close stale rows: %w", err) + } + } + + for i := 0; i < len(toUpsert); i += scdUpsertBatch { + end := min(i+scdUpsertBatch, len(toUpsert)) + batch := toUpsert[i:end] + + placeholders := make([]string, len(batch)) + args := make([]any, 0, len(batch)*4) + for j, r := range batch { + placeholders[j] = "(?, ?, ?, ?)" + args = append(args, dataset, r.entityID, r.bitmap, bucketStart) + } + // Concatenating hardcoded "(?,?,?,?)" placeholder strings, not user input. + stmt := `INSERT INTO host_scd_data (dataset, entity_id, host_bitmap, valid_from) VALUES ` + //nolint:gosec // G202 + strings.Join(placeholders, ", ") + + ` ON DUPLICATE KEY UPDATE host_bitmap = VALUES(host_bitmap)` + if _, err := db.Exec(stmt, args...); err != nil { + return fmt.Errorf("upsert rows: %w", err) + } + } + return nil +} + +// --- API helpers --- + +// fetchHostIDs pages through the hosts list endpoint. Pass extra query params +// (e.g. "status=online") or "" for all hosts. +func fetchHostIDs(api *apiClient, extraParams string) ([]uint, error) { + var all []uint + for page := 0; ; page++ { + path := fmt.Sprintf("/api/v1/fleet/hosts?per_page=%d&page=%d", perPage, page) + if extraParams != "" { + path += "&" + extraParams + } + + resp, err := api.get(path) + if err != nil { + return nil, err + } + + var result hostsResponse + err = json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("decode hosts page %d: %w", page, err) + } + + for _, h := range result.Hosts { + all = append(all, h.ID) + } + + if len(result.Hosts) < perPage { + break + } + } + return all, nil +} + +// fetchHostCVEs returns deduplicated CVE IDs for a single host. +func fetchHostCVEs(api *apiClient, hostID uint) ([]string, error) { + seen := make(map[string]struct{}) + for page := 0; ; page++ { + path := fmt.Sprintf("/api/v1/fleet/hosts/%d/software?vulnerable=true&per_page=%d&page=%d", hostID, perPage, page) + + resp, err := api.get(path) + if err != nil { + return nil, err + } + + var result hostSoftwareResponse + err = json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("decode software for host %d page %d: %w", hostID, page, err) + } + + for _, sw := range result.Software { + for _, iv := range sw.InstalledVersions { + for _, cve := range iv.Vulnerabilities { + seen[cve] = struct{}{} + } + } + } + + if result.Meta == nil || !result.Meta.HasNextResults { + break + } + } + + cves := make([]string, 0, len(seen)) + for cve := range seen { + cves = append(cves, cve) + } + return cves, nil +}