diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index ddcd1e140f..aca0eec207 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -697,6 +697,10 @@ func TestCronVulnerabilitiesSkipMkdirIfDisabled(t *testing.T) { return nil } + ds.InsertKernelSoftwareMappingFunc = func(ctx context.Context) error { + return nil + } + mockLocker := schedule.SetupMockLocker("vulnerabilities", "test_instance", time.Now().UTC()) ds.LockFunc = mockLocker.Lock ds.UnlockFunc = mockLocker.Unlock diff --git a/cmd/fleet/vuln_process.go b/cmd/fleet/vuln_process.go index 79a7ea0ee1..f0a2147be1 100644 --- a/cmd/fleet/vuln_process.go +++ b/cmd/fleet/vuln_process.go @@ -29,9 +29,9 @@ func createVulnProcessingCmd(configManager config.Manager) *cobra.Command { vulnProcessingCmd := &cobra.Command{ Use: "vuln_processing", Short: "Run the vulnerability processing features of Fleet", - Long: `The vuln_processing command is intended for advanced configurations that want to externally manage + Long: `The vuln_processing command is intended for advanced configurations that want to externally manage vulnerability processing. By default the Fleet server command internally manages vulnerability processing via scheduled -'cron' style jobs, but setting 'vulnerabilities.disable_schedule=true' or 'FLEET_VULNERABILITIES_DISABLE_SCHEDULE=true' +'cron' style jobs, but setting 'vulnerabilities.disable_schedule=true' or 'FLEET_VULNERABILITIES_DISABLE_SCHEDULE=true' will disable it on the server allowing the user configure their own 'cron' mechanism. Successful processing will be indicated by an exit code of zero.`, RunE: func(cmd *cobra.Command, args []string) (err error) { @@ -189,6 +189,12 @@ func getVulnFuncs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, return ds.UpdateHostIssuesVulnerabilities(ctx) }, }, + { + Name: "insert_kernel_software_mapping", + VulnFunc: func(ctx context.Context) error { + return ds.InsertKernelSoftwareMapping(ctx) + }, + }, } return vulnFuncs diff --git a/cmd/osquery-perf/ubuntu_2204-software.json.bz2 b/cmd/osquery-perf/ubuntu_2204-software.json.bz2 index ea9cc3399d..e04a0f838d 100644 Binary files a/cmd/osquery-perf/ubuntu_2204-software.json.bz2 and b/cmd/osquery-perf/ubuntu_2204-software.json.bz2 differ diff --git a/server/datastore/mysql/migrations/tables/20250813205039_SoftwareIsKernelColumn.go b/server/datastore/mysql/migrations/tables/20250813205039_SoftwareIsKernelColumn.go new file mode 100644 index 0000000000..fd13b9590c --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20250813205039_SoftwareIsKernelColumn.go @@ -0,0 +1,60 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20250813205039, Down_20250813205039) +} + +func Up_20250813205039(tx *sql.Tx) error { + if _, err := tx.Exec(` +ALTER TABLE software_titles + ADD COLUMN is_kernel TINYINT(1) NOT NULL DEFAULT '0'`); err != nil { + return fmt.Errorf("failed to add software_titles.is_kernel column: %w", err) + } + + // Backfill existing software titles + if _, err := tx.Exec(` +UPDATE software_titles +SET is_kernel = + -- Debian/Ubuntu + CASE WHEN name REGEXP '^linux-image-[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+-[[:digit:]]+-[[:alnum:]]+' THEN + 1 + -- Amazon Linux + WHEN name = 'kernel' THEN + 1 + -- RHEL + WHEN name = 'kernel-core' THEN + 1 + ELSE + 0 + END +WHERE source IN ('rpm_packages', 'deb_packages') + `); err != nil { + return fmt.Errorf("failed to backfill software_titles.is_kernel column: %w", err) + } + + if _, err := tx.Exec(` +CREATE TABLE kernel_host_counts ( + id int unsigned NOT NULL AUTO_INCREMENT, + software_title_id int unsigned DEFAULT NULL, + software_id int unsigned DEFAULT NULL, + os_version_id int unsigned DEFAULT NULL, + hosts_count int unsigned NOT NULL, + team_id int unsigned NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY idx_kernels_unique_mapping (os_version_id,team_id,software_id), + FOREIGN KEY (software_title_id) REFERENCES software_titles (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`); err != nil { + return fmt.Errorf("failed to create kernel_host_counts table: %w", err) + } + + return nil +} + +func Down_20250813205039(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20250813205039_SoftwareIsKernelColumn_test.go b/server/datastore/mysql/migrations/tables/20250813205039_SoftwareIsKernelColumn_test.go new file mode 100644 index 0000000000..1cb8c5e485 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20250813205039_SoftwareIsKernelColumn_test.go @@ -0,0 +1,76 @@ +package tables + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20250813205039(t *testing.T) { + db := applyUpToPrev(t) + + // Name as reported for Ubuntu + kernelID1 := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("linux-image-6.11.0-9-generic", "deb_packages", "")`) + // Name as reported for Debian + kernelID2 := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("linux-image-6.1.0-37-cloud-arm64", "deb_packages", "")`) + amazonKernelID := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("kernel", "rpm_packages", "")`) + rhelKernelID := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("kernel-core", "rpm_packages", "")`) + otherLinuxAppID := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("vim", "deb_packages", "")`) + otherAppMacOSID := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("Calculator", "apps", "")`) + otherAppWindowsID := execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, browser) VALUES ("Notepad", "programs", "")`) + + // Apply current migration. + applyNext(t, db) + + tests := []struct { + name string + titleID int64 + shouldBeKernel bool + }{ + { + name: "ubuntu kernel", + titleID: kernelID1, + shouldBeKernel: true, + }, + { + name: "debian kernel", + titleID: kernelID2, + shouldBeKernel: true, + }, + { + name: "amazon linuxkernel", + titleID: amazonKernelID, + shouldBeKernel: true, + }, + { + name: "rhel kernel", + titleID: rhelKernelID, + shouldBeKernel: true, + }, + { + name: "other linux title", + titleID: otherLinuxAppID, + shouldBeKernel: false, + }, + { + name: "other title macOS", + titleID: otherAppMacOSID, + shouldBeKernel: false, + }, + { + name: "other title Windows", + titleID: otherAppWindowsID, + shouldBeKernel: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var isKernel bool + err := db.Get(&isKernel, `SELECT is_kernel FROM software_titles WHERE id = ?`, tt.titleID) + require.NoError(t, err) + require.Equal(t, tt.shouldBeKernel, isKernel) + }) + } + +} diff --git a/server/datastore/mysql/operating_system_vulnerabilities.go b/server/datastore/mysql/operating_system_vulnerabilities.go index 97d8634918..eee50e6b8d 100644 --- a/server/datastore/mysql/operating_system_vulnerabilities.go +++ b/server/datastore/mysql/operating_system_vulnerabilities.go @@ -31,7 +31,7 @@ func (ds *Datastore) ListOSVulnerabilitiesByOS(ctx context.Context, osID uint) ( return r, nil } -func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, version string, includeCVSS bool) (fleet.Vulnerabilities, error) { +func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, version string, includeCVSS bool, teamID *uint) (fleet.Vulnerabilities, error) { r := fleet.Vulnerabilities{} stmt := ` @@ -42,6 +42,23 @@ func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, vers JOIN operating_systems os ON os.id = osv.operating_system_id AND os.name = ? AND os.version = ? GROUP BY osv.cve + + UNION + + SELECT DISTINCT + software_cve.cve, + MIN(software_cve.created_at) created_at + FROM + software_cve + JOIN kernel_host_counts ON kernel_host_counts.software_id = software_cve.software_id + JOIN operating_systems ON operating_systems.os_version_id = kernel_host_counts.os_version_id + WHERE + operating_systems.name = ? + AND operating_systems.version = ? + AND kernel_host_counts.hosts_count > 0 + %s + GROUP BY software_cve.cve + ` if includeCVSS { @@ -68,12 +85,40 @@ func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, vers JOIN operating_systems os ON os.id = v.operating_system_id AND os.name = ? AND os.version = ? GROUP BY v.cve + + UNION + + SELECT DISTINCT + software_cve.cve, + MIN(software_cve.created_at) created_at, + GROUP_CONCAT(DISTINCT software_cve.resolved_in_version SEPARATOR ',') resolved_in_version + FROM + software_cve + JOIN kernel_host_counts ON kernel_host_counts.software_id = software_cve.software_id + JOIN operating_systems ON operating_systems.os_version_id = kernel_host_counts.os_version_id + WHERE + operating_systems.name = ? + AND operating_systems.version = ? + AND kernel_host_counts.hosts_count > 0 + %s + GROUP BY software_cve.cve ) osv LEFT JOIN cve_meta cm ON cm.cve = osv.cve ` } - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &r, stmt, name, version); err != nil { + var tmID uint + var teamFilter string + args := []any{name, version, name, version} + if teamID != nil { + tmID = *teamID + teamFilter = "AND kernel_host_counts.team_id = ?" + args = append(args, tmID) + } + + stmt = fmt.Sprintf(stmt, teamFilter) + + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &r, stmt, args...); err != nil { return nil, ctxerr.Wrap(ctx, err, "error executing SQL statement") } @@ -168,3 +213,117 @@ func (ds *Datastore) DeleteOutOfDateOSVulnerabilities(ctx context.Context, src f } return nil } + +func (ds *Datastore) ListKernelsByOS(ctx context.Context, osVersionID uint, teamID *uint) ([]*fleet.Kernel, error) { + var kernels []*fleet.Kernel + + stmt := ` +SELECT DISTINCT + software.id AS id, + software_cve.cve AS cve, + software.version AS version, + SUM(kernel_host_counts.hosts_count) AS hosts_count +FROM + software + LEFT JOIN software_cve ON software.id = software_cve.software_id + JOIN kernel_host_counts ON kernel_host_counts.software_id = software.id +WHERE + kernel_host_counts.os_version_id = ? %s GROUP BY id, cve, version +` + + var tmID uint + var teamFilter string + args := []any{osVersionID} + if teamID != nil { + tmID = *teamID + teamFilter = "AND kernel_host_counts.team_id = ?" + args = append(args, tmID) + } + + stmt = fmt.Sprintf(stmt, teamFilter) + + var results []struct { + ID uint `db:"id"` + CVE *string `db:"cve"` + Version string `db:"version"` + HostsCount uint `db:"hosts_count"` + } + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "listing kernels by OS name") + } + + kernelSet := make(map[uint]*fleet.Kernel) + + for _, result := range results { + k, ok := kernelSet[result.ID] + if !ok { + kernel := &fleet.Kernel{ + ID: result.ID, + Version: result.Version, + HostsCount: result.HostsCount, + } + + kernelSet[kernel.ID] = kernel + k = kernel + } + + if result.CVE != nil { + k.Vulnerabilities = append(k.Vulnerabilities, *result.CVE) + } + + } + for _, kernel := range kernelSet { + kernels = append(kernels, kernel) + } + return kernels, nil +} + +func (ds *Datastore) InsertKernelSoftwareMapping(ctx context.Context) error { + _, err := ds.writer(ctx).ExecContext(ctx, `UPDATE kernel_host_counts SET hosts_count = 0`) + if err != nil { + return ctxerr.Wrap(ctx, err, "zero out existing kernel hosts counts") + } + + statsStmt := ` +INSERT INTO kernel_host_counts (software_title_id, software_id, os_version_id, hosts_count, team_id) + SELECT + software_titles.id AS software_title_id, + software.id AS software_id, + operating_systems.os_version_id AS os_version_id, + COUNT(host_operating_system.host_id) AS hosts_count, + COALESCE(hosts.team_id, 0) AS team_id + FROM + software_titles + JOIN software ON software.title_id = software_titles.id + JOIN host_software ON host_software.software_id = software.id + JOIN host_operating_system ON host_operating_system.host_id = host_software.host_id + JOIN operating_systems ON operating_systems.id = host_operating_system.os_id + JOIN hosts ON hosts.id = host_software.host_id + WHERE + software_titles.is_kernel = TRUE + GROUP BY + software_title_id, + software_id, + os_version_id, + team_id +ON DUPLICATE KEY UPDATE + hosts_count=VALUES(hosts_count) + ` + + _, err = ds.writer(ctx).ExecContext(ctx, statsStmt) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert kernel software mapping") + } + + _, err = ds.writer(ctx).ExecContext(ctx, `DELETE k FROM kernel_host_counts k LEFT JOIN software ON k.software_id = software.id WHERE software.id IS NULL`) + if err != nil { + return ctxerr.Wrap(ctx, err, "clean up orphan kernels by software id") + } + + _, err = ds.writer(ctx).ExecContext(ctx, `DELETE k FROM kernel_host_counts k LEFT JOIN operating_systems ON k.os_version_id = operating_systems.os_version_id WHERE operating_systems.id IS NULL`) + if err != nil { + return ctxerr.Wrap(ctx, err, "clean up orphan kernels by os version id") + } + + return nil +} diff --git a/server/datastore/mysql/operating_system_vulnerabilities_test.go b/server/datastore/mysql/operating_system_vulnerabilities_test.go index 3e6b8beccd..9c5db33dcd 100644 --- a/server/datastore/mysql/operating_system_vulnerabilities_test.go +++ b/server/datastore/mysql/operating_system_vulnerabilities_test.go @@ -2,11 +2,13 @@ package mysql import ( "context" + "sort" "testing" "time" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/test" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,6 +29,8 @@ func TestOperatingSystemVulnerabilities(t *testing.T) { {"DeleteOSVulnerabilitiesEmpty", testDeleteOSVulnerabilitiesEmpty}, {"DeleteOSVulnerabilities", testDeleteOSVulnerabilities}, {"DeleteOutOfDateOSVulnerabilities", testDeleteOutOfDateOSVulnerabilities}, + {"TestListKernelsByOS", testListKernelsByOS}, + {"TestKernelVulnsHostCount", testKernelVulnsHostCount}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -109,7 +113,7 @@ func testListVulnsByOsNameAndVersion(t *testing.T, ds *Datastore) { dbOS = append(dbOS, *os) } - cves, err := ds.ListVulnsByOsNameAndVersion(ctx, "Microsoft Windows 11 Pro 21H2", "10.0.22000.795", false) + cves, err := ds.ListVulnsByOsNameAndVersion(ctx, "Microsoft Windows 11 Pro 21H2", "10.0.22000.795", false, nil) require.NoError(t, err) require.Empty(t, cves) @@ -165,7 +169,7 @@ func testListVulnsByOsNameAndVersion(t *testing.T, ds *Datastore) { require.NoError(t, err) // test without CVS meta - cves, err = ds.ListVulnsByOsNameAndVersion(ctx, "Microsoft Windows 11 Pro 21H2", "10.0.22000.795", false) + cves, err = ds.ListVulnsByOsNameAndVersion(ctx, "Microsoft Windows 11 Pro 21H2", "10.0.22000.795", false, nil) require.NoError(t, err) expected := []string{"CVE-2021-1234", "CVE-2021-1235"} @@ -176,7 +180,7 @@ func testListVulnsByOsNameAndVersion(t *testing.T, ds *Datastore) { } // test with CVS meta - cves, err = ds.ListVulnsByOsNameAndVersion(ctx, "Microsoft Windows 11 Pro 21H2", "10.0.22000.795", true) + cves, err = ds.ListVulnsByOsNameAndVersion(ctx, "Microsoft Windows 11 Pro 21H2", "10.0.22000.795", true, nil) require.NoError(t, err) require.Len(t, cves, 2) @@ -348,3 +352,311 @@ func testDeleteOutOfDateOSVulnerabilities(t *testing.T, ds *Datastore) { require.Len(t, actual, 1) require.ElementsMatch(t, []fleet.OSVulnerability{newVuln}, actual) } + +func testListKernelsByOS(t *testing.T, ds *Datastore) { + ctx := context.Background() + + kernel1 := fleet.Software{Name: "linux-image-6.11.0-9-generic", Version: "6.11.0-9.9", Source: "deb_packages", IsKernel: true} + kernel2 := fleet.Software{Name: "linux-image-7.11.0-10-generic", Version: "7.11.0-10.10", Source: "deb_packages", IsKernel: true} + kernel3 := fleet.Software{Name: "linux-image-8.11.0-11-generic", Version: "8.11.0-11.11", Source: "deb_packages", IsKernel: true} + software := []fleet.Software{ + kernel1, + kernel2, + kernel3, // this one will have 0 vulns + } + + cases := []struct { + name string + team bool + host *fleet.Host + software []fleet.Software + vulns []fleet.SoftwareVulnerability + vulnsByKernelVersion map[string][]string + os fleet.OperatingSystem + }{ + { + name: "ubuntu no team", + team: false, + host: test.NewHost(t, ds, "host_ubuntu2410", "", "hostkey_ubuntu2410", "hostuuid_ubuntu2410", time.Now(), test.WithPlatform("linux")), + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0001"}, {CVE: "CVE-2025-0002"}, {CVE: "CVE-2025-0003"}}, + vulnsByKernelVersion: map[string][]string{ + kernel1.Version: {"CVE-2025-0001", "CVE-2025-0002"}, + kernel2.Version: {"CVE-2025-0003"}, + kernel3.Version: nil, + }, + software: software, + os: fleet.OperatingSystem{Name: "Ubuntu", Version: "24.10", Arch: "x86_64", KernelVersion: "6.11.0-9-generic", Platform: "ubuntu"}, + }, + { + name: "ubuntu with team", + team: true, + host: test.NewHost(t, ds, "host_ubuntu2404", "", "hostkey_ubuntu2404", "hostuuid_ubuntu2404", time.Now(), test.WithPlatform("linux")), + software: software[1:], + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0004"}, {CVE: "CVE-2025-0005"}, {CVE: "CVE-2025-0003"}}, // Note the overlap; kernel2 has 0003 from the previous test + vulnsByKernelVersion: map[string][]string{ + kernel2.Version: {"CVE-2025-0004", "CVE-2025-0005", "CVE-2025-0003"}, + kernel3.Version: nil, + }, + os: fleet.OperatingSystem{Name: "Ubuntu", Version: "24.04", Arch: "x86_64", KernelVersion: "6.11.0-9-generic", Platform: "ubuntu"}, + }, + { + name: "amazon linux with team", + team: true, + host: test.NewHost(t, ds, "host_amzn2023", "", "hostkey_amzn2023", "hostuuid_amzn2023", time.Now(), test.WithPlatform("fedora")), + software: []fleet.Software{{Name: "kernel", Version: "6.1.144", Arch: "x86_64", Source: "rpm_packages", IsKernel: true}}, + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0006"}}, + vulnsByKernelVersion: map[string][]string{ + "6.1.144": {"CVE-2025-0006"}, + }, + os: fleet.OperatingSystem{Name: "Amazon Linux", Version: "2023.0.0", Arch: "x86_64", KernelVersion: "6.1.144-170.251.amzn2023.x86_64", Platform: "amzn"}, + }, + { + name: "RHEL with team", + team: true, + host: test.NewHost(t, ds, "host_fedora41", "", "hostkey_fedora41", "hostuuid_fedora41", time.Now(), test.WithPlatform("rhel")), + software: []fleet.Software{{Name: "kernel-core", Version: "6.11.4", Arch: "aarch64", Source: "rpm_packages", IsKernel: true}}, + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0007"}}, + vulnsByKernelVersion: map[string][]string{ + "6.11.4": {"CVE-2025-0007"}, + }, + os: fleet.OperatingSystem{Name: "Fedora Linux", Version: "41.0.0", Arch: "aarch64", KernelVersion: "6.11.4-301.fc41.aarch64", Platform: "rhel"}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + + var teamID uint + if tt.team { + team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1_" + tt.name}) + require.NoError(t, err) + require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&team1.ID, []uint{tt.host.ID}))) + teamID = team1.ID + } + + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, tt.host.ID, tt.os)) + + os, err := ds.GetHostOperatingSystem(ctx, tt.host.ID) + require.NoError(t, err) + + _, err = ds.UpdateHostSoftware(ctx, tt.host.ID, tt.software) + require.NoError(t, err) + require.NoError(t, ds.LoadHostSoftware(ctx, tt.host, false)) + + // Sort the host software by name to enforce a deterministic order + sort.Slice(tt.host.Software, func(i, j int) bool { + return tt.host.Software[i].Name < tt.host.Software[j].Name + }) + + softwareIDByVersion := make(map[string]uint) + for _, s := range tt.host.Software { + softwareIDByVersion[s.Version] = s.ID + } + + cpes := []fleet.SoftwareCPE{ + {SoftwareID: tt.host.Software[0].ID, CPE: "somecpe"}, + } + _, err = ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + require.NoError(t, ds.LoadHostSoftware(ctx, tt.host, false)) + + var vulnsToInsert []fleet.SoftwareVulnerability + for k, v := range tt.vulnsByKernelVersion { + for _, s := range v { + vulnsToInsert = append(vulnsToInsert, fleet.SoftwareVulnerability{ + SoftwareID: softwareIDByVersion[k], + CVE: s, + }) + } + } + + for _, v := range vulnsToInsert { + _, err = ds.InsertSoftwareVulnerability(ctx, v, fleet.NVDSource) + require.NoError(t, err) + } + require.NoError(t, ds.LoadHostSoftware(ctx, tt.host, false)) + + require.NoError(t, ds.UpdateOSVersions(ctx)) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + require.NoError(t, ds.InsertKernelSoftwareMapping(ctx)) + + kernels, err := ds.ListKernelsByOS(ctx, os.OSVersionID, &teamID) + require.NoError(t, err) + + require.Len(t, kernels, len(tt.software)) + + for _, kernel := range kernels { + expectedVulns, ok := tt.vulnsByKernelVersion[kernel.Version] + require.True(t, ok) + require.ElementsMatchf(t, expectedVulns, kernel.Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernel.Version) + require.Equal(t, kernel.HostsCount, uint(1)) + } + + expectedSet := make(map[string]struct{}) + for _, v := range tt.vulns { + expectedSet[v.CVE] = struct{}{} + } + + cves, err := ds.ListVulnsByOsNameAndVersion(ctx, os.Name, os.Version, false, &teamID) + require.NoError(t, err) + for _, g := range cves { + _, ok := expectedSet[g.CVE] + assert.Truef(t, ok, "got unexpected CVE: %s", g.CVE) + } + + assert.Len(t, cves, len(tt.vulns)) + + cves, err = ds.ListVulnsByOsNameAndVersion(ctx, os.Name, "not_found", false, nil) + require.NoError(t, err) + require.Empty(t, cves) + + cves, err = ds.ListVulnsByOsNameAndVersion(ctx, os.Name, os.Version, true, nil) + require.NoError(t, err) + require.Len(t, cves, len(tt.vulns)) + for _, g := range cves { + _, ok := expectedSet[g.CVE] + assert.True(t, ok) + } + + cves, err = ds.ListVulnsByOsNameAndVersion(ctx, os.Name, "not_found", true, nil) + require.NoError(t, err) + require.Empty(t, cves) + + }) + } +} + +func testKernelVulnsHostCount(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host1 := test.NewHost(t, ds, "host_ubuntu2410", "", "hostkey_ubuntu2410", "hostuuid_ubuntu2410", time.Now(), test.WithPlatform("ubuntu")) + host2 := test.NewHost(t, ds, "host_ubuntu2404", "", "hostkey_ubuntu2404", "hostuuid_ubuntu2404", time.Now(), test.WithPlatform("ubuntu")) + host3 := test.NewHost(t, ds, "host_ubuntu2404_2", "", "hostkey_ubuntu2404_2", "hostuuid_ubuntu2404_2", time.Now(), test.WithPlatform("ubuntu")) + + // Same as host 2 and 3, but on a different team + host4 := test.NewHost(t, ds, "host_ubuntu2404_3", "", "hostkey_ubuntu2404_3", "hostuuid_ubuntu2404_3", time.Now(), test.WithPlatform("ubuntu")) + + os1 := &fleet.OperatingSystem{Name: "Ubuntu", Version: "24.10", Arch: "x86_64", KernelVersion: "6.11.0-9-generic", Platform: "ubuntu"} + os2 := &fleet.OperatingSystem{Name: "Ubuntu", Version: "24.04", Arch: "x86_64", KernelVersion: "6.11.0-9-generic", Platform: "ubuntu"} + + kernel := fleet.Software{Name: "linux-image-6.11.0-9-generic", Version: "6.11.0-9.9", Source: "deb_packages", IsKernel: true} + + team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1_" + t.Name()}) + require.NoError(t, err) + + team2, err := ds.NewTeam(ctx, &fleet.Team{Name: "team2_" + t.Name()}) + require.NoError(t, err) + require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&team1.ID, []uint{host1.ID, host2.ID, host3.ID}))) + require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&team2.ID, []uint{host4.ID}))) + + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, host1.ID, *os1)) + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, host2.ID, *os2)) + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, host3.ID, *os2)) + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, host4.ID, *os2)) + + os1, err = ds.GetHostOperatingSystem(ctx, host1.ID) + require.NoError(t, err) + + os2, err = ds.GetHostOperatingSystem(ctx, host2.ID) + require.NoError(t, err) + + addKernelToHost := func(h *fleet.Host) { + var vulnsToInsert []fleet.SoftwareVulnerability + _, err = ds.UpdateHostSoftware(ctx, h.ID, []fleet.Software{kernel}) + require.NoError(t, err) + require.NoError(t, ds.LoadHostSoftware(ctx, h, false)) + + _, err = ds.UpsertSoftwareCPEs(ctx, []fleet.SoftwareCPE{{SoftwareID: h.Software[0].ID, CPE: "somecpe"}}) + require.NoError(t, err) + + for _, cve := range []string{"CVE-2025-0001", "CVE-2025-0002"} { + vulnsToInsert = append(vulnsToInsert, fleet.SoftwareVulnerability{ + SoftwareID: h.Software[0].ID, + CVE: cve, + }) + } + + for _, v := range vulnsToInsert { + _, err = ds.InsertSoftwareVulnerability(ctx, v, fleet.NVDSource) + require.NoError(t, err) + } + } + + for _, h := range []*fleet.Host{host1, host2, host3, host4} { + addKernelToHost(h) + } + + for _, h := range []*fleet.Host{host1, host2, host3, host4} { + require.NoError(t, ds.LoadHostSoftware(ctx, h, false)) + } + + require.NoError(t, ds.UpdateOSVersions(ctx)) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + require.NoError(t, ds.InsertKernelSoftwareMapping(ctx)) + + expectedCVEs := []string{"CVE-2025-0001", "CVE-2025-0002"} + + kernels, err := ds.ListKernelsByOS(ctx, os1.OSVersionID, &team1.ID) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + assert.Equal(t, uint(1), kernels[0].HostsCount) // host1 + + kernels, err = ds.ListKernelsByOS(ctx, os2.OSVersionID, &team1.ID) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + require.Equal(t, uint(2), kernels[0].HostsCount) // host2, host3 + + kernels, err = ds.ListKernelsByOS(ctx, os2.OSVersionID, &team2.ID) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + assert.Equal(t, uint(1), kernels[0].HostsCount) // host4 + + // "All teams" (aka team ID is nil) + // For os2, should be 3 since it's on host2, host3, and host4 + kernels, err = ds.ListKernelsByOS(ctx, os2.OSVersionID, nil) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + assert.Equal(t, uint(3), kernels[0].HostsCount) + + // For os1, should be 1 since it's on host1 + kernels, err = ds.ListKernelsByOS(ctx, os1.OSVersionID, nil) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + assert.Equal(t, uint(1), kernels[0].HostsCount) + + // Add another host to team1, counts should update + host5 := test.NewHost(t, ds, "host_ubuntu2404_4", "", "hostkey_ubuntu2404_4", "hostuuid_ubuntu2404_4", time.Now(), test.WithPlatform("ubuntu")) + require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&team1.ID, []uint{host5.ID}))) + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, host5.ID, *os2)) + addKernelToHost(host5) + + require.NoError(t, ds.UpdateOSVersions(ctx)) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + require.NoError(t, ds.InsertKernelSoftwareMapping(ctx)) + + kernels, err = ds.ListKernelsByOS(ctx, os2.OSVersionID, &team1.ID) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + assert.Equal(t, uint(3), kernels[0].HostsCount) // host2, host3, host5 + + // "All teams" (aka team ID is nil) + // For os2, should be 4 since it's on host2, host3, host4, and now host5 + kernels, err = ds.ListKernelsByOS(ctx, os2.OSVersionID, nil) + require.NoError(t, err) + require.Len(t, kernels, 1) + assert.ElementsMatchf(t, expectedCVEs, kernels[0].Vulnerabilities, "unexpected vulnerabilities for kernel %s", kernels[0].Version) + assert.Equal(t, uint(4), kernels[0].HostsCount) + +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 6023483f32..0d58c2d100 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1030,6 +1030,21 @@ CREATE TABLE `jobs` ( INSERT INTO `jobs` VALUES (1,'2024-03-20 00:00:00','2024-03-20 00:00:00','macos_setup_assistant','{\"task\": \"update_all_profiles\"}','queued',0,'','2024-03-20 00:00:00'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `kernel_host_counts` ( + `id` int unsigned NOT NULL AUTO_INCREMENT, + `software_title_id` int unsigned DEFAULT NULL, + `software_id` int unsigned DEFAULT NULL, + `os_version_id` int unsigned DEFAULT NULL, + `hosts_count` int unsigned NOT NULL, + `team_id` int unsigned NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_kernels_unique_mapping` (`os_version_id`,`team_id`,`software_id`), + KEY `software_title_id` (`software_title_id`), + CONSTRAINT `kernel_host_counts_ibfk_1` FOREIGN KEY (`software_title_id`) REFERENCES `software_titles` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `label_membership` ( `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -1441,9 +1456,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=411 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=412 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -2373,6 +2388,7 @@ CREATE TABLE `software_titles` ( `bundle_identifier` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, `additional_identifier` tinyint unsigned GENERATED ALWAYS AS ((case when (`source` = _utf8mb4'ios_apps') then 1 when (`source` = _utf8mb4'ipados_apps') then 2 when (`bundle_identifier` is not null) then 0 else NULL end)) VIRTUAL, `unique_identifier` varchar(255) COLLATE utf8mb4_unicode_ci GENERATED ALWAYS AS (coalesce(`bundle_identifier`,`name`)) VIRTUAL, + `is_kernel` tinyint(1) NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `idx_software_titles_bundle_identifier` (`bundle_identifier`,`additional_identifier`), UNIQUE KEY `idx_unique_sw_titles` (`unique_identifier`,`source`,`browser`), diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 7542c4d700..dd0bbff91f 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -424,7 +424,7 @@ func updateExistingBundleIDs(ctx context.Context, tx sqlx.ExtContext, hostID uin updateSoftwareStmt := `UPDATE software SET software.name = ?, software.name_source = 'bundle_4.67' WHERE software.bundle_identifier = ?` hostSoftwareStmt := ` - INSERT IGNORE INTO host_software + INSERT IGNORE INTO host_software (host_id, software_id, last_opened_at) VALUES (?, (SELECT id FROM software WHERE bundle_identifier = ? AND name_source = 'bundle_4.67' ORDER BY id DESC LIMIT 1), ?)` @@ -820,9 +820,10 @@ func (ds *Datastore) insertNewInstalledHostSoftwareDB( titleID = &title.ID } else if _, ok := newTitlesNeeded[checksum]; !ok { st := fleet.SoftwareTitle{ - Name: sw.Name, - Source: sw.Source, - Browser: sw.Browser, + Name: sw.Name, + Source: sw.Source, + Browser: sw.Browser, + IsKernel: sw.IsKernel, } if sw.BundleIdentifier != "" { @@ -843,14 +844,14 @@ func (ds *Datastore) insertNewInstalledHostSoftwareDB( // Insert into software_titles totalTitlesToProcess := len(newTitlesNeeded) if totalTitlesToProcess > 0 { - const numberOfArgsPerSoftwareTitles = 4 // number of ? in each VALUES clause - titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?),", totalTitlesToProcess), ",") + const numberOfArgsPerSoftwareTitles = 5 // number of ? in each VALUES clause + titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?),", totalTitlesToProcess), ",") // INSERT IGNORE is used to avoid duplicate key errors, which may occur since our previous read came from the replica. - titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, browser, bundle_identifier) VALUES %s", titlesValues) + titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, browser, bundle_identifier, is_kernel) VALUES %s", titlesValues) titlesArgs := make([]interface{}, 0, totalTitlesToProcess*numberOfArgsPerSoftwareTitles) titleChecksums := make([]string, 0, totalTitlesToProcess) for checksum, title := range newTitlesNeeded { - titlesArgs = append(titlesArgs, title.Name, title.Source, title.Browser, title.BundleIdentifier) + titlesArgs = append(titlesArgs, title.Name, title.Source, title.Browser, title.BundleIdentifier, title.IsKernel) titleChecksums = append(titleChecksums, checksum) } if _, err := tx.ExecContext(ctx, titlesStmt, titlesArgs...); err != nil { @@ -2061,8 +2062,8 @@ DELETE st FROM software_titles st id DESC LIMIT 1 ) - WHERE - st.bundle_identifier IS NOT NULL AND + WHERE + st.bundle_identifier IS NOT NULL AND st.bundle_identifier != '' AND s.name_source = 'bundle_4.67' ` @@ -2424,7 +2425,7 @@ func hostInstalledSoftware(ds *Datastore, ctx context.Context, hostID uint) ([]* software.source AS software_source, software.version AS version, software.bundle_identifier AS bundle_identifier - FROM + FROM host_software INNER JOIN software ON host_software.software_id = software.id @@ -3910,7 +3911,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt FROM software_titles LEFT JOIN - software_installers ON software_titles.id = software_installers.title_id + software_installers ON software_titles.id = software_installers.title_id AND software_installers.global_or_team_id = :global_or_team_id LEFT JOIN software ON software_titles.id = software.title_id ` + installedSoftwareJoinsCondition + ` diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e17e517379..a125ac1958 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1095,7 +1095,7 @@ type Datastore interface { /////////////////////////////////////////////////////////////////////////////// // OperatingSystemVulnerabilities Store ListOSVulnerabilitiesByOS(ctx context.Context, osID uint) ([]OSVulnerability, error) - ListVulnsByOsNameAndVersion(ctx context.Context, name, version string, includeCVSS bool) (Vulnerabilities, error) + ListVulnsByOsNameAndVersion(ctx context.Context, name, version string, includeCVSS bool, teamID *uint) (Vulnerabilities, error) InsertOSVulnerabilities(ctx context.Context, vulnerabilities []OSVulnerability, source VulnerabilitySource) (int64, error) DeleteOSVulnerabilities(ctx context.Context, vulnerabilities []OSVulnerability) error // InsertOSVulnerability will either insert a new vulnerability in the datastore (in which @@ -1106,6 +1106,10 @@ type Datastore interface { // the updated_at timestamp is older than the supplied timestamp DeleteOutOfDateOSVulnerabilities(ctx context.Context, source VulnerabilitySource, olderThan time.Time) error + ListKernelsByOS(ctx context.Context, osID uint, teamID *uint) ([]*Kernel, error) + + InsertKernelSoftwareMapping(ctx context.Context) error + /////////////////////////////////////////////////////////////////////////////// // Vulnerabilities diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index e13509722e..a7f4eec1c5 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -1018,23 +1019,13 @@ var HostRpmPackageOSs = map[string]struct{}{ } func IsLinux(hostPlatform string) bool { - for _, linuxPlatform := range HostLinuxOSs { - if linuxPlatform == hostPlatform { - return true - } - } - return false + return slices.Contains(HostLinuxOSs, hostPlatform) } func IsUnixLike(hostPlatform string) bool { unixLikeOSs := HostLinuxOSs unixLikeOSs = append(unixLikeOSs, "darwin") - for _, p := range unixLikeOSs { - if p == hostPlatform { - return true - } - } - return false + return slices.Contains(unixLikeOSs, hostPlatform) } // PlatformFromHost converts the given host platform into @@ -1317,10 +1308,18 @@ type VulnerableOS struct { ResolvedInVersion *string `json:"resolved_in_version"` } +// Kernel represents a Linux kernel found on a host. +type Kernel struct { + ID uint `json:"id"` + Version string `json:"version"` + Vulnerabilities []string `json:"vulnerabilities"` + HostsCount uint `json:"hosts_count"` +} + type OSVersion struct { // ID is the unique id of the operating system. ID uint `json:"id,omitempty"` - // OSVersionID is a uniqe NameOnly/Version combination for the operating system. + // OSVersionID is a unique NameOnly/Version combination for the operating system. OSVersionID uint `json:"os_version_id"` // HostsCount is the number of hosts that have reported the operating system. HostsCount int `json:"hosts_count"` @@ -1339,7 +1338,13 @@ type OSVersion struct { // in NVD (macOS only) GeneratedCPEs []string `json:"generated_cpes,omitempty"` // Vulnerabilities are the vulnerabilities associated with the operating system. + // For Linux-based operating systems, these are vulnerabilities associated with the Linux kernel. Vulnerabilities Vulnerabilities `json:"vulnerabilities"` + // Kernels is a list of Linux kernels found on this operating system. + // This list is only populated for Linux-based operating systems. + // Vulnerabilities are pulled based on the software entries for the kernels. + // Kernels are associated based on enrolled hosts with the selected OS version. + Kernels []*Kernel `json:"kernels"` } type HostDetailOptions struct { diff --git a/server/fleet/software.go b/server/fleet/software.go index 5db1166287..dd3fc1cac9 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -94,6 +94,7 @@ type Software struct { // TODO: should we create a separate type? Feels like this field shouldn't be here since it's // just used for VPP install verification. Installed bool `json:"-"` + IsKernel bool `json:"-"` } func (Software) AuthzType() string { @@ -209,6 +210,8 @@ type SoftwareTitle struct { // the software installed. It's surfaced in software_titles to match // with existing software entries. BundleIdentifier *string `json:"bundle_identifier,omitempty" db:"bundle_identifier"` + // IsKernel indicates if the software title is a Linux kernel. + IsKernel bool `json:"-" db:"is_kernel"` } // This type is essentially the same as the above SoftwareTitle type. The only difference is that @@ -469,6 +472,7 @@ func SoftwareFromOsqueryRow( if !lastOpenedAtTime.IsZero() { software.LastOpenedAt = &lastOpenedAtTime } + return &software, nil } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 996601f3b8..6d6f60247f 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -783,7 +783,7 @@ type InsertWindowsUpdatesFunc func(ctx context.Context, hostID uint, updates []f type ListOSVulnerabilitiesByOSFunc func(ctx context.Context, osID uint) ([]fleet.OSVulnerability, error) -type ListVulnsByOsNameAndVersionFunc func(ctx context.Context, name string, version string, includeCVSS bool) (fleet.Vulnerabilities, error) +type ListVulnsByOsNameAndVersionFunc func(ctx context.Context, name string, version string, includeCVSS bool, teamID *uint) (fleet.Vulnerabilities, error) type InsertOSVulnerabilitiesFunc func(ctx context.Context, vulnerabilities []fleet.OSVulnerability, source fleet.VulnerabilitySource) (int64, error) @@ -793,6 +793,10 @@ type InsertOSVulnerabilityFunc func(ctx context.Context, vuln fleet.OSVulnerabil type DeleteOutOfDateOSVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, olderThan time.Time) error +type ListKernelsByOSFunc func(ctx context.Context, osID uint, teamID *uint) ([]*fleet.Kernel, error) + +type InsertKernelSoftwareMappingFunc func(ctx context.Context) error + type ListVulnerabilitiesFunc func(ctx context.Context, opt fleet.VulnListOptions) ([]fleet.VulnerabilityWithMetadata, *fleet.PaginationMetadata, error) type VulnerabilityFunc func(ctx context.Context, cve string, teamID *uint, includeCVEScores bool) (*fleet.VulnerabilityWithMetadata, error) @@ -2607,6 +2611,12 @@ type DataStore struct { DeleteOutOfDateOSVulnerabilitiesFunc DeleteOutOfDateOSVulnerabilitiesFunc DeleteOutOfDateOSVulnerabilitiesFuncInvoked bool + ListKernelsByOSFunc ListKernelsByOSFunc + ListKernelsByOSFuncInvoked bool + + InsertKernelSoftwareMappingFunc InsertKernelSoftwareMappingFunc + InsertKernelSoftwareMappingFuncInvoked bool + ListVulnerabilitiesFunc ListVulnerabilitiesFunc ListVulnerabilitiesFuncInvoked bool @@ -6257,11 +6267,11 @@ func (s *DataStore) ListOSVulnerabilitiesByOS(ctx context.Context, osID uint) ([ return s.ListOSVulnerabilitiesByOSFunc(ctx, osID) } -func (s *DataStore) ListVulnsByOsNameAndVersion(ctx context.Context, name string, version string, includeCVSS bool) (fleet.Vulnerabilities, error) { +func (s *DataStore) ListVulnsByOsNameAndVersion(ctx context.Context, name string, version string, includeCVSS bool, teamID *uint) (fleet.Vulnerabilities, error) { s.mu.Lock() s.ListVulnsByOsNameAndVersionFuncInvoked = true s.mu.Unlock() - return s.ListVulnsByOsNameAndVersionFunc(ctx, name, version, includeCVSS) + return s.ListVulnsByOsNameAndVersionFunc(ctx, name, version, includeCVSS, teamID) } func (s *DataStore) InsertOSVulnerabilities(ctx context.Context, vulnerabilities []fleet.OSVulnerability, source fleet.VulnerabilitySource) (int64, error) { @@ -6292,6 +6302,20 @@ func (s *DataStore) DeleteOutOfDateOSVulnerabilities(ctx context.Context, source return s.DeleteOutOfDateOSVulnerabilitiesFunc(ctx, source, olderThan) } +func (s *DataStore) ListKernelsByOS(ctx context.Context, osID uint, teamID *uint) ([]*fleet.Kernel, error) { + s.mu.Lock() + s.ListKernelsByOSFuncInvoked = true + s.mu.Unlock() + return s.ListKernelsByOSFunc(ctx, osID, teamID) +} + +func (s *DataStore) InsertKernelSoftwareMapping(ctx context.Context) error { + s.mu.Lock() + s.InsertKernelSoftwareMappingFuncInvoked = true + s.mu.Unlock() + return s.InsertKernelSoftwareMappingFunc(ctx) +} + func (s *DataStore) ListVulnerabilities(ctx context.Context, opt fleet.VulnListOptions) ([]fleet.VulnerabilityWithMetadata, *fleet.PaginationMetadata, error) { s.mu.Lock() s.ListVulnerabilitiesFuncInvoked = true diff --git a/server/service/hosts.go b/server/service/hosts.go index 2892122e63..3182b461d9 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -2155,7 +2155,7 @@ func (svc *Service) OSVersions(ctx context.Context, teamID *uint, platform *stri } for i := range osVersions.OSVersions { - if err := svc.populateOSVersionDetails(ctx, &osVersions.OSVersions[i], includeCVSS); err != nil { + if err := svc.populateOSVersionDetails(ctx, &osVersions.OSVersions[i], includeCVSS, teamID, false); err != nil { return nil, count, nil, err } } @@ -2269,7 +2269,7 @@ func (svc *Service) OSVersion(ctx context.Context, osID uint, teamID *uint, incl } if osVersion != nil { - if err = svc.populateOSVersionDetails(ctx, osVersion, includeCVSS); err != nil { + if err = svc.populateOSVersionDetails(ctx, osVersion, includeCVSS, teamID, true); err != nil { return nil, nil, err } } @@ -2278,7 +2278,7 @@ func (svc *Service) OSVersion(ctx context.Context, osID uint, teamID *uint, incl } // PopulateOSVersionDetails populates the GeneratedCPEs and Vulnerabilities for an OSVersion. -func (svc *Service) populateOSVersionDetails(ctx context.Context, osVersion *fleet.OSVersion, includeCVSS bool) error { +func (svc *Service) populateOSVersionDetails(ctx context.Context, osVersion *fleet.OSVersion, includeCVSS bool, teamID *uint, includeKernels bool) error { // Populate GeneratedCPEs if osVersion.Platform == "darwin" { osVersion.GeneratedCPEs = []string{ @@ -2288,16 +2288,27 @@ func (svc *Service) populateOSVersionDetails(ctx context.Context, osVersion *fle } // Populate Vulnerabilities - vulns, err := svc.ds.ListVulnsByOsNameAndVersion(ctx, osVersion.NameOnly, osVersion.Version, includeCVSS) + vulns, err := svc.ds.ListVulnsByOsNameAndVersion(ctx, osVersion.NameOnly, osVersion.Version, includeCVSS, teamID) if err != nil { return err } osVersion.Vulnerabilities = make(fleet.Vulnerabilities, 0) // avoid null in JSON + osVersion.Kernels = make([]*fleet.Kernel, 0) // avoid null in JSON for _, vuln := range vulns { vuln.DetailsLink = fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", vuln.CVE) osVersion.Vulnerabilities = append(osVersion.Vulnerabilities, vuln) + } + + if fleet.IsLinux(osVersion.Platform) && includeKernels { + kernels, err := svc.ds.ListKernelsByOS(ctx, osVersion.OSVersionID, teamID) + if err != nil { + return err + } + osVersion.Kernels = kernels + } + return nil } diff --git a/server/service/hosts_test.go b/server/service/hosts_test.go index 229e08db1c..acff70eb57 100644 --- a/server/service/hosts_test.go +++ b/server/service/hosts_test.go @@ -1295,7 +1295,7 @@ func TestEmptyTeamOSVersions(t *testing.T) { return nil, newNotFoundError() } - ds.ListVulnsByOsNameAndVersionFunc = func(ctx context.Context, name, version string, includeCVSS bool) (fleet.Vulnerabilities, error) { + ds.ListVulnsByOsNameAndVersionFunc = func(ctx context.Context, name, version string, includeCVSS bool, teamID *uint) (fleet.Vulnerabilities, error) { return fleet.Vulnerabilities{}, nil } @@ -1339,7 +1339,7 @@ func TestOSVersionsListOptions(t *testing.T) { return &fleet.OSVersions{CountsUpdatedAt: time.Now(), OSVersions: testVersions}, nil } - ds.ListVulnsByOsNameAndVersionFunc = func(ctx context.Context, name, version string, includeCVSS bool) (fleet.Vulnerabilities, error) { + ds.ListVulnsByOsNameAndVersionFunc = func(ctx context.Context, name, version string, includeCVSS bool, teamID *uint) (fleet.Vulnerabilities, error) { return fleet.Vulnerabilities{}, nil } diff --git a/server/service/integration_enterprise_vulns_test.go b/server/service/integration_enterprise_vulns_test.go new file mode 100644 index 0000000000..e2f0fb4145 --- /dev/null +++ b/server/service/integration_enterprise_vulns_test.go @@ -0,0 +1,166 @@ +package service + +import ( + "context" + "fmt" + "net/http" + "sort" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/datastore/mysql" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func (s *integrationEnterpriseTestSuite) TestLinuxOSVulns() { + t := s.T() + ctx := context.Background() + + kernel1 := fleet.Software{Name: "linux-image-6.11.0-9-generic", Version: "6.11.0-9.9", Source: "deb_packages", IsKernel: true} + kernel2 := fleet.Software{Name: "linux-image-7.11.0-10-generic", Version: "7.11.0-10.10", Source: "deb_packages", IsKernel: true} + kernel3 := fleet.Software{Name: "linux-image-8.11.0-11-generic", Version: "8.11.0-11.11", Source: "deb_packages", IsKernel: true} + software := []fleet.Software{ + kernel1, + kernel2, + kernel3, // this one will have 0 vulns + } + + cases := []struct { + name string + host *fleet.Host + software []fleet.Software + vulns []fleet.SoftwareVulnerability + vulnsByKernelVersion map[string][]string + os fleet.OperatingSystem + }{ + { + name: "ubuntu", + host: test.NewHost(t, s.ds, "host_ubuntu2410", "", "hostkey_ubuntu2410", "hostuuid_ubuntu2410", time.Now(), test.WithPlatform("ubuntu")), + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0001"}, {CVE: "CVE-2025-0002"}, {CVE: "CVE-2025-0003"}}, + vulnsByKernelVersion: map[string][]string{ + kernel1.Version: {"CVE-2025-0001", "CVE-2025-0002"}, + kernel2.Version: {"CVE-2025-0003"}, + kernel3.Version: nil, + }, + software: software, + os: fleet.OperatingSystem{Name: "Ubuntu", Version: "24.10", Arch: "x86_64", KernelVersion: "6.11.0-9-generic", Platform: "ubuntu"}, + }, + { + name: "amazon linux", + host: test.NewHost(t, s.ds, "host_amzn2023", "", "hostkey_amzn2023", "hostuuid_amzn2023", time.Now(), test.WithPlatform("fedora")), + software: []fleet.Software{{Name: "kernel", Version: "6.1.144", Arch: "x86_64", Source: "rpm_packages", IsKernel: true}}, + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0006"}}, + vulnsByKernelVersion: map[string][]string{ + "6.1.144": {"CVE-2025-0006"}, + }, + os: fleet.OperatingSystem{Name: "Amazon Linux", Version: "2023.0.0", Arch: "x86_64", KernelVersion: "6.1.144-170.251.amzn2023.x86_64", Platform: "amzn"}, + }, + { + name: "RHEL", + host: test.NewHost(t, s.ds, "host_fedora41", "", "hostkey_fedora41", "hostuuid_fedora41", time.Now(), test.WithPlatform("rhel")), + software: []fleet.Software{{Name: "kernel-core", Version: "6.11.4", Arch: "aarch64", Source: "rpm_packages", IsKernel: true}}, + vulns: []fleet.SoftwareVulnerability{{CVE: "CVE-2025-0007"}}, + vulnsByKernelVersion: map[string][]string{ + "6.11.4": {"CVE-2025-0007"}, + }, + os: fleet.OperatingSystem{Name: "Fedora Linux", Version: "41.0.0", Arch: "aarch64", KernelVersion: "6.11.4-301.fc41.aarch64", Platform: "rhel"}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + + require.NoError(t, s.ds.UpdateHostOperatingSystem(ctx, tt.host.ID, tt.os)) + var osinfo struct { + ID uint `db:"id"` + OSVersionID uint `db:"os_version_id"` + } + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &osinfo, + `SELECT id, os_version_id FROM operating_systems WHERE name = ? AND version = ? AND arch = ? AND kernel_version = ? AND platform = ?`, + tt.os.Name, tt.os.Version, tt.os.Arch, tt.os.KernelVersion, tt.os.Platform) + }) + require.Greater(t, osinfo.ID, uint(0)) + require.Greater(t, osinfo.OSVersionID, uint(0)) + + _, err := s.ds.UpdateHostSoftware(ctx, tt.host.ID, tt.software) + require.NoError(t, err) + require.NoError(t, s.ds.LoadHostSoftware(ctx, tt.host, false)) + + softwareIDByVersion := make(map[string]uint) + for _, s := range tt.host.Software { + softwareIDByVersion[s.Version] = s.ID + } + + cpes := []fleet.SoftwareCPE{{SoftwareID: tt.host.Software[0].ID, CPE: "somecpe"}} + _, err = s.ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + + // Reload software so that GeneratedCPEID is set. + require.NoError(t, s.ds.LoadHostSoftware(ctx, tt.host, false)) + + var vulnsToInsert []fleet.SoftwareVulnerability + for k, v := range tt.vulnsByKernelVersion { + for _, s := range v { + vulnsToInsert = append(vulnsToInsert, fleet.SoftwareVulnerability{ + SoftwareID: softwareIDByVersion[k], + CVE: s, + }) + } + } + + for _, v := range vulnsToInsert { + _, err = s.ds.InsertSoftwareVulnerability(ctx, v, fleet.NVDSource) + require.NoError(t, err) + } + + // Aggregate OS versions + require.NoError(t, s.ds.UpdateOSVersions(ctx)) + require.NoError(t, s.ds.UpdateOSVersions(ctx)) + require.NoError(t, s.ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, s.ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, s.ds.SyncHostsSoftwareTitles(ctx, time.Now())) + require.NoError(t, s.ds.InsertKernelSoftwareMapping(ctx)) + + var osVersionsResp osVersionsResponse + s.DoJSON("GET", "/api/latest/fleet/os_versions", nil, http.StatusOK, &osVersionsResp) + var osVersion *fleet.OSVersion + for _, os := range osVersionsResp.OSVersions { + if os.Version == tt.os.Version { + osVersion = &os + break + } + } + + assert.Equal(t, 1, osVersion.HostsCount) + assert.Equal(t, fmt.Sprintf("%s %s", tt.os.Name, tt.os.Version), osVersion.Name) + assert.Equal(t, tt.os.Name, osVersion.NameOnly) + assert.Equal(t, tt.os.Version, osVersion.Version) + assert.Equal(t, tt.os.Platform, osVersion.Platform) + assert.Len(t, osVersion.Vulnerabilities, len(tt.vulns)) + + // Test entity endpoint + var osVersionResp getOSVersionResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/os_versions/%d", osVersion.OSVersionID), nil, http.StatusOK, &osVersionResp, "team_id", fmt.Sprintf("%d", 0)) + assert.Len(t, osVersionResp.OSVersion.Kernels, len(tt.software)) + // Make sure the ordering is the same + sort.Slice(osVersionResp.OSVersion.Kernels, func(i, j int) bool { + return osVersionResp.OSVersion.Kernels[i].Version < osVersionResp.OSVersion.Kernels[j].Version + }) + sort.Slice(tt.software, func(i, j int) bool { + return tt.software[i].Version < tt.software[j].Version + }) + for i, k := range osVersionResp.OSVersion.Kernels { + assert.Equal(t, tt.software[i].Version, k.Version) + assert.Equal(t, uint(1), k.HostsCount) + assert.ElementsMatch(t, tt.vulnsByKernelVersion[k.Version], k.Vulnerabilities) + } + + }) + } + +} diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index 10c22fcf83..8d77b2d416 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -1789,6 +1789,12 @@ func directIngestScheduledQueryStats(ctx context.Context, logger log.Logger, hos return nil } +const linuxImageRegex = `^linux-image-[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+-[[:digit:]]+-[[:alnum:]]+` +const amazonLinuxKernelName = "kernel" +const rhelKernelName = "kernel-core" + +var kernelRegex = regexp.MustCompile(linuxImageRegex) + func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Host, ds fleet.Datastore, rows []map[string]string) error { var software []fleet.Software sPaths := map[string]struct{}{} @@ -1826,6 +1832,10 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho continue } + if fleet.IsLinux(host.Platform) && (kernelRegex.MatchString(s.Name) || s.Name == amazonLinuxKernelName || s.Name == rhelKernelName) { + s.IsKernel = true + } + MutateSoftwareOnIngestion(s, logger) if shouldRemoveSoftware(host, s) {