From 4ece43b1434a39d9ac862ceeebe9bdfee3176e85 Mon Sep 17 00:00:00 2001 From: Tim Lee Date: Thu, 30 Nov 2023 13:01:50 -0700 Subject: [PATCH] 14969 vuln software seeding tool (#15116) --- tools/seed_data/README.md | 26 +++ tools/seed_data/seed_vuln_data.go | 351 ++++++++++++++++++++++++++++ tools/seed_data/software-macos.csv | 14 ++ tools/seed_data/software-ubuntu.csv | 9 + tools/seed_data/software-win.csv | 12 + 5 files changed, 412 insertions(+) create mode 100644 tools/seed_data/README.md create mode 100644 tools/seed_data/seed_vuln_data.go create mode 100644 tools/seed_data/software-macos.csv create mode 100644 tools/seed_data/software-ubuntu.csv create mode 100644 tools/seed_data/software-win.csv diff --git a/tools/seed_data/README.md b/tools/seed_data/README.md new file mode 100644 index 0000000000..f65f49977d --- /dev/null +++ b/tools/seed_data/README.md @@ -0,0 +1,26 @@ +# Vulnerability Data Seeder + +The purpose of `seed_vuln_data.go` is to provide developers an easy way to insert software titles +into your local development Fleet server in order to troubleshoot false positive|negative vulnerabilities. + +## Usage + +1. Ensure your local development enviornment is running using `docker-compose up` and `fleet serve` + +2. Review and modify the software titles in the following files in this folder: + + - software_macos.csv + - software_ubuntu.csv + - software_win.csv + + Comma seperated values align directly with the columns in the `software` table in MySQL + +3. Run the data seeder + +```bash +go run ./tools/seed_data/seed_vuln_data.go +``` + +You should now see new hosts with the configured software attached in the UI and database. This +tool is idempotent as it will not create duplicate hosts or software titles if run multiple times, +however it will not delete software if removed from the CSV files. diff --git a/tools/seed_data/seed_vuln_data.go b/tools/seed_data/seed_vuln_data.go new file mode 100644 index 0000000000..503d1a835c --- /dev/null +++ b/tools/seed_data/seed_vuln_data.go @@ -0,0 +1,351 @@ +package main + +import ( + "database/sql" + "encoding/csv" + "errors" + "fmt" + "log" + "os" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" +) + +type Host struct { + ID int `db:"id"` + OsqueryHostID string `db:"osquery_host_id"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt *time.Time `db:"updated_at"` + DetailUpdatedAt time.Time `db:"detail_updated_at"` + NodeKey string `db:"node_key"` + Hostname string `db:"hostname"` + Uuid string `db:"uuid"` + Platform string `db:"platform"` + OsqueryVersion string `db:"osquery_version"` + OsVersion string `db:"os_version"` + Build string `db:"build"` + PlatformLike string `db:"platform_like"` + CodeName string `db:"code_name"` + Uptime int64 `db:"uptime"` + Memory int64 `db:"memory"` + CpuType string `db:"cpu_type"` + CpuSubtype string `db:"cpu_subtype"` + CpuBrand string `db:"cpu_brand"` + CpuPhysicalCores int `db:"cpu_physical_cores"` + CpuLogicalCores int `db:"cpu_logical_cores"` + HardwareVendor string `db:"hardware_vendor"` + HardwareModel string `db:"hardware_model"` + HardwareVersion string `db:"hardware_version"` + HardwareSerial string `db:"hardware_serial"` + ComputerName string `db:"computer_name"` + PrimaryIP string `db:"primary_ip"` + PrimaryMac string `db:"primary_mac"` + LabelUpdatedAt time.Time `db:"label_updated_at"` + LastEnrolledAt time.Time `db:"last_enrolled_at"` + RefetchRequested int `db:"refetch_requested"` + PublicIP string `db:"public_ip"` +} + +type HostDisplayName struct { + HostID int64 `db:"host_id"` + Name string `db:"display_name"` +} + +type Software struct { + ID int64 `db:"id"` + Name string `db:"name"` + Version string `db:"version"` + Source string `db:"source"` + BundleIdentifier string `db:"bundle_identifier"` + Release string `db:"release"` + VendorOld string `db:"vendor_old"` + Arch string `db:"arch"` + Vendor string `db:"vendor"` +} + +type HostSoftware struct { + HostID int64 `db:"host_id"` + SoftwareID int64 `db:"software_id"` +} + +func readCSVFile(filePath string) ([]Software, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + reader := csv.NewReader(file) + lines, err := reader.ReadAll() + if err != nil { + return nil, err + } + + var software []Software + for _, line := range lines[1:] { // Skip header line + software = append(software, Software{ + Name: line[0], + Version: line[1], + Source: line[2], + BundleIdentifier: line[3], + Release: line[4], + VendorOld: line[5], + Arch: line[6], + Vendor: line[7], + }) + } + return software, nil +} + +func insertOrUpdateSoftware(db *sqlx.DB, hostID int, software Software) error { + // Check if the software already exists + var existingID int64 + query := `SELECT id FROM software WHERE name = ? AND version = ? AND source = ?` + err := db.Get(&existingID, query, software.Name, software.Version, software.Source) + if err != nil && errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("select software: %w", err) + } + software.ID = existingID + + if existingID > 0 { + // Update existing record, set ID for the update + software.ID = existingID + updateQuery := `UPDATE software SET bundle_identifier = :bundle_identifier, ` + "`release` = :release, vendor_old = :vendor_old, arch = :arch, vendor = :vendor WHERE id = :id" + _, err := db.NamedExec(updateQuery, software) + if err != nil { + return fmt.Errorf("update software: %w", err) + } + } else { + // Insert new record + insertQuery := `INSERT INTO software (name, version, source, bundle_identifier, ` + "`release`," + ` vendor_old, arch, vendor) VALUES (:name, :version, :source, :bundle_identifier, :release, :vendor_old, :arch, :vendor)` + res, err := db.NamedExec(insertQuery, software) + if err != nil { + return fmt.Errorf("insert software: %w", err) + } + software.ID, err = res.LastInsertId() + if err != nil { + return fmt.Errorf("get last insert id: %w", err) + } + } + + err = insertOrUpdateHostSoftware(db, HostSoftware{ + HostID: int64(hostID), + SoftwareID: software.ID, + }) + if err != nil { + return fmt.Errorf("insert host software: %w", err) + } + + return nil +} + +func insertOrUpdateHost(db *sqlx.DB, host Host) (int, error) { + // Check if the host already exists + var existingID int + query := `SELECT id FROM hosts WHERE osquery_host_id = ?` + err := db.Get(&existingID, query, host.OsqueryHostID) + if err != nil && err != sql.ErrNoRows { + return existingID, fmt.Errorf("select host: %w", err) + } + + if existingID > 0 { + // Update existing record + updateQuery := `UPDATE hosts SET updated_at = :updated_at, detail_updated_at = :detail_updated_at, node_key = :node_key, hostname = :hostname, uuid = :uuid, platform = :platform, osquery_version = :osquery_version, os_version = :os_version, build = :build, platform_like = :platform_like, code_name = :code_name, uptime = :uptime, memory = :memory, cpu_type = :cpu_type, cpu_subtype = :cpu_subtype, cpu_brand = :cpu_brand, cpu_physical_cores = :cpu_physical_cores, cpu_logical_cores = :cpu_logical_cores, hardware_vendor = :hardware_vendor, hardware_model = :hardware_model, hardware_version = :hardware_version, hardware_serial = :hardware_serial, computer_name = :computer_name, primary_ip = :primary_ip, primary_mac = :primary_mac, label_updated_at = :label_updated_at, last_enrolled_at = :last_enrolled_at, refetch_requested = :refetch_requested, public_ip = :public_ip WHERE id = :id` + _, err := db.NamedExec(updateQuery, host) + if err != nil { + return 0, fmt.Errorf("update host: %w", err) + } + err = insertOrUpdateHostDisplayName(db, HostDisplayName{ + HostID: int64(existingID), + Name: host.Hostname, + }) + if err != nil { + return 0, fmt.Errorf("insert host display name: %w", err) + } + return existingID, nil + + } + + // Insert new record + insertQuery := `INSERT INTO hosts (osquery_host_id, created_at, detail_updated_at, node_key, hostname, uuid, platform, osquery_version, os_version, build, platform_like, code_name, uptime, memory, cpu_type, cpu_subtype, cpu_brand, cpu_physical_cores, cpu_logical_cores, hardware_vendor, hardware_model, hardware_version, hardware_serial, computer_name, primary_ip, primary_mac, label_updated_at, last_enrolled_at, refetch_requested, public_ip) VALUES (:osquery_host_id, :created_at, :detail_updated_at, :node_key, :hostname, :uuid, :platform, :osquery_version, :os_version, :build, :platform_like, :code_name, :uptime, :memory, :cpu_type, :cpu_subtype, :cpu_brand, :cpu_physical_cores, :cpu_logical_cores, :hardware_vendor, :hardware_model, :hardware_version, :hardware_serial, :computer_name, :primary_ip, :primary_mac, :label_updated_at, :last_enrolled_at, :refetch_requested, :public_ip)` + res, err := db.NamedExec(insertQuery, host) + if err != nil { + return 0, fmt.Errorf("insert host: %w", err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("get last insert id: %w", err) + } + + err = insertOrUpdateHostDisplayName(db, HostDisplayName{ + HostID: id, + Name: host.Hostname, + }) + if err != nil { + return 0, fmt.Errorf("insert host display name: %w", err) + } + + return int(id), nil +} + +func insertOrUpdateHostDisplayName(db *sqlx.DB, hdn HostDisplayName) error { + // Check if the host already exists + var foundDN HostDisplayName + query := `SELECT host_id, display_name FROM host_display_names WHERE host_id = ?` + err := db.Get(&foundDN, query, hdn.HostID) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("select host display name: %w", err) + } + + if err == sql.ErrNoRows { + // Insert new record + insertQuery := `INSERT INTO host_display_names (host_id, display_name) VALUES (:host_id, :display_name)` + _, err := db.NamedExec(insertQuery, hdn) + if err != nil { + return fmt.Errorf("insert host display name: %w", err) + } + } + + return nil +} + +func insertOrUpdateHostSoftware(db *sqlx.DB, hs HostSoftware) error { + // Check if the host already exists + var foundhs HostSoftware + query := `SELECT host_id, software_id FROM host_software WHERE host_id = ? AND software_id = ?` + err := db.Get(&foundhs, query, hs.HostID, hs.SoftwareID) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("select host software: %w", err) + } + + if err == sql.ErrNoRows { + // Insert new record + insertQuery := `INSERT INTO host_software (host_id, software_id) VALUES (:host_id, :software_id)` + _, err := db.NamedExec(insertQuery, hs) + if err != nil { + return fmt.Errorf("insert host software: %w", err) + } + } + + return nil +} + +func main() { + // Database connection string + dsn := "fleet:insecure@tcp(localhost:3306)/fleet" + + // Connect to database + db, err := sqlx.Connect("mysql", dsn) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Host data to insert + host := Host{ + OsqueryHostID: "sample_host_id", + CreatedAt: time.Now(), + DetailUpdatedAt: time.Now(), + NodeKey: "sample_node_key", + Hostname: "sample_hostname", + Uuid: "sample_uuid", + Platform: "sample_platform", + OsqueryVersion: "sample_version", + OsVersion: "sample_os_version", + Build: "sample_build", + PlatformLike: "sample_platform_like", + CodeName: "sample_code_name", + Uptime: 1000, + Memory: 8000, + CpuType: "sample_cpu_type", + CpuSubtype: "sample_cpu_subtype", + CpuBrand: "sample_cpu_brand", + CpuPhysicalCores: 4, + CpuLogicalCores: 8, + HardwareVendor: "sample_vendor", + HardwareModel: "sample_model", + HardwareVersion: "sample_hardware_version", + HardwareSerial: "sample_serial", + ComputerName: "sample_computer_name", + PrimaryIP: "192.168.1.1", + PrimaryMac: "00:11:22:33:44:55", + LabelUpdatedAt: time.Now(), + LastEnrolledAt: time.Now(), + RefetchRequested: 0, + PublicIP: "203.0.113.1", + } + + // macos Host + host.Platform = "darwin" + host.OsVersion = "Mac OS X 10.14.6" + host.Hostname = "macos-seed-host" + host.ComputerName = "macos-seed-host" + host.OsqueryHostID = "macos-seed-host" + host.NodeKey = "macos-seed-host" + macosID, err := insertOrUpdateHost(db, host) + if err != nil { + log.Fatal(err) + } + + // windows Host + host.Platform = "windows" + host.OsVersion = "Windows 11 Enterprise" + host.ComputerName = "windows-seed-host" + host.Hostname = "windows-seed-host" + host.OsqueryHostID = "windows-seed-host" + host.NodeKey = "windows-seed-host" + winID, err := insertOrUpdateHost(db, host) + if err != nil { + log.Fatal(err) + } + + // ubuntu Host + host.Platform = "debian" + host.OsVersion = "Ubuntu 22.04.1 LTS" + host.Hostname = "ubuntu-seed-host" + host.ComputerName = "ubuntu-seed-host" + host.OsqueryHostID = "ubuntu-seed-host" + host.NodeKey = "ubuntu-seed-host" + ubuntuID, err := insertOrUpdateHost(db, host) + if err != nil { + log.Fatal(err) + } + + // Insert macOS software + macSoftware, err := readCSVFile("./tools/seed_data/software-macos.csv") + if err != nil { + log.Fatal(err) + } + for _, s := range macSoftware { + err := insertOrUpdateSoftware(db, macosID, s) + if err != nil { + log.Fatal(err) + } + } + + // Insert win software + winSoftware, err := readCSVFile("./tools/seed_data/software-win.csv") + if err != nil { + log.Fatal(err) + } + for _, s := range winSoftware { + err := insertOrUpdateSoftware(db, winID, s) + if err != nil { + log.Fatal(err) + } + } + + // Insert ubuntu software + ubuntuSoftware, err := readCSVFile("./tools/seed_data/software-ubuntu.csv") + if err != nil { + log.Fatal(err) + } + for _, s := range ubuntuSoftware { + err := insertOrUpdateSoftware(db, ubuntuID, s) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/tools/seed_data/software-macos.csv b/tools/seed_data/software-macos.csv new file mode 100644 index 0000000000..2964d92471 --- /dev/null +++ b/tools/seed_data/software-macos.csv @@ -0,0 +1,14 @@ +name,version,source,bundle_identifier,release,vendor_old,arch,vendor +Example macos Software,1.0,apps,com.example.software,Stable,ExampleVendorOld,x86_64,ExampleVendor +Another macos Software,2.0,apps,com.another.software,Beta,AnotherVendorOld,arm64,AnotherVendor +Kubernetes Minikube 1.26.0,1.26.0,apps,,,,, +LibreOffice,7.4.0,apps,,,,, +Open Ticket Request System (OTRS) ITSM,6.0.32,apps,,,,, +Open Ticket Request System (OTRS),7.0.40,apps,,,,, +YugabyteDB,2.1.10,apps,,,,, +HYPR Workforce Access (WFA),7.7.1,apps,,,,, +MongoDB,4.4.4,apps,,,,, +ij_network_tool,4.7.3,apps,,,,, +Mozilla Thunderbird,115.2.0,apps,,,,, +Google Chrome,116.0.5845.109,apps,,,,, +Mozilla Firefox,1.5.2,apps,,,,, \ No newline at end of file diff --git a/tools/seed_data/software-ubuntu.csv b/tools/seed_data/software-ubuntu.csv new file mode 100644 index 0000000000..7face9449e --- /dev/null +++ b/tools/seed_data/software-ubuntu.csv @@ -0,0 +1,9 @@ +name,version,source,bundle_identifier,release,vendor_old,arch,vendor +example-ubuntu-software,1.0,Package (deb),,Stable,ExampleVendorOld,x86_64,ExampleVendor +another-ubuntu-software,2.0,Package (deb),,Beta,AnotherVendorOld,arm64,AnotherVendor +LibreOffice,7.4.0,Package (deb),,,,, +Open Ticket Request System (OTRS) ITSM,6.0.32,Package (deb),,,,, +Open Ticket Request System (OTRS),7.0.40,Package (deb),,,,, +YugabyteDB,2.1.10,Package (deb),,,,, +MongoDB,4.4.4,Package (deb),,,,, +Mozilla Firefox,1.5.2,Package (deb),,,,, \ No newline at end of file diff --git a/tools/seed_data/software-win.csv b/tools/seed_data/software-win.csv new file mode 100644 index 0000000000..ae66112ec1 --- /dev/null +++ b/tools/seed_data/software-win.csv @@ -0,0 +1,12 @@ +name,version,source,bundle_identifier,release,vendor_old,arch,vendor +Example Windows Software,1.0,programs,,Stable,ExampleVendorOld,x86_64,ExampleVendor +Another Windows Software,2.0,programs,,Beta,AnotherVendorOld,arm64,AnotherVendor +Open Ticket Request System (OTRS) ITSM,6.0.32,programs,,,,, +Open Ticket Request System (OTRS),7.0.40,programs,,,,, +YugabyteDB,2.1.10,programs,,,,, +HYPR Workforce Access (WFA),7.7.1,programs,,,,, +MongoDB 4.4.4,4.4.4,programs,,,,, +Microsoft Exchange Server,5.5,programs,,,,, +McAfee ePolicy Orchestrator,2.5.1,programs,,,,, +Microsoft Internet Explorer,5.5,programs,,,,, +Mozilla Firefox,1.5.2,programs,,,,, \ No newline at end of file