From 94eb5737365d52ea49e62fac046be62dfc2b4db9 Mon Sep 17 00:00:00 2001 From: Scott Gress Date: Wed, 5 Feb 2025 09:52:10 -0600 Subject: [PATCH] Add Fleet dev snapshot tool (#25909) For #23750 # Overview This PR adds a basic tool for creating and restoring Fleet dev snapshots. In this first iteration a snapshot is just a folder containing a MySQL db dump made using the existing backup/restore scripts, and the tool allows you to easily save and restore snapshots interactively. ## Usage * `make snapshot` to create a new snapshot * `make restore` to select and restore a snapshot ## Future plans Future iterations can add metadata to snapshots to integrate things like: * node keys from osquery-perf, so you can easily reconnect to hosts created in a previous session * env vars from when the snapshot was made * the branch from when the snapshot was made, to allow switching to that branch and restarting the server as part of the restore process * ![image](https://github.com/user-attachments/assets/6dc86581-5c12-4b57-b900-5034e00bc496) ## Demo https://github.com/user-attachments/assets/1590c37a-3df9-4201-a42b-ccd1a36cb6cf --- Makefile | 10 ++ tools/backup_db/backup.sh | 4 +- tools/backup_db/restore.sh | 4 +- tools/snapshot/go.mod | 10 ++ tools/snapshot/go.sum | 11 ++ tools/snapshot/snapshot.go | 254 +++++++++++++++++++++++++++++++++++++ 6 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 tools/snapshot/go.mod create mode 100644 tools/snapshot/go.sum create mode 100644 tools/snapshot/snapshot.go diff --git a/Makefile b/Makefile index 0f0b1c2316..d4655dba04 100644 --- a/Makefile +++ b/Makefile @@ -420,6 +420,16 @@ db-backup: db-restore: ./tools/backup_db/restore.sh + +# Interactive snapshot / restore +SNAPSHOT_BINARY = ./build/snapshot +snapshot: $(SNAPSHOT_BINARY) + @ $(SNAPSHOT_BINARY) snapshot +$(SNAPSHOT_BINARY): tools/snapshot/*.go + cd tools/snapshot && go build -o ../../build/snapshot +restore: $(SNAPSHOT_BINARY) + @ $(SNAPSHOT_BINARY) restore + # Generate osqueryd.app.tar.gz bundle from osquery.io. # # Usage: diff --git a/tools/backup_db/backup.sh b/tools/backup_db/backup.sh index 03c7a29b42..13c89ec551 100755 --- a/tools/backup_db/backup.sh +++ b/tools/backup_db/backup.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -euo pipefail - -docker run --rm --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:8.0.36} bash -c 'mysqldump -hmysql -uroot -ptoor --default-character-set=utf8mb4 fleet | gzip -' > backup.sql.gz +BACKUP_NAME="${1:-backup.sql.gz}" +docker run --rm --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:8.0.36} bash -c 'mysqldump -hmysql -uroot -ptoor --default-character-set=utf8mb4 fleet | gzip -' > $BACKUP_NAME \ No newline at end of file diff --git a/tools/backup_db/restore.sh b/tools/backup_db/restore.sh index a95dafc671..0d8f721368 100755 --- a/tools/backup_db/restore.sh +++ b/tools/backup_db/restore.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -euo pipefail - -docker run --rm -i --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:8.0.36} bash -c 'gzip -dc - | mysql -hmysql -uroot -ptoor fleet' < backup.sql.gz +BACKUP_NAME="${1:-backup.sql.gz}" +docker run --rm -i --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:8.0.36} bash -c 'gzip -dc - | mysql -hmysql -uroot -ptoor fleet' < ${BACKUP_NAME} diff --git a/tools/snapshot/go.mod b/tools/snapshot/go.mod new file mode 100644 index 0000000000..7257baef82 --- /dev/null +++ b/tools/snapshot/go.mod @@ -0,0 +1,10 @@ +module github.com/fleetdm/fleet/v4/tools/snapshot + +go 1.23.4 + +require ( + github.com/manifoldco/promptui v0.9.0 + golang.org/x/sys v0.28.0 +) + +require github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect diff --git a/tools/snapshot/go.sum b/tools/snapshot/go.sum new file mode 100644 index 0000000000..810f6681c3 --- /dev/null +++ b/tools/snapshot/go.sum @@ -0,0 +1,11 @@ +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= +github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/tools/snapshot/snapshot.go b/tools/snapshot/snapshot.go new file mode 100644 index 0000000000..68c1f2fcd0 --- /dev/null +++ b/tools/snapshot/snapshot.go @@ -0,0 +1,254 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/manifoldco/promptui" + // Force promptui to use our newer x/sys package, + // which doesn't have security vulnerabilities. + _ "golang.org/x/sys/unix" +) + +// Represents a snapshot. +// Snapshots are stored in folders named after the snapshot name. +// Each snapshot folder contains a db.sql.gz file. +type Snapshot struct { + Name string + Date string + Path string // The directory containing the snapshot. +} + +// Which command to run. +type Command int + +const ( + CMD_SNAPSHOT Command = iota + CMD_RESTORE +) + +func main() { + // Ensure there's a command specified. + // TODO - as we add more commands, we should probably use a library like spf13/cobra. + if len(os.Args) < 2 { + fmt.Println("Please specify whether to (b)ackup or (r)estore.") + os.Exit(1) + } + + // Determine the command. + var command Command + switch os.Args[1] { + case "s", "snap", "snapshot": + command = CMD_SNAPSHOT + case "r", "restore": + command = CMD_RESTORE + default: + fmt.Println("Please specify whether to (s)snapshot or (r)estore.") + } + + // Determine the path to the top-level directory (where the Makefile resides). + repoRoot, err := getRepoRoot() + if err != nil { + fmt.Printf("Error determining repo root: %v\n", err) + os.Exit(1) + } + + // Change the working directory to the repo root. + if err := os.Chdir(repoRoot); err != nil { + fmt.Printf("Error changing directory to repo root: %v\n", err) + os.Exit(1) + } + + // Get the home directory so we can get the snapshots dir. + homedir, err := os.UserHomeDir() + if err != nil { + fmt.Printf("Could not determine home directory: %v\n", err) + return + } + + // Run the command. + switch command { + case CMD_SNAPSHOT: + snapshot(homedir) + case CMD_RESTORE: + restore(homedir) + } +} + +// Restore a snapshot. +func restore(homedir string) error { + snapshotsDir := filepath.Join(homedir, ".fleet", "snapshots") + _, err := os.Lstat(snapshotsDir) + if err != nil { + if os.IsNotExist(err) { + fmt.Printf("You don't currently have any snapshots.\n") + } else { + // Handle other PathError-specific cases + fmt.Printf("Error reading snapshots directory (%s): %v\n", snapshotsDir, err) + } + return err + } + + // Walk the ~/.fleet/snapshots directory if it exists. + dirEntries, err := os.ReadDir(snapshotsDir) + var snapshots []Snapshot + for _, entry := range dirEntries { + if entry.IsDir() { + // Ensure there's a db backup file. + dbBackupFile := filepath.Join(snapshotsDir, entry.Name(), "db.sql.gz") + dbBackupFileInfo, err := os.Lstat(dbBackupFile) + if err != nil { + continue + } + snapshot := Snapshot{ + Name: entry.Name(), + Date: dbBackupFileInfo.ModTime().Format("Jan 02, 2006 03:04:05 PM"), + Path: dbBackupFile, + } + snapshots = append(snapshots, snapshot) + } + } + + // Set up and run the "Select snapshot" UI. + templates := &promptui.SelectTemplates{ + Label: "{{ .Name }}", + Active: "> {{ .Name }} ({{ .Date }})", + Inactive: "{{ .Name }} ({{ .Date }})", + Selected: "{{ .Name }} ({{ .Date }})", + } + prompt := promptui.Select{ + Label: "Select snapshot to restore", + Items: snapshots, + Templates: templates, + } + index, _, err := prompt.Run() + if err != nil { + fmt.Printf("Prompt failed %v\n", err) + return err + } + + // Prepare the restore script with the selected snapshot. + cmd := exec.Command("./tools/backup_db/restore.sh", snapshots[index].Path) + + // Use the same stdin, stdout, and stderr as the parent process. + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Run the command. + err = cmd.Run() + output, _ := cmd.CombinedOutput() + fmt.Println(string(output)) + if err != nil { + fmt.Printf("Error: %v\n", err) + return err + } + + return nil +} + +// Create a snapshot. +func snapshot(homedir string) error { + snapshotsDir := filepath.Join(homedir, ".fleet", "snapshots") + + // Ensure the snapshots directory exists. + _, err := os.Lstat(snapshotsDir) + if err != nil { + // If the directory doesn't exist, create it. + if os.IsNotExist(err) { + err = os.Mkdir(snapshotsDir, 0o755) + if err != nil { + fmt.Printf("Error creating snapshots directory (%s): %v\n", snapshotsDir, err) + } + } else { + fmt.Printf("Error reading snapshots directory (%s): %v\n", snapshotsDir, err) + } + return err + } + + // Prompt the user for a name for the snapshot. + prompt := promptui.Prompt{ + Label: "Enter a name for the snapshot", + } + result, err := prompt.Run() + if err != nil { + fmt.Printf("Prompt failed %v\n", err) + return err + } + snapshotPath := filepath.Join(snapshotsDir, result) + + // Check if the snapshot already exists. + _, err = os.Lstat(snapshotPath) + // If the file exists, prompt the user to overwrite it. + if err == nil { + prompt := promptui.Prompt{ + Label: "This snapshot already exists. Overwrite? (Y/n)", + } + result, err := prompt.Run() + if err != nil { + fmt.Printf("Prompt failed %v\n", err) + return err + } + switch result { + case "Y", "y", "": + err = os.Remove(filepath.Join(snapshotPath, "db.sql.gz")) + if err != nil { + fmt.Printf("Error removing existing snapshot (%s): %v\n", result, err) + return err + } + default: + return nil + } + } else if !os.IsNotExist(err) { + fmt.Printf("Error checking for existing snapshot (%s): %v\n", result, err) + return err + } + + // Create the snapshot directory + err = os.Mkdir(snapshotPath, 0o755) + if err != nil && !os.IsExist(err) { + fmt.Printf("Error creating snapshot directory (%s): %v\n", snapshotPath, err) + } + + // Prepare the backup script with the snapshot path. + cmd := exec.Command("./tools/backup_db/backup.sh", filepath.Join(snapshotPath, "db.sql.gz")) + + // Use the same stdin, stdout, and stderr as the parent process. + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Run the command. + err = cmd.Run() + output, _ := cmd.CombinedOutput() + fmt.Println(string(output)) + if err != nil { + fmt.Printf("Error: %v\n", err) + return err + } + + return nil +} + +// getRepoRoot determines the repo root (top-level directory) relative to this binary. +func getRepoRoot() (string, error) { + // Get the path of the currently executing binary + executable, err := os.Executable() + if err != nil { + return "", err + } + + // Get the path of the binary, following symlinks. + execDir, err := filepath.EvalSymlinks(executable) + if err != nil { + return "", err + } + // Get the directory. + execDir = filepath.Dir(execDir) + + // Compute the repo root relative to the binary's location. + repoRoot := filepath.Join(execDir, "../") + return filepath.Abs(repoRoot) +}