From 4f0682f34603d64a345311cac1aed2b70637f2f9 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 26 Feb 2024 14:29:04 -0300 Subject: [PATCH] New scripts to automate download of components from Github actions (#17144) #17054 This was used as part of the release of fleetd 1.22.0 to the `edge` channel. I added more automation to ease releasing fleetd. (They were too many manual clicks and error prone actions.) --- orbit/docs/TUF-Update-Guide.md | 74 +++-- .../download-artifacts/download-artifacts.go | 289 ++++++++++++++++++ 2 files changed, 344 insertions(+), 19 deletions(-) create mode 100644 tools/tuf/download-artifacts/download-artifacts.go diff --git a/orbit/docs/TUF-Update-Guide.md b/orbit/docs/TUF-Update-Guide.md index 81cb611b0f..9abefa9237 100644 --- a/orbit/docs/TUF-Update-Guide.md +++ b/orbit/docs/TUF-Update-Guide.md @@ -57,8 +57,34 @@ make changelog-orbit git tag orbit-v1.21.0 git push origin --tags ``` -6. Once the two github actions finish their runs, download the 6 artifacts (orbit and Fleet Desktop for macOS, Linux and Windows) on your workstation. +6. Once the two github actions finish their runs, use the following scripts that will download the artifacts to a folder in your workstation (on this guide we assume you are using `$HOME/release-friday`). NOTE: The `goreleaser-macos` job is unstable and may need several re-runs until it works. +```sh +go run ./tools/tuf/download-artifacts desktop \ + --git-branch release-fleetd-v1.21.0 \ + --output-directory $HOME/release-friday/desktop \ + --github-username $GITHUB_USERNAME --github-api-token $GITHUB_TOKEN +go run ./tools/tuf/download-artifacts orbit \ + --git-tag orbit-v1.21.0 \ + --output-directory $HOME/release-friday/orbit \ + --github-username $GITHUB_USERNAME --github-api-token $GITHUB_TOKEN +tree $HOME/release-friday +$HOME/release-friday +├── desktop +│   ├── linux +│   │   └── desktop.tar.gz +│   ├── macos +│   │   └── desktop.app.tar.gz +│   └── windows +│   └── fleet-desktop.exe +└── orbit + ├── linux + │   └── orbit + ├── macos + │   └── orbit + └── windows + └── orbit.exe +``` 7. With the executables on your workstation, proceed to [Pushing updates](#pushing-updates) (`edge`). ## Pushing updates @@ -75,30 +101,40 @@ NOTE: The `goreleaser-macos` job is unstable and may need several re-runs until ### Releasing to the `edge` channel -> Make sure to install fleetd components using the `edge` channels in the three supported OSs (this is useful to smoke test the update). +The commands shown here update the local repository. After you are done running the commands below for each component, see [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository) to push the updates to Fleet's TUF repository (https://tuf.fleetctl.com). -Following is the list of components and each command for each operating system. +#### Setup -The commands show here update the local repository. After you are done running the commands below for each component, see [Pushing releases to Fleet's TUF repository](#pushing-releases-to-fleets-tuf-repository) to push the updates to Fleet's TUF repository (https://tuf.fleetctl.com). +Make sure to install fleetd components using the `edge` channels in the three supported OSs (this is useful to smoke test the update). +Here's how to generate the packages: +```sh +# (The same for --type=deb and --type=msi.) +fleetctl package --type=pkg \ + --enable-scripts \ + --fleet-desktop \ + --fleet-url=... --enroll-secret=... \ + --update-interval 10s \ + --orbit-channel edge --desktop-channel edge --osqueryd-channel edge +``` #### orbit The `orbit` executables are downloaded from the [GoReleaser Orbit action](https://github.com/fleetdm/fleet/actions/workflows/goreleaser-orbit.yaml). -Such action is triggered when git tagging a new orbit version with a tag of the form: `orbit-v1.15.0`. +Such action is triggered when git tagging a new orbit version with a tag of the form: `orbit-v1.21.0`. > IMPORTANT: If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped (even if there are no changes in it). -> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.15.0"`. +> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.21.0"`. > Technical debt: We could improve this process to reduce the complexity of releasing fleetd when there are no Fleet Desktop changes. -> The following commands assume you are pushing version `1.15.0`. +> The following commands assume you are pushing version `1.21.0`. ```sh # macOS -fleetctl updates add --target /path/to/downloaded/macos/orbit --platform macos --name orbit --version 1.15.0 -t edge +fleetctl updates add --target $HOME/release-friday/orbit/macos/orbit --platform macos --name orbit --version 1.21.0 -t edge # Linux -fleetctl updates add --target /path/to/downloaded/linux/orbit --platform linux --name orbit --version 1.15.0 -t edge +fleetctl updates add --target $HOME/release-friday/orbit/linux/orbit --platform linux --name orbit --version 1.21.0 -t edge # Windows -fleetctl updates add --target /path/to/downloaded/windows/orbit.exe --platform windows --name orbit --version 1.15.0 -t edge +fleetctl updates add --target $HOME/release-friday/orbit/windows/orbit.exe --platform windows --name orbit --version 1.21.0 -t edge ``` #### desktop @@ -106,15 +142,15 @@ fleetctl updates add --target /path/to/downloaded/windows/orbit.exe --platform w The Fleet Desktop executables are downloaded from the [Generate Fleet Desktop targets for Orbit action](https://github.com/fleetdm/fleet/actions/workflows/generate-desktop-targets.yml). Such action is triggered by submitting a PR with the [following version string](https://github.com/fleetdm/fleet/blob/4a6bf0d447a2080f994da1e2f36ce6d51db88109/.github/workflows/generate-desktop-targets.yml#L27) changed. -> The following commands assume you are pushing version `1.15.0`. +> The following commands assume you are pushing version `1.21.0`. ```sh # macOS -fleetctl updates add --target /path/to/macos/downloaded/desktop.app.tar.gz --platform macos --name desktop --version 1.15.0 -t edge +fleetctl updates add --target $HOME/release-friday/desktop/macos/desktop.app.tar.gz --platform macos --name desktop --version 1.21.0 -t edge # Linux -fleetctl updates add --target /path/to/linux/downloaded/desktop.tar.gz --platform linux --name desktop --version 1.15.0 -t edge +fleetctl updates add --target $HOME/release-friday/desktop/linux/desktop.tar.gz --platform linux --name desktop --version 1.21.0 -t edge # Windows -fleetctl updates add --target /path/to/windows/downloaded/fleet-desktop.exe --platform windows --name desktop --version 1.15.0 -t edge +fleetctl updates add --target $HOME/release-friday/desktop/windows/fleet-desktop.exe --platform windows --name desktop --version 1.21.0 -t edge ``` #### swiftDialog @@ -175,19 +211,19 @@ The commands show here update the local repository. After you are done running t #### orbit > IMPORTANT: If there are only `orbit` changes on a release we still have to release the `desktop` component with its version string bumped (even if there are no changes in it). -> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.15.0"`. +> This is due to the fact that we want users to see the new version in the tray icon, e.g. `"Fleet Desktop v1.21.0"`. > Technical debt: We could improve this process to reduce the complexity of releasing fleetd when there are no Fleet Desktop changes. -> The following command assumes you are pushing version `1.15.0`: +> The following command assumes you are pushing version `1.21.0`: ```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh orbit 1.15.0 +/fleet/repo/tools/tuf/promote_edge_to_stable.sh orbit 1.21.0 ``` #### desktop -> The following command assumes you are pushing version `1.15.0`: +> The following command assumes you are pushing version `1.21.0`: ```sh -/fleet/repo/tools/tuf/promote_edge_to_stable.sh desktop 1.15.0 +/fleet/repo/tools/tuf/promote_edge_to_stable.sh desktop 1.21.0 ``` #### swiftDialog diff --git a/tools/tuf/download-artifacts/download-artifacts.go b/tools/tuf/download-artifacts/download-artifacts.go new file mode 100644 index 0000000000..bc06f53559 --- /dev/null +++ b/tools/tuf/download-artifacts/download-artifacts.go @@ -0,0 +1,289 @@ +package main + +import ( + "archive/zip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/fleetdm/fleet/v4/orbit/pkg/constant" + "github.com/fleetdm/fleet/v4/pkg/fleethttp" + "github.com/google/go-github/v37/github" + "github.com/urfave/cli/v2" +) + +func main() { + app := cli.NewApp() + app.Name = "download-artifacts" + app.Usage = "CLI to download TUF artifacts from Github Actions" + app.Commands = []*cli.Command{ + orbitCommand(), + desktopCommand(), + } + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stdout, "Error: %+v\n", err) + os.Exit(1) + } +} + +func orbitCommand() *cli.Command { + var ( + gitTag string + outputDirectory string + githubUsername string + githubAPIToken string + ) + return &cli.Command{ + Name: "orbit", + Usage: "Fetch orbit executables from the goreleaser-orbit.yaml action", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "git-tag", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_GIT_TAG"}, + Required: true, + Destination: &gitTag, + Usage: "git tag generated for the orbit release", + }, + &cli.StringFlag{ + Name: "output-directory", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_OUTPUT_DIRECTORY"}, + Required: true, + Destination: &outputDirectory, + Usage: "name of the output directory to create and download the orbit executables", + }, + &cli.StringFlag{ + Name: "github-username", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_GITHUB_USERNAME"}, + Required: true, + Destination: &githubUsername, + Usage: "Github username", + }, + &cli.StringFlag{ + Name: "github-api-token", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_GITHUB_API_TOKEN"}, + Required: true, + Destination: &githubAPIToken, + Usage: "Github API token (https://github.com/settings/tokens)", + }, + }, + Action: func(c *cli.Context) error { + return downloadComponents("goreleaser-orbit.yaml", gitTag, map[string]string{ + "macos": "orbit-macos", + "linux": "orbit-linux", + "windows": "orbit-windows", + }, outputDirectory, githubUsername, githubAPIToken) + }, + } +} + +func desktopCommand() *cli.Command { + var ( + gitBranch string + outputDirectory string + githubUsername string + githubAPIToken string + ) + return &cli.Command{ + Name: "desktop", + Usage: "Fetch Fleet Desktop executables from the generate-desktop-targets.yml action", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "git-branch", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_GIT_BRANCH"}, + Required: true, + Destination: &gitBranch, + Usage: "branch name used to bump the Fleet Desktop version", + }, + &cli.StringFlag{ + Name: "output-directory", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_OUTPUT_DIRECTORY"}, + Required: true, + Destination: &outputDirectory, + Usage: "name of the output directory to create and download the Fleet Desktop executables", + }, + &cli.StringFlag{ + Name: "github-username", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_GITHUB_USERNAME"}, + Required: true, + Destination: &githubUsername, + Usage: "Github username", + }, + &cli.StringFlag{ + Name: "github-api-token", + EnvVars: []string{"DOWNLOAD_ARTIFACTS_GITHUB_API_TOKEN"}, + Required: true, + Destination: &githubAPIToken, + Usage: "Github API token (https://github.com/settings/tokens)", + }, + }, + Action: func(c *cli.Context) error { + return downloadComponents("generate-desktop-targets.yml", gitBranch, map[string]string{ + "macos": "desktop.app.tar.gz", + "linux": "desktop.tar.gz", + "windows": "fleet-desktop.exe", + }, outputDirectory, githubUsername, githubAPIToken) + }, + } +} + +func downloadAndExtractZip(client *http.Client, githubUsername string, githubAPIToken string, urlPath string, destPath string) error { + zipFile, err := os.CreateTemp("", "file.zip") + if err != nil { + return fmt.Errorf("create file: %w", err) + } + defer zipFile.Close() + defer os.Remove(zipFile.Name()) + + req, err := http.NewRequest(http.MethodGet, urlPath, nil) + if err != nil { + return err + } + req.SetBasicAuth(githubUsername, githubAPIToken) + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("could not download %s: %w", urlPath, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("could not download %s: received http status code %s", urlPath, resp.Status) + } + _, err = io.Copy(zipFile, resp.Body) + if err != nil { + return fmt.Errorf("could not write %s: %w", zipFile.Name(), err) + } + + // Open the downloaded file for reading. With zip, we cannot unzip directly from resp.Body + zipReader, err := zip.OpenReader(zipFile.Name()) + if err != nil { + return fmt.Errorf("could not open %s: %w", zipFile.Name(), err) + } + defer zipReader.Close() + + err = os.MkdirAll(filepath.Dir(destPath), 0o755) + if err != nil { + return fmt.Errorf("could not create directory %s: %w", filepath.Dir(destPath), err) + } + + // Extract each file in the archive + for _, archiveReader := range zipReader.File { + err = extractZipFile(archiveReader, destPath) + if err != nil { + return err + } + } + + return nil +} + +func extractZipFile(archiveReader *zip.File, destPath string) error { + if archiveReader.FileInfo().Mode()&os.ModeSymlink != 0 { + // Skip symlinks for security reasons + return nil + } + + // Open the file in the archive + archiveFile, err := archiveReader.Open() + if err != nil { + return fmt.Errorf("could not open archive %s: %w", archiveReader.Name, err) + } + defer archiveFile.Close() + + // Clean the archive path to prevent extracting files outside the destination. + archivePath := filepath.Clean(archiveReader.Name) + if strings.HasPrefix(archivePath, ".."+string(filepath.Separator)) { + // Skip relative paths for security reasons + return nil + } + // Prepare to write the file + finalPath := filepath.Join(destPath, archivePath) + + // Check if the file to extract is just a directory + if archiveReader.FileInfo().IsDir() { + err = os.MkdirAll(finalPath, 0o755) + if err != nil { + return fmt.Errorf("could not create directory %s: %w", finalPath, err) + } + } else { + // Create all needed directories + if os.MkdirAll(filepath.Dir(finalPath), 0o755) != nil { + return fmt.Errorf("could not create directory %s: %w", filepath.Dir(finalPath), err) + } + + // Prepare to write the destination file + destinationFile, err := os.OpenFile(finalPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, archiveReader.Mode()) + if err != nil { + return fmt.Errorf("could not open file %s: %w", finalPath, err) + } + defer destinationFile.Close() + + // Write the destination file + // Ignoring gosec's G110 warning as these are artifacts downloaded from Fleet's Github. + if _, err = io.Copy(destinationFile, archiveFile); err != nil { //nolint:gosec + return fmt.Errorf("could not write file %s: %w", finalPath, err) + } + } + return nil +} + +func downloadComponents(workflowName string, headBranch string, artifactNames map[string]string, outputDirectory string, githubUsername string, githubAPIToken string) error { + if err := os.RemoveAll(outputDirectory); err != nil { + return err + } + for _, osPath := range []string{"macos", "windows", "linux"} { + if err := os.MkdirAll(filepath.Join(outputDirectory, osPath), constant.DefaultDirMode); err != nil { + return err + } + } + ctx := context.Background() + gc := github.NewClient(fleethttp.NewClient()) + workflow, _, err := gc.Actions.GetWorkflowByFileName(ctx, "fleetdm", "fleet", workflowName) + if err != nil { + return err + } + workflowRuns, _, err := gc.Actions.ListWorkflowRunsByID(ctx, "fleetdm", "fleet", *workflow.ID, nil) + if err != nil { + return err + } + var workflowRun *github.WorkflowRun + for _, wr := range workflowRuns.WorkflowRuns { + if headBranch == *wr.HeadBranch { + workflowRun = wr + break + } + } + if workflowRun == nil { + return fmt.Errorf("workflow with tag %s not found", headBranch) + } + artifactList, _, err := gc.Actions.ListWorkflowRunArtifacts(ctx, "fleetdm", "fleet", *workflowRun.ID, nil) + if err != nil { + return err + } + urls := make(map[string]string) + for _, artifact := range artifactList.Artifacts { + if *artifact.Name == artifactNames["linux"] { + urls["linux"] = *artifact.ArchiveDownloadURL + } else if *artifact.Name == artifactNames["macos"] { + urls["macos"] = *artifact.ArchiveDownloadURL + } else if *artifact.Name == artifactNames["windows"] { + urls["windows"] = *artifact.ArchiveDownloadURL + } else { + return fmt.Errorf("unknown artifact name: %s", *artifact.Name) + } + } + if len(urls) != 3 { + return fmt.Errorf("missing some artifact: %+v", urls) + } + for osName, downloadURL := range urls { + outputDir := filepath.Join(outputDirectory, osName) + fmt.Printf("Downloading and extracting %s into %s...\n", downloadURL, outputDir) + if err := downloadAndExtractZip(fleethttp.NewClient(), githubUsername, githubAPIToken, downloadURL, outputDir); err != nil { + return err + } + } + return nil +}