diff --git a/.github/actions/r2-upload/action.yml b/.github/actions/r2-upload/action.yml new file mode 100644 index 0000000000..60fb7e0786 --- /dev/null +++ b/.github/actions/r2-upload/action.yml @@ -0,0 +1,34 @@ +name: R2 upload +description: Upload a file to R2 +# Schema: https://json.schemastore.org/github-action.json + +# This action expects the following env vars to be set: +# - R2_ENDPOINT: The endpoint of the R2 instance to upload to +# - R2_ACCESS_KEY_ID: The access key ID to use for R2 +# - R2_ACCESS_KEY_SECRET: The access key secret to use for R2 +# - R2_BUCKET: The bucket to upload to + +inputs: + filename: + # Future improvement: accept array of filenames as JSON string, and loop over it like in https://www.starkandwayne.com/blog/bash-for-loop-over-json-array-using-jq/index.html + description: 'Name of the file to upload' + required: true + +runs: + using: 'composite' + steps: + - name: Upload file to R2 + shell: bash + run: | + sudo ./.github/scripts/rclone-install.sh + mkdir -p ~/.config/rclone + echo "[r2] + type = s3 + provider = Cloudflare + region = auto + no_check_bucket = true + access_key_id = $R2_ACCESS_KEY_ID + secret_access_key = $R2_ACCESS_KEY_SECRET + endpoint = $R2_ENDPOINT + " > ~/.config/rclone/rclone.conf + rclone copy --verbose ${{ inputs.filename }} r2:${R2_BUCKET}/ diff --git a/.github/scripts/rclone-install.sh b/.github/scripts/rclone-install.sh new file mode 100755 index 0000000000..da6a276253 --- /dev/null +++ b/.github/scripts/rclone-install.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash + +# This script is a modified version of MIT licensed script from https://github.com/rclone/rclone/blob/v1.66.0/docs/content/install.sh +# The script is used to install rclone on a GitHub Actions runner machine. +# We use a specific version of rclone for stability/security reasons. +download_version="v1.66.0" + +# error codes +# 0 - exited without problems +# 1 - parameters not supported were used or some unexpected error occurred +# 2 - OS not supported by this script +# 0 (was 3) - installed version of rclone is up to date +# 4 - supported unzip tools are not available + +set -e + +#when adding a tool to the list make sure to also add its corresponding command further in the script +unzip_tools_list=('unzip' '7z' 'busybox') + +usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; } + +#check for beta flag +if [ -n "$1" ] && [ "$1" != "beta" ]; then + usage +fi + +if [ -n "$1" ]; then + install_beta="beta " +fi + + +#create tmp directory and move to it with macOS compatibility fallback +tmp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'rclone-install.XXXXXXXXXX') +cd "$tmp_dir" + + +#make sure unzip tool is available and choose one to work with +set +e +for tool in ${unzip_tools_list[*]}; do + trash=$(hash "$tool" 2>>errors) + if [ "$?" -eq 0 ]; then + unzip_tool="$tool" + break + fi +done +set -e + +# exit if no unzip tools available +if [ -z "$unzip_tool" ]; then + printf "\nNone of the supported tools for extracting zip archives (${unzip_tools_list[*]}) were found. " + printf "Please install one of them and try again.\n\n" + exit 4 +fi + +# Make sure we don't create a root owned .config/rclone directory #2127 +export XDG_CONFIG_HOME=config + +#check installed version of rclone to determine if update is necessary +version=$(rclone --version 2>>errors | head -n 1) +if [ -z "$install_beta" ]; then + current_version=$(curl -fsS https://downloads.rclone.org/version.txt) +else + current_version=download_version + # current_version=$(curl -fsS https://beta.rclone.org/version.txt) +fi + +if [ "$version" = "$current_version" ]; then + printf "\nThe latest ${install_beta}version of rclone ${version} is already installed.\n\n" + exit 0 # originally 3 +fi + + +#detect the platform +OS="$(uname)" +case $OS in + Linux) + OS='linux' + ;; + FreeBSD) + OS='freebsd' + ;; + NetBSD) + OS='netbsd' + ;; + OpenBSD) + OS='openbsd' + ;; + Darwin) + OS='osx' + binTgtDir=/usr/local/bin + man1TgtDir=/usr/local/share/man/man1 + ;; + SunOS) + OS='solaris' + echo 'OS not supported' + exit 2 + ;; + *) + echo 'OS not supported' + exit 2 + ;; +esac + +OS_type="$(uname -m)" +case "$OS_type" in + x86_64|amd64) + OS_type='amd64' + ;; + i?86|x86) + OS_type='386' + ;; + aarch64|arm64) + OS_type='arm64' + ;; + armv7*) + OS_type='arm-v7' + ;; + armv6*) + OS_type='arm-v6' + ;; + arm*) + OS_type='arm' + ;; + *) + echo 'OS type not supported' + exit 2 + ;; +esac + + +#download and unzip +if [ -z "$install_beta" ]; then + download_link="https://downloads.rclone.org/${download_version}/rclone-${download_version}-${OS}-${OS_type}.zip" + rclone_zip="rclone-${download_version}-${OS}-${OS_type}.zip" + # download_link="https://downloads.rclone.org/rclone-current-${OS}-${OS_type}.zip" + # rclone_zip="rclone-current-${OS}-${OS_type}.zip" +else + download_link="https://beta.rclone.org/rclone-beta-latest-${OS}-${OS_type}.zip" + rclone_zip="rclone-beta-latest-${OS}-${OS_type}.zip" +fi + +curl -OfsS "$download_link" +unzip_dir="tmp_unzip_dir_for_rclone" +# there should be an entry in this switch for each element of unzip_tools_list +case "$unzip_tool" in + 'unzip') + unzip -a "$rclone_zip" -d "$unzip_dir" + ;; + '7z') + 7z x "$rclone_zip" "-o$unzip_dir" + ;; + 'busybox') + mkdir -p "$unzip_dir" + busybox unzip "$rclone_zip" -d "$unzip_dir" + ;; +esac + +cd $unzip_dir/* + +#mounting rclone to environment + +case "$OS" in + 'linux') + #binary + cp rclone /usr/bin/rclone.new + chmod 755 /usr/bin/rclone.new + chown root:root /usr/bin/rclone.new + mv /usr/bin/rclone.new /usr/bin/rclone + #manual +# if ! [ -x "$(command -v mandb)" ]; then +# echo 'mandb not found. The rclone man docs will not be installed.' +# else +# mkdir -p /usr/local/share/man/man1 +# cp rclone.1 /usr/local/share/man/man1/ +# mandb +# fi + ;; + 'freebsd'|'openbsd'|'netbsd') + #binary + cp rclone /usr/bin/rclone.new + chown root:wheel /usr/bin/rclone.new + mv /usr/bin/rclone.new /usr/bin/rclone + #manual + mkdir -p /usr/local/man/man1 + cp rclone.1 /usr/local/man/man1/ + makewhatis + ;; + 'osx') + #binary + mkdir -m 0555 -p ${binTgtDir} + cp rclone ${binTgtDir}/rclone.new + mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone + chmod a=x ${binTgtDir}/rclone + #manual +# mkdir -m 0555 -p ${man1TgtDir} +# cp rclone.1 ${man1TgtDir} +# chmod a=r ${man1TgtDir}/rclone.1 + ;; + *) + echo 'OS not supported' + exit 2 +esac + +#update version variable post install +version=$(rclone --version 2>>errors | head -n 1) + +#cleanup +rm -rf "$tmp_dir" + +printf "\n${version} has successfully installed." +printf '\nNow run "rclone config" for setup. Check https://rclone.org/docs/ for more details.\n\n' +exit 0 diff --git a/.github/workflows/release-fleetd-base.yml b/.github/workflows/release-fleetd-base.yml new file mode 100644 index 0000000000..6d3a0e2e61 --- /dev/null +++ b/.github/workflows/release-fleetd-base.yml @@ -0,0 +1,177 @@ +name: Upload fleetd base to https://download.fleetdm.com + +on: + workflow_dispatch: # Manual + schedule: + - cron: '0 3 * * *' # Nightly 3AM UTC + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}} + cancel-in-progress: true + +defaults: + run: + # fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference + shell: bash + +permissions: + contents: read + +env: + R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_DOWNLOAD_TESTING_ACCESS_KEY_ID }} # Production: ${{ secrets.R2_DOWNLOAD_ACCESS_KEY_ID }} | Testing: ${{ secrets.R2_DOWNLOAD_TESTING_ACCESS_KEY_ID }} + R2_ACCESS_KEY_SECRET: ${{ secrets.R2_DOWNLOAD_TESTING_ACCESS_KEY_SECRET }} # Production: ${{ secrets.R2_DOWNLOAD_ACCESS_KEY_SECRET }} | Testing: ${{ secrets.R2_DOWNLOAD_TESTING_ACCESS_KEY_SECRET }} + R2_BUCKET: download-testing # Production: download | Testing: download-testing + BASE_URL: https://download-testing.fleetdm.com # Production: https://download.fleetdm.com | Testing: https://download-testing.fleetdm.com + +jobs: + check-for-fleetd-component-updates: + runs-on: ubuntu-latest + outputs: + update_needed: ${{ steps.check-for-fleetd-component-updates.outputs.update_needed }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + egress-policy: audit + + - name: Install Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version: ${{ vars.GO_VERSION }} + + - name: Checkout Code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Check for fleetd component updates + id: check-for-fleetd-component-updates + run: | + go run tools/tuf/status/tuf-status.go channel-version -channel stable --components orbit,desktop,osqueryd --format json > latest-meta.json + curl -O $BASE_URL/meta.json + if diff latest-meta.json meta.json >/dev/null 2>&1 + then + echo "update_needed=false" >> $GITHUB_OUTPUT + else + echo "update_needed=true" >> $GITHUB_OUTPUT + fi + + - name: Upload latest meta.json artifact + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: latest-meta.json + path: latest-meta.json + + update-fleetd-base-pkg: + needs: [check-for-fleetd-component-updates] + if: needs.check-for-fleetd-component-updates.outputs.update_needed == 'true' + runs-on: macos-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + egress-policy: audit + + - name: Checkout code needed for R2 upload + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + sparse-checkout: | + .github/actions/r2-upload/action.yml + .github/scripts/rclone-install.sh + sparse-checkout-cone-mode: false + + - name: Install fleetctl + run: npm install -g fleetctl + + - name: Import package signing keys + env: + APPLE_INSTALLER_CERTIFICATE: ${{ secrets.APPLE_INSTALLER_CERTIFICATE }} + APPLE_INSTALLER_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_INSTALLER_CERTIFICATE_PASSWORD }} + KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} + run: | + echo "$APPLE_INSTALLER_CERTIFICATE" | base64 --decode > certificate.p12 + security create-keychain -p $KEYCHAIN_PASSWORD build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p $KEYCHAIN_PASSWORD build.keychain + security import certificate.p12 -k build.keychain -P $APPLE_INSTALLER_CERTIFICATE_PASSWORD -T /usr/bin/productsign + security set-key-partition-list -S apple-tool:,apple:,productsign: -s -k $KEYCHAIN_PASSWORD build.keychain + security find-identity -vv + rm certificate.p12 + + - name: Build PKG, sign, and notarize + env: + AC_USERNAME: ${{ secrets.APPLE_USERNAME }} + AC_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + AC_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + PACKAGE_SIGNING_IDENTITY_SHA1: D52080FD1F0941DE31346F06DA0F08AED6FACBBF + run: | + fleetctl package --type pkg --fleet-desktop --use-system-configuration --sign-identity $PACKAGE_SIGNING_IDENTITY_SHA1 --notarize + mv fleet-osquery*.pkg fleetd-base.pkg + + - name: Upload package + uses: ./.github/actions/r2-upload + with: + filename: fleetd-base.pkg + + update-fleetd-base-msi: + needs: [check-for-fleetd-component-updates] + if: needs.check-for-fleetd-component-updates.outputs.update_needed == 'true' + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + egress-policy: audit + + - name: Checkout code needed for R2 upload + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + sparse-checkout: | + .github/actions/r2-upload/action.yml + .github/scripts/rclone-install.sh + sparse-checkout-cone-mode: false + + - name: Install fleetctl + run: npm install -g fleetctl + + - name: Build MSI + run: | + fleetctl package --type msi --fleet-desktop --fleet-url dummy --enroll-secret dummy + mv fleet-osquery*.msi fleetd-base.msi + + - name: Upload package + uses: ./.github/actions/r2-upload + with: + filename: fleetd-base.msi + + update-meta-json: + needs: [update-fleetd-base-pkg, update-fleetd-base-msi] + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + egress-policy: audit + + - name: Checkout code needed for R2 upload + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + sparse-checkout: | + .github/actions/r2-upload/action.yml + .github/scripts/rclone-install.sh + sparse-checkout-cone-mode: false + + - name: Download latest-meta.json artifact + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 + with: + name: latest-meta.json + + - name: Rename latest-meta.json to meta.json + run: mv latest-meta.json meta.json + + - name: Upload meta.json + uses: ./.github/actions/r2-upload + with: + filename: meta.json