Ian Littman
7d4acdc5c4
Fixes #40975. 8.0.32 (was running in Aurora managed cloud at the time) -> 8.0.39 (what we're running now) 8.0.36 -> 8.0.44 (latest 8.0.x version supported by Aurora; holding off on 8.0.45 until Aurora supports it) 8.4.7 -> 8.4.8 9.5.0 -> 9.6.0 Also bumped the supported Aurora version from 3.07.0 to 3.08.2 to match what we're running in managed cloud right now Fleet might work on older patch versions but we'll no longer dev/test on them. MySQL 9.x not testing previous minor versions matches with our previous approach for that version. Since these are all patch/minor bumps (and the overnight build cases are patch bumps/are covered by AWS envs) automated testing should be sufficient here.
330 lines
12 KiB
YAML
330 lines
12 KiB
YAML
# Reusable workflow for running a single Go test suite.
|
|
# Called by test-go.yaml with different matrix configurations.
|
|
name: Test Go suite
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
suite:
|
|
description: 'Test suite name (e.g., "integration-core", "fast", "mysql")'
|
|
required: true
|
|
type: string
|
|
mysql:
|
|
description: 'MySQL Docker image (e.g., "mysql:8.0.44"). Leave empty for suites that do not need MySQL.'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
cover_pkg:
|
|
description: 'Go coverage package pattern (e.g., "github.com/fleetdm/fleet/v4/server/activity/..."). Defaults to all fleet packages.'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
generate_go:
|
|
description: 'Whether to run make generate-go before tests. Disable for suites that do not need generated static files.'
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
is_cron:
|
|
description: 'Whether this is a scheduled (cron) run. Enables race detector and longer timeouts.'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
defaults:
|
|
run:
|
|
# fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
|
|
shell: bash
|
|
|
|
jobs:
|
|
test:
|
|
# Don't cancel other jobs if one test suite fails. Since our product teams are tightly coupled, we never want to see our tests fail due
|
|
# to an unrelated issue in another product area.
|
|
continue-on-error: true
|
|
runs-on: ubuntu-latest
|
|
|
|
env:
|
|
RACE_ENABLED: false
|
|
GO_TEST_TIMEOUT: 20m
|
|
DOCKER_COMMAND: docker compose -f docker-compose.yml -f docker-compose-redis-cluster.yml up -d mysql_test mysql_replica_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup s3 saml_idp mailhog mailpit smtp4dev_test
|
|
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Configure job
|
|
run: |
|
|
echo "RUN_TESTS_ARG=" >> $GITHUB_ENV
|
|
if [[ "${{ inputs.suite }}" == "main" ]]; then
|
|
echo "CI_TEST_PKG=main" >> $GITHUB_ENV
|
|
echo "NEED_DOCKER=1" >> $GITHUB_ENV
|
|
echo "DOCKER_COMMAND=${{ env.DOCKER_COMMAND }} localstack" >> $GITHUB_ENV
|
|
elif [[ "${{ inputs.suite }}" == "fast" ]]; then
|
|
# DO NOT add any dependencies in this test suite.
|
|
echo "CI_TEST_PKG=${{ inputs.suite }}" >> $GITHUB_ENV
|
|
elif [[ "${{ inputs.suite }}" == "service" ]]; then
|
|
echo "CI_TEST_PKG=service" >> $GITHUB_ENV
|
|
echo "RUN_TESTS_ARG=-skip=^TestIntegrations" >> $GITHUB_ENV
|
|
echo "NEED_DOCKER=1" >> $GITHUB_ENV
|
|
elif [[ "${{ inputs.suite }}" == "integration-core" ]]; then
|
|
echo "CI_TEST_PKG=service" >> $GITHUB_ENV
|
|
echo "RUN_TESTS_ARG=-run=^TestIntegrations -skip '^(TestIntegrationsMDM|TestIntegrationsEnterprise)'" >> $GITHUB_ENV
|
|
# We re-generate test schema just in case there is an issue with the schema. We only do this for one test.
|
|
echo "GENERATE_TEST_SCHEMA=1" >> $GITHUB_ENV
|
|
echo "NEED_DOCKER=1" >> $GITHUB_ENV
|
|
elif [[ "${{ inputs.suite }}" == "integration-mdm" ]]; then
|
|
echo "CI_TEST_PKG=service" >> $GITHUB_ENV
|
|
echo "RUN_TESTS_ARG=-run=^TestIntegrationsMDM" >> $GITHUB_ENV
|
|
echo "NEED_DOCKER=1" >> $GITHUB_ENV
|
|
elif [[ "${{ inputs.suite }}" == "integration-enterprise" ]]; then
|
|
echo "CI_TEST_PKG=service" >> $GITHUB_ENV
|
|
echo "RUN_TESTS_ARG=-run=^TestIntegrationsEnterprise" >> $GITHUB_ENV
|
|
echo "NEED_DOCKER=1" >> $GITHUB_ENV
|
|
elif [[ "${{ inputs.suite }}" == "scripts" ]]; then
|
|
echo "CI_TEST_PKG=${{ inputs.suite }}" >> $GITHUB_ENV
|
|
echo "NEED_ZSH=1" >> $GITHUB_ENV
|
|
else
|
|
echo "CI_TEST_PKG=${{ inputs.suite }}" >> $GITHUB_ENV
|
|
echo "NEED_DOCKER=1" >> $GITHUB_ENV
|
|
fi
|
|
|
|
- name: Compute artifact prefix
|
|
run: |
|
|
if [[ -n "${{ inputs.mysql }}" ]]; then
|
|
MYSQL_ID=$(echo "${{ inputs.mysql }}" | tr -d ':')
|
|
echo "ARTIFACT_PREFIX=${{ inputs.suite }}-${MYSQL_ID}" >> $GITHUB_ENV
|
|
else
|
|
echo "ARTIFACT_PREFIX=${{ inputs.suite }}" >> $GITHUB_ENV
|
|
fi
|
|
|
|
- name: Set Go race setting on schedule
|
|
if: ${{ inputs.is_cron }}
|
|
run: |
|
|
echo "RACE_ENABLED=true" >> $GITHUB_ENV
|
|
echo "GO_TEST_TIMEOUT=1h" >> $GITHUB_ENV
|
|
|
|
- name: Checkout Code
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
|
|
|
- name: Install Go
|
|
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
- name: Install gotestsum
|
|
run: go install gotest.tools/gotestsum@latest
|
|
|
|
# Pre-starting dependencies here means they are ready to go when we need them.
|
|
- name: Start Infra Dependencies
|
|
if: ${{ env.NEED_DOCKER }}
|
|
# Use & to background this
|
|
run: FLEET_MYSQL_IMAGE=${{ inputs.mysql }} $DOCKER_COMMAND &
|
|
|
|
- name: Add TLS certificate for SMTP Tests
|
|
if: ${{ env.NEED_DOCKER }}
|
|
run: |
|
|
sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
|
|
sudo update-ca-certificates
|
|
|
|
- name: Install ZSH
|
|
if: ${{ env.NEED_ZSH }}
|
|
run: sudo apt update && sudo apt install -y zsh
|
|
|
|
- name: Generate static files
|
|
if: ${{ inputs.generate_go }}
|
|
run: |
|
|
export PATH=$PATH:~/go/bin
|
|
make generate-go
|
|
|
|
- name: Wait for mysql
|
|
if: ${{ env.NEED_DOCKER }}
|
|
run: |
|
|
# Function to wait for MySQL with timeout
|
|
wait_for_mysql() {
|
|
local container_name=$1
|
|
local timeout_seconds=60 # 1 minute
|
|
local start_time=$(date +%s)
|
|
local attempt_logs=""
|
|
|
|
echo "waiting for ${container_name}..."
|
|
while true; do
|
|
# Check if timeout has been reached
|
|
current_time=$(date +%s)
|
|
elapsed_time=$((current_time - start_time))
|
|
if [ $elapsed_time -ge $timeout_seconds ]; then
|
|
echo "Timeout reached (${timeout_seconds}s) while waiting for ${container_name}"
|
|
echo "Connection attempt logs:"
|
|
echo "$attempt_logs"
|
|
# Dump MySQL container logs
|
|
echo "Dumping ${container_name} logs:"
|
|
docker compose logs ${container_name}
|
|
return 1
|
|
fi
|
|
|
|
# Try to connect to MySQL
|
|
output=$(docker compose exec -T $container_name sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" 2>&1)
|
|
exit_code=$?
|
|
|
|
# Log the attempt
|
|
timestamp=$(date "+%Y-%m-%d %H:%M:%S")
|
|
attempt_logs="${attempt_logs}$(printf "\n%s - Exit code: %s - Output: %s" "$timestamp" "$exit_code" "$output")"
|
|
|
|
# If connection successful, break the loop
|
|
if [ $exit_code -eq 0 ]; then
|
|
echo "${container_name} is ready"
|
|
return 0
|
|
fi
|
|
|
|
echo "."
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
# Function to restart containers
|
|
restart_containers() {
|
|
echo "Stopping all containers..."
|
|
docker compose down
|
|
|
|
echo "Restarting containers..."
|
|
FLEET_MYSQL_IMAGE=${{ inputs.mysql }} $DOCKER_COMMAND &
|
|
|
|
# Give containers a moment to start
|
|
sleep 10
|
|
}
|
|
|
|
# Try up to 5 times to connect to MySQL
|
|
max_attempts=5
|
|
attempt=1
|
|
|
|
while [ $attempt -le $max_attempts ]; do
|
|
echo "Attempt $attempt of $max_attempts"
|
|
|
|
# Try to connect to MySQL
|
|
if wait_for_mysql "mysql_test"; then
|
|
# If MySQL is ready, try to connect to MySQL replica
|
|
if wait_for_mysql "mysql_replica_test"; then
|
|
# Both are ready, we're done
|
|
echo "All MySQL connections successful"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# If we get here, at least one connection failed
|
|
echo "Failed to connect to MySQL on attempt $attempt"
|
|
|
|
if [ $attempt -lt $max_attempts ]; then
|
|
echo "Restarting containers and trying again..."
|
|
restart_containers
|
|
else
|
|
echo "Maximum attempts reached. Failing the job."
|
|
exit 1
|
|
fi
|
|
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
- name: Generate test schema
|
|
if: ${{ env.GENERATE_TEST_SCHEMA }}
|
|
run: make test-schema
|
|
|
|
- name: Run Go Tests
|
|
run: |
|
|
USE_GOTESTSUM=1 \
|
|
GOTESTSUM_FORMAT=testdox \
|
|
GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT ${{ env.RUN_TESTS_ARG }}" \
|
|
TEST_LOCK_FILE_PATH=$(pwd)/lock \
|
|
TEST_CRON_NO_RECOVER=1 \
|
|
NETWORK_TEST=1 \
|
|
REDIS_TEST=1 \
|
|
MYSQL_TEST=1 \
|
|
MYSQL_REPLICA_TEST=1 \
|
|
S3_STORAGE_TEST=1 \
|
|
SAML_IDP_TEST=1 \
|
|
MAIL_TEST=1 \
|
|
AWS_ENDPOINT_URL="http://localhost:4566" \
|
|
AWS_REGION=us-east-1 \
|
|
NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
|
|
CI_TEST_PKG="${{ env.CI_TEST_PKG }}" \
|
|
COVER_PKG="${{ inputs.cover_pkg || 'github.com/fleetdm/fleet/v4/...' }}" \
|
|
make test-go 2>&1 | tee /tmp/gotest.log
|
|
|
|
- name: Save coverage
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ env.ARTIFACT_PREFIX }}-coverage
|
|
path: ./coverage.txt
|
|
if-no-files-found: error
|
|
|
|
- name: Generate summary of errors
|
|
if: failure()
|
|
run: |
|
|
c1grep() { grep "$@" || test $? = 1; }
|
|
c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
|
|
c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
|
|
GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
|
|
echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
|
|
if [[ -z "$GO_FAIL_SUMMARY" ]]; then
|
|
GO_FAIL_SUMMARY="unknown, please check the build URL"
|
|
fi
|
|
GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json
|
|
|
|
- name: Slack Notification
|
|
if: ${{ inputs.is_cron && failure() }}
|
|
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
|
|
with:
|
|
payload-file-path: ./payload.json
|
|
env:
|
|
JOB_STATUS: ${{ job.status }}
|
|
EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
|
|
RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
|
|
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
|
|
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
|
|
|
|
- name: Upload test log
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ env.ARTIFACT_PREFIX }}-test-log
|
|
path: /tmp/gotest.log
|
|
if-no-files-found: error
|
|
|
|
- name: Upload summary test log
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ env.ARTIFACT_PREFIX }}-summary-test-log
|
|
path: /tmp/summary.txt
|
|
|
|
- name: Upload JSON test output
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ env.ARTIFACT_PREFIX }}-test-json
|
|
path: /tmp/test-output.json
|
|
if-no-files-found: warn
|
|
|
|
- name: Set test status
|
|
if: always()
|
|
run: |
|
|
if [[ "${{ job.status }}" == "success" ]]; then
|
|
echo "success" > /tmp/status
|
|
else
|
|
echo "fail" > /tmp/status
|
|
fi
|
|
|
|
- name: Upload status indicator
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ env.ARTIFACT_PREFIX }}-status
|
|
path: /tmp/status
|
|
overwrite: true
|