From e6fb647d95dc12135d7c6ca239b512ae6e2fb899 Mon Sep 17 00:00:00 2001
From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com>
Date: Thu, 9 Jan 2025 13:38:24 -0500
Subject: [PATCH] Run CI tests in parallel (#25271)

#21774

Improves run time by about 30%.

Things have been arranged in such a way that splitting modules out
further will be trivial in the future, such as breaking the different
integration test suited into their own units.


![image](https://github.com/user-attachments/assets/ead46e4c-6f14-406d-a29b-b25abc79c384)


![image](https://github.com/user-attachments/assets/3f7fd7f3-d7a8-4ff8-a184-646a72f1d015)
---
 .github/workflows/test-go.yaml |  6 +++++-
 Makefile                       | 30 +++++++++++++++++++++++-------
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml
index 75c4800928..e347f72782 100644
--- a/.github/workflows/test-go.yaml
+++ b/.github/workflows/test-go.yaml
@@ -42,7 +42,7 @@ jobs:
   test-go:
     strategy:
       matrix:
-        suite: ["integration", "core"]
+        suite: ["integration", "core", "mysql", "fleetctl", "vuln"]
         os: [ubuntu-latest]
         mysql: ["mysql:8.0.36", "mysql:8.4.3", "mysql:9.1.0"] # make sure to update supported versions docs when this changes
         isCron:
@@ -118,10 +118,13 @@ jobs:
     - name: Run Go Tests
       run: |
         if [[ "${{ matrix.suite }}" == "core" ]]; then
+          CI_TEST_PKG=main
           RUN_TESTS_ARG='-skip=^TestIntegrations'
         elif [[ "${{ matrix.suite }}" == "integration" ]]; then
+          CI_TEST_PKG=main
           RUN_TESTS_ARG='-run=^TestIntegrations'
         else
+          CI_TEST_PKG="${{ matrix.suite }}"
           RUN_TESTS_ARG=''
         fi
         GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \
@@ -135,6 +138,7 @@ jobs:
           SAML_IDP_TEST=1 \
           MAIL_TEST=1 \
           NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
+          CI_TEST_PKG="${CI_TEST_PKG}" \
           make test-go 2>&1 | tee /tmp/gotest.log
 
     - name: Create mysql identifier without colon
diff --git a/Makefile b/Makefile
index 406d8486bf..6ffdca38de 100644
--- a/Makefile
+++ b/Makefile
@@ -150,14 +150,30 @@ dump-test-schema:
 # TESTS_TO_RUN: Name specific tests to run in the specified packages.  Leave blank to run all tests in the specified packages.
 # GO_TEST_EXTRA_FLAGS: Used to specify other arguments to `go test`.
 # GO_TEST_MAKE_FLAGS: Internal var used by other targets to add arguments to `go test`.
-#						 
+#
 PKG_TO_TEST := "" # default to empty string; can be overridden on command line.
 go_test_pkg_to_test := $(addprefix ./,$(PKG_TO_TEST)) # set paths for packages to test
 dlv_test_pkg_to_test := $(addprefix github.com/fleetdm/fleet/v4/,$(PKG_TO_TEST)) # set URIs for packages to debug
 
+DEFAULT_PKG_TO_TEST := ./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/...
+ifeq ($(CI_TEST_PKG), main)
+	CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | grep -v "server/datastore/mysql" | grep -v "cmd/fleetctl" | grep -v "server/vulnerabilities" | sed -e 's|github.com/fleetdm/fleet/v4/||g')
+else ifeq ($(CI_TEST_PKG), mysql)
+	CI_PKG_TO_TEST="server/datastore/mysql/..."
+else ifeq ($(CI_TEST_PKG), fleetctl)
+	CI_PKG_TO_TEST="cmd/fleetctl/..."
+else ifeq ($(CI_TEST_PKG), vuln)
+	CI_PKG_TO_TEST="server/vulnerabilities/..."
+else
+	CI_PKG_TO_TEST=$(DEFAULT_PKG_TO_TEST)
+endif
+
+ci-pkg-list:
+	@echo $(CI_PKG_TO_TEST)
+
 .run-go-tests:
 ifeq ($(PKG_TO_TEST), "")
-		@echo "Please specify one or more packages to test with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"..."; 
+		@echo "Please specify one or more packages to test with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"...";
 else
 		@echo Running Go tests with command:
 		go test -tags full,fts5,netgo -run=${TESTS_TO_RUN} ${GO_TEST_MAKE_FLAGS} ${GO_TEST_EXTRA_FLAGS} -parallel 8 -coverprofile=coverage.txt -covermode=atomic -coverpkg=github.com/fleetdm/fleet/v4/... $(go_test_pkg_to_test)
@@ -171,10 +187,10 @@ endif
 # GO_TEST_EXTRA_FLAGS: Used to specify other arguments to `go test`.
 .debug-go-tests:
 ifeq ($(PKG_TO_TEST), "")
-		@echo "Please specify one or more packages to debug with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"..."; 
+		@echo "Please specify one or more packages to debug with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"...";
 else
 		@echo Debugging tests with command:
-		dlv test ${dlv_test_pkg_to_test} --api-version=2 --listen=127.0.0.1:61179 ${DEBUG_TEST_EXTRA_FLAGS} -- -test.v -test.run=${TESTS_TO_RUN} ${GO_TEST_EXTRA_FLAGS} 
+		dlv test ${dlv_test_pkg_to_test} --api-version=2 --listen=127.0.0.1:61179 ${DEBUG_TEST_EXTRA_FLAGS} -- -test.v -test.run=${TESTS_TO_RUN} ${GO_TEST_EXTRA_FLAGS}
 endif
 
 # Command to run specific tests in development.  Can run all tests for one or more packages, or specific tests within packages.
@@ -182,11 +198,11 @@ run-go-tests:
 	@MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .run-go-tests GO_TEST_MAKE_FLAGS="-v"
 
 debug-go-tests:
-	@MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .debug-go-tests 
+	@MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .debug-go-tests
 
 # Command used in CI to run all tests.
-test-go: dump-test-schema generate-mock 
-	make .run-go-tests PKG_TO_TEST="./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/..."
+test-go: dump-test-schema generate-mock
+	make .run-go-tests PKG_TO_TEST="$(CI_PKG_TO_TEST)"
 
 analyze-go:
 	go test -tags full,fts5,netgo -race -cover ./...