Add fixes for running tests with mysql:8 and add mysql to test-go job matrix (#3627)

* Add fixes for running tests with mysql:8

* Add getServer function

* Test github matrix

* Add changes file for the user facing fix

* Remove unused mysql8 docker-compose

.github/workflows/test-go.yaml

@@ -17,6 +17,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         go-version: ['^1.17.0']
+        mysql: ["mysql:5.7", "mysql:8"]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -31,7 +32,7 @@ jobs:
     # Pre-starting dependencies here means they are ready to go when we need them.
     - name: Start Infra Dependencies
       # Use & to background this
-      run: docker-compose up -d mysql_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup &
+      run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker-compose up -d mysql_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup &
 
     # It seems faster not to cache Go dependencies
     - name: Install Go Dependencies

changes/issue-3345-run-tests-with-mysql8

@@ -0,0 +1 @@
+* Fix aggregated stats query to support MySQL 8 (#3219).

docker-compose-mysql8.yml

@@ -1,19 +0,0 @@
----
-version: '2'
-services:
-  # To test with MariaDB, set FLEET_MYSQL_IMAGE to mariadb:10.6 or the like.
-  mysql:
-    image: ${FLEET_MYSQL_IMAGE:-mysql:8}
-    platform: linux/x86_64
-    volumes:
-      - mysql-persistent-volume:/tmp
-    # see https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_sql_require_primary_key
-    command: mysqld --datadir=/tmp/mysqldata --event-scheduler=ON --sql-require-primary-key=ON
-    environment: &mysql-default-environment
-      MYSQL_ROOT_PASSWORD: toor
-      MYSQL_DATABASE: fleet
-      MYSQL_USER: fleet
-      MYSQL_PASSWORD: insecure
-    ports:
-      - "3306:3306"
-

server/datastore/mysql/aggregated_stats.go

@@ -17,7 +17,7 @@ const scheduledQueryPercentileQuery = `
 SELECT
 	coalesce((t1.%s / t1.executions), 0)
 FROM (
-	SELECT @rownum := @rownum + 1 AS row_number, mm.* FROM (
+	SELECT (@rownum := @rownum + 1) AS row_number_value, mm.* FROM (
 		SELECT d.scheduled_query_id, d.%s, d.executions
 		FROM scheduled_query_stats d
 		WHERE d.scheduled_query_id=?
@@ -30,13 +30,13 @@ FROM (
 	FROM scheduled_query_stats d
 	WHERE d.scheduled_query_id=?
 ) AS t2
-WHERE t1.row_number = floor(total_rows * %s) + 1;`
+WHERE t1.row_number_value = floor(total_rows * %s) + 1;`
 
 const queryPercentileQuery = `
 SELECT
 	coalesce((t1.%s / t1.executions), 0)
 FROM (
-	SELECT @rownum := @rownum + 1 AS row_number, mm.* FROM (
+	SELECT @rownum := @rownum + 1 AS row_number_value, mm.* FROM (
 		SELECT d.scheduled_query_id, d.%s, d.executions
 		FROM scheduled_query_stats d
 		JOIN scheduled_queries sq ON (sq.id=d.scheduled_query_id)
@@ -51,10 +51,12 @@ FROM (
 	JOIN scheduled_queries sq ON (sq.id=d.scheduled_query_id)
 	WHERE sq.query_id=?
 ) AS t2
-WHERE t1.row_number = floor(total_rows * %s) + 1;`
+WHERE t1.row_number_value = floor(total_rows * %s) + 1;`
 
-const scheduledQueryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats WHERE scheduled_query_id=?`
-const queryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats sqs JOIN scheduled_queries sq ON (sqs.scheduled_query_id=sq.id) JOIN queries q ON (q.id=sq.query_id) WHERE sq.query_id=?`
+const (
+	scheduledQueryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats WHERE scheduled_query_id=?`
+	queryTotalExecutions          = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats sqs JOIN scheduled_queries sq ON (sqs.scheduled_query_id=sq.id) JOIN queries q ON (q.id=sq.query_id) WHERE sq.query_id=?`
+)
 
 func getPercentileQuery(aggregate string, time string, percentile string) string {
 	switch aggregate {

server/datastore/mysql/locks_test.go

@@ -3,6 +3,7 @@ package mysql
 import (
 	"context"
 	"database/sql"
+	"strings"
 	"testing"
 	"time"
 
@@ -76,7 +77,37 @@ func testLocksLockUnlock(t *testing.T, ds *Datastore) {
 	assert.True(t, locked)
 }
 
+type mysqlServer int
+
+const (
+	unknownServer mysqlServer = 0
+	mysql5        mysqlServer = 1
+	mysql8        mysqlServer = 3
+	mariaDB_10_6  mysqlServer = 3
+)
+
+func getMySQLServer(t *testing.T, r dbReader) mysqlServer {
+	row := r.QueryRowxContext(context.Background(), "SELECT VERSION()")
+	var version string
+	require.NoError(t, row.Scan(&version))
+	switch {
+	case strings.Contains(version, "MariaDB") && strings.Contains(version, "10.6"):
+		return mariaDB_10_6
+	case strings.HasPrefix(version, "5."):
+		return mysql5
+	case strings.HasPrefix(version, "8."):
+		return mysql8
+	default:
+		t.Fatalf("unsupported mysql server: %s", version)
+		return unknownServer
+	}
+}
+
 func testLocksDBLocks(t *testing.T, ds *Datastore) {
+	if srv := getMySQLServer(t, ds.reader); srv == mysql8 {
+		t.Skip("#3626: DBLocks is not supported for mysql 8 yet.")
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	t.Cleanup(cancel)
 

server/datastore/mysql/software_test.go

@@ -459,7 +459,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 
 	t.Run("lists everything", func(t *testing.T) {
 		software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{})
-		expected := []fleet.Software{foo001, foo002, foo003, bar003}
+		expected := []fleet.Software{bar003, foo001, foo003, foo002}
 		test.ElementsMatchSkipID(t, software, expected)
 	})
 
@@ -542,5 +542,13 @@ func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int,
 	count, err := ds.CountSoftware(context.Background(), opts)
 	require.NoError(t, err)
 	require.Equal(t, expectedFullCount, count)
+	for _, s := range software {
+		sort.Slice(s.Vulnerabilities, func(i, j int) bool {
+			return s.Vulnerabilities[i].CVE < s.Vulnerabilities[j].CVE
+		})
+	}
+	sort.Slice(software, func(i, j int) bool {
+		return software[i].Name+software[i].Version < software[j].Name+software[j].Version
+	})
 	return software
 }

server/datastore/mysql/testing_utils.go

@@ -85,17 +85,17 @@ func setupReadReplica(t testing.TB, testName string, ds *Datastore, opts *Datast
 		// drop all foreign keys in the replica, as that causes issues even with
 		// FOREIGN_KEY_CHECKS=0
 		var fks []struct {
-			TableName      string `db:"table_name"`
-			ConstraintName string `db:"constraint_name"`
+			TableName      string `db:"TABLE_NAME"`
+			ConstraintName string `db:"CONSTRAINT_NAME"`
 		}
 		err := primary.SelectContext(ctx, &fks, `
           SELECT
-            table_name, constraint_name
+            TABLE_NAME, CONSTRAINT_NAME
           FROM
-            information_schema.key_column_usage
+            INFORMATION_SCHEMA.KEY_COLUMN_USAGE
           WHERE
-            table_schema = ? AND
-            referenced_table_name IS NOT NULL`, testName)
+            TABLE_SCHEMA = ? AND
+            REFERENCED_TABLE_NAME IS NOT NULL`, testName)
 		require.NoError(t, err)
 		for _, fk := range fks {
 			stmt := fmt.Sprintf(`ALTER TABLE %s.%s DROP FOREIGN KEY %s`, replicaDB, fk.TableName, fk.ConstraintName)