From 49ceee59aa3540d637d317ea4d32156eae46c9d2 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Tue, 11 Jan 2022 22:44:37 -0300 Subject: [PATCH] Add fixes for running tests with mysql:8 and add `mysql` to `test-go` job matrix (#3627) * Add fixes for running tests with mysql:8 * Add getServer function * Test github matrix * Add changes file for the user facing fix * Remove unused mysql8 docker-compose --- .github/workflows/test-go.yaml | 3 ++- changes/issue-3345-run-tests-with-mysql8 | 1 + docker-compose-mysql8.yml | 19 ------------- server/datastore/mysql/aggregated_stats.go | 14 +++++----- server/datastore/mysql/locks_test.go | 31 ++++++++++++++++++++++ server/datastore/mysql/software_test.go | 10 ++++++- server/datastore/mysql/testing_utils.go | 12 ++++----- 7 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 changes/issue-3345-run-tests-with-mysql8 delete mode 100644 docker-compose-mysql8.yml diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 961b6afe6d..a9a8a06a81 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -17,6 +17,7 @@ jobs: matrix: os: [ubuntu-latest] go-version: ['^1.17.0'] + mysql: ["mysql:5.7", "mysql:8"] runs-on: ${{ matrix.os }} steps: @@ -31,7 +32,7 @@ jobs: # Pre-starting dependencies here means they are ready to go when we need them. - name: Start Infra Dependencies # Use & to background this - run: docker-compose up -d mysql_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup & + run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker-compose up -d mysql_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup & # It seems faster not to cache Go dependencies - name: Install Go Dependencies diff --git a/changes/issue-3345-run-tests-with-mysql8 b/changes/issue-3345-run-tests-with-mysql8 new file mode 100644 index 0000000000..b82489b805 --- /dev/null +++ b/changes/issue-3345-run-tests-with-mysql8 @@ -0,0 +1 @@ +* Fix aggregated stats query to support MySQL 8 (#3219). diff --git a/docker-compose-mysql8.yml b/docker-compose-mysql8.yml deleted file mode 100644 index 51f9a4c7e3..0000000000 --- a/docker-compose-mysql8.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -version: '2' -services: - # To test with MariaDB, set FLEET_MYSQL_IMAGE to mariadb:10.6 or the like. - mysql: - image: ${FLEET_MYSQL_IMAGE:-mysql:8} - platform: linux/x86_64 - volumes: - - mysql-persistent-volume:/tmp - # see https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_sql_require_primary_key - command: mysqld --datadir=/tmp/mysqldata --event-scheduler=ON --sql-require-primary-key=ON - environment: &mysql-default-environment - MYSQL_ROOT_PASSWORD: toor - MYSQL_DATABASE: fleet - MYSQL_USER: fleet - MYSQL_PASSWORD: insecure - ports: - - "3306:3306" - diff --git a/server/datastore/mysql/aggregated_stats.go b/server/datastore/mysql/aggregated_stats.go index cc2305c4f0..a297b35d6b 100644 --- a/server/datastore/mysql/aggregated_stats.go +++ b/server/datastore/mysql/aggregated_stats.go @@ -17,7 +17,7 @@ const scheduledQueryPercentileQuery = ` SELECT coalesce((t1.%s / t1.executions), 0) FROM ( - SELECT @rownum := @rownum + 1 AS row_number, mm.* FROM ( + SELECT (@rownum := @rownum + 1) AS row_number_value, mm.* FROM ( SELECT d.scheduled_query_id, d.%s, d.executions FROM scheduled_query_stats d WHERE d.scheduled_query_id=? @@ -30,13 +30,13 @@ FROM ( FROM scheduled_query_stats d WHERE d.scheduled_query_id=? ) AS t2 -WHERE t1.row_number = floor(total_rows * %s) + 1;` +WHERE t1.row_number_value = floor(total_rows * %s) + 1;` const queryPercentileQuery = ` SELECT coalesce((t1.%s / t1.executions), 0) FROM ( - SELECT @rownum := @rownum + 1 AS row_number, mm.* FROM ( + SELECT @rownum := @rownum + 1 AS row_number_value, mm.* FROM ( SELECT d.scheduled_query_id, d.%s, d.executions FROM scheduled_query_stats d JOIN scheduled_queries sq ON (sq.id=d.scheduled_query_id) @@ -51,10 +51,12 @@ FROM ( JOIN scheduled_queries sq ON (sq.id=d.scheduled_query_id) WHERE sq.query_id=? ) AS t2 -WHERE t1.row_number = floor(total_rows * %s) + 1;` +WHERE t1.row_number_value = floor(total_rows * %s) + 1;` -const scheduledQueryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats WHERE scheduled_query_id=?` -const queryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats sqs JOIN scheduled_queries sq ON (sqs.scheduled_query_id=sq.id) JOIN queries q ON (q.id=sq.query_id) WHERE sq.query_id=?` +const ( + scheduledQueryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats WHERE scheduled_query_id=?` + queryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats sqs JOIN scheduled_queries sq ON (sqs.scheduled_query_id=sq.id) JOIN queries q ON (q.id=sq.query_id) WHERE sq.query_id=?` +) func getPercentileQuery(aggregate string, time string, percentile string) string { switch aggregate { diff --git a/server/datastore/mysql/locks_test.go b/server/datastore/mysql/locks_test.go index 565399618d..e36eb8efd4 100644 --- a/server/datastore/mysql/locks_test.go +++ b/server/datastore/mysql/locks_test.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "strings" "testing" "time" @@ -76,7 +77,37 @@ func testLocksLockUnlock(t *testing.T, ds *Datastore) { assert.True(t, locked) } +type mysqlServer int + +const ( + unknownServer mysqlServer = 0 + mysql5 mysqlServer = 1 + mysql8 mysqlServer = 3 + mariaDB_10_6 mysqlServer = 3 +) + +func getMySQLServer(t *testing.T, r dbReader) mysqlServer { + row := r.QueryRowxContext(context.Background(), "SELECT VERSION()") + var version string + require.NoError(t, row.Scan(&version)) + switch { + case strings.Contains(version, "MariaDB") && strings.Contains(version, "10.6"): + return mariaDB_10_6 + case strings.HasPrefix(version, "5."): + return mysql5 + case strings.HasPrefix(version, "8."): + return mysql8 + default: + t.Fatalf("unsupported mysql server: %s", version) + return unknownServer + } +} + func testLocksDBLocks(t *testing.T, ds *Datastore) { + if srv := getMySQLServer(t, ds.reader); srv == mysql8 { + t.Skip("#3626: DBLocks is not supported for mysql 8 yet.") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) t.Cleanup(cancel) diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 71bef8b377..4e4651d3fd 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -459,7 +459,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) { t.Run("lists everything", func(t *testing.T) { software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{}) - expected := []fleet.Software{foo001, foo002, foo003, bar003} + expected := []fleet.Software{bar003, foo001, foo003, foo002} test.ElementsMatchSkipID(t, software, expected) }) @@ -542,5 +542,13 @@ func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int, count, err := ds.CountSoftware(context.Background(), opts) require.NoError(t, err) require.Equal(t, expectedFullCount, count) + for _, s := range software { + sort.Slice(s.Vulnerabilities, func(i, j int) bool { + return s.Vulnerabilities[i].CVE < s.Vulnerabilities[j].CVE + }) + } + sort.Slice(software, func(i, j int) bool { + return software[i].Name+software[i].Version < software[j].Name+software[j].Version + }) return software } diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index d3f3620b38..41743d782e 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -85,17 +85,17 @@ func setupReadReplica(t testing.TB, testName string, ds *Datastore, opts *Datast // drop all foreign keys in the replica, as that causes issues even with // FOREIGN_KEY_CHECKS=0 var fks []struct { - TableName string `db:"table_name"` - ConstraintName string `db:"constraint_name"` + TableName string `db:"TABLE_NAME"` + ConstraintName string `db:"CONSTRAINT_NAME"` } err := primary.SelectContext(ctx, &fks, ` SELECT - table_name, constraint_name + TABLE_NAME, CONSTRAINT_NAME FROM - information_schema.key_column_usage + INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE - table_schema = ? AND - referenced_table_name IS NOT NULL`, testName) + TABLE_SCHEMA = ? AND + REFERENCED_TABLE_NAME IS NOT NULL`, testName) require.NoError(t, err) for _, fk := range fks { stmt := fmt.Sprintf(`ALTER TABLE %s.%s DROP FOREIGN KEY %s`, replicaDB, fk.TableName, fk.ConstraintName)