MySQL 8.0 Migration (#20225)

#17249

.github/workflows/fleet-and-orbit.yml

@@ -63,7 +63,7 @@ jobs:
     strategy:
       matrix:
         go-version: ["${{ vars.GO_VERSION }}"]
-        mysql: ["mysql:5.7"]
+        mysql: ["mysql:8.0.36"]
     runs-on: ubuntu-latest
     needs: gen
     steps:

.github/workflows/test-go.yaml

@@ -45,7 +45,7 @@ jobs:
         suite: ["integration", "core"]
         os: [ubuntu-latest]
         go-version: ['${{ vars.GO_VERSION }}']
-        mysql: ["mysql:5.7.21", "mysql:8.0.28"]
+        mysql: ["mysql:8.0.36"]
     continue-on-error: ${{ matrix.suite == 'integration' }} # Since integration tests have a higher chance of failing, often for unrelated reasons, we don't want to fail the whole job if they fail
     runs-on: ${{ matrix.os }}
 

articles/deploy-fleet-on-hetzner-cloud.md

@@ -69,14 +69,14 @@ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docke
 apt update
 apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
-docker pull mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67 # mysql:5.7.38 as of 2022/05/19
+docker pull mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1 # mysql:8.0.36 as of 2024/07/04
 
 ######################
 # MySQL (dockerized) #
 ######################
 
-# mysql:5.7.38 as of 2022/05/19
-docker pull mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67
+# mysql:8.0.36 as of 2024/07/04
+docker pull mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1
 
 # Create the Fleet MySQL data folder
 mkdir -p /etc/fleet
@@ -102,14 +102,14 @@ Restart=always
 
 ExecStartPre=-/usr/bin/docker exec %n stop
 ExecStartPre=-/usr/bin/docker rm %n
-ExecStartPre=-/usr/bin/docker pull mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67
+ExecStartPre=-/usr/bin/docker pull mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1
 
 ExecStart=/usr/bin/docker run --rm \
     --name %n \
     -p 127.0.0.1:3306:3306 \
     -v /etc/fleet/mysql:/var/lib/mysql \
     --env-file /etc/fleet/mysql.env \
-    mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67
+    mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1
 
 ExecStop=/usr/bin/docker stop %n
 
@@ -435,7 +435,7 @@ To run MySQL, we’ll have to do the following:
 We can pull the [official MySQL docker image](https://hub.docker.com/_/mysql) like so:
 
 ```sh
-$ docker pull mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67 # mysql:5.7.38 as of 2022/05/19
+$ docker pull mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1 # mysql:8.0.36 as of 2024/07/04
 ```
 
 ### Create & enable a systemd unit for MySQL
@@ -472,14 +472,14 @@ Restart=always
 
 ExecStartPre=-/usr/bin/docker exec %n stop
 ExecStartPre=-/usr/bin/docker rm %n
-ExecStartPre=-/usr/bin/docker pull mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67
+ExecStartPre=-/usr/bin/docker pull mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1
 
 ExecStart=/usr/bin/docker run --rm \
     --name %n \
     -p 127.0.0.1:3306:3306 \
     -v /etc/fleet/mysql:/var/lib/mysql \
     --env-file /etc/fleet/mysql.env \
-    mysql@sha256:16e159331007eccc069822f7b731272043ed572a79a196a05ffa2ea127caaf67
+    mysql@sha256:134e2d1c7c517d05e5328a77aa5a165a314dc4c4116503e7e089494f4e398ab1
 
 ExecStop=/usr/bin/docker stop %n
 
@@ -712,4 +712,4 @@ Now that you’re ready to use Fleet and have a host installed. Here's some next
 <meta name="publishedOn" value="2022-06-27">
 <meta name="category" value="guides">
 <meta name="articleImageUrl" value="../website/assets/images/articles/deploy-fleet-on-hetzner-cloud-800x450@2x.png">
-<meta name="description" value="Learn how to deploy Fleet on Hetzner Cloud using cloud-init and Docker.">
+<meta name="description" value="Learn how to deploy Fleet on Hetzner Cloud using cloud-init and Docker.">

changes/17249-mysql-8

@@ -0,0 +1,2 @@
+* Drop support for MySQL 5.7
+* Minimum requirements raised to MySQL 8.0

docker-compose.yml

@@ -5,7 +5,7 @@ services:
   # officially supported).
   # To run in macOS M1, set FLEET_MYSQL_IMAGE=arm64v8/mysql:oracle FLEET_MYSQL_PLATFORM=linux/arm64/v8
   mysql:
-    image: ${FLEET_MYSQL_IMAGE:-mysql:5.7.21}
+    image: ${FLEET_MYSQL_IMAGE:-mysql:8.0.36}
     platform: ${FLEET_MYSQL_PLATFORM:-linux/x86_64}
     volumes:
       - mysql-persistent-volume:/tmp
@@ -30,7 +30,7 @@ services:
       - "3306:3306"
 
   mysql_test:
-    image: ${FLEET_MYSQL_IMAGE:-mysql:5.7.21}
+    image: ${FLEET_MYSQL_IMAGE:-mysql:8.0.36}
     platform: ${FLEET_MYSQL_PLATFORM:-linux/x86_64}
     # innodb-file-per-table=OFF gives ~20% speedup for test runs.
     command: [
@@ -56,7 +56,7 @@ services:
       - /tmpfs
 
   mysql_replica_test:
-    image: ${FLEET_MYSQL_IMAGE:-mysql:5.7.21}
+    image: ${FLEET_MYSQL_IMAGE:-mysql:8.0.36}
     platform: ${FLEET_MYSQL_PLATFORM:-linux/x86_64}
     # innodb-file-per-table=OFF gives ~20% speedup for test runs.
     command: [

docs/Configuration/fleet-server-configuration.md

@@ -236,7 +236,7 @@ The maximum amount of time, in seconds, a connection may be reused.
 
 ##### mysql_sql_mode
 
-Sets the connection `sql_mode`. See [MySQL Reference](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html) for more details.
+Sets the connection `sql_mode`. See [MySQL Reference](https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html) for more details.
 This setting should not usually be used.
 
 - Default value: `""`

docs/Contributing/File-carving.md

@@ -23,8 +23,7 @@ The default flagfile provided in the "Add new host" dialog also includes this co
 The `carver_block_size` flag should be configured in osquery.
 
 For the (default) MySQL Backend, the configured value must be less than the value of
-`max_allowed_packet` in the MySQL connection, allowing for some overhead. The default for [MySQL 5.7](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_allowed_packet)
-is 4MB and for [MySQL 8](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet) it is 64MB.
+`max_allowed_packet` in the MySQL connection, allowing for some overhead. The default for [MySQL 8](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet) it is 64MB.
 
 For the S3/Minio backend, this value must be set to at least 5MiB (`5242880`) due to the
 [constraints of S3's multipart

docs/Deploy/Reference-Architectures.md

@@ -26,7 +26,7 @@ Fleet currently has three infrastructure dependencies: MySQL, Redis, and a TLS c
 Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well-supported MySQL [Docker image](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. 
 For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](https://fleetdm.com/docs/deploying/configuration) document.
 
-Fleet requires at least MySQL version 5.7, and is tested using the InnoDB storage engine. 
+Fleet requires at least MySQL version 8.0, and is tested using the InnoDB storage engine. 
 
 There are many "drop-in replacements" for MySQL available. If you'd like to experiment with some bleeding-edge technology and use Fleet with one of these alternative database servers, we think that's awesome! Please be aware they are not officially supported and that it is very important to set up a dev environment to thoroughly test new releases. 
 
@@ -277,7 +277,7 @@ The following permissions are the minimum required to apply AWS terraform resour
 GCP reference architecture can be found in [the Fleet repository](https://github.com/fleetdm/fleet/tree/main/infrastructure/dogfood/terraform/gcp). This configuration includes:
 
 - Cloud Run (Fleet backend)
-- Cloud SQL MySQL 5.7 (Fleet database)
+- Cloud SQL MySQL 8.0 (Fleet database)
 - Memorystore Redis (Fleet cache & live query orchestrator)
 
 GCP support for add/install software and file carve features is coming soon. Get [commmunity support](https://chat.osquery.io/c/fleet).

docs/Get started/FAQ.md

@@ -594,7 +594,7 @@ This could be caused by a mismatched connection limit between the Fleet server a
 
 ### Why am I receiving a database connection error when attempting to "prepare" the database?
 
-First, check if you have a version of MySQL installed that is at least 5.7. Then, make sure that you currently have a MySQL server running.
+First, check if you have a version of MySQL installed that is at least 8.0. Then, make sure that you currently have a MySQL server running.
 
 The next step is to make sure the credentials for the database match what is expected. Test your ability to connect to the database with `mysql -u<username> -h<hostname_or_ip> -P<port> -D<database_name> -p`.
 
@@ -614,7 +614,7 @@ Yes! Please sign up for the [Fleet Cloud Beta](https://kqphpqst851.typeform.com/
 
 ### What MySQL versions are supported?
 
-Fleet is tested with MySQL 5.7.21 and 8.0.28. Newer versions of MySQL 5.7 and MySQL 8 typically work well. AWS Aurora requires at least version 2.10.0. Please avoid using MariaDB or other MySQL variants that are not officially supported. Compatibility issues have been identified with MySQL variants, and these may not be addressed in future Fleet releases.
+Fleet is tested with MySQL 8.0.36. Newer versions of MySQL 8 typically work well. AWS Aurora requires at least version 2.10.0. Please avoid using MariaDB or other MySQL variants that are not officially supported. Compatibility issues have been identified with MySQL variants, and these may not be addressed in future Fleet releases.
 
 ### What are the MySQL user requirements?
 
@@ -677,7 +677,7 @@ If you would like to use Fleet's MDM features, the following endpoints need to b
 
 ### What is the minimum version of MySQL required by Fleet?
 
-Fleet requires at least MySQL version 5.7.
+Fleet requires at least MySQL version 8.0.
 
 ### How do I migrate from Fleet Free to Fleet Premium?
 

server/service/handler_test.go

@@ -37,7 +37,7 @@ func TestAPIRoutesConflicts(t *testing.T) {
 	}
 	var cases []testCase
 
-	// build the test cases: for each route, generate a request designed to match
+	// Build the test cases: for each route, generate a request designed to match
 	// it, and override its handler to return a unique status code. If the
 	// request doesn't result in that status code, then some other route
 	// conflicts with it and took precedence - a route conflict. The route's name

server/service/hosts.go

@@ -994,7 +994,9 @@ type refetchHostResponse struct {
 	Err error `json:"error,omitempty"`
 }
 
-func (r refetchHostResponse) error() error { return r.Err }
+func (r refetchHostResponse) error() error {
+	return r.Err
+}
 
 func refetchHostEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) {
 	req := request.(*refetchHostRequest)

test/upgrade/docker-compose.yaml

@@ -1,7 +1,7 @@
 services:
   mysql:
     platform: ${FLEET_MYSQL_PLATFORM:-linux/x86_64}
-    image: mysql:5.7
+    image: mysql:8.0.36
     environment:
       MYSQL_ROOT_PASSWORD: toor
       MYSQL_DATABASE: fleet

tools/backup_db/backup.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-docker run --rm --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:5.7} bash -c 'mysqldump -hmysql -uroot -ptoor --default-character-set=utf8mb4 fleet | gzip -' > backup.sql.gz
+docker run --rm --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:8.0.36} bash -c 'mysqldump -hmysql -uroot -ptoor --default-character-set=utf8mb4 fleet | gzip -' > backup.sql.gz

tools/backup_db/restore.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-docker run --rm -i --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:5.7} bash -c 'gzip -dc - | mysql -hmysql -uroot -ptoor fleet' < backup.sql.gz
+docker run --rm -i --network fleet_default ${FLEET_MYSQL_IMAGE:-mysql:8.0.36} bash -c 'gzip -dc - | mysql -hmysql -uroot -ptoor fleet' < backup.sql.gz

tools/backup_db/restore_e2e_software_test.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-docker run --rm -i --network fleet_default mysql:5.7 bash -c 'gzip -kdc - | mysql -hmysql_test -uroot -ptoor e2e'  < tools/testdata/e2e_software_test.sql.gz
-
+docker run --rm -i --network fleet_default mysql:8.0.36 bash -c 'gzip -kdc - | mysql -hmysql_test -uroot -ptoor e2e'  < tools/testdata/e2e_software_test.sql.gz

tools/mysql-replica-testing/docker-compose.yml

@@ -2,7 +2,7 @@
 version: "2"
 services:
   mysql_main:
-    image: ${FLEET_MYSQL_IMAGE:-mysql:5.7}
+    image: ${FLEET_MYSQL_IMAGE:-mysql:8.0.36}
     platform: ${FLEET_MYSQL_PLATFORM:-linux/x86_64}
     volumes:
       - mysql-persistent-volume-replica-main:/tmp
@@ -27,7 +27,7 @@ services:
       - "3308:3306"
 
   mysql_read_replica:
-    image: ${FLEET_MYSQL_IMAGE:-mysql:5.7}
+    image: ${FLEET_MYSQL_IMAGE:-mysql:8.0.36}
     platform: ${FLEET_MYSQL_PLATFORM:-linux/x86_64}
     volumes:
       - mysql-persistent-volume-replica-read:/tmp

tools/osquery/in-a-box/docker-compose.yml

@@ -8,7 +8,7 @@ networks:
 
 services:
   mysql01:
-    image: mysql:5.7
+    image: mysql:8.0.36
     platform: linux/x86_64
     volumes:
       - .:/data

tools/percona/test/upgrade.sh

@@ -12,7 +12,7 @@ docker volume rm fleet_mysql-persistent-volume
 
 # Start dependencies using Percona XtraDB as MySQL server.
 # NOTE: To troubleshoot, remove `>/dev/null`.
-FLEET_MYSQL_IMAGE=percona/percona-xtradb-cluster:5.7.25 docker compose up >/dev/null 2>&1 &
+FLEET_MYSQL_IMAGE=percona/percona-xtradb-cluster:8.0.36 docker compose up >/dev/null 2>&1 &
 
 export MYSQL_PWD=toor
 
@@ -39,4 +39,4 @@ mysql --host 127.0.0.1 --port 3306 -uroot -e 'SET GLOBAL pxc_strict_mode=ENFORCI
 # Run migrations from fleet-v4.42.0 up to latest to catch any future bugs when running with `pxc_strict_mode=ENFORCING`.
 git checkout main
 make generate && make fleet
-./build/fleet prepare db --dev --logging_debug
+./build/fleet prepare db --dev --logging_debug