Run go tests with -race at night (#4090)

* Run go tests with race at night

* Add missing run on step

* Fix boolean logic

* Allow manual run for test-go.yaml

* Add slack notification step

* Try global environment variable for cron schedule

* Fix indentation

* Try number 2

* Try setting cron as usual

* Remove global env

* Only send notification in case of failure when running schedule

* Run with race enable to test

* Add more fixes

* Fix github event variable name

* Set timeouts

* Fix slack notification link

* Re-enable if clause

* Last try on Github Actions

* Re-enable the if clause

.github/workflows/test-go.yaml

@@ -11,6 +11,8 @@ on:
     paths:
       - '**.go'
   workflow_dispatch: # Manual
+  schedule:
+    - cron: '0 4 * * *'
 
 jobs:
   test-go:
@@ -21,6 +23,10 @@ jobs:
         mysql: ["mysql:5.7", "mysql:8"]
     runs-on: ${{ matrix.os }}
 
+    env:
+      RACE_ENABLED: false
+      GO_TEST_TIMEOUT: 10m
+
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
@@ -44,11 +50,39 @@ jobs:
         export PATH=$PATH:~/go/bin
         make generate-go
 
+    - name: Set Go race setting on schedule
+      if: github.event.schedule == '0 4 * * *'
+      run: echo "Running go tests with race enabled"
+      env:
+        RACE_ENABLED: true
+        GO_TEST_TIMEOUT: 1h
+
     - name: Run Go Tests
       run: |
-        NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 make test-go
+        NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 RACE_ENABLED=$RACE_ENABLED GO_TEST_TIMEOUT=$GO_TEST_TIMEOUT make test-go
 
     - name: Upload to Codecov
       uses: codecov/codecov-action@v2
       with:
         files: coverage.txt
+
+    - name: Slack Notification
+      if: github.event.schedule == '0 4 * * *' && failure()
+      uses: slackapi/slack-github-action@v1.18.0
+      with:
+        payload: |
+          {
+            "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Go tests result: ${{ job.status }}\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}"
+                }
+              }
+            ]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CORE_ENGINEERING_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK