Orbit to wait for osqueryd extension manager socket to be ready (#3836)

* Wait for osquery extension manager socket to be ready

* PR review feedback

* Add WIP Github Action to test orbit

* Set fleet address on fleetctl

* Add logging for troubleshooting

* Add prepare db statement

* Use tls-skip-verify on fleetctl

* Move steps around

* Fix addresses

* Fix fleetctl path

* Add certs.pem to orbit root dir

* Add orbit logs

* Increase timeout

* Add proper orbit log paths

* Fix tmp path orbit

* Add get hosts command to troubleshoot

* Fix orbit job termination

* Add comments to workflows

* Wait for server to go down

* Add orbit Windows job

* Use bash on windws-latest

* Fix missing quote

* Run orbit on Ubuntu

* Bump host count

* Increase timeout for extensions and not terminate on err

* Add comment to clarify high timeout value

* Revert change, we do want to exit in case of err

.github/workflows/fleet-and-orbit.yml

@@ -0,0 +1,254 @@
+name: Test Orbit & Fleet
+
+# This workflow tests orbit code changes (compiles and runs orbit from source).
+# It uses a fleet instance also built and executed from source.
+#
+# It tests that orbit osquery agents enroll successfully to Fleet.
+
+on:
+  push:
+    branches:
+      - main
+      - patch-*
+    paths:
+      - 'orbit/**.go'
+  pull_request:
+    paths:
+      - 'orbit/**.go'
+  workflow_dispatch: # Manual
+
+jobs:
+  gen:
+    runs-on: ubuntu-latest
+    outputs:
+      subdomain: ${{ steps.gen.outputs.subdomain }}
+      domain: ${{ steps.gen.outputs.domain }}
+      address: ${{ steps.gen.outputs.address }}
+    steps:
+    - id: gen
+      run: |
+        UUID=$(uuidgen)
+        echo "::set-output name=subdomain::fleet-test-$UUID"
+        echo "::set-output name=domain::fleet-test-$UUID.fleetuem.com"
+        echo "::set-output name=address::https://fleet-test-$UUID.fleetuem.com"
+  
+  run-server:
+    strategy:
+      matrix:
+        go-version: ['^1.17.0']
+        mysql: ['mysql:5.7']
+    runs-on: ubuntu-latest
+    needs: gen
+    steps:
+
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Start tunnel
+      env: 
+        CERT_PEM: ${{ secrets.CLOUDFLARE_TUNNEL_FLEETUEM_CERT_B64 }}
+      run: |
+        # Install cloudflared
+        wget https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb
+        sudo dpkg -i cloudflared-linux-amd64.deb
+        # Add secret
+        echo "$CERT_PEM" | base64 -d > cert.pem
+        # Start tunnel
+        cloudflared tunnel --origincert cert.pem --hostname ${{ needs.gen.outputs.subdomain }} --url http://localhost:1337 --name ${{ needs.gen.outputs.subdomain }} &
+        until [ $(cloudflared tunnel --origincert cert.pem info -o json ${{ needs.gen.outputs.subdomain }} | jq '.conns[0].conns[0].is_pending_reconnect') = false ]; do
+          echo "Awaiting tunnel ready..."
+          sleep 5
+        done
+
+    - name: Start Infra Dependencies
+      run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker-compose up -d mysql redis &
+
+    - name: Install JS Dependencies
+      run: make deps-js
+
+    - name: Generate and bundle go & js code
+      run: make generate
+
+    - name: Build fleet and fleetctl
+      run: make fleet fleetctl
+
+    - name: Run Fleet server
+      timeout-minutes: 10
+      env:
+        FLEET_OSQUERY_HOST_IDENTIFIER: instance # use instance identifier to allow for duplicate UUIDs
+        FLEET_SERVER_ADDRESS: 0.0.0.0:1337
+        FLEET_SERVER_TLS: false
+        FLEET_LOGGING_DEBUG: true
+      run: |
+        mkdir ./fleet_log
+        make db-reset
+        ./build/fleet serve --dev --dev_license 1>./fleet_log/stdout.log 2>./fleet_log/stderr.log &
+        ./build/fleetctl config set --address http://localhost:1337 --tls-skip-verify
+        until ./build/fleetctl setup --email admin@example.com --name Admin --password admin123# --org-name Example
+        do
+          echo "Retrying setup in 5s..."
+          sleep 5
+        done
+        # Wait for all of the hosts to be enrolled
+        EXPECTED=3
+        until [ $(./build/fleetctl get hosts --json | grep "hostname" | wc -l | tee hostcount) -ge $EXPECTED ]; do
+          echo -n "Waiting for hosts to enroll: "
+          cat hostcount | xargs echo -n
+          echo " / $EXPECTED"
+          sleep 10
+        done
+        ./build/fleetctl get hosts
+        echo "Success! $EXPECTED hosts enrolled."
+
+    - name: Cleanup tunnel
+      if: always()
+      run: cloudflared tunnel --origincert cert.pem delete --force ${{ needs.gen.outputs.subdomain }}
+
+    - name: Upload fleet logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: fleet-logs
+        path: |
+          fleet_log
+
+  get-enroll-secret:
+    strategy:
+      matrix:
+        go-version: ['^1.17.0']
+    runs-on: ubuntu-latest
+    needs: gen
+    outputs:
+      enroll_secret: ${{ steps.enroll.outputs.enroll_secret }}
+    steps:
+
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Build Fleetctl
+      run: make fleetctl
+
+    - id: enroll
+      name: Fetch enroll secret
+      timeout-minutes: 10
+      run: |
+        ./build/fleetctl config set --address ${{ needs.gen.outputs.address }}
+        until ./build/fleetctl login --email admin@example.com --password admin123#
+        do
+          echo "Retrying in 10s..."
+          sleep 10
+        done
+        SECRET_JSON=$(./build/fleetctl get enroll_secret --json --debug)
+        echo $SECRET_JSON
+        SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret')
+        echo "::set-output name=enroll_secret::$SECRET"
+      
+  orbit-macos-and-ubuntu:
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        # TODO(lucas): Add edge channel for osqueryd.
+        osqueryd-channel: ['stable']
+        go-version: ['^1.17.0']
+        os: ['macos-latest', 'ubuntu-latest']
+    runs-on: ${{ matrix.os }}
+    needs: [gen, get-enroll-secret]
+    steps:
+
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Build and Run Orbit
+      run: |
+        sudo hostname ${{ matrix.os }}-orbit-dev-osqueryd-${{ matrix.osqueryd-channel }}
+        echo "Hostname: $(hostname -s)"
+        mkdir /tmp/orbit
+        cp ./orbit/pkg/packaging/certs.pem /tmp/orbit
+        mkdir orbit_logs
+        go run github.com/fleetdm/fleet/v4/orbit/cmd/orbit \
+          --dev-mode \
+          --disable-updates \
+          --root-dir /tmp/orbit \
+          --fleet-url ${{ needs.gen.outputs.address }} \
+          --enroll-secret ${{ needs.get-enroll-secret.outputs.enroll_secret }} \
+          --osqueryd-channel ${{ matrix.osqueryd-channel }} \
+          -- --verbose 1>./orbit_logs/stdout.log 2>./orbit_logs/stderr.log &
+        # TODO(lucas): Improve checking of "enrolled".
+        # This waits until the server goes down.
+        while curl --fail ${{ needs.gen.outputs.address }};
+        do
+          echo "Retrying in 10s..."
+          sleep 10
+        done
+
+    - name: Upload orbit logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: orbit-${{ matrix.os }}-logs-${{ matrix.osqueryd-channel }}
+        path: |
+          orbit_logs
+
+  orbit-windows:
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        # TODO(lucas): Add edge channel for osqueryd.
+        osqueryd-channel: ['stable']
+        go-version: ['^1.17.0']
+    needs: [gen, get-enroll-secret]
+    runs-on: windows-latest
+    steps:
+    
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Build and Run Orbit
+      shell: bash
+      run: |
+        mkdir "/C/Program Files/Orbit"
+        cp ./orbit/pkg/packaging/certs.pem "/C/Program Files/Orbit"
+        mkdir orbit_logs
+        go run github.com/fleetdm/fleet/v4/orbit/cmd/orbit \
+          --dev-mode \
+          --disable-updates \
+          --root-dir "/C/Program Files/Orbit" \
+          --fleet-url ${{ needs.gen.outputs.address }} \
+          --enroll-secret ${{ needs.get-enroll-secret.outputs.enroll_secret }} \
+          --osqueryd-channel ${{ matrix.osqueryd-channel }} \
+          -- --verbose 1>./orbit_logs/stdout.log 2>./orbit_logs/stderr.log &
+        # TODO(lucas): Improve checking of "enrolled".
+        # This waits until the server goes down.
+        while curl --fail ${{ needs.gen.outputs.address }};
+        do
+          echo "Retrying in 10s..."
+          sleep 10
+        done
+
+    - name: Upload orbit logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: orbit-windows-logs-${{ matrix.osqueryd-channel }}
+        path: |
+          orbit_logs

.github/workflows/integration.yml

@@ -1,4 +1,11 @@
-name: Test Orbit & Preview
+name: Test Fleetctl, Orbit & Preview
+
+# This workflow tests enrolling of agents on the supported platforms,
+# using the latest version of fleet, fleetctl and orbit.
+#
+# It starts the latest release of fleet with the "fleetctl preview" command.
+# It generates the installers for the latest version of Orbit with the
+# "fleetctl package" command.
 
 on:
   workflow_dispatch: # Manual

orbit/changes/3679-orbit-wait-for-osquery-ext-manager-socket

@@ -0,0 +1 @@
+* Orbit to wait for osqueryd's extension manager socket to be ready.

orbit/cmd/orbit/orbit.go

@@ -396,11 +396,8 @@ func main() {
 		}
 		g.Add(r.Execute, r.Interrupt)
 
-		if runtime.GOOS != "windows" {
-			// We are disabling extensions for Windows until #3679 is fixed.
-			ext := table.NewRunner(r.ExtensionSocketPath())
-			g.Add(ext.Execute, ext.Interrupt)
-		}
+		ext := table.NewRunner(r.ExtensionSocketPath())
+		g.Add(ext.Execute, ext.Interrupt)
 
 		// Install a signal handler
 		ctx, cancel := context.WithCancel(context.Background())

orbit/pkg/table/extension.go

@@ -2,6 +2,7 @@ package table
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -31,7 +32,7 @@ func NewRunner(socket string) *Runner {
 func (r *Runner) Execute() error {
 	log.Debug().Msg("start osquery extension")
 
-	if err := waitForSocket(r.socket, 1*time.Minute); err != nil {
+	if err := waitExtensionSocket(r.socket, 1*time.Minute); err != nil {
 		return err
 	}
 
@@ -44,7 +45,12 @@ func (r *Runner) Execute() error {
 		r.srv, err = osquery.NewExtensionManagerServer(
 			"com.fleetdm.orbit.osquery_extension.v1",
 			r.socket,
-			osquery.ServerTimeout(3*time.Second))
+			// This timeout is only used for registering the extension tables
+			// and for the heartbeat ping requests in r.srv.Run().
+			//
+			// On some systems, registering tables takes more than a couple
+			// of seconds, thus set timeout to minutes instead (see #3878).
+			osquery.ServerTimeout(5*time.Minute))
 		if err == nil {
 			ticker.Stop()
 			break
@@ -85,19 +91,33 @@ func (r *Runner) Interrupt(err error) {
 	}
 }
 
-// waitForSocket waits for the osquery socket to exist.
+// waitExtensionSocket waits until the osquery extension manager socket is ready.
+// First, it waits for the unix socket/Windows pipe to be available.
+// Then, it tries connecting as a client and sending a ping to ensure that osquery
+// is ready for extensions.
 //
-// This method was copied from osquery-go because we can't rely on option.ServerTimeout.
-// Such timeout is used both for waiting for the socket and for the thrift transport.
-func waitForSocket(sockPath string, timeout time.Duration) error {
-	ticker := time.NewTicker(200 * time.Millisecond)
-	defer ticker.Stop()
+// This method is a workaround for https://github.com/osquery/osquery-go/issues/80.
+func waitExtensionSocket(sockPath string, timeout time.Duration) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
+
+	if err := waitSocketExists(ctx, sockPath); err != nil {
+		return err
+	}
+	if err := waitSocketReady(ctx, sockPath); err != nil {
+		return err
+	}
+	return nil
+}
+
+func waitSocketExists(ctx context.Context, sockPath string) error {
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():
-			return ctx.Err()
+			return errors.New("extension socket stat timeout")
 		case <-ticker.C:
 			switch _, err := os.Stat(sockPath); {
 			case err == nil:
@@ -110,3 +130,36 @@ func waitForSocket(sockPath string, timeout time.Duration) error {
 		}
 	}
 }
+
+func waitSocketReady(ctx context.Context, sockPath string) error {
+	serverClient, err := osquery.NewClient(sockPath, 3*time.Second)
+	if err != nil {
+		return err
+	}
+	defer serverClient.Close()
+
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return errors.New("extension socket ping timeout")
+		case <-ticker.C:
+			status, err := serverClient.Ping()
+			if err != nil {
+				log.Debug().Err(err).Msgf(
+					"failed to ping extension socket, retrying...",
+				)
+				continue
+			}
+			if status.Code == 0 {
+				log.Debug().Msg("extension manager checked")
+				return nil
+			}
+			log.Debug().Int32("status_code", status.Code).Str("status_message", status.Message).Msgf(
+				"extension socket not ready, retrying...",
+			)
+		}
+	}
+}

tools/backup_db/backup.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+docker run --rm --network fleet_default mysql:5.7 bash -c 'mysqldump -hmysql -uroot -ptoor fleet | gzip -' > backup.sql.gz