Switch preflights from Depot to our own BuildKit

Author: mattp-fly

.github/PREFLIGHT_SETUP.md

@@ -0,0 +1,53 @@
+# Preflight Test CI Setup
+
+## Required GitHub Secrets
+
+The preflight tests require the following GitHub repository secrets to be configured:
+
+### `FLYCTL_PREFLIGHT_CI_USER_TOKEN` (Required for deploy token tests)
+
+This must be a **user token** (not a limited access token) with permissions to:
+- Create apps in the `flyctl-ci-preflight` organization
+- Create deploy tokens (requires user-level permissions)
+- Manage machines, volumes, and other resources
+
+**How to create:**
+1. Log in to Fly.io with a user account that has access to the `flyctl-ci-preflight` org
+2. Run: `flyctl auth token`
+3. Copy the token (it should NOT end with `@tokens.fly.io`)
+4. Add it to GitHub Secrets as `FLYCTL_PREFLIGHT_CI_USER_TOKEN`
+
+### `FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN` (Fallback)
+
+This is the fallback token used when `FLYCTL_PREFLIGHT_CI_USER_TOKEN` is not available. It can be either a user token or a limited access token.
+
+**Current behavior:**
+- If `FLYCTL_PREFLIGHT_CI_USER_TOKEN` exists, it will be used (preferred)
+- If not, falls back to `FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN`
+- Deploy token tests will fail if neither is a user token
+
+## Why Two Tokens?
+
+We use two separate tokens for security reasons:
+
+1. **Most tests** can run with limited access tokens (more secure, limited blast radius)
+2. **Deploy token tests** require user tokens (can create other tokens)
+
+By having both, we can use the least privileged token for most operations while still supporting the full test suite.
+
+## Verifying Token Type
+
+To check if a token is a user token vs limited access token:
+
+```bash
+# Set the token
+export FLY_API_TOKEN="your-token-here"
+
+# Check the user
+flyctl auth whoami
+```
+
+**User token output:** `user@example.com` or `uuid@some-domain.com`
+**Limited access token output:** `uuid@tokens.fly.io` (ends with `@tokens.fly.io`)
+
+Deploy token tests **require** a token that does NOT end with `@tokens.fly.io`.

.github/workflows/preflight.yml

@@ -12,14 +12,25 @@ on:
 
 jobs:
   preflight-tests:
+    name: "preflight-tests (${{ matrix.group }})"
     if: ${{ github.repository == 'superfly/flyctl' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        parallelism: [20]
-        index:
-          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+        group:
+          - apps
+          - deploy
+          - launch
+          - scale
+          - volume
+          - console
+          - logs
+          - machine
+          - postgres
+          - tokens
+          - wireguard
+          - misc
     steps:
       - uses: actions/checkout@v6
       - uses: actions/setup-go@v6
@@ -53,37 +64,19 @@ jobs:
       - name: Run preflight tests
         id: preflight
         env:
-          FLY_PREFLIGHT_TEST_ACCESS_TOKEN: ${{ secrets.FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN }}
+          # Use user token if available (required for deploy token tests), otherwise fall back to limited token
+          FLY_PREFLIGHT_TEST_ACCESS_TOKEN: ${{ secrets.FLYCTL_PREFLIGHT_CI_USER_TOKEN || secrets.FLYCTL_PREFLIGHT_CI_FLY_API_TOKEN }}
           FLY_PREFLIGHT_TEST_FLY_ORG: flyctl-ci-preflight
           FLY_PREFLIGHT_TEST_FLY_REGIONS: ${{ inputs.region }}
           FLY_PREFLIGHT_TEST_NO_PRINT_HISTORY_ON_FAIL: 'true'
           FLY_FORCE_TRACE: 'true'
         run: |
           mkdir -p bin
-          if [ -e master-build/flyctl ]; then
-            mv master-build/flyctl bin/flyctl
-          fi
-          if [ -e bin/flyctl ]; then
-            chmod +x bin/flyctl
-          fi
+          (test -e master-build/flyctl) && mv master-build/flyctl bin/flyctl
+          chmod +x bin/flyctl
           export PATH=$PWD/bin:$PATH
-          test_opts=""
-          if [[ "${{ github.ref }}" != "refs/heads/master" ]]; then
-            test_opts="-short"
-          fi
-          test_log="$(mktemp)"
-          function finish {
-            rm "$test_log"
-          }
-          trap finish EXIT
-          set +e
-          go test ./test/preflight/... --tags=integration -v -timeout=15m $test_opts -run "${{ steps.test_split.outputs.run }}" | tee "$test_log"
-          test_status=$?
-          set -e
           echo -n failed= >> $GITHUB_OUTPUT
-          awk '/^--- FAIL:/{ printf("%s ", $3) }' "$test_log" >> $GITHUB_OUTPUT
-          echo >> $GITHUB_OUTPUT
-          exit $test_status
+          ./scripts/preflight.sh -r "${{ github.ref }}" -g "${{ matrix.group }}" -o $GITHUB_OUTPUT
       - name: Post failure to slack
         if: ${{ github.ref == 'refs/heads/master' && failure() }}
         uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a

internal/build/imgsrc/docker.go

@@ -301,6 +301,7 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, flapsC
 
 	if !connectOverWireguard && !wglessCompatible {
 		client := &http.Client{
+			Timeout: 30 * time.Second, // Add timeout for each request
 			Transport: &http.Transport{
 				DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 					return tls.Dial("tcp", fmt.Sprintf("%s.fly.dev:443", app.Name), &tls.Config{})
@@ -322,9 +323,29 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, flapsC
 		fmt.Fprintln(streams.Out, streams.ColorScheme().Yellow("👀 checking remote builder compatibility with wireguardless deploys ..."))
 		span.AddEvent("checking remote builder compatibility with wireguardless deploys")
 
-		res, err := client.Do(req)
+		// Retry with backoff to allow DNS propagation time
+		var res *http.Response
+		b := &backoff.Backoff{
+			Min:    2 * time.Second,
+			Max:    30 * time.Second,
+			Factor: 2,
+			Jitter: true,
+		}
+		maxRetries := 10 // Up to ~5 minutes total with backoff
+		for attempt := 0; attempt < maxRetries; attempt++ {
+			res, err = client.Do(req)
+			if err == nil {
+				break
+			}
+
+			if attempt < maxRetries-1 {
+				dur := b.Duration()
+				terminal.Debugf("Remote builder compatibility check failed (attempt %d/%d), retrying in %s (err: %v)\n", attempt+1, maxRetries, dur, err)
+				pause.For(ctx, dur)
+			}
+		}
 		if err != nil {
-			tracing.RecordError(span, err, "failed to get remote builder settings")
+			tracing.RecordError(span, err, "failed to get remote builder settings after retries")
 			return nil, err
 		}
 
@@ -594,7 +615,7 @@ func buildRemoteClientOpts(ctx context.Context, apiClient flyutil.Client, appNam
 }
 
 func waitForDaemon(parent context.Context, client *dockerclient.Client) (up bool, err error) {
-	ctx, cancel := context.WithTimeout(parent, 2*time.Minute)
+	ctx, cancel := context.WithTimeout(parent, 5*time.Minute) // 5 minutes for daemon to become responsive (includes DNS propagation time)
 	defer cancel()
 
 	b := &backoff.Backoff{

internal/build/imgsrc/ensure_builder.go

@@ -531,7 +531,7 @@ func (p *Provisioner) createBuilder(ctx context.Context, region, builderName str
 		return nil, nil, retErr
 	}
 
-	retErr = flapsClient.Wait(ctx, builderName, mach, "started", 60*time.Second)
+	retErr = flapsClient.Wait(ctx, builderName, mach, "started", 180*time.Second) // 3 minutes for machine start + DNS propagation
 	if retErr != nil {
 		tracing.RecordError(span, retErr, "error waiting for builder machine to start")
 		return nil, nil, retErr
@@ -582,7 +582,7 @@ func restartBuilderMachine(ctx context.Context, appName string, builderMachine *
 		return err
 	}
 
-	if err := flapsClient.Wait(ctx, appName, builderMachine, "started", time.Second*60); err != nil {
+	if err := flapsClient.Wait(ctx, appName, builderMachine, "started", time.Second*180); err != nil { // 3 minutes for restart + DNS propagation
 		tracing.RecordError(span, err, "error waiting for builder machine to start")
 		return err
 	}

internal/command/console/console.go

@@ -231,7 +231,11 @@ func runConsole(ctx context.Context) error {
 		consoleCommand = flag.GetString(ctx, "command")
 	}
 
-	return ssh.Console(ctx, sshClient, consoleCommand, true, params.Container)
+	// Allocate PTY only when no command is specified or when explicitly requested
+	// This matches the behavior of `fly ssh console`
+	allocPTY := consoleCommand == "" || flag.GetBool(ctx, "pty")
+
+	return ssh.Console(ctx, sshClient, consoleCommand, allocPTY, params.Container)
 }
 
 func selectMachine(ctx context.Context, app *fly.AppCompact, appConfig *appconfig.Config) (*fly.Machine, func(), error) {

internal/command/deploy/machines_deploymachinesapp.go

@@ -107,7 +107,9 @@ func (md *machineDeployment) DeployMachinesApp(ctx context.Context) error {
 
 	if updateErr := md.updateReleaseInBackend(ctx, status, metadata); updateErr != nil {
 		if err == nil {
-			err = fmt.Errorf("failed to set final release status: %w", updateErr)
+			// Deployment succeeded, but we couldn't update the release status
+			// This is not critical enough to fail the entire deployment
+			terminal.Warnf("failed to set final release status after successful deployment: %v\n", updateErr)
 		} else {
 			terminal.Warnf("failed to set final release status after deployment failure: %v\n", updateErr)
 		}

scanner/rails_dockerfile_test.go

@@ -46,13 +46,8 @@ CMD ["rails", "server"]
 		err = os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(customDockerfile), 0644)
 		require.NoError(t, err)
 
-		// Change to test directory
-		originalDir, _ := os.Getwd()
-		defer os.Chdir(originalDir)
-		err = os.Chdir(dir)
-		require.NoError(t, err)
-
 		// Run the scanner - it should detect the Rails app
+		// No need to change directories, configureRails accepts a directory path
 		si, err := configureRails(dir, &ScannerConfig{SkipHealthcheck: true})
 		drainHealthcheckChannel() // Wait for goroutine to complete before cleanup
 
@@ -89,11 +84,7 @@ CMD ["rails", "server"]`
 		err = os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(customDockerfile), 0644)
 		require.NoError(t, err)
 
-		originalDir, _ := os.Getwd()
-		defer os.Chdir(originalDir)
-		err = os.Chdir(dir)
-		require.NoError(t, err)
-
+		// No need to change directories, configureRails accepts a directory path
 		si, err := configureRails(dir, &ScannerConfig{SkipHealthcheck: true})
 		drainHealthcheckChannel() // Wait for goroutine to complete before cleanup
 		require.NoError(t, err)
@@ -123,11 +114,7 @@ CMD ["rails", "server"]`
 		err = os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(customDockerfile), 0644)
 		require.NoError(t, err)
 
-		originalDir, _ := os.Getwd()
-		defer os.Chdir(originalDir)
-		err = os.Chdir(dir)
-		require.NoError(t, err)
-
+		// No need to change directories, configureRails accepts a directory path
 		si, err := configureRails(dir, &ScannerConfig{SkipHealthcheck: true})
 		drainHealthcheckChannel() // Wait for goroutine to complete before cleanup
 		require.NoError(t, err)
@@ -150,12 +137,8 @@ CMD ["rails", "server"]`
 
 		// Note: No Dockerfile created
 
-		originalDir, _ := os.Getwd()
-		defer os.Chdir(originalDir)
-		err = os.Chdir(dir)
-		require.NoError(t, err)
-
 		// This test would need bundle to not be available, which is hard to simulate
+		// No need to change directories, configureRails accepts a directory path
 		// The scanner will either find bundle (and try to use it) or not find it
 		// If bundle is not found and no Dockerfile exists, it should fail
 
@@ -199,11 +182,7 @@ EXPOSE 3000`
 		err = os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(customDockerfile), 0644)
 		require.NoError(t, err)
 
-		originalDir, _ := os.Getwd()
-		defer os.Chdir(originalDir)
-		err = os.Chdir(dir)
-		require.NoError(t, err)
-
+		// No need to change directories, configureRails accepts a directory path
 		si, err := configureRails(dir, &ScannerConfig{SkipHealthcheck: true})
 		drainHealthcheckChannel() // Wait for goroutine to complete before cleanup
 		require.NoError(t, err)

scripts/preflight.sh

@@ -1,17 +1,22 @@
-#! /bin/bash
+#!/bin/bash
 set -euo pipefail
 
 ref=
+group=
+# Legacy support for numeric sharding (deprecated)
 total=
 index=
 out=
 
-while getopts r:t:i:o: name
+while getopts r:g:t:i:o: name
 do
     case "$name" in
         r)
 	    ref="$OPTARG"
 	    ;;
+        g)
+	    group="$OPTARG"
+	    ;;
         t)
 	    total="$OPTARG"
 	    ;;
@@ -22,7 +27,7 @@ do
 	    out="$OPTARG"
 	    ;;
         ?)
-	    printf "Usage: %s: [-r REF] [-t TOTAL] [-i INDEX] [-o FILE]\n" $0
+	    printf "Usage: %s: [-r REF] [-g GROUP] [-t TOTAL] [-i INDEX] [-o FILE]\n" $0
             exit 2
 	    ;;
     esac
@@ -43,12 +48,66 @@ trap finish EXIT
 
 set +e
 
-gotesplit \
-    -total "$total" \
-    -index "$index" \
-    github.com/superfly/flyctl/test/preflight/... \
-    -- --tags=integration -v -timeout=15m $test_opts | tee "$test_log"
-test_status=$?
+# Define test groups based on logical groupings
+if [[ -n "$group" ]]; then
+    case "$group" in
+        apps)
+            test_pattern="^TestAppsV2"
+            ;;
+        deploy)
+            test_pattern="^Test(FlyDeploy|Deploy)"
+            ;;
+        launch)
+            test_pattern="^Test(FlyLaunch|Launch)"
+            ;;
+        scale)
+            test_pattern="^TestFlyScale"
+            ;;
+        volume)
+            test_pattern="^TestVolume"
+            ;;
+        console)
+            test_pattern="^TestFlyConsole"
+            ;;
+        logs)
+            test_pattern="^TestFlyLogs"
+            ;;
+        machine)
+            test_pattern="^TestFlyMachine"
+            ;;
+        postgres)
+            test_pattern="^TestPostgres"
+            ;;
+        tokens)
+            test_pattern="^TestTokens"
+            ;;
+        wireguard)
+            test_pattern="^TestFlyWireguard"
+            ;;
+        misc)
+            test_pattern="^Test(ErrOutput|ImageLabel|NoPublicIP)"
+            ;;
+        *)
+            echo "Unknown test group: $group"
+            echo "Available groups: apps, deploy, launch, scale, volume, console, logs, machine, postgres, tokens, wireguard, misc"
+            exit 1
+            ;;
+    esac
+
+    go test -tags=integration -v -timeout=15m $test_opts -run "$test_pattern" github.com/superfly/flyctl/test/preflight/... | tee "$test_log"
+    test_status=$?
+# Legacy numeric sharding using gotesplit (deprecated)
+elif [[ -n "$total" && -n "$index" ]]; then
+    gotesplit \
+        -total "$total" \
+        -index "$index" \
+        github.com/superfly/flyctl/test/preflight/... \
+        -- --tags=integration -v -timeout=15m $test_opts | tee "$test_log"
+    test_status=$?
+else
+    echo "Error: Must specify either -g GROUP or both -t TOTAL and -i INDEX"
+    exit 1
+fi
 
 set -e