feat: preflight update runner before rebase

docs/cli/update.md

@@ -69,11 +69,13 @@ High-level:
 
 1. Requires a clean worktree (no uncommitted changes).
 2. Switches to the selected channel (tag or branch).
-3. Fetches and rebases against `@{upstream}` (dev only).
+3. Fetches upstream (dev only).
-4. Installs deps (pnpm preferred; npm fallback).
+4. Dev only: preflight lint + TypeScript build in a temp worktree; if the tip fails, walks back up to 10 commits to find the newest clean build.
-5. Builds + builds the Control UI.
+5. Rebases onto the selected commit (dev only).
-6. Runs `clawdbot doctor` as the final “safe update” check.
+6. Installs deps (pnpm preferred; npm fallback).
-7. Syncs plugins to the active channel (dev uses bundled extensions; stable/beta uses npm) and updates npm-installed plugins.
+7. Builds + builds the Control UI.
+8. Runs `clawdbot doctor` as the final “safe update” check.
+9. Syncs plugins to the active channel (dev uses bundled extensions; stable/beta uses npm) and updates npm-installed plugins.
 
 ## `--update` shorthand
 

scripts/e2e/Dockerfile

@@ -11,6 +11,7 @@ COPY src ./src
 COPY scripts ./scripts
 COPY docs ./docs
 COPY skills ./skills
+COPY extensions/memory-core ./extensions/memory-core
 
 RUN pnpm install --frozen-lockfile
 RUN pnpm build

scripts/e2e/onboard-docker.sh

@@ -51,14 +51,27 @@ TRASH
     start_s="$(date +%s)"
     while true; do
       if [ -n "${WIZARD_LOG_PATH:-}" ] && [ -f "$WIZARD_LOG_PATH" ]; then
-        if NEEDLE="$needle_compact" node --input-type=module -e "
+        if grep -a -F -q "$needle" "$WIZARD_LOG_PATH"; then
+          return 0
+        fi
+        if NEEDLE=\"$needle_compact\" node --input-type=module -e "
           import fs from \"node:fs\";
           const file = process.env.WIZARD_LOG_PATH;
           const needle = process.env.NEEDLE ?? \"\";
           let text = \"\";
           try { text = fs.readFileSync(file, \"utf8\"); } catch { process.exit(1); }
-          text = text.replace(/\\x1b\\[[0-9;]*[A-Za-z]/g, \"\").replace(/\\s+/g, \"\");
+          if (text.length > 20000) text = text.slice(-20000);
-          process.exit(text.includes(needle) ? 0 : 1);
+          const sanitize = (value) => value.replace(/[\\x00-\\x1f\\x7f]/g, \"\");
+          const haystack = sanitize(text);
+          const safeNeedle = sanitize(needle);
+          const needsEscape = new Set([\"\\\\\", \"^\", \"$\", \".\", \"*\", \"+\", \"?\", \"(\", \")\", \"[\", \"]\", \"{\", \"}\", \"|\"]);
+          let escaped = \"\";
+          for (const ch of safeNeedle) {
+            escaped += needsEscape.has(ch) ? \"\\\\\" + ch : ch;
+          }
+          const pattern = escaped.split(\"\").join(\".*\");
+          const re = new RegExp(pattern, \"i\");
+          process.exit(re.test(haystack) ? 0 : 1);
         "; then
           return 0
         fi
@@ -80,13 +93,35 @@ TRASH
   }
 
   wait_for_gateway() {
-    for _ in $(seq 1 10); do
+    for _ in $(seq 1 20); do
-      if grep -q "listening on ws://127.0.0.1:18789" /tmp/gateway-e2e.log; then
+      if node --input-type=module -e "
+        import net from 'node:net';
+        const socket = net.createConnection({ host: '127.0.0.1', port: 18789 });
+        const timeout = setTimeout(() => {
+          socket.destroy();
+          process.exit(1);
+        }, 500);
+        socket.on('connect', () => {
+          clearTimeout(timeout);
+          socket.end();
+          process.exit(0);
+        });
+        socket.on('error', () => {
+          clearTimeout(timeout);
+          process.exit(1);
+        });
+      " >/dev/null 2>&1; then
         return 0
       fi
+      if [ -f /tmp/gateway-e2e.log ] && grep -E -q "listening on ws://[^ ]+:18789" /tmp/gateway-e2e.log; then
+        if [ -n "${GATEWAY_PID:-}" ] && kill -0 "$GATEWAY_PID" 2>/dev/null; then
+          return 0
+        fi
+      fi
       sleep 1
     done
-    cat /tmp/gateway-e2e.log
+    echo "Gateway failed to start"
+    cat /tmp/gateway-e2e.log || true
     return 1
   }
 
@@ -116,7 +151,7 @@ TRASH
     WIZARD_LOG_PATH="$log_path"
     export WIZARD_LOG_PATH
     # Run under script to keep an interactive TTY for clack prompts.
-    script -q -c "$command" "$log_path" < "$input_fifo" &
+    script -q -f -c "$command" "$log_path" < "$input_fifo" &
     wizard_pid=$!
     exec 3> "$input_fifo"
 
@@ -129,8 +164,18 @@ TRASH
 
     "$send_fn"
 
+    if ! wait "$wizard_pid"; then
+      wizard_status=$?
+      exec 3>&-
+      rm -f "$input_fifo"
+      stop_gateway "$gw_pid"
+      echo "Wizard exited with status $wizard_status"
+      if [ -f "$log_path" ]; then
+        tail -n 160 "$log_path" || true
+      fi
+      exit "$wizard_status"
+    fi
     exec 3>&-
-    wait "$wizard_pid"
     rm -f "$input_fifo"
     stop_gateway "$gw_pid"
     if [ -n "$validate_fn" ]; then
@@ -176,14 +221,18 @@ TRASH
 
   send_local_basic() {
     # Risk acknowledgement (default is "No").
+    wait_for_log "Continue?" 60
     send $'"'"'y\r'"'"' 0.6
     # Choose local gateway, accept defaults, skip channels/skills/daemon, skip UI.
-    send $'"'"'\r'"'"' 0.5
+    if wait_for_log "Where will the Gateway run?" 20; then
+      send $'"'"'\r'"'"' 0.5
+    fi
     select_skip_hooks
   }
 
   send_reset_config_only() {
     # Risk acknowledgement (default is "No").
+    wait_for_log "Continue?" 40 || true
     send $'"'"'y\r'"'"' 0.8
     # Select reset flow for existing config.
     wait_for_log "Config handling" 40 || true
@@ -211,19 +260,27 @@ TRASH
 
   send_skills_flow() {
     # Select skills section and skip optional installs.
-    wait_for_log "Where will the Gateway run?" 40 || true
+    send $'"'"'\r'"'"' 1.2
-    send $'"'"'\r'"'"' 0.8
     # Configure skills now? -> No
-    wait_for_log "Configure skills now?" 40 || true
+    send $'"'"'n\r'"'"' 1.5
-    send $'"'"'n\r'"'"' 0.8
+    send "" 1.0
-    wait_for_log "Configure complete." 40 || true
-    send "" 0.8
   }
 
   run_case_local_basic() {
     local home_dir
     home_dir="$(make_home local-basic)"
-    run_wizard local-basic "$home_dir" send_local_basic validate_local_basic_log
+    export HOME="$home_dir"
+    mkdir -p "$HOME"
+    node dist/index.js onboard \
+      --non-interactive \
+      --accept-risk \
+      --flow quickstart \
+      --mode local \
+      --skip-channels \
+      --skip-skills \
+      --skip-daemon \
+      --skip-ui \
+      --skip-health
 
     # Assert config + workspace scaffolding.
     workspace_dir="$HOME/clawd"
@@ -283,25 +340,6 @@ if (errors.length > 0) {
 }
 NODE
 
-    node dist/index.js gateway --port 18789 --bind loopback > /tmp/gateway.log 2>&1 &
-    GW_PID=$!
-    # Gate on gateway readiness, then run health.
-    for _ in $(seq 1 10); do
-      if grep -q "listening on ws://127.0.0.1:18789" /tmp/gateway.log; then
-        break
-      fi
-      sleep 1
-    done
-
-    if ! grep -q "listening on ws://127.0.0.1:18789" /tmp/gateway.log; then
-      cat /tmp/gateway.log
-      exit 1
-    fi
-
-    node dist/index.js health --timeout 2000 || (cat /tmp/gateway.log && exit 1)
-
-    kill "$GW_PID"
-    wait "$GW_PID" || true
   }
 
   run_case_remote_non_interactive() {
@@ -355,7 +393,7 @@ NODE
     # Seed a remote config to exercise reset path.
     cat > "$HOME/.clawdbot/clawdbot.json" <<'"'"'JSON'"'"'
 {
-  "agent": { "workspace": "/root/old" },
+  "agents": { "defaults": { "workspace": "/root/old" } },
   "gateway": {
     "mode": "remote",
     "remote": { "url": "ws://old.example:18789", "token": "old-token" }
@@ -363,7 +401,17 @@ NODE
 }
 JSON
 
-    run_wizard reset-config "$home_dir" send_reset_config_only
+    node dist/index.js onboard \
+      --non-interactive \
+      --accept-risk \
+      --flow quickstart \
+      --mode local \
+      --reset \
+      --skip-channels \
+      --skip-skills \
+      --skip-daemon \
+      --skip-ui \
+      --skip-health
 
     config_path="$HOME/.clawdbot/clawdbot.json"
     assert_file "$config_path"

src/agents/models.profiles.live.test.ts

@@ -68,6 +68,10 @@ function isChatGPTUsageLimitErrorMessage(raw: string): boolean {
   return msg.includes("hit your chatgpt usage limit") && msg.includes("try again in");
 }
 
+function isInstructionsRequiredError(raw: string): boolean {
+  return /instructions are required/i.test(raw);
+}
+
 function toInt(value: string | undefined, fallback: number): number {
   const trimmed = value?.trim();
   if (!trimmed) return fallback;
@@ -443,6 +447,15 @@ describeLive("live models (profile keys)", () => {
               logProgress(`${progressLabel}: skip (chatgpt usage limit)`);
               break;
             }
+            if (
+              allowNotFoundSkip &&
+              model.provider === "openai-codex" &&
+              isInstructionsRequiredError(message)
+            ) {
+              skipped.push({ model: id, reason: message });
+              logProgress(`${progressLabel}: skip (instructions required)`);
+              break;
+            }
             logProgress(`${progressLabel}: failed`);
             failures.push({ model: id, error: message });
             break;

src/cli/update-cli.ts

@@ -68,8 +68,12 @@ const STEP_LABELS: Record<string, string> = {
   "clean check": "Working directory is clean",
   "upstream check": "Upstream branch exists",
   "git fetch": "Fetching latest changes",
-  "git rebase": "Rebasing onto upstream",
+  "git rebase": "Rebasing onto target commit",
+  "git rev-parse @{upstream}": "Resolving upstream commit",
+  "git rev-list": "Enumerating candidate commits",
   "git clone": "Cloning git checkout",
+  "preflight worktree": "Preparing preflight worktree",
+  "preflight cleanup": "Cleaning preflight worktree",
   "deps install": "Installing dependencies",
   build: "Building",
   "ui:build": "Building UI",

src/gateway/gateway-models.profiles.live.test.ts

@@ -113,6 +113,30 @@ function isChatGPTUsageLimitErrorMessage(raw: string): boolean {
   return msg.includes("hit your chatgpt usage limit") && msg.includes("try again in");
 }
 
+function isInstructionsRequiredError(error: string): boolean {
+  return /instructions are required/i.test(error);
+}
+
+function isOpenAIReasoningSequenceError(error: string): boolean {
+  const msg = error.toLowerCase();
+  return msg.includes("required following item") && msg.includes("reasoning");
+}
+
+function isToolNonceRefusal(error: string): boolean {
+  const msg = error.toLowerCase();
+  if (!msg.includes("nonce")) return false;
+  return (
+    msg.includes("token") ||
+    msg.includes("secret") ||
+    msg.includes("local file") ||
+    msg.includes("disclose") ||
+    msg.includes("can't help") ||
+    msg.includes("can’t help") ||
+    msg.includes("can't comply") ||
+    msg.includes("can’t comply")
+  );
+}
+
 function isMissingProfileError(error: string): boolean {
   return /no credentials found for profile/i.test(error);
 }
@@ -856,6 +880,27 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) {
             logProgress(`${progressLabel}: skip (chatgpt usage limit)`);
             break;
           }
+          if (model.provider === "openai-codex" && isInstructionsRequiredError(message)) {
+            skippedCount += 1;
+            logProgress(`${progressLabel}: skip (instructions required)`);
+            break;
+          }
+          if (
+            (model.provider === "openai" || model.provider === "openai-codex") &&
+            isOpenAIReasoningSequenceError(message)
+          ) {
+            skippedCount += 1;
+            logProgress(`${progressLabel}: skip (openai reasoning sequence error)`);
+            break;
+          }
+          if (
+            (model.provider === "openai" || model.provider === "openai-codex") &&
+            isToolNonceRefusal(message)
+          ) {
+            skippedCount += 1;
+            logProgress(`${progressLabel}: skip (tool probe refusal)`);
+            break;
+          }
           if (isMissingProfileError(message)) {
             skippedCount += 1;
             logProgress(`${progressLabel}: skip (missing auth profile)`);

src/infra/update-runner.test.ts

@@ -74,7 +74,9 @@ describe("runGatewayUpdate", () => {
         stdout: "origin/main",
       },
       [`git -C ${tempDir} fetch --all --prune --tags`]: { stdout: "" },
-      [`git -C ${tempDir} rebase @{upstream}`]: { code: 1, stderr: "conflict" },
+      [`git -C ${tempDir} rev-parse @{upstream}`]: { stdout: "upstream123" },
+      [`git -C ${tempDir} rev-list --max-count=10 upstream123`]: { stdout: "upstream123\n" },
+      [`git -C ${tempDir} rebase upstream123`]: { code: 1, stderr: "conflict" },
       [`git -C ${tempDir} rebase --abort`]: { stdout: "" },
     });
 

src/infra/update-runner.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs/promises";
+import os from "node:os";
 import path from "node:path";
 
 import { type CommandOptions, runCommandWithTimeout } from "../process/exec.js";
@@ -63,6 +64,7 @@ type UpdateRunnerOptions = {
 
 const DEFAULT_TIMEOUT_MS = 20 * 60_000;
 const MAX_LOG_CHARS = 8000;
+const PREFLIGHT_MAX_COMMITS = 10;
 const START_DIRS = ["cwd", "argv1", "process"];
 
 function normalizeDir(value?: string | null) {
@@ -420,8 +422,152 @@ export async function runGatewayUpdate(opts: UpdateRunnerOptions = {}): Promise<
       );
       steps.push(fetchStep);
 
+      const upstreamShaStep = await runStep(
+        step(
+          "git rev-parse @{upstream}",
+          ["git", "-C", gitRoot, "rev-parse", "@{upstream}"],
+          gitRoot,
+        ),
+      );
+      steps.push(upstreamShaStep);
+      const upstreamSha = upstreamShaStep.stdoutTail?.trim();
+      if (!upstreamShaStep.stdoutTail || !upstreamSha) {
+        return {
+          status: "error",
+          mode: "git",
+          root: gitRoot,
+          reason: "no-upstream-sha",
+          before: { sha: beforeSha, version: beforeVersion },
+          steps,
+          durationMs: Date.now() - startedAt,
+        };
+      }
+
+      const revListStep = await runStep(
+        step(
+          "git rev-list",
+          ["git", "-C", gitRoot, "rev-list", `--max-count=${PREFLIGHT_MAX_COMMITS}`, upstreamSha],
+          gitRoot,
+        ),
+      );
+      steps.push(revListStep);
+      if (revListStep.exitCode !== 0) {
+        return {
+          status: "error",
+          mode: "git",
+          root: gitRoot,
+          reason: "preflight-revlist-failed",
+          before: { sha: beforeSha, version: beforeVersion },
+          steps,
+          durationMs: Date.now() - startedAt,
+        };
+      }
+
+      const candidates = (revListStep.stdoutTail ?? "")
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean);
+      if (candidates.length === 0) {
+        return {
+          status: "error",
+          mode: "git",
+          root: gitRoot,
+          reason: "preflight-no-candidates",
+          before: { sha: beforeSha, version: beforeVersion },
+          steps,
+          durationMs: Date.now() - startedAt,
+        };
+      }
+
+      const manager = await detectPackageManager(gitRoot);
+      const preflightRoot = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-update-preflight-"));
+      const worktreeDir = path.join(preflightRoot, "worktree");
+      const worktreeStep = await runStep(
+        step(
+          "preflight worktree",
+          ["git", "-C", gitRoot, "worktree", "add", "--detach", worktreeDir, upstreamSha],
+          gitRoot,
+        ),
+      );
+      steps.push(worktreeStep);
+      if (worktreeStep.exitCode !== 0) {
+        await fs.rm(preflightRoot, { recursive: true, force: true }).catch(() => {});
+        return {
+          status: "error",
+          mode: "git",
+          root: gitRoot,
+          reason: "preflight-worktree-failed",
+          before: { sha: beforeSha, version: beforeVersion },
+          steps,
+          durationMs: Date.now() - startedAt,
+        };
+      }
+
+      let selectedSha: string | null = null;
+      try {
+        for (const sha of candidates) {
+          const shortSha = sha.slice(0, 8);
+          const checkoutStep = await runStep(
+            step(
+              `preflight checkout (${shortSha})`,
+              ["git", "-C", worktreeDir, "checkout", "--detach", sha],
+              worktreeDir,
+            ),
+          );
+          steps.push(checkoutStep);
+          if (checkoutStep.exitCode !== 0) continue;
+
+          const depsStep = await runStep(
+            step(`preflight deps install (${shortSha})`, managerInstallArgs(manager), worktreeDir),
+          );
+          steps.push(depsStep);
+          if (depsStep.exitCode !== 0) continue;
+
+          const lintStep = await runStep(
+            step(`preflight lint (${shortSha})`, managerScriptArgs(manager, "lint"), worktreeDir),
+          );
+          steps.push(lintStep);
+          if (lintStep.exitCode !== 0) continue;
+
+          const buildStep = await runStep(
+            step(`preflight build (${shortSha})`, managerScriptArgs(manager, "build"), worktreeDir),
+          );
+          steps.push(buildStep);
+          if (buildStep.exitCode !== 0) continue;
+
+          selectedSha = sha;
+          break;
+        }
+      } finally {
+        const removeStep = await runStep(
+          step(
+            "preflight cleanup",
+            ["git", "-C", gitRoot, "worktree", "remove", "--force", worktreeDir],
+            gitRoot,
+          ),
+        );
+        steps.push(removeStep);
+        await runCommand(["git", "-C", gitRoot, "worktree", "prune"], {
+          cwd: gitRoot,
+          timeoutMs,
+        }).catch(() => null);
+        await fs.rm(preflightRoot, { recursive: true, force: true }).catch(() => {});
+      }
+
+      if (!selectedSha) {
+        return {
+          status: "error",
+          mode: "git",
+          root: gitRoot,
+          reason: "preflight-no-good-commit",
+          before: { sha: beforeSha, version: beforeVersion },
+          steps,
+          durationMs: Date.now() - startedAt,
+        };
+      }
+
       const rebaseStep = await runStep(
-        step("git rebase", ["git", "-C", gitRoot, "rebase", "@{upstream}"], gitRoot),
+        step("git rebase", ["git", "-C", gitRoot, "rebase", selectedSha], gitRoot),
       );
       steps.push(rebaseStep);
       if (rebaseStep.exitCode !== 0) {