fix(recovery): show backgrounded command when gateway restart fails (#2426)

src/lib/agent-runtime.test.ts

@@ -3,7 +3,11 @@
 
 import { describe, it, expect } from "vitest";
 // Import from compiled dist/ so coverage is attributed correctly.
-import { buildOpenClawRecoveryScript, buildRecoveryScript } from "../../dist/lib/agent-runtime";
+import {
+  buildManualRecoveryCommand,
+  buildOpenClawRecoveryScript,
+  buildRecoveryScript,
+} from "../../dist/lib/agent-runtime";
 import type { AgentDefinition } from "./agent-defs";
 
 function makeAgent(overrides: Partial<AgentDefinition> = {}): AgentDefinition {
@@ -257,3 +261,67 @@ describe("buildRecoveryScript", () => {
     });
   });
 });
+
+describe("buildManualRecoveryCommand (#2426)", () => {
+  const hermesAgent = {
+    name: "hermes",
+    displayName: "Hermes Agent",
+    gateway_command: "hermes gateway run",
+  } as unknown as AgentDefinition;
+
+  it("backgrounds the process with nohup and '&'", () => {
+    const cmd = buildManualRecoveryCommand(minimalAgent, 19000);
+    expect(cmd.startsWith("nohup ")).toBe(true);
+    expect(cmd.endsWith(" &")).toBe(true);
+  });
+
+  it("embeds the port, matching buildRecoveryScript", () => {
+    const cmd = buildManualRecoveryCommand(minimalAgent, 19000);
+    expect(cmd).toContain("--port 19000");
+  });
+
+  it("redirects stdout and stderr to /tmp/gateway.log", () => {
+    const cmd = buildManualRecoveryCommand(minimalAgent, 19000);
+    expect(cmd).toContain(">/tmp/gateway.log 2>&1");
+  });
+
+  it("prefixes HERMES_HOME for the hermes agent so the gateway can find its config", () => {
+    const cmd = buildManualRecoveryCommand(hermesAgent, 8642);
+    expect(cmd).toContain("HERMES_HOME=/sandbox/.hermes-data");
+    expect(cmd).toContain("nohup hermes gateway run >/tmp/gateway.log");
+  });
+
+  it("omits --port for hermes so it reads the listen port from HERMES_HOME/config.yaml (NemoClaw provisions 18642, socat bridges 0.0.0.0:8642)", () => {
+    const cmd = buildManualRecoveryCommand(hermesAgent, 8642);
+    expect(cmd).not.toContain("--port");
+    expect(cmd).toBe("HERMES_HOME=/sandbox/.hermes-data nohup hermes gateway run >/tmp/gateway.log 2>&1 &");
+  });
+
+  it("does not prefix HERMES_HOME for non-hermes agents", () => {
+    const cmd = buildManualRecoveryCommand(minimalAgent, 19000);
+    expect(cmd).not.toContain("HERMES_HOME");
+  });
+
+  it("falls back to openclaw gateway run for null agent (OpenClaw path)", () => {
+    const cmd = buildManualRecoveryCommand(null, 18789);
+    expect(cmd).toBe("nohup openclaw gateway run --port 18789 >/tmp/gateway.log 2>&1 &");
+  });
+
+  it("derives the default gateway command from binary_path when gateway_command is whitespace-only (mirrors buildRecoveryScript)", () => {
+    const agent = makeAgent({ gateway_command: "   " });
+    const cmd = buildManualRecoveryCommand(agent, 19000);
+    expect(cmd).toBe("nohup test-agent gateway run --port 19000 >/tmp/gateway.log 2>&1 &");
+  });
+
+  it("derives the default gateway command from binary_path when gateway_command is undefined (mirrors buildRecoveryScript)", () => {
+    const agent = makeAgent({ gateway_command: undefined });
+    const cmd = buildManualRecoveryCommand(agent, 19000);
+    expect(cmd).toBe("nohup test-agent gateway run --port 19000 >/tmp/gateway.log 2>&1 &");
+  });
+
+  it("falls back to openclaw gateway run when both binary_path and gateway_command are absent", () => {
+    const agent = makeAgent({ binary_path: undefined, gateway_command: undefined });
+    const cmd = buildManualRecoveryCommand(agent, 19000);
+    expect(cmd).toBe("nohup openclaw gateway run --port 19000 >/tmp/gateway.log 2>&1 &");
+  });
+});

src/lib/agent-runtime.ts

@@ -238,3 +238,36 @@ export function getAgentDisplayName(agent: AgentDefinition | null): string {
 export function getGatewayCommand(agent: AgentDefinition | null): string {
   return agent?.gateway_command || "openclaw gateway run";
 }
+
+/**
+ * Build a single copy-pasteable command that starts the agent gateway as a
+ * backgrounded, persistent process, the manual equivalent of
+ * {@link buildRecoveryScript}. Shown to the user when automatic recovery
+ * fails.
+ *
+ * The raw `gateway_command` (e.g. `hermes gateway run`) is a foreground
+ * debugging command; it does not survive a disconnect and lacks the
+ * agent-specific env vars that the sandbox boot sequence sets.
+ *
+ * Port source varies by agent. OpenClaw reads the port from the `--port`
+ * flag; Hermes reads it from HERMES_HOME/config.yaml (NemoClaw provisions
+ * `platforms.api_server.extra.port: 18642` with socat forwarding
+ * 0.0.0.0:8642 → 127.0.0.1:18642 from agents/hermes/start.sh). Passing
+ * `--port 8642` to `hermes gateway run` would override the config and
+ * break the forwarder, so we omit the flag for hermes and mirror what
+ * start.sh does: set HERMES_HOME and let config.yaml drive the port.
+ *
+ * This helper mirrors nemoclaw-start.sh / buildRecoveryScript for
+ * non-hermes agents: `env vars → nohup → gateway_command → --port →
+ * log redirect → backgrounded`.
+ */
+export function buildManualRecoveryCommand(agent: AgentDefinition | null, port: number): string {
+  const binaryPath = agent?.binary_path || "/usr/local/bin/openclaw";
+  const binaryName = binaryPath.split("/").pop() ?? "openclaw";
+  const defaultGatewayCommand = `${binaryName} gateway run`;
+  const gatewayCmd = agent?.gateway_command?.trim() || defaultGatewayCommand;
+  const isHermes = agent?.name === "hermes";
+  const envPrefix = isHermes ? "HERMES_HOME=/sandbox/.hermes-data " : "";
+  const portFlag = isHermes ? "" : ` --port ${port}`;
+  return `${envPrefix}nohup ${gatewayCmd}${portFlag} >/tmp/gateway.log 2>&1 &`;
+}

src/nemoclaw.ts

@@ -458,8 +458,13 @@ function checkAndRecoverSandboxProcesses(
     // recovered process can be alive before the OpenAI-compatible API is ready.
     if (!waitForRecoveredSandboxGateway(sandboxName)) {
       if (!quiet) {
+        const _recoveryPort = _recoveryAgent?.forwardPort ?? DASHBOARD_PORT;
         console.error("  Gateway process started but is not responding.");
         console.error("  Check /tmp/gateway.log inside the sandbox for details.");
+        console.error("  Connect to the sandbox and run manually:");
+        console.error(
+          `    ${agentRuntime.buildManualRecoveryCommand(_recoveryAgent, _recoveryPort)}`,
+        );
       }
       return { checked: true, wasRunning: false, recovered: false };
     }
@@ -471,11 +476,14 @@ function checkAndRecoverSandboxProcesses(
       console.log(`  ${G}✓${R} Dashboard port forward re-established.`);
     }
   } else if (!quiet) {
+    const _recoveryPort = _recoveryAgent?.forwardPort ?? DASHBOARD_PORT;
     console.error(
       `  Could not restart ${agentRuntime.getAgentDisplayName(_recoveryAgent)} gateway automatically.`,
     );
     console.error("  Connect to the sandbox and run manually:");
-    console.error(`    ${agentRuntime.getGatewayCommand(_recoveryAgent)}`);
+    console.error(
+      `    ${agentRuntime.buildManualRecoveryCommand(_recoveryAgent, _recoveryPort)}`,
+    );
   }
 
   return { checked: true, wasRunning: false, recovered };