feat: Smart Remote Control (Approvals & Action Timeline)

README.md

@@ -15,7 +15,9 @@ Start an agent on your laptop, walk away, and check in from your phone or tablet
 ## Why CloudCode?
 
 - **Agent agnostic:** Works with Claude Code, Gemini CLI, OpenAI Codex, GitHub Copilot CLI, or any CLI tool.
-- **Task launch:** Start tasks instantly from your mobile dashboard without opening a full terminal. CloudCode defaults to your recent projects and drops you into the live session view.
+- **Smart Remote Control:** Detects agent activity and transforms it into a **Task Timeline** of collapsible cards.
+- **Instant Approvals:** Automatically pops up native **Approval Modals** when an agent requests permission, saving you from opening the mobile keyboard.
+- **Task launch:** Start tasks instantly from your mobile dashboard without opening a full terminal.
 - **Transcript logs:** Shows the full session output in a scrollable, timestamped transcript view.
 - **QR code pairing:** Scan a QR code from your terminal to authenticate your phone. No passwords or SSH keys.
 - **Persistent sessions:** Sessions run inside `tmux`. Your agent keeps working if your laptop sleeps or your connection drops. Reconnect and pick up where you left off.

backend/src/terminal/heuristics.ts

@@ -0,0 +1,210 @@
+import xtermHeadless from '@xterm/headless';
+import { nanoid } from 'nanoid';
+
+// Handle ESM / CJS interop for @xterm/headless
+const Terminal = (xtermHeadless as any).Terminal || (xtermHeadless as any).default?.Terminal;
+
+export interface PromptState {
+  isActive: boolean;
+  type: 'yesno' | 'enter' | null;
+  text?: string;
+}
+
+export interface TimelineAction {
+  id: string;
+  type: 'bash' | 'read' | 'edit' | 'grep' | 'ls' | 'custom';
+  label: string;
+  status: 'running' | 'completed' | 'error';
+  content?: string;
+  startTime: string;
+  endTime?: string;
+}
+
+export interface HeuristicsResult {
+  prompt?: PromptState;
+  action?: TimelineAction;
+}
+
+export class HeuristicsEngine {
+  private term: any;
+  private lastPromptState: PromptState = { isActive: false, type: null };
+  private activeAction: TimelineAction | null = null;
+
+  constructor() {
+    this.term = new Terminal({
+      cols: 1000,
+      rows: 100,
+      scrollback: 500, // Reduced from 100000 to save memory in headless mode
+      allowProposedApi: true,
+    });
+
+    if (this.term._core?.optionsService?.options) {
+      this.term._core.optionsService.options.allowProposedApi = true;
+    }
+  }
+
+  /**
+   * Process a chunk of raw PTY data.
+   */
+  public process(chunkBase64: string): HeuristicsResult {
+    if (!chunkBase64) return {};
+
+    const chunk = Buffer.from(chunkBase64, 'base64').toString('utf8');
+    this.term.write(chunk);
+
+    return {
+      prompt: this.detectPrompt() || undefined,
+      action: this.detectAction() || undefined
+    };
+  }
+
+  /**
+   * Cleanup resources.
+   */
+  public dispose(): void {
+    if (this.term) {
+      this.term.dispose();
+      this.term = null;
+    }
+  }
+
+  private detectPrompt(): PromptState | null {
+    if (!this.term) return null;
+
+    const activeBuffer = this.term.buffer.active;
+    const end = activeBuffer.cursorY + activeBuffer.baseY;
+    const start = Math.max(0, end - 5); // Prompts are usually very near the cursor
+
+    let text = '';
+    for (let i = start; i <= end; i++) {
+      const line = activeBuffer.getLine(i);
+      if (line) {
+        text += line.translateToString(true) + '\n';
+      }
+    }
+
+    const trimmed = text.trim();
+
+    // Improved regex for prompt detection
+    // Matches patterns like: "Do you want to continue? [Y/n]" or "Overwrite file? (y/N)"
+    const yesnoMatch = trimmed.match(/(.*?)(?:\[|\()([yY]\/[nN])(?:\]|\))\s*\??\s*$/s);
+
+    // Matches: "Press Enter to continue..." or "Press [Enter] to exit"
+    const enterMatch = trimmed.match(/(.*?)Press (?:\[?Enter\]?|any key) to (?:continue|exit)\.*$/is);
+
+    let newState: PromptState = { isActive: false, type: null };
+
+    if (yesnoMatch) {
+      const precedingLines = yesnoMatch[1].trim().split('\n');
+      const context = precedingLines.slice(-2).join('\n').trim() || 'Permission requested';
+      newState = { isActive: true, type: 'yesno', text: context };
+    } else if (enterMatch) {
+      const precedingLines = enterMatch[1].trim().split('\n');
+      const context = precedingLines.slice(-1).join('\n').trim();
+      newState = { isActive: true, type: 'enter', text: context || 'Action required' };
+    }
+
+    // Only return if the prompt state has changed significantly
+    if (this.lastPromptState.isActive !== newState.isActive || 
+        this.lastPromptState.type !== newState.type ||
+        (newState.isActive && this.lastPromptState.text !== newState.text)) {
+      this.lastPromptState = newState;
+      return newState;
+    }
+
+    return null;
+  }
+
+  private detectAction(): TimelineAction | null {
+    if (!this.term) return null;
+
+    const activeBuffer = this.term.buffer.active;
+    const end = activeBuffer.cursorY + activeBuffer.baseY;
+    const start = Math.max(0, end - 15);
+
+    let lines: string[] = [];
+    for (let i = start; i <= end; i++) {
+      const line = activeBuffer.getLine(i);
+      if (line) {
+        lines.push(line.translateToString(true));
+      }
+    }
+
+    const text = lines.join('\n');
+
+    // 1. Detect Tool Use Start (Claude / Gemini Style boxes)
+    const toolStartMatch = text.match(/[┌╭]─+ Tool Use: ([\w]+) ─+┐/i);
+    if (toolStartMatch) {
+      const toolName = toolStartMatch[1].toLowerCase();
+      const type = this.mapToolType(toolName);
+
+      // Look for the specific argument/command line
+      const startIndex = lines.findIndex(l => l.match(/[┌╭]─+ Tool Use:/i));
+      let label = 'Working...';
+      if (startIndex !== -1 && lines[startIndex + 1]) {
+        // Clean up common box characters
+        label = lines[startIndex + 1].replace(/[│|┃]/g, '').trim();
+      }
+
+      // Avoid creating a new action if the label is exactly the same as the current running one
+      if (!this.activeAction || this.activeAction.label !== label || this.activeAction.status !== 'running') {
+        this.activeAction = {
+          id: nanoid(8),
+          type,
+          label,
+          status: 'running',
+          startTime: new Date().toISOString()
+        };
+        return this.activeAction;
+      }
+    }
+
+    // 2. Detect Tool Completion (Bottom of box)
+    if (this.activeAction && this.activeAction.status === 'running') {
+      const lastLine = lines[lines.length - 1] || '';
+      // Looks for └──────────┘ or ╰──────────╯
+      if (lastLine.match(/[└╰]─+┘/)) {
+        this.activeAction.status = 'completed';
+        this.activeAction.endTime = new Date().toISOString();
+        const completedAction = { ...this.activeAction };
+        this.activeAction = null; 
+        return completedAction;
+      }
+    }
+
+    // 3. Fallback: Generic Shell Command Detection
+    // Detects lines starting with $ or > followed by a command
+    if (!this.activeAction) {
+      for (let i = lines.length - 1; i >= Math.max(0, lines.length - 3); i--) {
+        const line = lines[i];
+        const bashMatch = line.match(/^[\$]\s+([a-zA-Z0-9].+)$/);
+        if (bashMatch) {
+          const cmd = bashMatch[1].trim();
+          // Filter out common interactive shells/prompts that aren't actions
+          if (['bash', 'zsh', 'sh', 'node', 'python'].includes(cmd)) continue;
+
+          this.activeAction = {
+            id: nanoid(8),
+            type: 'bash',
+            label: cmd,
+            status: 'running',
+            startTime: new Date().toISOString()
+          };
+          return this.activeAction;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private mapToolType(tool: string): TimelineAction['type'] {
+    const t = tool.toLowerCase();
+    if (t.includes('bash') || t.includes('shell') || t.includes('run')) return 'bash';
+    if (t.includes('read') || t.includes('cat')) return 'read';
+    if (t.includes('edit') || t.includes('write') || t.includes('patch')) return 'edit';
+    if (t.includes('grep') || t.includes('search') || t.includes('find')) return 'grep';
+    if (t.includes('ls') || t.includes('list')) return 'ls';
+    return 'custom';
+  }
+}

backend/src/terminal/routes.ts

@@ -3,6 +3,7 @@ import { getSession, getSessionByPublicId, hasTranscriptRecorder } from '../sess
 import { validateSession } from '../auth/service.js';
 import { sidecarManager, type SidecarStreamHandle } from './sidecar-manager.js';
 import * as tmux from '../tmux/adapter.js';
+import { HeuristicsEngine } from './heuristics.js';
 import {
   appendTranscript,
   appendTranscriptResize,
@@ -141,6 +142,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => {
     let attachedSession: ReturnType<typeof getSession> | null = null;
     let attachPromise: Promise<void> | null = null;
     let lastSize = { cols: 80, rows: 24 };
+    const heuristics = new HeuristicsEngine();
 
     // Heartbeat: detect silent/dead connections (e.g. phone sleep, network change).
     // Server pings every 15s; if no pong arrives before the next ping, the connection
@@ -201,6 +203,14 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => {
         onData: (dataBase64) => {
           if (ws.readyState !== 1) return;
           ws.send(JSON.stringify({ type: 'terminal.output', dataBase64 }));
+
+          const result = heuristics.process(dataBase64);
+          if (result.prompt) {
+            ws.send(JSON.stringify({ type: 'prompt.state', promptState: result.prompt }));
+          }
+          if (result.action) {
+            ws.send(JSON.stringify({ type: 'timeline.action', action: result.action }));
+          }
         },
         onText: (text) => {
           if (session && !isMirrorOnly && !hasTranscriptRecorder(session.id)) {
@@ -324,13 +334,15 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => {
 
     ws.on('close', () => {
       cleanupHeartbeat();
+      heuristics.dispose();
       void ptySession?.close().catch(() => {});
       ptySession = null;
       attachedSession = null;
     });
 
     ws.on('error', () => {
       cleanupHeartbeat();
+      heuristics.dispose();
       void ptySession?.close().catch(() => {});
       ptySession = null;
       attachedSession = null;

frontend/src/components/ActionTimeline.tsx

@@ -0,0 +1,112 @@
+import { TimelineAction } from '../hooks/useTerminal'
+import { useState } from 'react'
+
+interface ActionTimelineProps {
+  actions: TimelineAction[]
+}
+
+export function ActionTimeline({ actions }: ActionTimelineProps) {
+  const [expandedId, setExpandedId] = useState<string | null>(null)
+
+  if (actions.length === 0) {
+    return (
+      <div className="flex flex-col items-center justify-center py-12 px-4 text-center">
+        <div className="w-12 h-12 rounded-2xl bg-zinc-900 border border-zinc-800 flex items-center justify-center mb-4 text-zinc-600">
+          <svg className="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z" />
+          </svg>
+        </div>
+        <h3 className="text-sm font-bold text-zinc-400 tracking-tight">Timeline is empty</h3>
+        <p className="text-xs text-zinc-500 mt-1 max-w-[200px]">Agent actions will appear here as they happen.</p>
+      </div>
+    )
+  }
+
+  return (
+    <div className="flex flex-col gap-3 p-4">
+      {actions.slice().reverse().map((action) => (
+        <div 
+          key={action.id}
+          className={`group bg-zinc-900 border rounded-2xl transition-all duration-200 ${
+            expandedId === action.id ? 'border-indigo-500/50 shadow-lg shadow-indigo-500/10' : 'border-zinc-800 hover:border-zinc-700'
+          }`}
+        >
+          <button
+            onClick={() => setExpandedId(expandedId === action.id ? null : action.id)}
+            className="w-full text-left p-4 flex items-center gap-3"
+          >
+            <ActionIcon type={action.type} status={action.status} />
+
+            <div className="flex-1 min-w-0">
+              <div className="flex items-center justify-between gap-2">
+                <span className="text-[10px] font-black uppercase tracking-[0.15em] text-indigo-400">
+                  {action.type}
+                </span>
+                <span className="text-[9px] font-medium text-zinc-500 font-mono">
+                  {new Date(action.startTime).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' })}
+                </span>
+              </div>
+              <h4 className="text-sm font-bold text-zinc-100 truncate mt-0.5 tracking-tight">
+                {action.label}
+              </h4>
+            </div>
+
+            <div className="flex-shrink-0">
+              {action.status === 'running' ? (
+                <div className="w-5 h-5 flex items-center justify-center">
+                  <div className="w-1.5 h-1.5 rounded-full bg-emerald-500 animate-pulse" />
+                </div>
+              ) : action.status === 'completed' ? (
+                <svg className="w-5 h-5 text-emerald-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={3} d="M5 13l4 4L19 7" />
+                </svg>
+              ) : (
+                <svg className="w-5 h-5 text-rose-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={3} d="M6 18L18 6M6 6l12 12" />
+                </svg>
+              )}
+            </div>
+          </button>
+
+          {expandedId === action.id && action.content && (
+            <div className="px-4 pb-4 pt-0 animate-in fade-in slide-in-from-top-1 duration-200">
+              <div className="p-3 bg-black/40 rounded-xl border border-white/5 font-mono text-[11px] leading-relaxed text-zinc-300 overflow-x-auto whitespace-pre-wrap">
+                {action.content}
+              </div>
+            </div>
+          )}
+        </div>
+      ))}
+    </div>
+  )
+}
+
+function ActionIcon({ type, status }: { type: TimelineAction['type'], status: TimelineAction['status'] }) {
+  const baseClasses = "w-9 h-9 rounded-xl flex items-center justify-center flex-shrink-0 border"
+
+  const styles = {
+    bash: "bg-zinc-800/50 border-zinc-700/50 text-zinc-300",
+    read: "bg-blue-500/10 border-blue-500/20 text-blue-400",
+    edit: "bg-amber-500/10 border-amber-500/20 text-amber-400",
+    grep: "bg-purple-500/10 border-purple-500/20 text-purple-400",
+    ls: "bg-emerald-500/10 border-emerald-500/20 text-emerald-400",
+    custom: "bg-zinc-800/50 border-zinc-700/50 text-zinc-300",
+  }
+
+  const iconD = {
+    bash: "M8 9l3 3-3 3m5 0h3",
+    read: "M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.084.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253",
+    edit: "M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z",
+    grep: "M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z",
+    ls: "M3 7v10a2 2 0 002 2h14a2 2 0 002-2V9a2 2 0 00-2-2h-6l-2-2H5a2 2 0 00-2 2z",
+    custom: "M13 10V3L4 14h7v7l9-11h-7z",
+  }
+
+  return (
+    <div className={`${baseClasses} ${styles[type]}`}>
+      <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+        <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d={iconD[type]} />
+      </svg>
+    </div>
+  )
+}