e2e: overhaul test suite — consolidate daemons, property-based testing, vmproof quality

docs/dev/testing/formal-verification-matrix.md

@@ -81,6 +81,14 @@ version.
 inside the guest VM and can run containers. Evidence MUST include `docker info` succeeding and a
 `docker run` command executing a container.
 
+**`VM-PROOF-015 (Docker Buildx Plugin, E2E)`** — Tests MUST demonstrate that the Docker buildx
+plugin is available inside a docker-enabled workspace. Evidence MUST include `docker buildx version`
+succeeding and listing the default builder.
+
+**`VM-PROOF-016 (Docker Compose Build, E2E)`** — Tests MUST demonstrate that Docker compose build
+works inside a workspace. Evidence MUST include a `docker compose up` with a multi-service stack
+succeeding and all services reachable.
+
 ---
 
 ## macOS App Proof Obligations — `MACAPP-PROOF-001`–`MACAPP-PROOF-003`

docs/spec/07-invariants.md

@@ -215,6 +215,13 @@ empty agent, but MUST NOT fail with a connection-refused or socket-not-found err
 independent SSH agent proxy socket. A forked workspace's SSH agent MUST NOT share state with the
 parent workspace's SSH agent.
 
+**`VM-029` (SSH Bootstrap Socket)** — The SSH bootstrap socket MUST exist inside the workspace
+guest. The socket file MUST be present at the expected path and usable for SSH authentication.
+
+**`VM-029d` (GIT_SSH_COMMAND)** — The `GIT_SSH_COMMAND` environment variable MUST be set in user
+shell sessions inside the workspace. The value MUST point to the SSH agent socket for Git SSH
+operations.
+
 ---
 
 ## macOS App — `MACAPP-001`–`MACAPP-004`

packages/nexus/cmd/nexus-guest-agent/ptyproxy.go

@@ -160,6 +160,8 @@ func handleShellOpen(req execRequest, encoder *json.Encoder) {
 
 	_ = encoder.Encode(execResponse{ID: req.ID, Type: "ack", ExitCode: 0})
 
+	var encMu sync.Mutex
+
 	var readWg sync.WaitGroup
 	readWg.Add(1)
 	go func() {
@@ -168,7 +170,9 @@ func handleShellOpen(req execRequest, encoder *json.Encoder) {
 		for {
 			n, err := ptmx.Read(buf)
 			if n > 0 {
+				encMu.Lock()
 				_ = encoder.Encode(execResponse{ID: req.ID, Type: "chunk", Stream: "stdout", Data: string(buf[:n])})
+				encMu.Unlock()
 			}
 			if err != nil {
 				return
@@ -190,14 +194,18 @@ func handleShellOpen(req execRequest, encoder *json.Encoder) {
 				exitCode = 1
 			}
 		}
-		// Send result immediately — the daemon/CLI needs the exit status.
-		// This must happen BEFORE readWg.Wait() to avoid deadlock when
-		// the vsock connection has backpressure (read goroutine blocked on Encode).
-		_ = encoder.Encode(execResponse{ID: req.ID, Type: "result", ExitCode: exitCode})
 		// Wait for the read goroutine to finish draining all output
-		// from the PTY master. The read goroutine exits when the slave
-		// is closed (shell exits) or when EOF is reached.
+		// from the PTY master before sending the result. This ensures
+		// all "chunk" messages are delivered before the "result" message,
+		// preventing the daemon from tearing down the session (on pty.exit)
+		// before all pty.data notifications have been sent.
+		// The encMu mutex guarantees the read goroutine will always
+		// eventually acquire the lock and complete its Encode call,
+		// so there is no deadlock risk from waiting first.
 		readWg.Wait()
+		encMu.Lock()
+		_ = encoder.Encode(execResponse{ID: req.ID, Type: "result", ExitCode: exitCode})
+		encMu.Unlock()
 		shellSessionsMu.Lock()
 		if !s.closed {
 			s.closed = true

packages/nexus/cmd/nexus/commands/workspace/run.go

@@ -36,6 +36,7 @@ type ptyExitParams struct {
 
 func runCommand() *cobra.Command {
 	var workDir string
+	var execTimeout time.Duration
 
 	cmd := &cobra.Command{
 		Use:   "exec <workspace> -- <command> [args...]",
@@ -132,18 +133,19 @@ func runCommand() *cobra.Command {
 				}
 			}()
 
-			return runExecEventLoop(cmd.Context(), conn, session, dataCh, exitCh)
+			return runExecEventLoop(cmd.Context(), conn, session, dataCh, exitCh, execTimeout)
 		},
 	}
 	cmd.Flags().StringVar(&workDir, "workdir", "/workspace", "working directory inside the workspace")
+	cmd.Flags().DurationVar(&execTimeout, "timeout", 120*time.Second, "maximum duration for the exec session")
 	cmd.Aliases = []string{"run"}
 	return cmd
 }
 
-func runExecEventLoop(ctx context.Context, conn *rpc.MuxConn, session ptySessionInfo, dataCh, exitCh <-chan json.RawMessage) error {
+func runExecEventLoop(ctx context.Context, conn *rpc.MuxConn, session ptySessionInfo, dataCh, exitCh <-chan json.RawMessage, execTimeout time.Duration) error {
 	// Safety timeout to prevent indefinite hangs if the guest agent deadlocks
 	// or the vsock connection has backpressure.
-	timeout := time.After(120 * time.Second)
+	timeout := time.After(execTimeout)
 
 	for {
 		select {
@@ -198,7 +200,7 @@ func runExecEventLoop(ctx context.Context, conn *rpc.MuxConn, session ptySession
 			return context.Canceled
 		case <-timeout:
 			_ = conn.Send("pty.close", map[string]any{"sessionId": session.ID})
-			return fmt.Errorf("exec timed out after 120s")
+			return fmt.Errorf("exec timed out after %s", execTimeout)
 		}
 	}
 }

packages/nexus/internal/app/workspace/service_create.go

@@ -3,7 +3,6 @@ package workspace
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -90,15 +89,9 @@ func (s *Service) Create(ctx context.Context, spec workspace.CreateSpec) (*works
 		return nil, err
 	}
 
-	// Auto-start: begin booting the VM in the background.
-	// Skip auto-start when no runtime registry is available (e.g. integration tests).
-	if s.registry != nil {
-		go func() {
-			if _, err := s.Start(context.Background(), ws.ID); err != nil {
-				log.Printf("[workspace] auto-start %s: %v", ws.ID, err)
-			}
-		}()
-	}
+	// Auto-start removed: all callers (CLI, TUI, e2e tests) explicitly call
+	// Start after Create. Background auto-start raced with explicit Start calls,
+	// causing duplicate VM boots and state transition conflicts.
 
 	return ws, nil
 }

packages/nexus/internal/app/workspace/service_fork.go

@@ -165,12 +165,8 @@ func (s *Service) runForkAsync(parent, child *workspace.Workspace, parentWasRunn
 	// Broadcast fork completion
 	s.broadcastForkCompleted(parent.ID, child.ID, child.WorkspaceName)
 
-	// Auto-start the child workspace
-	go func() {
-		if _, err := s.Start(context.Background(), child.ID); err != nil {
-			log.Printf("[workspace] auto-start fork %s: %v", child.ID, err)
-		}
-	}()
+	// Auto-start removed: callers explicitly call Start after Fork.
+	// Background auto-start raced with explicit Start calls.
 }
 
 // waitAndMarkParentRunning polls the driver until the parent VM is reachable

packages/nexus/internal/infra/runtime/libkrun/manager_stop.go

@@ -13,6 +13,9 @@ import (
 )
 
 // Stop terminates a running VM.
+// libkrun VMs are stateless processes (no ACPI shutdown), so we SIGKILL
+// directly — matching the forceStopGitVM pattern in manager_git.go.
+// A brief poll follows as a safety net to confirm exit.
 func (m *Manager) Stop(_ context.Context, workspaceID string) error {
 	m.mu.Lock()
 	inst, exists := m.instances[workspaceID]
@@ -22,60 +25,42 @@ func (m *Manager) Stop(_ context.Context, workspaceID string) error {
 	m.mu.Unlock()
 
 	if !exists {
-		// Workspace has no running VM (never started or still booting before
-		// the process was registered). Nothing to stop.
 		return nil
 	}
 
-	// Gracefully stop VM and passt child processes concurrently.
 	var stopWg sync.WaitGroup
 	if inst.Process != nil {
 		stopWg.Add(1)
 		go func() {
 			defer stopWg.Done()
-			if err := inst.Process.Signal(os.Interrupt); err != nil {
-				_ = inst.Process.Kill()
-			}
-			deadline := time.Now().Add(8 * time.Second)
+			_ = inst.Process.Signal(syscall.SIGKILL)
+			deadline := time.Now().Add(2 * time.Second)
 			for time.Now().Before(deadline) {
 				if err := inst.Process.Signal(syscall.Signal(0)); err != nil {
 					break
 				}
-				time.Sleep(200 * time.Millisecond)
-			}
-			if err := inst.Process.Signal(syscall.Signal(0)); err == nil {
-				_ = inst.Process.Kill()
+				time.Sleep(50 * time.Millisecond)
 			}
 		}()
 	}
 	if inst.PasstProcess != nil {
 		stopWg.Add(1)
 		go func() {
 			defer stopWg.Done()
-			if err := inst.PasstProcess.Signal(os.Interrupt); err != nil {
-				_ = inst.PasstProcess.Kill()
-			}
-			deadline := time.Now().Add(3 * time.Second)
+			_ = inst.PasstProcess.Signal(syscall.SIGKILL)
+			deadline := time.Now().Add(1 * time.Second)
 			for time.Now().Before(deadline) {
 				if err := inst.PasstProcess.Signal(syscall.Signal(0)); err != nil {
 					break
 				}
-				time.Sleep(150 * time.Millisecond)
-			}
-			if err := inst.PasstProcess.Signal(syscall.Signal(0)); err == nil {
-				_ = inst.PasstProcess.Kill()
+				time.Sleep(50 * time.Millisecond)
 			}
 		}()
 	}
 	stopWg.Wait()
 	_ = os.Remove(filepath.Join(inst.WorkDir, libkrunPIDFileName))
 	_ = os.Remove(filepath.Join(inst.WorkDir, passtPIDFileName))
-	// Clear the dirty flag so it doesn't accumulate across normal stop/start
-	// cycles. Note: Stop() sends SIGINT and does NOT guarantee a clean guest
-	// unmount, so ForkWorkspaceImage always fsyncs regardless of this flag.
 	_ = os.Remove(filepath.Join(inst.WorkDir, vmDirtyFlagFileName))
-	// Unmount bind-mounted host-config directories before the workdir can be
-	// removed by a subsequent CleanupWorkspaceByID call.
 	cleanupHostConfigBinds(inst.WorkDir)
 	return nil
 }

packages/nexus/internal/infra/runtime/libkrun/workspace_rootfs.go

@@ -77,12 +77,12 @@ func (d *Driver) EnsureWorkspaceRootfs(ctx context.Context, tools workspace.Tool
 		}
 	}
 
-	// Cache miss or no local image — pull the image
-	pullDigest, err := d.podman.Pull(ctx, workspaceBaseImage)
-	if err != nil {
-		return "", fmt.Errorf("pull image %q: %w", workspaceBaseImage, err)
-	}
+	// Cache miss or no local image — pull only if image is not available locally
 	if digest == "" {
+		pullDigest, err := d.podman.Pull(ctx, workspaceBaseImage)
+		if err != nil {
+			return "", fmt.Errorf("pull image %q: %w", workspaceBaseImage, err)
+		}
 		digest = pullDigest
 	}