Support custom subject commands

cmd/harness/main.go

@@ -170,7 +170,10 @@ func testCmd() *cobra.Command {
 							continue
 						}
 						for _, s := range subs {
-							pairs = append(pairs, runPair{tc: tc, subject: applyCasePin(s, tc)})
+							// Own copy per pair — the runner mutates the case
+							// in place (see TestCase.CloneForRun), and -s a,b
+							// puts several subjects on one loaded case here.
+							pairs = append(pairs, runPair{tc: tc.CloneForRun(), subject: applyCasePin(s, tc)})
 						}
 					}
 					if len(subjectsToLoop) > 1 {
@@ -215,7 +218,11 @@ func testCmd() *cobra.Command {
 						}
 					}
 					for _, s := range subjects {
-						pairs = append(pairs, runPair{tc: tc, subject: applyCasePin(s, tc)})
+						// Every subject gets its own copy of the case: the
+						// runner rewrites endpoint commands/env in place while
+						// resolving `${NAME}`, so a shared pointer would leave
+						// the second subject running the first one's command.
+						pairs = append(pairs, runPair{tc: tc.CloneForRun(), subject: applyCasePin(s, tc)})
 					}
 				}
 				if len(pairs) == 0 {

internal/config/case.go

@@ -2,11 +2,13 @@ package config
 
 import (
 	"fmt"
+	"maps"
 	"net"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -48,6 +50,19 @@ type TestCase struct {
 	// drives data purely through endpoints.
 	Endpoints []Endpoint `yaml:"endpoints"`
 
+	// Resolve computes values from the subject BEFORE the topology starts, so a
+	// case can drive the subject with a string the subject itself produces
+	// rather than one the case author transcribed. Each entry runs a one-shot
+	// container and captures its stdout; `${NAME}` then expands in every
+	// endpoint command and endpoint env value.
+	//
+	// The point is contract coverage. When a case hard-codes a value that some
+	// component generates at runtime, it stops testing the generator: the case
+	// keeps passing against a copy of what the author believed the format to be,
+	// and drifts silently when the real one changes. Resolving it from the
+	// subject makes the producer part of the test.
+	Resolve []ResolveValue `yaml:"resolve"`
+
 	// Agent, when set, adds an external agent container to the test topology.
 	// The agent connects INTO the subject (director) rather than being connected
 	// to by it — useful for testing agent-mode device collection. The compose
@@ -608,6 +623,30 @@ type AgentConfig struct {
 	MountsSharedData bool `yaml:"mounts_shared_data"`
 }
 
+// ResolveValue asks the subject to produce a value the case then uses (see
+// TestCase.Resolve). The harness runs a one-shot container, trims the trailing
+// newline from its stdout, and substitutes the result for `${Name}` in every
+// endpoint command and endpoint env value.
+//
+// The container runs to completion before the topology starts, so the command
+// must be self-contained — it cannot reach a running subject, a generator, or
+// the bench network, and it must not need any of them. Anything that only makes
+// sense against a live topology belongs in an endpoint command instead.
+type ResolveValue struct {
+	// Name is the placeholder this value fills, referenced as ${Name}. Use
+	// uppercase with underscores. Must be unique within the case.
+	Name string `yaml:"name"`
+	// Image is the container image to run. The literal "subject" means the
+	// image under test, including any --image/--version override, which is what
+	// makes the value come from the build being tested rather than a fixed one.
+	Image string `yaml:"image"`
+	// Command is the command to run. Its stdout (trailing newline trimmed) is
+	// the value. A non-zero exit fails the run: a value that could not be
+	// produced must stop the case, never silently expand to an empty string and
+	// let a downstream assertion pass for the wrong reason.
+	Command []string `yaml:"command"`
+}
+
 // Endpoint is an auxiliary container in the test topology (see
 // TestCase.Endpoints). It's a host the subject reaches on the bench network —
 // not a generator or receiver.
@@ -1105,6 +1144,45 @@ func (tc *TestCase) Validate() error {
 		}
 		epNames[e.Name] = struct{}{}
 	}
+	// Resolved values must be well formed and actually referenced. An unused or
+	// misspelled entry is the failure mode worth catching here: the ${NAME} it
+	// was meant to fill would stay literal in the command, the container would
+	// run something meaningless, and the case would report on that instead.
+	resolveNames := map[string]struct{}{}
+	for i, rv := range tc.Resolve {
+		if rv.Name == "" {
+			return fmt.Errorf("case %q: resolve[%d] missing required `name`", tc.Name, i)
+		}
+		if rv.Image == "" {
+			return fmt.Errorf("case %q: resolve %q missing required `image` (use \"subject\" for the image under test)", tc.Name, rv.Name)
+		}
+		if len(rv.Command) == 0 {
+			return fmt.Errorf("case %q: resolve %q missing required `command`", tc.Name, rv.Name)
+		}
+		if _, dup := resolveNames[rv.Name]; dup {
+			return fmt.Errorf("case %q: duplicate resolve name %q", tc.Name, rv.Name)
+		}
+		resolveNames[rv.Name] = struct{}{}
+
+		placeholder := "${" + rv.Name + "}"
+		used := false
+		for _, e := range tc.Endpoints {
+			for _, part := range e.Command {
+				if strings.Contains(part, placeholder) {
+					used = true
+				}
+			}
+			for _, val := range e.Env {
+				if strings.Contains(val, placeholder) {
+					used = true
+				}
+			}
+		}
+		if !used {
+			return fmt.Errorf("case %q: resolve %q is never referenced as %s in any endpoint command or env",
+				tc.Name, rv.Name, placeholder)
+		}
+	}
 	// Kafka types require the broker block + a generator producing in kafka mode.
 	if tc.IsKafkaType() {
 		if tc.Kafka == nil {
@@ -2678,6 +2756,44 @@ func LoadCase(casesDir, name string) (*TestCase, error) {
 	return &tc, nil
 }
 
+// CloneForRun returns a copy of tc that one run may mutate in place without
+// affecting any other. One loaded case is shared by every (case, subject) pair
+// the CLI queues, and the runner rewrites parts of it per run: resolveValues
+// expands `${NAME}` into endpoint commands and env, and the Vault cert-rotation
+// flow seeds freshly generated certs into vault.secrets. Sharing one pointer
+// bakes the first subject's values in for every subject after it — the
+// placeholder is already gone, so the later runs silently execute the earlier
+// subject's command and still report a verdict on it.
+//
+// Only the fields the runner writes are deep-copied; the rest stays shared
+// because it is read-only for the duration of a run. Extend this when a new
+// mutation site appears.
+func (tc *TestCase) CloneForRun() *TestCase {
+	if tc == nil {
+		return nil
+	}
+	cp := *tc
+	if tc.Endpoints != nil {
+		cp.Endpoints = make([]Endpoint, len(tc.Endpoints))
+		for i, ep := range tc.Endpoints {
+			cp.Endpoints[i] = ep
+			cp.Endpoints[i].Command = slices.Clone(ep.Command)
+			cp.Endpoints[i].Env = maps.Clone(ep.Env)
+		}
+	}
+	if tc.Vault != nil {
+		v := *tc.Vault
+		if tc.Vault.Secrets != nil {
+			v.Secrets = make(map[string]map[string]string, len(tc.Vault.Secrets))
+			for secretPath, fields := range tc.Vault.Secrets {
+				v.Secrets[secretPath] = maps.Clone(fields)
+			}
+		}
+		cp.Vault = &v
+	}
+	return &cp
+}
+
 // ListCases returns all case names found in casesDir.
 func ListCases(casesDir string) ([]string, error) {
 	entries, err := os.ReadDir(casesDir)

internal/config/clone_test.go

@@ -0,0 +1,56 @@
+package config
+
+import "testing"
+
+// TestCloneForRunIsolatesMutatedFields pins the invariant the CLI relies on:
+// each (case, subject) pair runs against its own copy, so the runner rewriting
+// `${NAME}` into an endpoint command — or seeding a rotated cert into
+// vault.secrets — cannot leak into the next subject's run.
+func TestCloneForRunIsolatesMutatedFields(t *testing.T) {
+	orig := &TestCase{
+		Name: "resolve-case",
+		Endpoints: []Endpoint{{
+			Name:    "device",
+			Command: []string{"send", "--token", "${TOKEN}"},
+			Env:     map[string]string{"TOKEN": "${TOKEN}"},
+		}},
+		Vault: &VaultConfig{Secrets: map[string]map[string]string{
+			"tls": {"cert": "original"},
+		}},
+	}
+
+	cp := orig.CloneForRun()
+
+	// Same rewrites the runner performs, applied to the copy only.
+	cp.Endpoints[0].Command[2] = "subject-a-token"
+	cp.Endpoints[0].Env["TOKEN"] = "subject-a-token"
+	cp.Vault.Secrets["tls"]["cert"] = "rotated"
+
+	if got := orig.Endpoints[0].Command[2]; got != "${TOKEN}" {
+		t.Errorf("original endpoint command mutated: got %q, want %q", got, "${TOKEN}")
+	}
+	if got := orig.Endpoints[0].Env["TOKEN"]; got != "${TOKEN}" {
+		t.Errorf("original endpoint env mutated: got %q, want %q", got, "${TOKEN}")
+	}
+	if got := orig.Vault.Secrets["tls"]["cert"]; got != "original" {
+		t.Errorf("original vault secret mutated: got %q, want %q", got, "original")
+	}
+	if cp.Name != orig.Name {
+		t.Errorf("clone lost scalar fields: got name %q, want %q", cp.Name, orig.Name)
+	}
+}
+
+// TestCloneForRunHandlesEmptyCase covers the nil/empty shapes most cases have —
+// no endpoints and no vault block — where the clone must not invent one.
+func TestCloneForRunHandlesEmptyCase(t *testing.T) {
+	if got := (*TestCase)(nil).CloneForRun(); got != nil {
+		t.Fatalf("nil case cloned to %v, want nil", got)
+	}
+	cp := (&TestCase{Name: "bare"}).CloneForRun()
+	if cp.Endpoints != nil {
+		t.Errorf("endpoints became %v, want nil", cp.Endpoints)
+	}
+	if cp.Vault != nil {
+		t.Errorf("vault became %v, want nil", cp.Vault)
+	}
+}

internal/runner/runner.go

@@ -157,8 +157,78 @@ func (r *Runner) applySubjectOverrides(subject config.Subject) config.Subject {
 	return subject
 }
 
+// resolveValues runs each tc.Resolve entry and expands `${NAME}` in every
+// endpoint command and env value. It mutates tc, so it must run before any
+// driver reads the endpoints — i.e. before Run dispatches on tc.Type.
+//
+// A failure here aborts the run. The alternative — leaving the placeholder
+// literal or expanding it to "" — would let the case proceed and report a
+// verdict on a command that is not the one it meant to run.
+func (r *Runner) resolveValues(tc *config.TestCase, subject config.Subject) error {
+	if len(tc.Resolve) == 0 {
+		return nil
+	}
+
+	subject = r.applySubjectOverrides(subject)
+
+	// A resolve step is a `docker run` on an image that may not be on the host
+	// yet, so the wall has to cover a cold pull, not just the command — a fixed
+	// minute fails the whole case on a first-run pull that was going to
+	// succeed. Reuse the run timeout: it is the budget the operator already
+	// sized for this run, and r.ctx still cancels immediately on SIGINT.
+	resolveTimeout := r.opts.Timeout
+	if resolveTimeout <= 0 {
+		resolveTimeout = 10 * time.Minute
+	}
+
+	for _, rv := range tc.Resolve {
+		image := rv.Image
+		if image == "subject" {
+			image = subject.Image
+			if subject.Version != "" {
+				image += ":" + subject.Version
+			}
+		}
+
+		ctx, cancel := context.WithTimeout(r.ctx, resolveTimeout)
+		args := append([]string{"run", "--rm", "--entrypoint", rv.Command[0], image}, rv.Command[1:]...)
+		out, err := exec.CommandContext(ctx, "docker", args...).Output()
+		cancel()
+		if err != nil {
+			stderr := ""
+			var exitErr *exec.ExitError
+			if errors.As(err, &exitErr) {
+				stderr = strings.TrimSpace(string(exitErr.Stderr))
+			}
+			return fmt.Errorf("resolve %q via %s: %w (stderr: %s)", rv.Name, image, err, stderr)
+		}
+
+		value := strings.TrimRight(string(out), "\r\n")
+		if value == "" {
+			return fmt.Errorf("resolve %q via %s produced no output; a case must not run on an empty value", rv.Name, image)
+		}
+
+		fmt.Printf("  resolved ${%s} from %s\n", rv.Name, image)
+
+		placeholder := "${" + rv.Name + "}"
+		for i := range tc.Endpoints {
+			for j, part := range tc.Endpoints[i].Command {
+				tc.Endpoints[i].Command[j] = strings.ReplaceAll(part, placeholder, value)
+			}
+			for k, val := range tc.Endpoints[i].Env {
+				tc.Endpoints[i].Env[k] = strings.ReplaceAll(val, placeholder, value)
+			}
+		}
+	}
+
+	return nil
+}
+
 // Run executes the test and returns the persisted result.
 func (r *Runner) Run(tc *config.TestCase, subject config.Subject) (results.RunResult, error) {
+	if err := r.resolveValues(tc, subject); err != nil {
+		return results.RunResult{}, err
+	}
 	if tc.Type == "persistence_correctness" {
 		return r.runPersistenceCorrectness(tc, subject)
 	}