diff --git a/cmd/harness/main.go b/cmd/harness/main.go index dd778af..2c065cc 100644 --- a/cmd/harness/main.go +++ b/cmd/harness/main.go @@ -170,7 +170,10 @@ func testCmd() *cobra.Command { continue } for _, s := range subs { - pairs = append(pairs, runPair{tc: tc, subject: applyCasePin(s, tc)}) + // Own copy per pair — the runner mutates the case + // in place (see TestCase.CloneForRun), and -s a,b + // puts several subjects on one loaded case here. + pairs = append(pairs, runPair{tc: tc.CloneForRun(), subject: applyCasePin(s, tc)}) } } if len(subjectsToLoop) > 1 { @@ -215,7 +218,11 @@ func testCmd() *cobra.Command { } } for _, s := range subjects { - pairs = append(pairs, runPair{tc: tc, subject: applyCasePin(s, tc)}) + // Every subject gets its own copy of the case: the + // runner rewrites endpoint commands/env in place while + // resolving `${NAME}`, so a shared pointer would leave + // the second subject running the first one's command. + pairs = append(pairs, runPair{tc: tc.CloneForRun(), subject: applyCasePin(s, tc)}) } } if len(pairs) == 0 { diff --git a/internal/config/case.go b/internal/config/case.go index e8f709e..dcfab7e 100644 --- a/internal/config/case.go +++ b/internal/config/case.go @@ -2,11 +2,13 @@ package config import ( "fmt" + "maps" "net" "os" "path" "path/filepath" "regexp" + "slices" "sort" "strings" "time" @@ -48,6 +50,19 @@ type TestCase struct { // drives data purely through endpoints. Endpoints []Endpoint `yaml:"endpoints"` + // Resolve computes values from the subject BEFORE the topology starts, so a + // case can drive the subject with a string the subject itself produces + // rather than one the case author transcribed. Each entry runs a one-shot + // container and captures its stdout; `${NAME}` then expands in every + // endpoint command and endpoint env value. + // + // The point is contract coverage. When a case hard-codes a value that some + // component generates at runtime, it stops testing the generator: the case + // keeps passing against a copy of what the author believed the format to be, + // and drifts silently when the real one changes. Resolving it from the + // subject makes the producer part of the test. + Resolve []ResolveValue `yaml:"resolve"` + // Agent, when set, adds an external agent container to the test topology. // The agent connects INTO the subject (director) rather than being connected // to by it — useful for testing agent-mode device collection. The compose @@ -608,6 +623,30 @@ type AgentConfig struct { MountsSharedData bool `yaml:"mounts_shared_data"` } +// ResolveValue asks the subject to produce a value the case then uses (see +// TestCase.Resolve). The harness runs a one-shot container, trims the trailing +// newline from its stdout, and substitutes the result for `${Name}` in every +// endpoint command and endpoint env value. +// +// The container runs to completion before the topology starts, so the command +// must be self-contained — it cannot reach a running subject, a generator, or +// the bench network, and it must not need any of them. Anything that only makes +// sense against a live topology belongs in an endpoint command instead. +type ResolveValue struct { + // Name is the placeholder this value fills, referenced as ${Name}. Use + // uppercase with underscores. Must be unique within the case. + Name string `yaml:"name"` + // Image is the container image to run. The literal "subject" means the + // image under test, including any --image/--version override, which is what + // makes the value come from the build being tested rather than a fixed one. + Image string `yaml:"image"` + // Command is the command to run. Its stdout (trailing newline trimmed) is + // the value. A non-zero exit fails the run: a value that could not be + // produced must stop the case, never silently expand to an empty string and + // let a downstream assertion pass for the wrong reason. + Command []string `yaml:"command"` +} + // Endpoint is an auxiliary container in the test topology (see // TestCase.Endpoints). It's a host the subject reaches on the bench network — // not a generator or receiver. @@ -1105,6 +1144,45 @@ func (tc *TestCase) Validate() error { } epNames[e.Name] = struct{}{} } + // Resolved values must be well formed and actually referenced. An unused or + // misspelled entry is the failure mode worth catching here: the ${NAME} it + // was meant to fill would stay literal in the command, the container would + // run something meaningless, and the case would report on that instead. + resolveNames := map[string]struct{}{} + for i, rv := range tc.Resolve { + if rv.Name == "" { + return fmt.Errorf("case %q: resolve[%d] missing required `name`", tc.Name, i) + } + if rv.Image == "" { + return fmt.Errorf("case %q: resolve %q missing required `image` (use \"subject\" for the image under test)", tc.Name, rv.Name) + } + if len(rv.Command) == 0 { + return fmt.Errorf("case %q: resolve %q missing required `command`", tc.Name, rv.Name) + } + if _, dup := resolveNames[rv.Name]; dup { + return fmt.Errorf("case %q: duplicate resolve name %q", tc.Name, rv.Name) + } + resolveNames[rv.Name] = struct{}{} + + placeholder := "${" + rv.Name + "}" + used := false + for _, e := range tc.Endpoints { + for _, part := range e.Command { + if strings.Contains(part, placeholder) { + used = true + } + } + for _, val := range e.Env { + if strings.Contains(val, placeholder) { + used = true + } + } + } + if !used { + return fmt.Errorf("case %q: resolve %q is never referenced as %s in any endpoint command or env", + tc.Name, rv.Name, placeholder) + } + } // Kafka types require the broker block + a generator producing in kafka mode. if tc.IsKafkaType() { if tc.Kafka == nil { @@ -2678,6 +2756,44 @@ func LoadCase(casesDir, name string) (*TestCase, error) { return &tc, nil } +// CloneForRun returns a copy of tc that one run may mutate in place without +// affecting any other. One loaded case is shared by every (case, subject) pair +// the CLI queues, and the runner rewrites parts of it per run: resolveValues +// expands `${NAME}` into endpoint commands and env, and the Vault cert-rotation +// flow seeds freshly generated certs into vault.secrets. Sharing one pointer +// bakes the first subject's values in for every subject after it — the +// placeholder is already gone, so the later runs silently execute the earlier +// subject's command and still report a verdict on it. +// +// Only the fields the runner writes are deep-copied; the rest stays shared +// because it is read-only for the duration of a run. Extend this when a new +// mutation site appears. +func (tc *TestCase) CloneForRun() *TestCase { + if tc == nil { + return nil + } + cp := *tc + if tc.Endpoints != nil { + cp.Endpoints = make([]Endpoint, len(tc.Endpoints)) + for i, ep := range tc.Endpoints { + cp.Endpoints[i] = ep + cp.Endpoints[i].Command = slices.Clone(ep.Command) + cp.Endpoints[i].Env = maps.Clone(ep.Env) + } + } + if tc.Vault != nil { + v := *tc.Vault + if tc.Vault.Secrets != nil { + v.Secrets = make(map[string]map[string]string, len(tc.Vault.Secrets)) + for secretPath, fields := range tc.Vault.Secrets { + v.Secrets[secretPath] = maps.Clone(fields) + } + } + cp.Vault = &v + } + return &cp +} + // ListCases returns all case names found in casesDir. func ListCases(casesDir string) ([]string, error) { entries, err := os.ReadDir(casesDir) diff --git a/internal/config/clone_test.go b/internal/config/clone_test.go new file mode 100644 index 0000000..5386fc3 --- /dev/null +++ b/internal/config/clone_test.go @@ -0,0 +1,56 @@ +package config + +import "testing" + +// TestCloneForRunIsolatesMutatedFields pins the invariant the CLI relies on: +// each (case, subject) pair runs against its own copy, so the runner rewriting +// `${NAME}` into an endpoint command — or seeding a rotated cert into +// vault.secrets — cannot leak into the next subject's run. +func TestCloneForRunIsolatesMutatedFields(t *testing.T) { + orig := &TestCase{ + Name: "resolve-case", + Endpoints: []Endpoint{{ + Name: "device", + Command: []string{"send", "--token", "${TOKEN}"}, + Env: map[string]string{"TOKEN": "${TOKEN}"}, + }}, + Vault: &VaultConfig{Secrets: map[string]map[string]string{ + "tls": {"cert": "original"}, + }}, + } + + cp := orig.CloneForRun() + + // Same rewrites the runner performs, applied to the copy only. + cp.Endpoints[0].Command[2] = "subject-a-token" + cp.Endpoints[0].Env["TOKEN"] = "subject-a-token" + cp.Vault.Secrets["tls"]["cert"] = "rotated" + + if got := orig.Endpoints[0].Command[2]; got != "${TOKEN}" { + t.Errorf("original endpoint command mutated: got %q, want %q", got, "${TOKEN}") + } + if got := orig.Endpoints[0].Env["TOKEN"]; got != "${TOKEN}" { + t.Errorf("original endpoint env mutated: got %q, want %q", got, "${TOKEN}") + } + if got := orig.Vault.Secrets["tls"]["cert"]; got != "original" { + t.Errorf("original vault secret mutated: got %q, want %q", got, "original") + } + if cp.Name != orig.Name { + t.Errorf("clone lost scalar fields: got name %q, want %q", cp.Name, orig.Name) + } +} + +// TestCloneForRunHandlesEmptyCase covers the nil/empty shapes most cases have — +// no endpoints and no vault block — where the clone must not invent one. +func TestCloneForRunHandlesEmptyCase(t *testing.T) { + if got := (*TestCase)(nil).CloneForRun(); got != nil { + t.Fatalf("nil case cloned to %v, want nil", got) + } + cp := (&TestCase{Name: "bare"}).CloneForRun() + if cp.Endpoints != nil { + t.Errorf("endpoints became %v, want nil", cp.Endpoints) + } + if cp.Vault != nil { + t.Errorf("vault became %v, want nil", cp.Vault) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 295b824..4be2247 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -157,8 +157,78 @@ func (r *Runner) applySubjectOverrides(subject config.Subject) config.Subject { return subject } +// resolveValues runs each tc.Resolve entry and expands `${NAME}` in every +// endpoint command and env value. It mutates tc, so it must run before any +// driver reads the endpoints — i.e. before Run dispatches on tc.Type. +// +// A failure here aborts the run. The alternative — leaving the placeholder +// literal or expanding it to "" — would let the case proceed and report a +// verdict on a command that is not the one it meant to run. +func (r *Runner) resolveValues(tc *config.TestCase, subject config.Subject) error { + if len(tc.Resolve) == 0 { + return nil + } + + subject = r.applySubjectOverrides(subject) + + // A resolve step is a `docker run` on an image that may not be on the host + // yet, so the wall has to cover a cold pull, not just the command — a fixed + // minute fails the whole case on a first-run pull that was going to + // succeed. Reuse the run timeout: it is the budget the operator already + // sized for this run, and r.ctx still cancels immediately on SIGINT. + resolveTimeout := r.opts.Timeout + if resolveTimeout <= 0 { + resolveTimeout = 10 * time.Minute + } + + for _, rv := range tc.Resolve { + image := rv.Image + if image == "subject" { + image = subject.Image + if subject.Version != "" { + image += ":" + subject.Version + } + } + + ctx, cancel := context.WithTimeout(r.ctx, resolveTimeout) + args := append([]string{"run", "--rm", "--entrypoint", rv.Command[0], image}, rv.Command[1:]...) + out, err := exec.CommandContext(ctx, "docker", args...).Output() + cancel() + if err != nil { + stderr := "" + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + stderr = strings.TrimSpace(string(exitErr.Stderr)) + } + return fmt.Errorf("resolve %q via %s: %w (stderr: %s)", rv.Name, image, err, stderr) + } + + value := strings.TrimRight(string(out), "\r\n") + if value == "" { + return fmt.Errorf("resolve %q via %s produced no output; a case must not run on an empty value", rv.Name, image) + } + + fmt.Printf(" resolved ${%s} from %s\n", rv.Name, image) + + placeholder := "${" + rv.Name + "}" + for i := range tc.Endpoints { + for j, part := range tc.Endpoints[i].Command { + tc.Endpoints[i].Command[j] = strings.ReplaceAll(part, placeholder, value) + } + for k, val := range tc.Endpoints[i].Env { + tc.Endpoints[i].Env[k] = strings.ReplaceAll(val, placeholder, value) + } + } + } + + return nil +} + // Run executes the test and returns the persisted result. func (r *Runner) Run(tc *config.TestCase, subject config.Subject) (results.RunResult, error) { + if err := r.resolveValues(tc, subject); err != nil { + return results.RunResult{}, err + } if tc.Type == "persistence_correctness" { return r.runPersistenceCorrectness(tc, subject) }