diff --git a/cmd/artifact/code/checkout/cmd_test.go b/cmd/artifact/code/checkout/cmd_test.go index 860a61940..d4e0ecdad 100644 --- a/cmd/artifact/code/checkout/cmd_test.go +++ b/cmd/artifact/code/checkout/cmd_test.go @@ -525,3 +525,58 @@ func TestCheckout_PromptsForVersionWhenMissing(t *testing.T) { assert.Contains(t, buf.String(), "dr artifact code versions") assert.DirExists(t, wapi.CheckoutDir(dir, verA)) } + +// The download stages into a .tmp--* sibling of the final snapshot +// dir and swaps it into place. Regression guard for the surrounding-state +// contract: whether the checkout succeeds or fails mid-download, the staging +// directory must be gone from the checkouts parent — renamed away on success, +// removed by the failure defer otherwise. +func TestCheckout_LeavesNoTempDirsInCheckoutsParent(t *testing.T) { + // Failure first: the download dies mid-way and the staging dir must be + // cleaned up rather than stranded next to the (absent) snapshot. + failDir := initLinkedDir(t, "cat-1") + + failing := &fakeClient{ + versions: []filesapi.CatalogVersion{{ID: verA}}, + content: map[string][]byte{"a.txt": []byte("a")}, + downloadErr: errors.New("network died"), + } + + cmd, _ := newTestCmd(t, failDir, fakeDeps(draftArtifact("art-abc-123"), failing), []string{verA}) + + require.Error(t, cmd.Execute()) + + assertNoTmpStagingDirs(t, failDir) + + // Then success: the staging dir is renamed to the snapshot, so the + // parent must hold only the snapshot — no staging residue. + okDir := initLinkedDir(t, "cat-1") + + fc := &fakeClient{ + versions: []filesapi.CatalogVersion{{ID: verA}}, + content: map[string][]byte{"a.txt": []byte("a")}, + } + + cmd2, _ := newTestCmd(t, okDir, fakeDeps(draftArtifact("art-abc-123"), fc), []string{verA}) + + require.NoError(t, cmd2.Execute()) + + assertNoTmpStagingDirs(t, okDir) +} + +// assertNoTmpStagingDirs fails when any .tmp- prefixed staging directory +// survived a checkout in the project's checkouts parent. +func assertNoTmpStagingDirs(t *testing.T, dir string) { + t.Helper() + + entries, err := os.ReadDir(wapi.CheckoutsDir(dir)) + if os.IsNotExist(err) { + return + } + + require.NoError(t, err) + + for _, e := range entries { + assert.NotContains(t, e.Name(), ".tmp-", "checkout staging directory must not survive the command") + } +} diff --git a/cmd/artifact/code/codesync/cmd.go b/cmd/artifact/code/codesync/cmd.go index 7b5753c4c..0ce6e9b70 100644 --- a/cmd/artifact/code/codesync/cmd.go +++ b/cmd/artifact/code/codesync/cmd.go @@ -22,6 +22,8 @@ import ( "errors" "fmt" "io" + "sort" + "strings" "github.com/datarobot/cli/cmd/artifact/code/internal/dirprompt" "github.com/datarobot/cli/cmd/artifact/code/internal/format" @@ -50,6 +52,8 @@ type engineRunner interface { StateMigrationNotice() string IgnoreFileNotice() string LockedNotice() string + Divergences() []sync.Divergence + SkippedSymlinks() []sync.SkippedSymlink Fetcher() display.ContentFetcher } @@ -71,11 +75,15 @@ type Deps struct { // runFlags is the parsed view of the boolean flags that gate // finishSync's render/prompt/execute decisions. Grouped so the inner -// helpers don't carry a three-bool tail through every signature. +// helpers don't carry a bool tail through every signature. Verify is +// carried here for the engine Options and telemetry only: it must never +// gate the render/prompt/execute decisions, because a verify run that +// is not a preview still has to execute. type runFlags struct { DryRun bool Diff bool Yes bool + Verify bool } func defaultDeps() Deps { @@ -118,8 +126,11 @@ versioned step. Use --dry-run to preview the plan without writing anything; --diff to also print per-file unified diffs. Both modes exit before any remote -write. --yes auto-confirms the post-plan prompt and skips any -interactive directory prompt. +write. --verify forces a remote round-trip that surfaces BASE-vs-REMOTE +divergence (composes with --dry-run and --diff); on an applying run it +also re-fetches after upload to verify what was uploaded. --yes +auto-confirms the post-plan prompt and skips any interactive directory +prompt. Run 'dr artifact code init ' first to link a project directory to an artifact. @@ -129,6 +140,7 @@ Example: dr artifact code sync --dry-run dr artifact code sync --diff dr artifact code sync --yes + dr artifact code sync --verify dr artifact code sync --output-format json`, PreRunE: auth.EnsureAuthenticatedE, RunE: func(cmd *cobra.Command, _ []string) error { @@ -144,6 +156,15 @@ Example: c.Flags().Bool("dry-run", false, "Show plan, no writes.") c.Flags().Bool("diff", false, "Show plan + per-file unified diffs, no writes.") c.Flags().BoolP(cli.YesFlagName, "y", false, "Skip interactive prompts; auto-confirm.") + + // Transient flag, like --yes: read straight from cobra in parseRunFlags, + // never bound into viper and never persisted to drconfig.yaml. + // + // It must not join the dry-run/diff mutual-exclusion group: verify is a + // diagnostic intensity switch, not a third preview mode, and a verify + // run without --dry-run still applies its plan. + c.Flags().Bool("verify", false, "Force a remote round-trip to surface BASE-vs-REMOTE divergence; on an applying run also re-fetch after upload to verify what was uploaded.") + c.MarkFlagsMutuallyExclusive("dry-run", "diff") telemetry.TrackWith(c, func(cmd *cobra.Command, _ []string) map[string]any { @@ -153,6 +174,7 @@ Example: "dry_run": flags.DryRun, "diff": flags.Diff, "yes": flags.Yes, + "verify": flags.Verify, "output_format": string(outputFormat), } }) @@ -174,7 +196,12 @@ func runSync(cmd *cobra.Command, outputFormat outputformat.OutputFormat, deps De return errors.New("not linked: run 'dr artifact code init ' first") } - engine, err := deps.NewEngine(dir, sync.Options{DryRun: flags.DryRun, ShowDiffs: flags.Diff, Yes: flags.Yes}) + engine, err := deps.NewEngine(dir, sync.Options{ + DryRun: flags.DryRun, + ShowDiffs: flags.Diff, + Yes: flags.Yes, + Verify: flags.Verify, + }) if err != nil { return err } @@ -195,6 +222,8 @@ func runSync(cmd *cobra.Command, outputFormat outputformat.OutputFormat, deps De format.StateNotice(cmd.ErrOrStderr(), engine.StateMigrationNotice()) format.StateNotice(cmd.ErrOrStderr(), engine.IgnoreFileNotice()) format.StateNotice(cmd.ErrOrStderr(), engine.LockedNotice()) + format.StateNotice(cmd.ErrOrStderr(), divergenceSummaryNotice(flags, plan, engine.Divergences())) + format.StateNotice(cmd.ErrOrStderr(), skippedSymlinkSummaryNotice(engine.SkippedSymlinks())) if engine.StaleRollbackRestored() { fmt.Fprintln(cmd.ErrOrStderr(), tui.DimStyle.Render("Recovered from interrupted sync. Working tree restored.")) @@ -205,18 +234,120 @@ func runSync(cmd *cobra.Command, outputFormat outputformat.OutputFormat, deps De // parseRunFlags reads the cobra flags once and folds the // DATAROBOT_CLI_NON_INTERACTIVE env-var override into Yes, so the -// downstream helpers see a single source of truth. +// downstream helpers see a single source of truth. Every flag here is +// transient: read directly from cobra, never through viper. func parseRunFlags(cmd *cobra.Command) runFlags { dryRun, _ := cmd.Flags().GetBool("dry-run") diff, _ := cmd.Flags().GetBool("diff") + verify, _ := cmd.Flags().GetBool("verify") + return runFlags{ DryRun: dryRun, Diff: diff, Yes: cli.IsNonInteractive(cmd), + Verify: verify, } } +// divergenceSummaryNotice renders the one-line --verify summary that runs +// alongside the other state notices: what diverged and what happens next. +// The per-path detail (with hashes) is logged from Phase 2 itself, so this +// summary stays stream-agnostic and lists paths up to DivergenceNoticeBound, +// summarizing any remainder as a count. +// +// The "what happens next" half is mode- and plan-aware because the honest +// answer differs by what the run actually does: a preview writes nothing, an +// empty-plan repair reconciles through the Phase 6 manifest rewrite rather +// than through plan rows, and only a non-empty applying plan reconciles the +// divergences through its rows. +func divergenceSummaryNotice(flags runFlags, plan *sync.SyncPlan, divergences []sync.Divergence) string { + if len(divergences) == 0 { + return "" + } + + paths := make([]string, len(divergences)) + + for i, d := range divergences { + paths[i] = d.Path + } + + // Bound the joined path list the same way the per-symlink summary does: + // individual names up to DivergenceNoticeBound, then a count. The full + // list always travels in the plan JSON. + shown := paths + + if len(paths) > sync.DivergenceNoticeBound { + shown = paths[:sync.DivergenceNoticeBound] + } + + list := strings.Join(shown, ", ") + + if len(paths) > sync.DivergenceNoticeBound { + list += fmt.Sprintf(", and %d more", len(paths)-sync.DivergenceNoticeBound) + } + + head := fmt.Sprintf( + "--verify found %d divergence(s) between manifest.json (BASE) and the server (REMOTE): %s.", + len(divergences), list) + + switch { + case flags.DryRun: + return head + " This was a preview; nothing was written. Run without --dry-run to reconcile." + case flags.Diff: + return head + " This was a preview; nothing was written. Run without --diff to reconcile." + case plan.IsEmpty(): + return head + " The plan is empty, but manifest.json is being rewritten from the server's state to repair them." + default: + return head + " The plan reconciles them." + } +} + +// skippedSymlinkSummaryNotice renders the one-line summary of skipped +// symlinks that runs alongside the other state notices. The per-symlink +// detail (with kind-specific wording) is logged from Phase 2 itself, so +// this summary stays stream-agnostic and names each symlink with its kind. +// The prose is bounded at SymlinkNoticeBound entries; the structured JSON +// field on the plan document carries every symlink regardless. +func skippedSymlinkSummaryNotice(symlinks []sync.SkippedSymlink) string { + if len(symlinks) == 0 { + return "" + } + + // Sort by path for deterministic output across runs. + sorted := make([]sync.SkippedSymlink, len(symlinks)) + copy(sorted, symlinks) + + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Path < sorted[j].Path }) + + shown := sorted + + if len(sorted) > sync.SymlinkNoticeBound { + shown = sorted[:sync.SymlinkNoticeBound] + } + + parts := make([]string, len(shown)) + + for i, s := range shown { + kind := "file" + if s.IsDir { + kind = "directory" + } + + parts[i] = fmt.Sprintf("%s (%s)", s.Path, kind) + } + + if len(sorted) > sync.SymlinkNoticeBound { + return fmt.Sprintf( + "%d symlink(s) were not uploaded or synced: %s, and %d more.", + len(sorted), strings.Join(parts, ", "), len(sorted)-sync.SymlinkNoticeBound) + } + + return fmt.Sprintf( + "%d symlink(s) were not uploaded or synced: %s.", + len(sorted), strings.Join(parts, ", ")) +} + // finishSync handles the render → optional prompt → execute → render // tail of the command. Pulled out so runSync's early-return paths // (auth, lock, plan errors) stay flat. @@ -227,11 +358,11 @@ func finishSync(cmd *cobra.Command, engine engineRunner, plan *sync.SyncPlan, ou return finishJSON(engine, plan, out, flags) } - if err := renderHumanPlan(cmd, engine, plan, flags.Diff); err != nil { + if err := renderHumanPlan(cmd, engine, plan, flags); err != nil { return err } - if flags.DryRun || flags.Diff || plan.IsEmpty() { + if skipsExecute(flags, plan, engine.Divergences()) { return nil } @@ -255,20 +386,45 @@ func finishSync(cmd *cobra.Command, engine engineRunner, plan *sync.SyncPlan, ou } // renderHumanPlan prints the plan and optional per-file diffs. -func renderHumanPlan(cmd *cobra.Command, engine engineRunner, plan *sync.SyncPlan, diffFlag bool) error { +func renderHumanPlan(cmd *cobra.Command, engine engineRunner, plan *sync.SyncPlan, flags runFlags) error { out := cmd.OutOrStdout() + // An empty plan normally prints "Up to date." — truthful when disk and + // server genuinely agree. On an applying run whose --verify findings + // force the empty-plan repair, though, Execute is about to rewrite + // manifest.json from the server's state, and "Up to date." immediately + // before that rewrite would claim nothing is being fixed. Previews keep + // the ordinary line: nothing is written, so the divergence summary's + // preview wording is what carries the honesty there. + if plan.IsEmpty() && len(engine.Divergences()) > 0 && !flags.DryRun && !flags.Diff { + return display.PrintEmptyPlanRepair(out) + } + if err := display.PrintPlan(out, plan); err != nil { return err } - if !diffFlag { + if !flags.Diff { return nil } return display.PrintDiffs(out, plan, engine.Fetcher()) } +// skipsExecute encapsulates the decision to stop after rendering the plan. +// A preview always stops. An empty plan stops too — except when a --verify +// run recorded BASE-vs-REMOTE divergences: then Execute must run, because +// Phase 5 no-ops on an empty plan and the Execute is what drives Phase 6's +// rewrite of the poisoned manifest from the real remote. Skipping Execute +// there is how the poison survives the run that caught it. +func skipsExecute(flags runFlags, plan *sync.SyncPlan, divergences []sync.Divergence) bool { + if flags.DryRun || flags.Diff { + return true + } + + return plan.IsEmpty() && len(divergences) == 0 +} + // shouldPromptConflicts encapsulates the decision: prompt only when // the user has not passed --yes and the plan actually has conflicts. func shouldPromptConflicts(plan *sync.SyncPlan, yes bool) bool { @@ -283,11 +439,14 @@ func shouldPromptConflicts(plan *sync.SyncPlan, yes bool) bool { // plan is emitted and no Execute is run, so callers can inspect the // plan and re-invoke with --yes if they want to proceed. func finishJSON(engine engineRunner, plan *sync.SyncPlan, out io.Writer, flags runFlags) error { - if err := display.RenderPlanJSON(out, plan, engine.LockedNotice() != ""); err != nil { + if err := display.RenderPlanJSON(out, plan, engine.LockedNotice() != "", engine.Divergences(), engine.SkippedSymlinks()); err != nil { return err } - if flags.DryRun || flags.Diff || plan.IsEmpty() { + // Same skip rule as the human path, including the empty-plan exception: + // divergence findings from --verify mean Execute must run so Phase 6 + // repairs the manifest, even though the plan itself has no rows. + if skipsExecute(flags, plan, engine.Divergences()) { return nil } diff --git a/cmd/artifact/code/codesync/cmd_test.go b/cmd/artifact/code/codesync/cmd_test.go index 2e9a3c698..a05fc0582 100644 --- a/cmd/artifact/code/codesync/cmd_test.go +++ b/cmd/artifact/code/codesync/cmd_test.go @@ -34,16 +34,18 @@ import ( // flag to assert the cmd's branch decisions (dry-run, diff, conflict // prompt, JSON output). type fakeEngine struct { - plan *sync.SyncPlan - planErr error - result *sync.Result - executeErr error - stale bool - migrationNote string - ignoreNotice string - lockedNote string - fetcher display.ContentFetcher - closeErr error + plan *sync.SyncPlan + planErr error + result *sync.Result + executeErr error + stale bool + migrationNote string + ignoreNotice string + lockedNote string + divergences []sync.Divergence + skippedSymlinks []sync.SkippedSymlink + fetcher display.ContentFetcher + closeErr error executed bool closed bool @@ -71,6 +73,10 @@ func (f *fakeEngine) IgnoreFileNotice() string { return f.ignoreNotice } func (f *fakeEngine) LockedNotice() string { return f.lockedNote } +func (f *fakeEngine) Divergences() []sync.Divergence { return f.divergences } + +func (f *fakeEngine) SkippedSymlinks() []sync.SkippedSymlink { return f.skippedSymlinks } + func (f *fakeEngine) Fetcher() display.ContentFetcher { return f.fetcher } // fakeEngineDeps returns a Deps that hands fe back from NewEngine and diff --git a/cmd/artifact/code/codesync/combined_integration_test.go b/cmd/artifact/code/codesync/combined_integration_test.go new file mode 100644 index 000000000..36b1b974e --- /dev/null +++ b/cmd/artifact/code/codesync/combined_integration_test.go @@ -0,0 +1,334 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codesync + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/datarobot/cli/internal/workload/sync" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests prove the display layer composes all three fixes: when +// symlinks, divergence, and other notices fire simultaneously, stdout stays +// pure JSON (or carries only the plan in human mode), stderr carries every +// notice, and no notice suppresses another. They use the fakeEngine so the +// display layer is what is under test, not the engine's detection logic +// (which is covered by the engine-level combined_integration_test.go). +// +// Fulfills the display-layer portions of VAL-CROSS-004(b), VAL-CROSS-008, +// and VAL-CROSS-009. + +// --------------------------------------------------------------------------- +// VAL-CROSS-004(b): Symlink + divergence in JSON mode — pure stdout +// --------------------------------------------------------------------------- + +// TestCmd_Combined_JSON_SymlinkAndDivergence_PureStdout proves that a +// --verify --dry-run --output-format json run with both a skipped symlink +// and a divergence produces stdout that decodes as valid JSON to EOF with +// both the skippedSymlinks field (non-empty) and the divergence field +// (naming the path) populated, while stderr carries both notices and no +// prose reaches stdout. +func TestCmd_Combined_JSON_SymlinkAndDivergence_PureStdout(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: []sync.SkippedSymlink{ + {Path: "link_to_file.py", IsDir: false}, + }, + divergences: []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + } + + flags := map[string]string{ + "dir": dir, + "yes": "true", + "dry-run": "true", + "verify": "true", + "output-format": "json", + } + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + // stdout decodes as valid JSON to EOF. + assertOnlyJSON(t, strings.NewReader(raw)) + + // The JSON plan document contains both the skippedSymlinks field + // (non-empty) and the divergence field (naming the path). + dec := json.NewDecoder(strings.NewReader(raw)) + + var doc struct { + SkippedSymlinks []struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` + } `json:"skippedSymlinks"` + Divergence []struct { + Path string `json:"path"` + Kind string `json:"kind"` + } `json:"divergence"` + } + + require.NoError(t, dec.Decode(&doc), "stdout must decode as the plan JSON document") + + require.Len(t, doc.SkippedSymlinks, 1, "the skippedSymlinks field must be non-empty") + assert.Equal(t, "link_to_file.py", doc.SkippedSymlinks[0].Path) + assert.False(t, doc.SkippedSymlinks[0].IsDir) + + require.Len(t, doc.Divergence, 1, "the divergence field must be non-empty") + assert.Equal(t, "app.py", doc.Divergence[0].Path) + assert.Equal(t, "hash_mismatch", doc.Divergence[0].Kind) + + // stderr carries both the symlink notice and the divergence prose. + errText := stderr.String() + assert.Contains(t, errText, "link_to_file.py", "stderr must carry the symlink notice") + assert.Contains(t, errText, "app.py", "stderr must carry the divergence notice naming the path") + assert.Contains(t, errText, "divergence", "stderr must carry the divergence prose") + + // No prose reaches stdout. + assert.NotContains(t, raw, "was not uploaded", "no symlink prose on stdout") + assert.NotContains(t, raw, "divergence:", "no divergence prose on stdout") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-008: Maximal mixed plan in JSON mode — every field populated +// --------------------------------------------------------------------------- + +// TestCmd_Combined_JSON_MaximalMixedPlan proves that a --verify --yes +// --output-format json run with every action type and both diagnostic types +// produces stdout that is pure JSON with every field populated, stderr +// carrying every notice, and exit code 0. +func TestCmd_Combined_JSON_MaximalMixedPlan(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{ + Uploads: []sync.FileAction{{Path: "upload_me.py"}}, + Downloads: []sync.FileAction{{Path: "download_me.py"}}, + Deletes: []sync.FileAction{{Path: "delete_me.py"}}, + Conflicts: []sync.FileAction{{Path: "conflict_me.py"}}, + }, + result: &sync.Result{ + NewVersion: "ver-new", + UploadedCount: 1, + DownloadedCount: 1, + DeletedCount: 1, + ConflictCount: 1, + }, + skippedSymlinks: []sync.SkippedSymlink{ + {Path: "link_to_file.py", IsDir: false}, + {Path: "link_to_dir", IsDir: true}, + }, + divergences: []sync.Divergence{ + {Path: "diverge_me.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + } + + flags := map[string]string{ + "dir": dir, + "yes": "true", + "verify": "true", + "output-format": "json", + } + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err, "exit code must be 0 for a successful sync with diagnostics") + + raw := stdout.String() + + // stdout is pure JSON to EOF (two documents: plan + result). + assertOnlyJSON(t, strings.NewReader(raw)) + + // Decode the first document (the plan). + dec := json.NewDecoder(strings.NewReader(raw)) + + var planDoc struct { + Uploads []map[string]any `json:"uploads"` + Downloads []map[string]any `json:"downloads"` + Deletes []map[string]any `json:"deletes"` + Conflicts []map[string]any `json:"conflicts"` + SkippedSymlinks []map[string]any `json:"skippedSymlinks"` + Divergence []map[string]any `json:"divergence"` + } + + require.NoError(t, dec.Decode(&planDoc), "the first document must be the plan") + + // Every JSON field is populated. + assert.NotEmpty(t, planDoc.Uploads, "uploads field must be populated") + assert.NotEmpty(t, planDoc.Downloads, "downloads field must be populated") + assert.NotEmpty(t, planDoc.Deletes, "deletes field must be populated") + assert.NotEmpty(t, planDoc.Conflicts, "conflicts field must be populated") + assert.NotEmpty(t, planDoc.SkippedSymlinks, "skippedSymlinks field must be populated") + assert.NotEmpty(t, planDoc.Divergence, "divergence field must be populated") + + // Decode the second document (the result). + var resultDoc map[string]any + require.NoError(t, dec.Decode(&resultDoc), "the second document must be the result") + assert.Equal(t, "ver-new", resultDoc["newVersion"]) + + // stderr carries the symlink, divergence, and conflict-resolution notices. + errText := stderr.String() + assert.Contains(t, errText, "link_to_file.py", "stderr must carry the file symlink notice") + assert.Contains(t, errText, "link_to_dir", "stderr must carry the directory symlink notice") + assert.Contains(t, errText, "divergence", "stderr must carry the divergence notice") + assert.Contains(t, errText, "diverge_me.py", "stderr must name the divergent path") + + // No prose reaches stdout. + assert.NotContains(t, raw, "was not uploaded", "no symlink prose on stdout") + assert.NotContains(t, raw, "divergence:", "no divergence prose on stdout") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-009: All notices at once — JSON and human mode +// --------------------------------------------------------------------------- + +// TestCmd_Combined_AllNotices_JSONMode proves that in JSON mode, every notice +// type (state migration, .wapiignore shadow warning, symlinks, divergence) +// reaches stderr while stdout stays pure JSON. The fakeEngine carries all +// four notice types simultaneously, so the display layer is what is under +// test: no notice suppresses another, and stdout carries only JSON. +func TestCmd_Combined_AllNotices_JSONMode(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "a.py"}}}, + migrationNote: "Moved local state from .wapi/ to .datarobot/workload/.", + ignoreNotice: "Using .wapiignore for sync ignore patterns.", + skippedSymlinks: []sync.SkippedSymlink{ + {Path: "link_to_file.py", IsDir: false}, + {Path: "link_to_dir", IsDir: true}, + }, + divergences: []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + } + + flags := map[string]string{ + "dir": dir, + "yes": "true", + "dry-run": "true", + "verify": "true", + "output-format": "json", + } + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + // stdout is pure JSON to EOF. + assertOnlyJSON(t, strings.NewReader(raw)) + + // stderr carries every notice — no notice suppresses another. + errText := stderr.String() + assert.Contains(t, errText, "Moved local state", "state migration notice must reach stderr") + assert.Contains(t, errText, ".wapiignore", "ignore file notice must reach stderr") + assert.Contains(t, errText, "link_to_file.py", "file symlink notice must reach stderr") + assert.Contains(t, errText, "link_to_dir", "directory symlink notice must reach stderr") + assert.Contains(t, errText, "divergence", "divergence notice must reach stderr") + assert.Contains(t, errText, "app.py", "divergence notice must name the path") + + // No notice leaks onto stdout. + assert.NotContains(t, raw, "Moved local state", "no migration prose on stdout") + assert.NotContains(t, raw, ".wapiignore", "no ignore prose on stdout") + assert.NotContains(t, raw, "was not uploaded", "no symlink prose on stdout") + assert.NotContains(t, raw, "divergence:", "no divergence prose on stdout") + + // stdout is independently reconstructable: it contains only the plan JSON. + dec := json.NewDecoder(strings.NewReader(raw)) + + var doc struct { + Uploads []map[string]any `json:"uploads"` + SkippedSymlinks []map[string]any `json:"skippedSymlinks"` + Divergence []map[string]any `json:"divergence"` + } + + require.NoError(t, dec.Decode(&doc)) + assert.NotEmpty(t, doc.Uploads, "the plan must carry the upload row") + assert.Len(t, doc.SkippedSymlinks, 2, "the plan must carry both symlinks") + assert.Len(t, doc.Divergence, 1, "the plan must carry the divergence") + + // stderr is independently reconstructable: every notice is present and + // none is missing. The plan is NOT on stderr. + assert.NotContains(t, errText, "a.py", "the plan must not leak onto stderr") +} + +// TestCmd_Combined_AllNotices_HumanMode proves that in human mode, stdout +// carries only the plan and stderr carries every notice, so each stream is +// independently reconstructable. +func TestCmd_Combined_AllNotices_HumanMode(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "a.py"}}}, + migrationNote: "Moved local state from .wapi/ to .datarobot/workload/.", + ignoreNotice: "Using .wapiignore for sync ignore patterns.", + skippedSymlinks: []sync.SkippedSymlink{ + {Path: "link_to_file.py", IsDir: false}, + {Path: "link_to_dir", IsDir: true}, + }, + divergences: []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + } + + flags := map[string]string{ + "dir": dir, + "yes": "true", + "dry-run": "true", + "verify": "true", + } + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + outText := stdout.String() + errText := stderr.String() + + // stdout carries only the plan (the upload row). The negative checks pin + // the actual command-summary substrings (not the per-phase prose the + // fakeEngine never emits), so a StateNotice misrouted to stdout fails. + assert.Contains(t, outText, "a.py", "stdout must carry the plan") + assert.NotContains(t, outText, "Moved local state", "no migration notice on stdout") + assert.NotContains(t, outText, ".wapiignore", "no ignore notice on stdout") + assert.NotContains(t, outText, "were not uploaded or synced", "no symlink summary on stdout") + assert.NotContains(t, outText, "--verify found", "no divergence summary on stdout") + + // stderr carries every notice — no notice suppresses another. + assert.Contains(t, errText, "Moved local state", "state migration notice must reach stderr") + assert.Contains(t, errText, ".wapiignore", "ignore file notice must reach stderr") + assert.Contains(t, errText, "link_to_file.py", "file symlink notice must reach stderr") + assert.Contains(t, errText, "link_to_dir", "directory symlink notice must reach stderr") + assert.Contains(t, errText, "divergence", "divergence notice must reach stderr") + assert.Contains(t, errText, "app.py", "divergence notice must name the path") + + // The plan is NOT on stderr. + assert.NotContains(t, errText, "a.py", "the plan must not leak onto stderr") + + // Each stream is independently reconstructable: stdout has the plan, + // stderr has every notice. Together they carry the full picture, but + // neither duplicates the other. +} diff --git a/cmd/artifact/code/codesync/symlink_notice_test.go b/cmd/artifact/code/codesync/symlink_notice_test.go new file mode 100644 index 000000000..b7e993610 --- /dev/null +++ b/cmd/artifact/code/codesync/symlink_notice_test.go @@ -0,0 +1,355 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codesync + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/datarobot/cli/internal/workload/sync" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sampleSkippedSymlinks is a file symlink plus a directory symlink, so every +// test exercises a notice that carries both kind distinctions. +func sampleSkippedSymlinks() []sync.SkippedSymlink { + return []sync.SkippedSymlink{ + {Path: "link_to_file.py", IsDir: false}, + {Path: "link_to_dir", IsDir: true}, + } +} + +// The symlink notice is a diagnostic, and diagnostics never belong on the +// data stream: it must reach stderr in human mode and stay off stdout. +func TestRunE_SymlinkNotice_ReachesStderrOnly(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: sampleSkippedSymlinks(), + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true"} + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + errText := stderr.String() + + // The summary names every symlink with its kind. + assert.Contains(t, errText, "link_to_file.py") + assert.Contains(t, errText, "link_to_dir") + assert.Contains(t, errText, "file") + assert.Contains(t, errText, "directory") + + assert.NotContains(t, stdout.String(), "link_to_file.py", + "symlink prose must stay off the data stream") + assert.NotContains(t, stdout.String(), "link_to_dir", + "symlink prose must stay off the data stream") +} + +// Under --output-format json the symlink data flows into the plan document +// as a structured field, and every trace of the prose stays on stderr so +// stdout decodes as JSON to EOF. +func TestRunE_JSON_SkippedSymlinkFieldPopulated(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: sampleSkippedSymlinks(), + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "output-format": "json"} + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + assertOnlyJSON(t, strings.NewReader(raw)) + + assert.Contains(t, stderr.String(), "link_to_file.py", + "the symlink notice must remain on stderr in JSON mode") + + dec := json.NewDecoder(strings.NewReader(raw)) + + var doc struct { + SkippedSymlinks []struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` + } `json:"skippedSymlinks"` + } + + require.NoError(t, dec.Decode(&doc), "the first stdout document must be the plan") + require.Len(t, doc.SkippedSymlinks, 2, "every skipped symlink must be listed in the plan document") + + assert.Equal(t, "link_to_file.py", doc.SkippedSymlinks[0].Path) + assert.False(t, doc.SkippedSymlinks[0].IsDir) + + assert.Equal(t, "link_to_dir", doc.SkippedSymlinks[1].Path) + assert.True(t, doc.SkippedSymlinks[1].IsDir) +} + +// A project with no symlinks leaves the field explicitly empty rather than +// omitted: a script must be able to tell "no symlinks" from "the field was +// never emitted" (the locked:false precedent). +func TestRunE_JSON_SkippedSymlinksExplicitlyEmptyWhenNone(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{plan: &sync.SyncPlan{}} + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "output-format": "json"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + assertOnlyJSON(t, strings.NewReader(raw)) + assert.Contains(t, raw, `"skippedSymlinks": []`) +} + +// The notice must fire in every mode — dry-run, diff, and a real sync — and +// even when the plan is empty. The exit code must be 0 in all of these. +func TestRunE_SymlinkNotice_AllModes_ExitZero(t *testing.T) { + for _, tc := range []struct { + name string + flags map[string]string + }{ + {name: "dry-run", flags: map[string]string{"yes": "true", "dry-run": "true"}}, + {name: "diff", flags: map[string]string{"yes": "true", "diff": "true"}}, + {name: "real-sync", flags: map[string]string{"yes": "true"}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: sampleSkippedSymlinks(), + result: &sync.Result{NewVersion: "v1"}, + } + + flags := map[string]string{"dir": dir} + for k, v := range tc.flags { + flags[k] = v + } + + _, _, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err, "exit code must be 0 when the project contains skipped symlinks") + + assert.Contains(t, stderr.String(), "link_to_file.py", + "the symlink notice must fire in %s mode", tc.name) + }) + } +} + +// The notice must fire even when the plan is empty / "Up to date.". +func TestRunE_SymlinkNotice_EmptyPlan(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: sampleSkippedSymlinks(), + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true"} + + _, _, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + assert.Contains(t, stderr.String(), "link_to_file.py", + "the symlink notice must fire even when the plan is empty") +} + +// On a locked artifact containing a file symlink, --dry-run --yes emits both +// the symlink notice and the locked notice on stderr and prints the plan on +// stdout. +func TestRunE_SymlinkNotice_FiresAlongsideLockedNotice(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{ + Uploads: []sync.FileAction{{Path: "a.py"}}, + }, + lockedNote: "Artifact art-1 is locked, so this is a preview only.", + skippedSymlinks: []sync.SkippedSymlink{{Path: "link.py", IsDir: false}}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true"} + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + assert.Contains(t, stderr.String(), "is locked", "the locked notice must appear on stderr") + assert.Contains(t, stderr.String(), "link.py", "the symlink notice must also appear on stderr") + assert.Contains(t, stdout.String(), "a.py", "the plan must appear on stdout") +} + +// When more than SymlinkNoticeBound symlinks are skipped, the stderr prose +// lists the first 5 in deterministic order followed by a count of the +// remainder, while the JSON field still lists every one. +func TestRunE_SymlinkNotice_BoundedProse_CompleteJSON(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + // Create 7 symlinks so the bound (5) is exceeded. + symlinks := make([]sync.SkippedSymlink, 7) + for i := range symlinks { + symlinks[i] = sync.SkippedSymlink{ + Path: "link" + string(rune('a'+i)) + ".py", + IsDir: false, + } + } + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: symlinks, + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "output-format": "json"} + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + assertOnlyJSON(t, strings.NewReader(raw)) + + // The JSON field lists every symlink. + dec := json.NewDecoder(strings.NewReader(raw)) + + var doc struct { + SkippedSymlinks []struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` + } `json:"skippedSymlinks"` + } + + require.NoError(t, dec.Decode(&doc)) + require.Len(t, doc.SkippedSymlinks, 7, "the JSON field must list every symlink even when prose is bounded") + + // The stderr prose is bounded: mentions the first 5 and the count of the remainder. + errText := stderr.String() + assert.Contains(t, errText, "linka.py") + assert.Contains(t, errText, "linkb.py") + assert.Contains(t, errText, "linkc.py") + assert.Contains(t, errText, "linkd.py") + assert.Contains(t, errText, "linke.py") + assert.NotContains(t, errText, "linkf.py", "the 6th symlink must not appear in the bounded prose") + assert.NotContains(t, errText, "linkg.py", "the 7th symlink must not appear in the bounded prose") + assert.Contains(t, errText, "2 more", "the count of the remainder must appear") +} + +// A symlink path never appears in any action list; it exists only in the +// skipped-symlink notice/field. +func TestRunE_JSON_SymlinkPathNotInActionLists(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{ + Uploads: []sync.FileAction{{Path: "real.py"}}, + }, + skippedSymlinks: []sync.SkippedSymlink{{Path: "link.py", IsDir: false}}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "output-format": "json"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + dec := json.NewDecoder(strings.NewReader(raw)) + + var doc struct { + Uploads []map[string]any `json:"uploads"` + Downloads []map[string]any `json:"downloads"` + Deletes []map[string]any `json:"deletes"` + Conflicts []map[string]any `json:"conflicts"` + SkippedSymlinks []map[string]any `json:"skippedSymlinks"` + } + + require.NoError(t, dec.Decode(&doc)) + + // The symlink path is only in skippedSymlinks. + require.Len(t, doc.SkippedSymlinks, 1) + assert.Equal(t, "link.py", doc.SkippedSymlinks[0]["path"]) + + // It is not in any action list. + for _, u := range doc.Uploads { + assert.NotEqual(t, "link.py", u["path"], "symlink path must not appear in uploads") + } + + for _, d := range doc.Deletes { + assert.NotEqual(t, "link.py", d["path"], "symlink path must not appear in deletes") + } + + for _, d := range doc.Downloads { + assert.NotEqual(t, "link.py", d["path"], "symlink path must not appear in downloads") + } + + for _, c := range doc.Conflicts { + assert.NotEqual(t, "link.py", c["path"], "symlink path must not appear in conflicts") + } +} + +// Two runs against an unchanged project produce the same set, order and +// notice text (deterministic across runs). +func TestRunE_SymlinkNotice_DeterministicAcrossRuns(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + symlinks := []sync.SkippedSymlink{ + {Path: "z_link.py", IsDir: false}, + {Path: "a_link.py", IsDir: false}, + {Path: "m_link", IsDir: true}, + } + + makeEngine := func() *fakeEngine { + return &fakeEngine{ + plan: &sync.SyncPlan{}, + skippedSymlinks: symlinks, + } + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true"} + + _, _, stderr1, err := runWithDeps(t, fakeEngineDeps(makeEngine()), flags) + require.NoError(t, err) + + _, _, stderr2, err := runWithDeps(t, fakeEngineDeps(makeEngine()), flags) + require.NoError(t, err) + + // The notice text is identical across runs (sorted by path). + assert.Equal(t, stderr1.String(), stderr2.String(), + "two runs against an unchanged project must produce the same notice text") + + // The sorted order is a_link.py, m_link, z_link.py. + errText := stderr1.String() + idxA := strings.Index(errText, "a_link.py") + idxM := strings.Index(errText, "m_link") + idxZ := strings.Index(errText, "z_link.py") + require.True(t, idxA >= 0 && idxM >= 0 && idxZ >= 0, "all three symlinks must appear") + assert.True(t, idxA < idxM && idxM < idxZ, "symlinks must be listed in sorted path order") +} diff --git a/cmd/artifact/code/codesync/telemetry_test.go b/cmd/artifact/code/codesync/telemetry_test.go index 9198df05d..96a772c0b 100644 --- a/cmd/artifact/code/codesync/telemetry_test.go +++ b/cmd/artifact/code/codesync/telemetry_test.go @@ -39,3 +39,30 @@ func TestTelemetry_ExtractorPropertiesAfterFlagParse(t *testing.T) { assert.Equal(t, true, event.EventProperties["yes"]) assert.Equal(t, "json", event.EventProperties["output_format"]) } + +// The verify attribute must make --verify runs distinguishable from plain +// runs. It is emitted unconditionally (false without the flag) rather than +// omitted, so a missing key reads as a wiring bug instead of quietly +// blending verify runs into the plain-run population. +func TestTelemetry_VerifyAttribute(t *testing.T) { + t.Run("false without the flag", func(t *testing.T) { + cmd := Cmd() + require.NoError(t, cmd.ParseFlags([]string{"--yes"})) + + event, ok := telemetry.EventFor(cmd, nil) + require.True(t, ok, "EventFor must return ok=true for an annotated command") + + assert.Contains(t, event.EventProperties, "verify", "the attribute must always be emitted, never omitted") + assert.Equal(t, false, event.EventProperties["verify"]) + }) + + t.Run("true with --verify", func(t *testing.T) { + cmd := Cmd() + require.NoError(t, cmd.ParseFlags([]string{"--verify", "--yes"})) + + event, ok := telemetry.EventFor(cmd, nil) + require.True(t, ok, "EventFor must return ok=true for an annotated command") + + assert.Equal(t, true, event.EventProperties["verify"]) + }) +} diff --git a/cmd/artifact/code/codesync/verify_flag_test.go b/cmd/artifact/code/codesync/verify_flag_test.go new file mode 100644 index 000000000..f978b38d9 --- /dev/null +++ b/cmd/artifact/code/codesync/verify_flag_test.go @@ -0,0 +1,494 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codesync + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/datarobot/cli/internal/workload/sync" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCmd_VerifyFlagRegistered pins the --verify registration: a boolean +// flag whose usage string tells the user what the flag buys them — the +// forced remote round-trip and the post-apply verification of what was +// uploaded. A flag that is present but unexplained reads as plumbing to +// nobody, so the usage string is part of the contract. +func TestCmd_VerifyFlagRegistered(t *testing.T) { + cmd := Cmd() + + flag := cmd.Flags().Lookup("verify") + require.NotNil(t, flag, "--verify must be registered on the sync command") + assert.Equal(t, "bool", flag.Value.Type()) + assert.NotEmpty(t, flag.Usage, "usage string must be non-empty") + assert.Contains(t, flag.Usage, "round-trip", "usage must describe the forced remote round-trip") + assert.Contains(t, flag.Usage, "verif", "usage must describe the post-apply verification of what was uploaded") + + verified, err := cmd.Flags().GetBool("verify") + require.NoError(t, err) + assert.False(t, verified, "--verify must default to false") +} + +// TestCmd_VerifyComposesWithOtherFlags: --verify gates network-cost checks, +// not rendering or confirmation, so it must parse alongside every other +// flag the command accepts — in particular it must NOT join the +// dry-run/diff mutual-exclusion group, and no combination here may be +// rejected by the parser. +func TestCmd_VerifyComposesWithOtherFlags(t *testing.T) { + for _, args := range [][]string{ + {"--verify", "--dry-run", "--yes"}, + {"--verify", "--diff", "--yes"}, + {"--verify", "--yes", "--output-format", "json"}, + } { + cmd := Cmd() + + require.NoError(t, cmd.ParseFlags(args), "flag set %v must be accepted with no flag-conflict error", args) + + verified, err := cmd.Flags().GetBool("verify") + require.NoError(t, err) + + assert.True(t, verified, "--verify must parse as true for args %v", args) + } +} + +// capturingEngineDeps returns Deps whose NewEngine records the options it +// was called with before handing back fe, so a test can assert the flag +// actually reached the engine rather than stopping at the command layer. +func capturingEngineDeps(fe *fakeEngine, captured *sync.Options) Deps { + return Deps{ + NewEngine: func(_ string, opts sync.Options) (engineRunner, error) { + *captured = opts + + return fe, nil + }, + } +} + +// TestCmd_VerifyThreadedIntoEngineOptions: the flag value must arrive in +// sync.Options.Verify — set with --verify, clear without it — for a test +// to catch the value stopping anywhere between cobra and the engine. +func TestCmd_VerifyThreadedIntoEngineOptions(t *testing.T) { + for _, tc := range []struct { + name string + extraFlag string + want bool + }{ + {name: "with --verify", extraFlag: "verify", want: true}, + {name: "without --verify", extraFlag: "", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "a.py"}}}, + result: &sync.Result{NewVersion: "v2", UploadedCount: 1}, + } + + var captured sync.Options + + flags := map[string]string{"dir": dir, "yes": "true"} + if tc.extraFlag != "" { + flags[tc.extraFlag] = "true" + } + + _, _, _, err := runWithDeps(t, capturingEngineDeps(fe, &captured), flags) + require.NoError(t, err) + assert.Equal(t, tc.want, captured.Verify, "Options.Verify must mirror the --verify flag") + }) + } +} + +// TestCmd_VerifyStillExecutes_NonDryRun: --verify is a diagnostic switch, +// not a preview mode. A non-dry-run verify run must reach Execute, create +// its version, and print the completion summary — the failure mode this +// guards is --verify being absorbed into the preview-only predicate and +// silently turning every verify user's sync into a preview. +func TestCmd_VerifyStillExecutes_NonDryRun(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "a.py"}}}, + result: &sync.Result{NewVersion: "v2", UploadedCount: 1}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + assert.True(t, fe.executed, "a non-dry-run --verify run must still Execute the plan") + assert.Contains(t, stdout.String(), "Sync complete", "the completion summary must be printed") +} + +// sampleDivergences is the flagship-shaped finding set: a hash mismatch plus +// both one-sided kinds, so every test below exercises a notice that carries +// all three distinctions. +func sampleDivergences() []sync.Divergence { + return []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + {Path: "gone.py", Kind: sync.DivergenceBaseOnly, BaseHash: "cccc"}, + {Path: "stray.py", Kind: sync.DivergenceRemoteOnly, RemoteHash: "dddd"}, + } +} + +// The divergence notice is a diagnostic, and diagnostics never belong on the +// data stream: it must reach stderr in human mode and stay off stdout. +func TestRunE_DivergenceNotice_ReachesStderrOnly(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: sampleDivergences(), + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "verify": "true"} + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + errText := stderr.String() + + // The summary names every divergent path, not just the first. + assert.Contains(t, errText, "app.py") + assert.Contains(t, errText, "gone.py") + assert.Contains(t, errText, "stray.py") + + assert.NotContains(t, stdout.String(), "app.py", "divergence prose must stay off the data stream") +} + +// Under --output-format json the divergence data flows into the plan document +// as a structured field, and every trace of the prose stays on stderr so +// stdout decodes as JSON to EOF. +func TestRunE_JSON_DivergenceFieldPopulated(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: sampleDivergences(), + } + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "verify": "true", "output-format": "json"} + + _, stdout, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + // Capture before any reader drains the buffer: assertOnlyJSON decodes to + // EOF, and a bytes.Buffer stays empty afterwards. + raw := stdout.String() + + assertOnlyJSON(t, strings.NewReader(raw)) + + assert.Contains(t, stderr.String(), "app.py", "the divergence notice must remain on stderr in JSON mode") + + dec := json.NewDecoder(strings.NewReader(raw)) + + var doc struct { + Divergence []struct { + Path string `json:"path"` + Kind string `json:"kind"` + BaseHash string `json:"baseHash"` + RemoteHash string `json:"remoteHash"` + } `json:"divergence"` + } + + require.NoError(t, dec.Decode(&doc), "the first stdout document must be the plan") + require.Len(t, doc.Divergence, 3, "every divergent path must be listed in the plan document") + + assert.Equal(t, "app.py", doc.Divergence[0].Path) + assert.Equal(t, "hash_mismatch", doc.Divergence[0].Kind) + assert.Equal(t, "aaaa", doc.Divergence[0].BaseHash) + assert.Equal(t, "bbbb", doc.Divergence[0].RemoteHash) + + assert.Equal(t, "gone.py", doc.Divergence[1].Path) + assert.Equal(t, "base_only", doc.Divergence[1].Kind) + + assert.Equal(t, "stray.py", doc.Divergence[2].Path) + assert.Equal(t, "remote_only", doc.Divergence[2].Kind) +} + +// A healthy --verify run leaves the field explicitly empty rather than +// omitted: a script must be able to tell "no divergence" from "the field was +// never emitted" (the locked:false precedent). +func TestRunE_JSON_DivergenceExplicitlyEmptyWhenClean(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{plan: &sync.SyncPlan{}} + + flags := map[string]string{"dir": dir, "yes": "true", "dry-run": "true", "verify": "true", "output-format": "json"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + // Same capture-before-drain discipline as above. + raw := stdout.String() + + assertOnlyJSON(t, strings.NewReader(raw)) + assert.Contains(t, raw, `"divergence": []`) +} + +// The empty-plan repair: a verify run whose plan came back empty but whose +// divergence findings are non-empty must still reach Execute. The engine's +// Phase 5 no-ops on an empty plan, so the Execute is what drives the Phase 6 +// state write that rewrites the poisoned manifest from the real remote; +// skipping Execute here is how the poison survives the run that caught it. +func TestRunE_EmptyPlan_WithDivergence_ExecutesForRepair(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + result: &sync.Result{NewVersion: "v1"}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true"} + + _, _, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + assert.True(t, fe.executed, + "an empty plan with divergence findings must still Execute so Phase 6 repairs the manifest") + assert.Contains(t, stderr.String(), "app.py", "the divergence summary must still reach stderr") +} + +// JSON-mode twin of the repair path: the plan document is emitted first (its +// divergence field populated), then Execute runs and the result document +// follows — the same two-document shape as a normal applying sync. +func TestRunE_JSON_EmptyPlan_WithDivergence_ExecutesAndEmitsResult(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + result: &sync.Result{NewVersion: "v1"}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true", "output-format": "json"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + raw := stdout.String() + + assertOnlyJSON(t, strings.NewReader(raw)) + assert.True(t, fe.executed, "the JSON path must Execute for the empty-plan repair too") + assert.Contains(t, raw, `"v1"`, "the result document must follow the plan document") +} + +// The one-line divergence summary must not claim a reconciliation the run +// does not perform. In a preview (--dry-run/--diff) nothing is written at +// all, so the summary has to say so and point at the mode flag instead of +// asserting that the plan reconciles anything. +func TestRunE_DivergenceSummary_PreviewSaysNothingWritten(t *testing.T) { + for _, tc := range []struct { + name string + previewFlag string + runWithout string + }{ + {name: "--dry-run", previewFlag: "dry-run", runWithout: "--dry-run"}, + {name: "--diff", previewFlag: "diff", runWithout: "--diff"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: sampleDivergences(), + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true", tc.previewFlag: "true"} + + _, _, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + errText := stderr.String() + + assert.Contains(t, errText, "This was a preview; nothing was written.", + "a preview must not read as a run that reconciles anything") + assert.Contains(t, errText, fmt.Sprintf("Run without %s to reconcile.", tc.runWithout), + "the summary must say how to actually reconcile") + + assert.NotContains(t, errText, "The plan reconciles them.", + "nothing is reconciled in a preview; the applying wording must not appear") + }) + } +} + +// The empty-plan repair run reconciles nothing through plan rows — the plan +// has none. The reconciliation is the Phase 6 manifest rewrite, so the +// summary must say the manifest is being rewritten from the server's state +// rather than repeating "The plan reconciles them.". +func TestRunE_DivergenceSummary_EmptyPlanRepair_SaysManifestRewritten(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: sampleDivergences(), + result: &sync.Result{NewVersion: "v1"}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true"} + + _, _, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + errText := stderr.String() + + assert.True(t, fe.executed, "the repair Execute must still run") + assert.Contains(t, errText, + "The plan is empty, but manifest.json is being rewritten from the server's state", + "the summary must name the manifest rewrite, not plan rows") + assert.NotContains(t, errText, "The plan reconciles them.", + "an empty plan reconciles no rows; the applying wording would mislead") +} + +// The ordinary applying wording is load-bearing and unchanged: a non-empty +// plan really does reconcile the divergences through its rows. +func TestRunE_DivergenceSummary_ApplyingKeepsReconcilesWording(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "app.py"}}}, + divergences: sampleDivergences(), + result: &sync.Result{NewVersion: "v2", UploadedCount: 1}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true"} + + _, _, stderr, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + assert.True(t, fe.executed) + assert.Contains(t, stderr.String(), "The plan reconciles them.", + "the non-empty applying wording must stay as it is") +} + +// The empty-plan repair run prints its honest line to stdout in place of +// "Up to date.": the plan really is empty, but the run is about to repair +// the manifest, and claiming the project is up to date immediately before +// that rewrite is the lie scrutiny flagged. The completion summary from the +// state write must still follow. +func TestRunE_EmptyPlanRepair_NeverSaysUpToDate(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{ + plan: &sync.SyncPlan{}, + divergences: []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "aaaa", RemoteHash: "bbbb"}, + }, + result: &sync.Result{NewVersion: "v1"}, + } + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + out := stdout.String() + + assert.NotContains(t, out, "Up to date.", + "the repair run must not claim the project is up to date") + assert.Contains(t, out, + "The plan is empty, but manifest.json is being rewritten from the server's state to repair the divergences found by --verify.", + "stdout must say the plan is empty and the manifest is being repaired") + assert.Contains(t, out, "Sync complete", + "the state-write completion summary must still follow") +} + +// The genuinely converged case keeps the pre-existing line: with no +// divergence findings there is nothing to repair, the empty plan +// short-circuits before Execute, and "Up to date." is the truth. +func TestRunE_EmptyPlan_Converged_KeepsUpToDate(t *testing.T) { + dir := t.TempDir() + linkProject(t, dir) + + fe := &fakeEngine{plan: &sync.SyncPlan{}} + + flags := map[string]string{"dir": dir, "yes": "true", "verify": "true"} + + _, stdout, _, err := runWithDeps(t, fakeEngineDeps(fe), flags) + require.NoError(t, err) + + assert.False(t, fe.executed, "no divergences means the empty-plan short-circuit stands") + assert.Contains(t, stdout.String(), "Up to date.", + "a genuinely converged empty plan must keep its existing line") +} + +// The summary is keyed off divergence findings alone: with none, every mode +// and plan shape must render nothing, so a plain sync gains no extra prose. +func TestDivergenceSummaryNotice_EmptyWhenNoDivergences(t *testing.T) { + for _, tc := range []struct { + name string + flags runFlags + plan *sync.SyncPlan + }{ + {name: "applying", flags: runFlags{}, plan: &sync.SyncPlan{}}, + {name: "applying non-empty", flags: runFlags{}, plan: &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "a.py"}}}}, + {name: "dry-run", flags: runFlags{DryRun: true}, plan: &sync.SyncPlan{}}, + {name: "diff", flags: runFlags{Diff: true}, plan: &sync.SyncPlan{}}, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Empty(t, divergenceSummaryNotice(tc.flags, tc.plan, nil)) + }) + } +} + +// The path list in the one-line summary is bounded at DivergenceNoticeBound, +// mirroring the per-symlink summary: the first five paths are named and the +// remainder is summarized as "and N more", while the full list still travels +// in the plan JSON. The mode/branch wording stays byte-identical. +func TestDivergenceSummaryNotice_BoundsPathListAtFive(t *testing.T) { + divs := make([]sync.Divergence, 0, 7) + + for i := 0; i < 7; i++ { + divs = append(divs, sync.Divergence{ + Path: fmt.Sprintf("%c.py", rune('a'+i)), + Kind: sync.DivergenceRemoteOnly, + }) + } + + plan := &sync.SyncPlan{Uploads: []sync.FileAction{{Path: "a.py"}}} + + notice := divergenceSummaryNotice(runFlags{}, plan, divs) + + assert.Contains(t, notice, "--verify found 7 divergence(s)", + "the full count must stay in the head") + assert.Contains(t, notice, "a.py") + assert.Contains(t, notice, "e.py", "the fifth path must be named") + assert.NotContains(t, notice, "f.py", "the sixth path must not be named") + assert.NotContains(t, notice, "g.py", "the seventh path must not be named") + assert.Contains(t, notice, ", and 2 more", + "the remainder must be summarized as a count") + assert.Contains(t, notice, "The plan reconciles them.", + "the applying sentence must stay intact") +} diff --git a/cmd/artifact/code/versions/cmd_test.go b/cmd/artifact/code/versions/cmd_test.go index 8d5e9b258..b027c9d2b 100644 --- a/cmd/artifact/code/versions/cmd_test.go +++ b/cmd/artifact/code/versions/cmd_test.go @@ -199,6 +199,29 @@ func TestVersions_TextOutput(t *testing.T) { assert.Contains(t, out, "* v3aaaaaaaaaaaa") assert.Contains(t, out, "v2bbbbbbbbbbbb") assert.Contains(t, out, "* = current") + assert.Contains(t, out, "Local synced to: v2", "the config's LastSyncedVersionID must be reported in text mode") +} + +// Text mode must stay silent about the synced version when the project has +// never synced: the line is conditional on config.LastSyncedVersionID, and a +// regression printing it empty would imply state the project does not have. +func TestVersions_TextOutput_NoSyncedVersionOmitsLine(t *testing.T) { + dir := initLinkedDir(t, "cat-1", "") + + deps := fakeDeps( + draftArtifact("art-abc-123", "my-agent", "v3aaaaaaaaaaaa"), + &fakeClient{ + versions: []filesapi.CatalogVersion{ + {ID: "v3aaaaaaaaaaaa", CreatedAt: "2026-04-10T14:30:00Z", NumFiles: 1, TotalSize: 10}, + }, + }, + ) + + cmd, buf := newTestCmd(t, dir, deps) + + require.NoError(t, cmd.Execute()) + + assert.NotContains(t, buf.String(), "Local synced to:") } func TestVersions_LimitFlagPropagates(t *testing.T) { diff --git a/docs/commands/artifact.md b/docs/commands/artifact.md index 396e9d500..21dae3804 100644 --- a/docs/commands/artifact.md +++ b/docs/commands/artifact.md @@ -175,7 +175,9 @@ dr artifact code checkout [] [--dir ] [--clean] - `init` creates the `.datarobot/workload/` state directory and binds it to an existing draft artifact. The artifact must already exist (`dr artifact create` or the DataRobot UI); these commands manage an artifact's code, not its lifecycle. It also drops a starter `.drignore` at the project root, in gitignore syntax, listing what `sync` should leave out. Edit it and commit it. A project that already has an ignore file under either name keeps it, and no new one is written. - Projects created before the file was renamed have a `.wapiignore` instead. It is still read when there is no `.drignore` beside it, and `sync` says so once per run, but the name is deprecated: rename the file when convenient. If both are present, `.drignore` is the one that applies and `sync` warns that the other file's patterns are not in effect. Merge them and delete the old one: two ignore files at a project root is a state where patterns you wrote silently stop filtering. The ignore file is uploaded with your code, so if you work with others, agree on the rename rather than doing it alone. -- `sync` computes a three-way diff against the last synced state and applies it in one versioned step. Conflicts resolve to the remote copy, and your version is kept as a `*.LOCAL.` file. Preview with `--dry-run`, or use `--diff` to also see per-file diffs. Both exit before any remote write. +- `sync` computes a three-way diff against the last synced state and applies it in one versioned step. Conflicts resolve to the remote copy, and your version is kept as a `*.LOCAL.` file. Preview with `--dry-run`, or use `--diff` to also see per-file diffs. Both exit before any remote write. The upload requests `overwrite=REPLACE` on the multipart form, so re-syncing an existing artifact replaces the prior version instead of the server renaming the upload to `name (2).zip`. +- `--verify` forces a remote round-trip before planning even when the artifact has not drifted, surfacing BASE-vs-REMOTE differences the fast path cannot see as `divergence` entries in the plan and in the plan JSON; this part composes with `--dry-run` and `--diff`. On an applying run `--verify` also re-fetches after upload and compares each uploaded path's checksum against what this run streamed, failing the run on a mismatch; that post-apply check never runs under `--dry-run` or `--diff`, which stop before executing. +- Symlinks in the source tree are not followed. Each skipped symlink prints a stderr notice (worded differently for file vs directory symlinks, since a directory symlink omits an entire subtree); the prose is bounded at five paths, with any remainder summarized as "and N more". The plan JSON carries the full list in `skippedSymlinks`, always emitted and `[]` when none. Symlinks excluded by `.drignore` or by the hardcoded system excludes are not reported. - For Python projects, the image build requires a `uv.lock` next to `pyproject.toml`. When your project has `pyproject.toml` but no `uv.lock`, `sync` generates one automatically by running your local `uv lock` (your uv configuration, private indexes, and credentials apply) and uploads it with the rest of your code — commit the generated file to your repo. If `uv` is not installed or lock generation fails, sync still completes and prints what to do (`uv lock`, then re-sync); the image build will fail until a lock file is added. This also happens on `--dry-run`/`--diff`, so the preview matches what a real sync would upload. An existing `uv.lock` is never modified, and sync warns if your `.drignore` excludes it. - `versions` lists the artifact's catalog versions, marking the one the artifact currently points at (`*`) and noting the one you last synced. - `checkout` downloads a version into `.datarobot/workload/.checkouts//` for read-only inspection; your working directory is left untouched. `--clean` removes checkout directories instead of downloading. diff --git a/internal/drapi/filesapi/client_test.go b/internal/drapi/filesapi/client_test.go index 151a8a497..11aca826f 100644 --- a/internal/drapi/filesapi/client_test.go +++ b/internal/drapi/filesapi/client_test.go @@ -364,25 +364,38 @@ func TestPollStatus_CompletedRedirect(t *testing.T) { assert.True(t, IsTerminalStatus(resp.Status)) } +// TestUploadFromZipExisting proves the REPLACE request rides in the +// multipart form body. The server's /files//fromFile/ route binds its +// validator fields from the parsed form only and silently defaults to +// RENAME when the overwrite field is absent — a query parameter is never +// read, and RENAME would store re-uploaded paths as "name (2).ext" while +// the original path keeps its stale bytes. Form fields must also precede +// the file part: streaming parsers collect fields as they arrive, before +// committing to a potentially huge file stream. func TestUploadFromZipExisting(t *testing.T) { startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/v2/files/cid-1/fromFile/", r.URL.Path) assert.Equal(t, "true", r.URL.Query().Get("useArchiveContents")) - assert.Equal(t, "REPLACE", r.URL.Query().Get("overwrite")) + assert.Empty(t, r.URL.Query().Get("overwrite"), + "overwrite in the URL query is silently ignored by the server") assert.Contains(t, r.Header.Get("Content-Type"), "multipart/form-data") - mr, err := r.MultipartReader() + parts, err := readMultipartParts(r) if !assert.NoError(t, err) { return } - part, err := mr.NextPart() - if !assert.NoError(t, err) { + if !assert.Len(t, parts, 2) { return } - assert.Equal(t, "file", part.FormName()) - assert.Equal(t, "changes.zip", part.FileName()) + assert.Equal(t, "overwrite", parts[0].Name, "form fields must precede the file part") + assert.Equal(t, "REPLACE", string(parts[0].Content)) + assert.Empty(t, parts[0].FileName) + + assert.Equal(t, "file", parts[1].Name) + assert.Equal(t, "changes.zip", parts[1].FileName) + assert.Equal(t, "PK\x03\x04fake-zip", string(parts[1].Content)) w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"catalogId":"cid-1","catalogVersionId":"v9","statusId":"sid-9"}`)) diff --git a/internal/drapi/filesapi/fromfile.go b/internal/drapi/filesapi/fromfile.go index e2f0d9a57..d319c61a5 100644 --- a/internal/drapi/filesapi/fromfile.go +++ b/internal/drapi/filesapi/fromfile.go @@ -33,7 +33,7 @@ func (c *httpClient) UploadFromZipNew(name string, size int64, body io.Reader) ( return nil, fmt.Errorf("build files url: %w", err) } - return uploadZipMultipart(requestURL, name, size, body) + return uploadZipMultipart(requestURL, nil, name, size, body) } func (c *httpClient) UploadFromZipExisting(catalogID, name, overwrite string, size int64, body io.Reader) (*FromFileResp, error) { @@ -43,18 +43,29 @@ func (c *httpClient) UploadFromZipExisting(catalogID, name, overwrite string, si q := url.Values{} q.Set("useArchiveContents", "true") - q.Set("overwrite", overwrite) requestURL, err := drapi.EndpointURL("/files/"+url.PathEscape(catalogID)+"/fromFile/", q) if err != nil { return nil, fmt.Errorf("build fromFile url: %w", err) } - return uploadZipMultipart(requestURL, name, size, body) + // overwrite must ride in the multipart form body: the server's + // /files//fromFile/ route binds its validator fields from the + // parsed form only, never from the query string, and silently defaults + // to RENAME when the field is absent. RENAME stores a re-uploaded path + // as "name (2).ext" while the original path keeps its stale bytes. + // useArchiveContents stays in the query: the server also ignores it + // there, but its declared form default is 'True' (archive extraction), + // so extraction happens either way and the request is unchanged + // apart from the overwrite fix. + fields := url.Values{} + fields.Set("overwrite", overwrite) + + return uploadZipMultipart(requestURL, fields, name, size, body) } -func uploadZipMultipart(requestURL, name string, size int64, body io.Reader) (*FromFileResp, error) { - req, err := newStreamingMultipartRequest(requestURL, nil, name, size, body) +func uploadZipMultipart(requestURL string, fields url.Values, name string, size int64, body io.Reader) (*FromFileResp, error) { + req, err := newStreamingMultipartRequest(requestURL, fields, name, size, body) if err != nil { return nil, err } diff --git a/internal/drapi/filesapi/fromfile_test.go b/internal/drapi/filesapi/fromfile_test.go new file mode 100644 index 000000000..a284cad28 --- /dev/null +++ b/internal/drapi/filesapi/fromfile_test.go @@ -0,0 +1,203 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesapi + +import ( + "bytes" + "errors" + "io" + "mime" + "mime/multipart" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// multipartPart is one decoded part of a multipart request body, in wire +// order. FileName is non-empty for file parts. +type multipartPart struct { + Name string + FileName string + Content []byte +} + +// decodeMultipart walks a multipart reader and returns its parts in wire +// order. Form fields written before the file part appear before it here, +// which is how tests pin the fields-first convention. +func decodeMultipart(mr *multipart.Reader) ([]multipartPart, error) { + var parts []multipartPart + + for { + part, err := mr.NextPart() + if errors.Is(err, io.EOF) { + return parts, nil + } + + if err != nil { + return nil, err + } + + content, err := io.ReadAll(part) + if err != nil { + return nil, err + } + + parts = append(parts, multipartPart{ + Name: part.FormName(), + FileName: part.FileName(), + Content: content, + }) + } +} + +// readMultipartParts parses an UNCONSUMED multipart request body with +// mime/multipart and returns its parts in wire order. Tests assert on +// these decoded parts rather than raw bytes so the framing stays free to +// evolve. Reading r.Body first (e.g. for a Content-Length check) exhausts +// the stream — parse the buffered copy via multipart.NewReader instead. +func readMultipartParts(r *http.Request) ([]multipartPart, error) { + mr, err := r.MultipartReader() + if err != nil { + return nil, err + } + + return decodeMultipart(mr) +} + +// TestUploadFromZipExisting_UseArchiveContentsStaysInQuery pins the +// placement of useArchiveContents. The monorepo fromFile validator +// declares it as a multipart form field whose default is 'True' (archive +// extraction), and the route binds validator fields from the parsed form +// only — request.args is never consulted. Extraction therefore happens via +// the server default regardless of what the CLI sends, so leaving the +// flag in the query string (where it is ignored) changes nothing +// observable. This test documents that decision so any future move into +// the form is a conscious one. +func TestUploadFromZipExisting_UseArchiveContentsStaysInQuery(t *testing.T) { + startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "true", r.URL.Query().Get("useArchiveContents")) + + parts, err := readMultipartParts(r) + if !assert.NoError(t, err) { + return + } + + for _, part := range parts { + assert.NotEqual(t, "useArchiveContents", part.Name, + "useArchiveContents is intentionally left out of the form body") + } + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"catalogId":"cid-1","catalogVersionId":"v9","statusId":"sid-9"}`)) + })) + + c := New() + + body := bytes.NewReader([]byte("PK\x03\x04fake-zip")) + _, err := c.UploadFromZipExisting("cid-1", "changes.zip", OverwriteReplace, int64(body.Len()), body) + require.NoError(t, err) +} + +// TestUploadFromZipNew_NoOverwriteField guards the shared-framing parity: +// a first sync has no pre-existing paths, so it must send no overwrite +// form field and no overwrite query parameter — the server's RENAME +// default is correct there and the request should stay minimal. +func TestUploadFromZipNew_NoOverwriteField(t *testing.T) { + startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.URL.Query().Get("overwrite")) + + parts, err := readMultipartParts(r) + if !assert.NoError(t, err) { + return + } + + for _, part := range parts { + assert.NotEqual(t, "overwrite", part.Name) + } + + if !assert.Len(t, parts, 1) { + return + } + + assert.Equal(t, "file", parts[0].Name) + assert.Equal(t, "wapi-sync.zip", parts[0].FileName) + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"catalogId":"new-cid","catalogVersionId":"new-ver","statusId":"sid-new"}`)) + })) + + c := New() + + body := bytes.NewReader([]byte("PK\x03\x04fake-zip")) + resp, err := c.UploadFromZipNew("wapi-sync.zip", int64(body.Len()), body) + require.NoError(t, err) + assert.Equal(t, "new-ver", resp.CatalogVersionID) +} + +// TestUploadFromZipExisting_ContentLengthMatchesBodyWithFormFields +// verifies the Content-Length accounting survives folding form fields +// into the prologue: the advertised length must equal the received body +// exactly, and the streamed file must still decode intact. An off-by-N +// here surfaces as a transport error or a truncated multipart stream, +// never as a silent pass. +func TestUploadFromZipExisting_ContentLengthMatchesBodyWithFormFields(t *testing.T) { + // Long enough to cross io.Copy's internal buffer more than once. + payload := strings.Repeat("0123456789abcdef", 4096) + + startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(r.Body) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, r.ContentLength, int64(len(raw)), + "advertised Content-Length must match the received body byte-for-byte") + + // The body is consumed by the ReadAll above, so parse the buffered + // copy rather than asking the request for a fresh MultipartReader. + _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if !assert.NoError(t, err) { + return + } + + parts, err := decodeMultipart(multipart.NewReader(bytes.NewReader(raw), params["boundary"])) + if !assert.NoError(t, err) { + return + } + + if !assert.Len(t, parts, 2) { + return + } + + assert.Equal(t, "REPLACE", string(parts[0].Content)) + assert.Equal(t, payload, string(parts[1].Content)) + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"catalogId":"cid-1","catalogVersionId":"v9","statusId":"sid-9"}`)) + })) + + c := New() + + body := bytes.NewReader([]byte(payload)) + _, err := c.UploadFromZipExisting("cid-1", "changes.zip", OverwriteReplace, int64(body.Len()), body) + require.NoError(t, err) +} + +// The package's mime/multipart types stay referenced even if helper +// implementations drift; mirrors the guard at the bottom of client_test.go. +var _ = multipart.ErrMessageTooLarge diff --git a/internal/drapi/filesapi/multipart.go b/internal/drapi/filesapi/multipart.go index 904a1b098..71ea3749c 100644 --- a/internal/drapi/filesapi/multipart.go +++ b/internal/drapi/filesapi/multipart.go @@ -22,6 +22,7 @@ import ( "net/http" "net/textproto" "net/url" + "sort" "github.com/datarobot/cli/internal/drapi" ) @@ -34,22 +35,25 @@ const multipartFormField = "file" // by the pipe (one chunk in flight) plus the small envelope, regardless // of file size — important because the engine may upload multi-GiB zips. // +// requestURL is used as-is: callers build any query parameters into it +// before calling. Fields ride in the multipart body, not the URL query: +// some server routes (fromFile) bind their validator fields from the +// parsed form only and silently ignore query parameters. Fields are +// framed as complete parts BEFORE the file part so a streaming parser +// collects them without buffering the file. +// // Trade-off: the request has no GetBody, so http.Transport cannot // transparently retry the body on connection reset. Callers needing // retry must redo the call from scratch (re-opening the source if it // isn't seekable). func newStreamingMultipartRequest( requestURL string, - query url.Values, + fields url.Values, filename string, size int64, body io.Reader, ) (*http.Request, error) { - if len(query) > 0 { - requestURL += "?" + query.Encode() - } - - contentType, prologue, epilogue, err := multipartFraming(filename) + contentType, prologue, epilogue, err := multipartFraming(fields, filename) if err != nil { return nil, err } @@ -65,6 +69,9 @@ func newStreamingMultipartRequest( return nil, fmt.Errorf("build multipart request: %w", err) } + // ContentLength stays exact because the form fields are folded into + // the prologue; the file bytes still contribute exactly size, and the + // epilogue is unchanged. if size >= 0 { req.ContentLength = int64(len(prologue)) + size + int64(len(epilogue)) } @@ -80,14 +87,35 @@ func newStreamingMultipartRequest( return req, nil } -// multipartFraming returns the prologue and epilogue around a single -// file part. Going through multipart.Writer keeps the framing -// RFC-2046-correct even though we stream the body separately. -func multipartFraming(filename string) (string, []byte, []byte, error) { +// multipartFraming returns the prologue and epilogue around the streamed +// file part, with any extra form fields framed as complete parts first. +// Fields must precede the file part: streaming parsers read form fields +// as they arrive, so a server can collect its parameters before +// committing to an arbitrarily large file stream. Field names are sorted +// so the framing is deterministic. Going through multipart.Writer keeps +// the framing RFC-2046-correct even though we stream the file content +// separately. +func multipartFraming(fields url.Values, filename string) (string, []byte, []byte, error) { var head bytes.Buffer w := multipart.NewWriter(&head) + names := make([]string, 0, len(fields)) + + for name := range fields { + names = append(names, name) + } + + sort.Strings(names) + + for _, name := range names { + for _, value := range fields[name] { + if err := w.WriteField(name, value); err != nil { + return "", nil, nil, fmt.Errorf("write multipart field %s: %w", name, err) + } + } + } + hdr := make(textproto.MIMEHeader) hdr.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, multipartFormField, filename)) hdr.Set("Content-Type", "application/octet-stream") diff --git a/internal/workload/fileops/walk.go b/internal/workload/fileops/walk.go index 0695b80fe..50222ebe7 100644 --- a/internal/workload/fileops/walk.go +++ b/internal/workload/fileops/walk.go @@ -15,6 +15,7 @@ package fileops import ( + "errors" "fmt" "io/fs" "os" @@ -26,7 +27,14 @@ import ( type IgnoreFunc func(relPath string, isDir bool) bool // SymlinkLogger is called once per skipped symlink. nil disables it. -type SymlinkLogger func(relPath, target string) +// +// isDir reports whether the link target resolves to a directory (resolved +// via os.Stat, which follows the link). A dangling symlink reports an empty +// target, isDir false, and dangling true: its target does not exist, so its +// kind is unknowable rather than "not a directory". Callers that filter by +// ignore patterns can use dangling to check both spellings of a pattern +// (file and directory) instead of only the file spelling. +type SymlinkLogger func(relPath, target string, isDir, dangling bool) type Entry struct { AbsPath string @@ -95,13 +103,31 @@ func walkVisit( return nil } +// notifySymlink resolves the link target and its kind, then delivers the +// callback. os.Stat follows the symlink, so a dangling link fails here and is +// reported with an empty target and isDir false — the user sees that a link +// points nowhere rather than getting a walk error. os.Readlink is still +// attempted first so a resolvable link reports the link text the user wrote; +// only when Stat fails (the target does not exist) is the target cleared. func notifySymlink(onSymlink SymlinkLogger, absPath, relPath string) { if onSymlink == nil { return } target, _ := os.Readlink(absPath) - onSymlink(relPath, target) + + // Stat follows the link. A failure means the target does not exist + // (dangling) or is inaccessible. The two are kept apart: a dangling + // link's kind is unknowable (dangling=true), while other Stat errors + // keep the historical empty-target, isDir=false report. + info, err := os.Stat(absPath) + if err != nil { + onSymlink(relPath, "", false, errors.Is(err, fs.ErrNotExist)) + + return + } + + onSymlink(relPath, target, info.IsDir(), false) } func dirAction(relPath string, ignore IgnoreFunc) error { diff --git a/internal/workload/fileops/walk_symlink_test.go b/internal/workload/fileops/walk_symlink_test.go new file mode 100644 index 000000000..357791d97 --- /dev/null +++ b/internal/workload/fileops/walk_symlink_test.go @@ -0,0 +1,273 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileops + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// symlinkTestSkipReason is the visible reason emitted when symlink tests are +// skipped on Windows. os.Symlink needs Developer Mode or elevated privileges +// there, and junctions are a different mechanism entirely — no attempt is made. +const symlinkTestSkipReason = "symlink tests skipped on Windows: os.Symlink needs Developer Mode; junctions are not equivalent" + +// skipNonWindowsSymlink skips on Windows following the existing +// runtime.GOOS == "windows" precedent in walk_test.go. The skip reason is +// visible so a CI log shows why the test did not run rather than appearing to +// have passed vacuously. +func skipNonWindowsSymlink(t *testing.T) { + t.Helper() + + if runtime.GOOS == "windows" { + t.Skip(symlinkTestSkipReason) + } +} + +// symlinkReport captures what the SymlinkLogger callback observed, so a test +// can assert on path, target, isDir, and dangling together. +type symlinkReport struct { + rel string + target string + isDir bool + dangling bool +} + +// TestWalk_SymlinkedDirectory_Pruned verifies that a symlink to a directory +// produces exactly one callback for the symlink path and zero callbacks for +// any path beneath it, and that no entry is emitted for the symlink or its +// children. filepath.WalkDir tests ModeSymlink before IsDir and never descends +// into a non-directory, so children are never visited. +// +// Fulfills VAL-SYMLINK-011(a) and VAL-SYMLINK-011(e). +func TestWalk_SymlinkedDirectory_Pruned(t *testing.T) { + skipNonWindowsSymlink(t) + + root := t.TempDir() + writeFile(t, root, "realdir/inner.py", "x") + writeFile(t, root, "realdir/sub/deep.py", "y") + + // link_to_dir -> realdir (a directory symlink) + require.NoError(t, os.Symlink( + filepath.Join(root, "realdir"), + filepath.Join(root, "link_to_dir"))) + + var reports []symlinkReport + + got, err := Walk(root, nil, func(rel, target string, isDir, dangling bool) { + reports = append(reports, symlinkReport{rel: rel, target: target, isDir: isDir, dangling: dangling}) + }) + require.NoError(t, err) + + // Exactly one callback for the symlink path itself. + require.Len(t, reports, 1, "a symlinked directory must produce exactly one callback") + assert.Equal(t, "link_to_dir", reports[0].rel) + assert.True(t, reports[0].isDir, "isDir must be true when the link target is a directory") + assert.False(t, reports[0].dangling, "a resolvable link must not be reported as dangling") + + // No entry for the symlink or anything beneath it. The real directory's + // children ARE present (the symlink is a separate path). + paths := relPaths(got) + assert.NotContains(t, paths, "link_to_dir") + assert.NotContains(t, paths, "link_to_dir/inner.py") + assert.NotContains(t, paths, "link_to_dir/sub/deep.py") + assert.Contains(t, paths, "realdir/inner.py") + assert.Contains(t, paths, "realdir/sub/deep.py") +} + +// TestWalk_Symlink_ReportsIsDir verifies that the callback reports isDir true +// when the link target is a directory and false when it is a regular file. +// The kind is resolved via os.Stat (which follows the link), the syscall +// already available on the notify path. +// +// Fulfills VAL-SYMLINK-011(b). +func TestWalk_Symlink_ReportsIsDir(t *testing.T) { + skipNonWindowsSymlink(t) + + root := t.TempDir() + writeFile(t, root, "realfile.py", "x") + writeFile(t, root, "realdir/inner.py", "y") + + // link_to_file -> realfile.py (a file symlink) + require.NoError(t, os.Symlink( + filepath.Join(root, "realfile.py"), + filepath.Join(root, "link_to_file"))) + + // link_to_dir -> realdir (a directory symlink) + require.NoError(t, os.Symlink( + filepath.Join(root, "realdir"), + filepath.Join(root, "link_to_dir"))) + + var reports []symlinkReport + + _, err := Walk(root, nil, func(rel, target string, isDir, dangling bool) { + reports = append(reports, symlinkReport{rel: rel, target: target, isDir: isDir, dangling: dangling}) + }) + require.NoError(t, err) + + // Two callbacks, one per symlink, each with the correct kind. + require.Len(t, reports, 2) + + byRel := make(map[string]symlinkReport, len(reports)) + for _, r := range reports { + byRel[r.rel] = r + } + + fileRpt, ok := byRel["link_to_file"] + require.True(t, ok, "file symlink must be reported") + assert.False(t, fileRpt.isDir, "isDir must be false for a file symlink") + assert.NotEmpty(t, fileRpt.target, "target must be the link text") + + dirRpt, ok := byRel["link_to_dir"] + require.True(t, ok, "directory symlink must be reported") + assert.True(t, dirRpt.isDir, "isDir must be true for a directory symlink") + assert.NotEmpty(t, dirRpt.target, "target must be the link text") +} + +// TestWalk_DanglingSymlink verifies that a symlink whose target does not exist +// produces a callback with an empty target and isDir false, and that the walk +// returns no error. The target is cleared because os.Stat fails on a dangling +// link — the user sees "this link points nowhere" rather than a walk crash. +// +// Fulfills VAL-SYMLINK-011(c). +func TestWalk_DanglingSymlink(t *testing.T) { + skipNonWindowsSymlink(t) + + root := t.TempDir() + writeFile(t, root, "agent.py", "x") + + // A symlink to a path that does not exist. + require.NoError(t, os.Symlink( + filepath.Join(root, "nonexistent"), + filepath.Join(root, "dangling.lnk"))) + + var reports []symlinkReport + + got, err := Walk(root, nil, func(rel, target string, isDir, dangling bool) { + reports = append(reports, symlinkReport{rel: rel, target: target, isDir: isDir, dangling: dangling}) + }) + require.NoError(t, err, "a dangling symlink must not cause a walk error") + + require.Len(t, reports, 1, "dangling symlink must produce exactly one callback") + assert.Equal(t, "dangling.lnk", reports[0].rel) + assert.Empty(t, reports[0].target, "dangling symlink must report an empty target") + assert.False(t, reports[0].isDir, "dangling symlink must report isDir false") + assert.True(t, reports[0].dangling, "dangling symlink must report dangling true") + + // The dangling symlink is not in entries. + paths := relPaths(got) + assert.NotContains(t, paths, "dangling.lnk") + assert.Contains(t, paths, "agent.py") +} + +// TestWalk_SymlinkChain_NotFollowed verifies that a symlink chain (outer -> +// inner -> realfile, both outer and inner top-level) reports each top-level +// symlink exactly once and follows neither. The walker never follows symlinks +// (it tests ModeSymlink before descending), so a chain is reported per link +// without traversing to the final target. +// +// Fulfills VAL-SYMLINK-011(d). +func TestWalk_SymlinkChain_NotFollowed(t *testing.T) { + skipNonWindowsSymlink(t) + + root := t.TempDir() + writeFile(t, root, "realfile.py", "x") + + // inner -> realfile.py + require.NoError(t, os.Symlink( + filepath.Join(root, "realfile.py"), + filepath.Join(root, "inner.lnk"))) + + // outer -> inner.lnk (a chain: outer -> inner -> realfile.py) + require.NoError(t, os.Symlink( + filepath.Join(root, "inner.lnk"), + filepath.Join(root, "outer.lnk"))) + + var reports []symlinkReport + + got, err := Walk(root, nil, func(rel, target string, isDir, dangling bool) { + reports = append(reports, symlinkReport{rel: rel, target: target, isDir: isDir, dangling: dangling}) + }) + require.NoError(t, err) + + // Each top-level symlink is reported exactly once. The walker does not + // follow the chain: outer reports its own link text (inner.lnk), not + // realfile.py, and neither is uploaded. + require.Len(t, reports, 2, "each top-level symlink must be reported exactly once") + + byRel := make(map[string]int) + for _, r := range reports { + byRel[r.rel]++ + } + + assert.Equal(t, 1, byRel["outer.lnk"], "outer must be reported exactly once") + assert.Equal(t, 1, byRel["inner.lnk"], "inner must be reported exactly once") + assert.NotContains(t, byRel, "realfile.py", "the chain target must not be reported as a symlink") + + // Neither symlink is in entries; the real file IS. + paths := relPaths(got) + assert.NotContains(t, paths, "outer.lnk") + assert.NotContains(t, paths, "inner.lnk") + assert.Contains(t, paths, "realfile.py") + + // Neither link resolves to a directory (realfile.py is a regular file). + for _, r := range reports { + assert.False(t, r.isDir, "chain links to a regular file must report isDir false") + } +} + +// TestWalk_SymlinkedDirectory_ExternalTarget verifies that a symlink whose +// target is an absolute path outside the project directory is reported as a +// skipped symlink, is not uploaded, and no file outside the project root is +// read. The plan contains only paths under the project root. +// +// Fulfills VAL-SYMLINK-004(b) at the walk level. +func TestWalk_SymlinkedDirectory_ExternalTarget(t *testing.T) { + skipNonWindowsSymlink(t) + + root := t.TempDir() + writeFile(t, root, "agent.py", "x") + + // A file outside the project root. + external := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(external, "secret.txt"), []byte("secret"), 0o644)) + + // link_to_external -> /tmp/.../secret.txt (absolute, outside project) + require.NoError(t, os.Symlink( + filepath.Join(external, "secret.txt"), + filepath.Join(root, "link_to_external"))) + + var reports []symlinkReport + + got, err := Walk(root, nil, func(rel, target string, isDir, dangling bool) { + reports = append(reports, symlinkReport{rel: rel, target: target, isDir: isDir, dangling: dangling}) + }) + require.NoError(t, err) + + require.Len(t, reports, 1, "external-target symlink must be reported") + assert.Equal(t, "link_to_external", reports[0].rel) + assert.False(t, reports[0].isDir, "external file target must report isDir false") + assert.False(t, reports[0].dangling, "an external file target must not be reported as dangling") + + // The symlink is not in entries; only the real project file is. + paths := relPaths(got) + assert.NotContains(t, paths, "link_to_external") + assert.Contains(t, paths, "agent.py") +} diff --git a/internal/workload/fileops/walk_test.go b/internal/workload/fileops/walk_test.go index 10b112bbe..1f352f504 100644 --- a/internal/workload/fileops/walk_test.go +++ b/internal/workload/fileops/walk_test.go @@ -107,8 +107,10 @@ func TestWalk_SkipsSymlinkAndNotifies(t *testing.T) { var seen []string - got, err := Walk(root, nil, func(rel, target string) { + got, err := Walk(root, nil, func(rel, target string, isDir, dangling bool) { _ = target + _ = isDir + _ = dangling seen = append(seen, rel) }) diff --git a/internal/workload/sync/classify_truth_table_test.go b/internal/workload/sync/classify_truth_table_test.go new file mode 100644 index 000000000..7bce46bb1 --- /dev/null +++ b/internal/workload/sync/classify_truth_table_test.go @@ -0,0 +1,269 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestClassifyTruthTable is a comprehensive table-driven test over every +// meaningful (base, local, remote) hash triple, covering all fourteen +// three-way classifications and their mapped actions. Each classification is +// exercised with multiple distinct hash values so the logic is proven +// value-agnostic, not merely correct for one chosen triple. +// +// An empty hash ("") means absent on that side. Non-empty hashes are compared +// by equality only; the actual hex content is irrelevant. The table is +// transcribed from the current Classify implementation (classify.go) and +// ActionFor, not from memory, so any later change to either function is +// caught. +// +// Fulfills VAL-REGRESSION-006. +func TestClassifyTruthTable(t *testing.T) { + // Four distinct non-empty hash values plus "" for absent. Using multiple + // values per branch proves the logic is value-agnostic. + const ( + A = "aaaa1111" + B = "bbbb2222" + C = "cccc3333" + D = "dddd4444" + ) + + cases := []struct { + name string + base string + local string + remote string + wantCls Classification + wantAct Action + }{ + // --- BASE ABSENT (classifyAbsentBase) --- + + // Both sides absent: nothing to do. + { + name: "absent_both_absent/empty_empty_empty", base: "", local: "", remote: "", + wantCls: ClsUnchanged, wantAct: ActSkip, + }, + + // Local added, remote absent. + { + name: "absent_local_added/empty_A_empty", base: "", local: A, remote: "", + wantCls: ClsLocalAdded, wantAct: ActUploadAdd, + }, + { + name: "absent_local_added/empty_B_empty", base: "", local: B, remote: "", + wantCls: ClsLocalAdded, wantAct: ActUploadAdd, + }, + + // Remote added, local absent. + { + name: "absent_remote_added/empty_empty_A", base: "", local: "", remote: A, + wantCls: ClsRemoteAdded, wantAct: ActDownloadAdd, + }, + { + name: "absent_remote_added/empty_empty_B", base: "", local: "", remote: B, + wantCls: ClsRemoteAdded, wantAct: ActDownloadAdd, + }, + + // Both added the same content: no conflict, skip. + { + name: "absent_both_added_same/empty_A_A", base: "", local: A, remote: A, + wantCls: ClsBothAddedSame, wantAct: ActSkip, + }, + { + name: "absent_both_added_same/empty_B_B", base: "", local: B, remote: B, + wantCls: ClsBothAddedSame, wantAct: ActSkip, + }, + + // Both added different content: conflict. + { + name: "absent_add_conflict/empty_A_B", base: "", local: A, remote: B, + wantCls: ClsAddConflict, wantAct: ActConflictCopy, + }, + { + name: "absent_add_conflict/empty_C_D", base: "", local: C, remote: D, + wantCls: ClsAddConflict, wantAct: ActConflictCopy, + }, + + // --- BASE PRESENT, BOTH PRESENT (classifyBothPresentWithBase) --- + + // All three identical: unchanged. + { + name: "present_unchanged/A_A_A", base: A, local: A, remote: A, + wantCls: ClsUnchanged, wantAct: ActSkip, + }, + { + name: "present_unchanged/B_B_B", base: B, local: B, remote: B, + wantCls: ClsUnchanged, wantAct: ActSkip, + }, + + // Local changed, remote unchanged: upload. + { + name: "present_local_modified/A_B_A", base: A, local: B, remote: A, + wantCls: ClsLocalModified, wantAct: ActUploadModify, + }, + { + name: "present_local_modified/A_C_A", base: A, local: C, remote: A, + wantCls: ClsLocalModified, wantAct: ActUploadModify, + }, + + // Remote changed, local unchanged: download. + { + name: "present_remote_modified/A_A_B", base: A, local: A, remote: B, + wantCls: ClsRemoteModified, wantAct: ActDownloadModify, + }, + { + name: "present_remote_modified/A_A_C", base: A, local: A, remote: C, + wantCls: ClsRemoteModified, wantAct: ActDownloadModify, + }, + + // Both changed to the same content: converged, skip. + { + name: "present_converged/A_B_B", base: A, local: B, remote: B, + wantCls: ClsConverged, wantAct: ActSkip, + }, + { + name: "present_converged/A_C_C", base: A, local: C, remote: C, + wantCls: ClsConverged, wantAct: ActSkip, + }, + + // Both changed differently: conflict. + { + name: "present_conflict/A_B_C", base: A, local: B, remote: C, + wantCls: ClsConflict, wantAct: ActConflictCopy, + }, + { + name: "present_conflict/B_C_D", base: B, local: C, remote: D, + wantCls: ClsConflict, wantAct: ActConflictCopy, + }, + + // --- BASE PRESENT, DELETION INVOLVED (classifyDeletionInvolvedWithBase) --- + + // Both deleted: skip. + { + name: "present_both_deleted/A_empty_empty", base: A, local: "", remote: "", + wantCls: ClsBothDeleted, wantAct: ActSkip, + }, + { + name: "present_both_deleted/B_empty_empty", base: B, local: "", remote: "", + wantCls: ClsBothDeleted, wantAct: ActSkip, + }, + + // Local deleted, remote unchanged from base: local delete. + { + name: "present_local_deleted/A_empty_A", base: A, local: "", remote: A, + wantCls: ClsLocalDeleted, wantAct: ActUploadDelete, + }, + { + name: "present_local_deleted/B_empty_B", base: B, local: "", remote: B, + wantCls: ClsLocalDeleted, wantAct: ActUploadDelete, + }, + + // Local deleted, remote changed from base: edit-del conflict. + // The user deleted the file, the teammate edited it. + { + name: "present_edit_del_conflict/A_empty_B", base: A, local: "", remote: B, + wantCls: ClsEditDelConflict, wantAct: ActDownloadOverDel, + }, + { + name: "present_edit_del_conflict/A_empty_C", base: A, local: "", remote: C, + wantCls: ClsEditDelConflict, wantAct: ActDownloadOverDel, + }, + + // Remote deleted, local unchanged from base: remote delete. + { + name: "present_remote_deleted/A_A_empty", base: A, local: A, remote: "", + wantCls: ClsRemoteDeleted, wantAct: ActDownloadDelete, + }, + { + name: "present_remote_deleted/B_B_empty", base: B, local: B, remote: "", + wantCls: ClsRemoteDeleted, wantAct: ActDownloadDelete, + }, + + // Remote deleted, local changed from base: del-edit conflict. + // The user edited the file, the teammate deleted it. + { + name: "present_del_edit_conflict/A_B_empty", base: A, local: B, remote: "", + wantCls: ClsDelEditConflict, wantAct: ActConflictCopy, + }, + { + name: "present_del_edit_conflict/A_C_empty", base: A, local: C, remote: "", + wantCls: ClsDelEditConflict, wantAct: ActConflictCopy, + }, + } + + // Verify we cover all fourteen classifications. + seenCls := make(map[Classification]struct{}) + for _, tc := range cases { + seenCls[tc.wantCls] = struct{}{} + } + + allCls := []Classification{ + ClsUnchanged, ClsLocalModified, ClsRemoteModified, ClsConverged, ClsConflict, + ClsLocalAdded, ClsRemoteAdded, ClsAddConflict, ClsLocalDeleted, ClsRemoteDeleted, + ClsBothDeleted, ClsDelEditConflict, ClsEditDelConflict, ClsBothAddedSame, + } + + for _, c := range allCls { + _, ok := seenCls[c] + assert.True(t, ok, "truth table must cover classification %s", c) + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + gotCls := Classify(tc.base, tc.local, tc.remote) + assert.Equal(t, tc.wantCls, gotCls, + "classification for (base=%q, local=%q, remote=%q)", tc.base, tc.local, tc.remote) + + gotAct := ActionFor(gotCls) + assert.Equal(t, tc.wantAct, gotAct, + "action for classification %s", gotCls) + }) + } +} + +// TestClassifyActionMappingIsExhaustive asserts that every classification has +// a defined action and that the action mapping is a total function: no +// classification maps to the zero-value ActSkip by falling through the switch +// default. This catches a refactor that adds a new classification without +// updating ActionFor. +func TestClassifyActionMappingIsExhaustive(t *testing.T) { + allCls := []Classification{ + ClsUnchanged, ClsLocalModified, ClsRemoteModified, ClsConverged, ClsConflict, + ClsLocalAdded, ClsRemoteAdded, ClsAddConflict, ClsLocalDeleted, ClsRemoteDeleted, + ClsBothDeleted, ClsDelEditConflict, ClsEditDelConflict, ClsBothAddedSame, + } + + // The skip classifications are the ones that legitimately map to ActSkip. + skipCls := map[Classification]bool{ + ClsUnchanged: true, + ClsConverged: true, + ClsBothDeleted: true, + ClsBothAddedSame: true, + } + + for _, c := range allCls { + act := ActionFor(c) + + if skipCls[c] { + assert.Equal(t, ActSkip, act, "%s must map to ActSkip", c) + } else { + assert.NotEqual(t, ActSkip, act, + "%s must map to a non-skip action; ActSkip here means it fell through the switch", c) + } + } +} diff --git a/internal/workload/sync/combined_integration_test.go b/internal/workload/sync/combined_integration_test.go new file mode 100644 index 000000000..011fbcf00 --- /dev/null +++ b/internal/workload/sync/combined_integration_test.go @@ -0,0 +1,1247 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests prove the three fixes compose: hash-while-streaming, --verify +// divergence detection and repair, and symlink surfacing all fire in a +// single run without interfering. Each fix is individually covered by its +// own test file; these are the integration cases that exercise multiple +// fixes at once, driven through Plan/Execute against the self-consistent +// fake. +// +// Fulfills VAL-CROSS-004, VAL-CROSS-006, VAL-CROSS-007, VAL-CROSS-008, +// VAL-CROSS-009, VAL-CROSS-010, VAL-CROSS-012 at the go test level. + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// addFileSymlink creates a file symlink (linkTo -> target) in dir, skipping +// on Windows where os.Symlink needs Developer Mode. +func addFileSymlink(t *testing.T, dir, target, linkTo string) { + t.Helper() + + require.NoError(t, os.WriteFile(filepath.Join(dir, target), []byte("target content\n"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, target), + filepath.Join(dir, linkTo))) +} + +// addDirSymlink creates a directory symlink (linkTo -> targetDir) in dir, +// with a child file inside the target so the subtree is non-empty. +func addDirSymlink(t *testing.T, dir, targetDir, child, linkTo string) { + t.Helper() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, targetDir), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, targetDir, child), []byte("child content\n"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, targetDir), + filepath.Join(dir, linkTo))) +} + +// manyFiles builds a map of n small files (file_00.py … file_NN-1.py) for +// tests that need to cross the zip-path threshold (>20 uploads). +func manyFiles(n int) map[string]string { + out := make(map[string]string, n) + for i := 0; i < n; i++ { + out[fmt.Sprintf("file_%02d.py", i)] = fmt.Sprintf("content %d\n", i) + } + + return out +} + +// sha256HexOf computes the SHA-256 hex of a string, matching what the fake +// records and what hashEntries produces. +func sha256HexOf(s string) string { + h := sha256.Sum256([]byte(s)) + return hex.EncodeToString(h[:]) +} + +// manifestHashes returns the map of path→hash from a loaded manifest, for +// easy comparison against the fake's AllFiles. +func manifestHashes(t *testing.T, dir string) map[string]string { + t.Helper() + + m, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + out := make(map[string]string, len(m.Files)) + for p, fm := range m.Files { + out[p] = fm.Hash + } + + return out +} + +// serverHashes returns the map of path→hash from the fake's AllFiles, for +// comparison against the manifest. +func serverHashes(t *testing.T, fake *fakeFilesClient, catalogID, versionID string) map[string]string { + t.Helper() + + files, err := fake.AllFiles(catalogID, versionID) + require.NoError(t, err) + + out := make(map[string]string, len(files)) + for p, fm := range files { + out[p] = fm.Hash + } + + return out +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-004(a): Symlink + mid-stream rewrite +// --------------------------------------------------------------------------- + +// TestCombined_SymlinkAndMidStreamRewrite proves that a file symlink and a +// regular file rewritten mid-stream both fire in one run: the symlink notice +// appears, the symlink is absent from uploads, and the raced file's manifest +// entry holds the streamed hash (not the Phase-2 planned hash). +func TestCombined_SymlinkAndMidStreamRewrite(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-x4a" + versionID = "ver-x4a-synced" + newVer = "ver-x4a-new" + ) + + original := "content original\n" + planContent := "content plantime\n" // same length as original + streamContent := "content streamed\n" // same length, different content + + require.Len(t, planContent, len(original), "fixture: same-size rewrite") + require.Len(t, streamContent, len(original), "fixture: same-size rewrite") + + // Synced project: BASE == LOCAL == REMOTE for app.py. + dir := syncedProject(t, map[string]string{"app.py": original}, catalogID, versionID) + + // Add a file symlink so both fixes fire in the same run. + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + + // Modify app.py so it appears as LOCAL_MODIFIED (needs upload). + modifyFile(t, dir, "app.py", planContent) + + // Seed the server with the ORIGINAL content (before the disk edit below), + // so BASE == REMOTE and the plan has an upload row for app.py. The seed + // is built explicitly because seededServerContents reads from disk, and + // app.py has already been modified to planContent. + seed := map[string][]byte{ + ignore.FileName: diskBytes(t, dir, ignore.FileName), + "app.py": []byte(original), + "realfile.py": []byte("target content\n"), + } + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-x4a", + versionID: newVer, + }).withVersionContent(catalogID, versionID, seed) + + // Add realfile.py to the manifest so BASE == REMOTE for it (it was added + // after syncedProject, so the manifest doesn't have it yet). + m, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + rh, _, err := hashLocal(t, dir, "realfile.py") + require.NoError(t, err) + + m.Files["realfile.py"] = wapi.FileMeta{Hash: rh, Size: int64(len("target content\n"))} + require.NoError(t, wapi.SaveManifest(dir, m)) + + var logged string + + var execErr error + + var plan *SyncPlan + + // Plan: walks, finds the symlink, hashes app.py with planContent. + logged = captureWarnLog(t, func() { + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, versionID) + + plan, err = e.Plan() + require.NoError(t, err) + + // Between Plan and Execute: rewrite app.py to different bytes of the + // SAME size. The streamed hash will differ from the Phase-2 planned + // hash — this is the TOCTOU the upload-integrity fix closes. + modifyFile(t, dir, "app.py", streamContent) + + _, execErr = e.Execute(plan) + }) + + require.NoError(t, execErr, "the sync must succeed despite the mid-stream rewrite") + + // The symlink notice fires. + assert.Contains(t, logged, "link_to_file.py", + "the symlink notice must fire in the same run as the mid-stream rewrite") + assert.Contains(t, logged, "was not uploaded", + "the symlink notice must say the symlink was not uploaded") + + // The symlink is absent from the upload plan. + for _, fa := range plan.Uploads { + assert.NotEqual(t, "link_to_file.py", fa.Path, + "the symlink must not appear in the upload list") + } + + // The raced file's manifest entry holds the streamed hash, not the + // Phase-2 planned hash. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + streamedHash := sha256HexOf(streamContent) + planHash := sha256HexOf(planContent) + + assert.Equal(t, streamedHash, manifest.Files["app.py"].Hash, + "the manifest must hold the streamed hash, not the Phase-2 planned hash") + assert.NotEqual(t, planHash, manifest.Files["app.py"].Hash, + "the Phase-2 planned hash must not survive the run") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-004(b): Symlink + poisoned BASE + --verify --dry-run +// --------------------------------------------------------------------------- + +// TestCombined_SymlinkAndPoisonedBase_VerifyDryRun proves that a --verify +// --dry-run run with both a symlink and a poisoned BASE produces both the +// skipped-symlink field and the divergence field on the engine, with both +// notices reaching the warn log (stderr). The JSON purity of stdout is +// asserted at the cmd level (TestCmd_Combined_JSON_SymlinkAndDivergence). +func TestCombined_SymlinkAndPoisonedBase_VerifyDryRun(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-x4b" + versionID = "ver-x4b-synced" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + // Synced project: disk and manifest both hold A. + dir := syncedProject(t, map[string]string{"app.py": contentA}, catalogID, versionID) + + // Add a file symlink. + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + + // Poison the manifest to a hash that is neither the disk content (A) nor + // the server content (B): syncedProject already wrote A's hash into the + // manifest, so this overwrite is the only poisoning the fixture needs. + // --verify then has a real BASE-vs-REMOTE mismatch to report. + poisonedHash := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + + poisonManifestHash(t, dir, "app.py", poisonedHash) + + // Seed the server with B (different from the poisoned BASE). + seed := seededServerContents(t, dir, map[string]string{"app.py": contentA}, map[string][]byte{ + "app.py": []byte(contentB), + }) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + var logged string + + e := engineFor(t, dir, Options{Verify: true, DryRun: true, Yes: true}, fake, catalogID, versionID) + + logged = captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + // Both the symlink notice and the divergence notice fire. + assert.Contains(t, logged, "link_to_file.py", + "the symlink notice must fire alongside the divergence notice") + assert.Contains(t, logged, "was not uploaded", + "the symlink notice must say the symlink was not uploaded") + assert.Contains(t, logged, "divergence", + "the divergence notice must fire alongside the symlink notice") + assert.Contains(t, logged, "app.py", + "the divergence notice must name the divergent path") + + // Both structured fields are populated on the engine. + require.Len(t, e.SkippedSymlinks(), 1, "the skipped-symlink field must be populated") + assert.Equal(t, "link_to_file.py", e.SkippedSymlinks()[0].Path) + assert.False(t, e.SkippedSymlinks()[0].IsDir) + + require.Len(t, e.Divergences(), 1, "the divergence field must be populated") + assert.Equal(t, "app.py", e.Divergences()[0].Path) + assert.Equal(t, DivergenceHashMismatch, e.Divergences()[0].Kind) +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-006: Symlink-replacement delete is NOT divergence +// --------------------------------------------------------------------------- + +// TestCombined_SymlinkReplacementDelete_NotDivergence proves that a path +// which was a real synced file and is now a symlink produces a delete row +// and a symlink notice, but is NOT reported as BASE-vs-REMOTE divergence. +// A false positive there would train users to ignore the notice. +func TestCombined_SymlinkReplacementDelete_NotDivergence(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-x6" + versionID = "ver-x6-synced" + ) + + // Synced project with a real file that will be replaced by a symlink. + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "oldfile.py": "old content\n", + }, catalogID, versionID) + + // Seed the server with the original content (BASE == REMOTE). + seed := seededServerContents(t, dir, map[string]string{ + "app.py": "print('hi')\n", + "oldfile.py": "old content\n", + }, nil) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + // Replace oldfile.py with a symlink. The walk will not follow it, so + // LOCAL is absent for oldfile.py. BASE and REMOTE still agree (both + // have the original hash), so this is a LOCAL_DELETED, not a divergence. + require.NoError(t, os.Remove(filepath.Join(dir, "oldfile.py"))) + require.NoError(t, os.Symlink( + filepath.Join(dir, "app.py"), + filepath.Join(dir, "oldfile.py"))) + + var logged string + + e := engineFor(t, dir, Options{Verify: true, DryRun: true, Yes: true}, fake, catalogID, versionID) + + logged = captureWarnLog(t, func() { + plan, err := e.Plan() + require.NoError(t, err) + + // The delete row is present. + deletePaths := make([]string, 0, len(plan.Deletes)) + for _, fa := range plan.Deletes { + deletePaths = append(deletePaths, fa.Path) + } + + assert.Contains(t, deletePaths, "oldfile.py", + "a path replaced by a symlink must produce a delete row") + }) + + // The symlink notice is present. + assert.Contains(t, logged, "oldfile.py", + "the symlink notice must name the replaced path") + assert.Contains(t, logged, "was not uploaded", + "the symlink notice must say the symlink was not uploaded") + + // The path is NOT reported as divergence. BASE and REMOTE agree (both + // have the original hash), so there is no BASE-vs-REMOTE divergence. + for _, d := range e.Divergences() { + assert.NotEqual(t, "oldfile.py", d.Path, + "a symlink-replacement delete must NOT be reported as divergence (false positive)") + } + + // The engine's skippedSymlinks includes the replaced path. + assert.Contains(t, symlinkPaths(e.skippedSymlinks), "oldfile.py") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-007: Zip path + divergence + both symlink kinds +// --------------------------------------------------------------------------- + +// TestCombined_ZipPathWithDivergenceAndSymlinks proves that a 21+ file zip-path +// project with a divergence and both symlink kinds: every one of the 23 +// uploaded files' manifest hashes equals the server checksum, one file is +// rewritten between Plan and Execute so the recorded hash provably comes from +// the bytes that entered the archive (not the Phase-2 planned hash), the +// divergent file's entry reflects the true server state after apply, both +// symlinks are reported with correct kinds, and neither enters the archive. +func TestCombined_ZipPathWithDivergenceAndSymlinks(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-x7" + versionID = "ver-x7-synced" + newVer = "ver-x7-new" + ) + + // 22 files: 21 will be modified (forcing the zip path with >20 uploads), + // and the 22nd will have its manifest poisoned (divergence). + files := manyFiles(22) + dir := syncedProject(t, files, catalogID, versionID) + + // Add both symlink kinds. + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + addDirSymlink(t, dir, "realdir", "inner.py", "link_to_dir") + + // Seed the server with the original content (BASE == REMOTE before + // poisoning). Include .drignore so it doesn't show up as a divergence. + seed := seededServerContents(t, dir, files, nil) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-x7", + versionID: newVer, + }).withVersionContent(catalogID, versionID, seed) + + // Modify 21 files (file_00 through file_20) so there are 21 uploads, + // forcing the zip path (>20 threshold). + for i := 0; i < 21; i++ { + rel := fmt.Sprintf("file_%02d.py", i) + modifyFile(t, dir, rel, fmt.Sprintf("modified %d\n", i)) + } + + // Poison file_21's manifest hash (divergence on a separate path that + // is NOT being uploaded — disk and server agree, so CONVERGED → skip). + poisonedHash := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + poisonManifestHash(t, dir, "file_21.py", poisonedHash) + + var logged string + + var execErr error + + e := engineFor(t, dir, Options{Verify: true, Yes: true}, fake, catalogID, versionID) + + logged = captureWarnLog(t, func() { + plan, planErr := e.Plan() + require.NoError(t, planErr) + + // Between Plan and Execute: rewrite one zip-bound file to different + // bytes of the SAME size. buildZip reads from disk at Execute time, + // so the archive carries these bytes and the streamed hash differs + // from the Phase-2 planned hash — the zip-path counterpart of the + // stage-path rewrite in TestCombined_SymlinkAndMidStreamRewrite. + // Without this, manifest==server would hold equally under an + // implementation that recorded the planned hash, so the streamed-hash + // claim below would not be discriminating. + modifyFile(t, dir, "file_10.py", "modified !!\n") + + _, execErr = e.Execute(plan) + }) + + require.NoError(t, execErr, "the zip-path sync with divergence must succeed (exit 0)") + + // Both symlink notices and the divergence notice fire in the warn log. + assert.Contains(t, logged, "link_to_file.py", "file symlink notice must fire in the zip-path run") + assert.Contains(t, logged, "link_to_dir", "directory symlink notice must fire in the zip-path run") + assert.Contains(t, logged, "divergence", "divergence notice must fire in the zip-path run") + assert.Contains(t, logged, "file_21.py", "divergence must name the poisoned path") + + // Both symlinks are reported with correct kinds. + skipped := e.SkippedSymlinks() + assert.Contains(t, symlinkPaths(skipped), "link_to_file.py") + assert.False(t, symlinkIsDir(skipped, "link_to_file.py"), + "file symlink must report isDir false") + assert.Contains(t, symlinkPaths(skipped), "link_to_dir") + assert.True(t, symlinkIsDir(skipped, "link_to_dir"), + "directory symlink must report isDir true") + + // The divergence is detected for file_21.py. + foundDiv := false + + for _, d := range e.Divergences() { + if d.Path == "file_21.py" { + foundDiv = true + + assert.Equal(t, DivergenceHashMismatch, d.Kind) + } + } + + assert.True(t, foundDiv, "the divergence must name file_21.py") + + // Neither symlink enters the archive (not in the server's AllFiles). + server, err := fake.AllFiles(catalogID, newVer) + require.NoError(t, err) + + _, hasFileSymlink := server["link_to_file.py"] + _, hasDirSymlink := server["link_to_dir"] + + assert.False(t, hasFileSymlink, "the file symlink must not enter the archive") + assert.False(t, hasDirSymlink, "the directory symlink must not enter the archive") + + // Every one of the 23 uploads — the 21 modified files plus realfile.py + // and realdir/inner.py (created alongside the symlinks, so LOCAL_ADDED) — + // must record a manifest hash equal to the server checksum. + manifest := manifestHashes(t, dir) + + require.Len(t, e.plan.Uploads, 23, + "precondition: 21 modified files plus realfile.py and realdir/inner.py must all upload") + + for _, fa := range e.plan.Uploads { + assert.Equal(t, server[fa.Path].Hash, manifest[fa.Path], + "uploaded file %s: manifest hash must equal server checksum", fa.Path) + } + + // The mid-stream rewrite is what makes the streamed-hash claim + // discriminating: file_10.py's Phase-2 planned hash describes bytes that + // never entered the archive, so only the hash of the bytes actually + // zipped can appear in both the manifest and the server listing. + streamed := sha256HexOf("modified !!\n") + planned := sha256HexOf("modified 10\n") + + assert.Equal(t, streamed, manifest["file_10.py"], + "the manifest must hold the hash of the bytes that entered the archive") + assert.NotEqual(t, planned, manifest["file_10.py"], + "the Phase-2 planned hash must not survive the run") + + // The divergent file's entry reflects the true server state after apply + // (the poisoned hash is gone; the manifest holds the real remote hash). + assert.NotEqual(t, poisonedHash, manifest["file_21.py"], + "the poisoned hash must not survive the run") + assert.Equal(t, server["file_21.py"].Hash, manifest["file_21.py"], + "the divergent file's manifest entry must reflect the true server state") + + // The zip path was actually taken (not the stage path). + assert.Positive(t, fake.UploadFromZipCalls(), + "the zip path must be taken for >20 uploads") + assert.Zero(t, fake.UploadToStageCalls(), + "the stage path must not be used when the zip path is taken") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-008: Maximal mixed plan under --verify +// --------------------------------------------------------------------------- + +// TestCombined_MaximalMixedPlan_Verify proves that a single --verify --yes run +// with an upload, a download, a delete, a conflict, a file symlink, a +// directory symlink, and a divergence on a separate path: every plan field +// is populated, every notice reaches stderr, the manifest matches the server +// for every path, and the run exits 0. +// +// The setup uses a poisoned BASE to create the download, conflict, and +// divergence simultaneously on a non-drifted artifact (the server holds the +// same version BASE claims to describe, but BASE is wrong for some paths). +func TestCombined_MaximalMixedPlan_Verify(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-x8" + versionID = "ver-x8-synced" + newVer = "ver-x8-new" + ) + + // All files start synced (BASE == LOCAL == REMOTE). + files := map[string]string{ + "upload_me.py": "upload original\n", + "download_me.py": "download original\n", + "delete_me.py": "delete original\n", + "conflict_me.py": "conflict original\n", + "diverge_me.py": "diverge original\n", + } + + dir := syncedProject(t, files, catalogID, versionID) + + // Add both symlink kinds. + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + addDirSymlink(t, dir, "realdir", "inner.py", "link_to_dir") + + // Seed the server with the original content (the real REMOTE). + seed := seededServerContents(t, dir, files, nil) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-x8", + versionID: newVer, + }).withVersionContent(catalogID, versionID, seed) + + // --- Create each plan shape --- + + // Upload: modify upload_me.py locally. BASE == REMOTE (no divergence + // for this path), LOCAL differs → LOCAL_MODIFIED → upload. + modifyFile(t, dir, "upload_me.py", "upload modified\n") + + // Download: rewrite download_me.py to content A, poison BASE to A's + // hash. Now BASE = A, LOCAL = A, REMOTE = original → REMOTE_MODIFIED → + // download. Divergence: BASE = A != REMOTE = original. + downloadA := "download rewritten A\n" + modifyFile(t, dir, "download_me.py", downloadA) + poisonManifestHash(t, dir, "download_me.py", sha256HexOf(downloadA)) + + // Delete: remove delete_me.py locally. BASE == REMOTE (no divergence), + // LOCAL absent → LOCAL_DELETED → delete. + require.NoError(t, os.Remove(filepath.Join(dir, "delete_me.py"))) + + // Conflict: rewrite conflict_me.py to content C locally, poison BASE to + // hash A (different from both original and C). Now BASE = A, LOCAL = C, + // REMOTE = original → CONFLICT. Divergence: BASE = A != REMOTE = original. + conflictC := "conflict rewritten C\n" + modifyFile(t, dir, "conflict_me.py", conflictC) + + poisonedA := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + poisonManifestHash(t, dir, "conflict_me.py", poisonedA) + + // Divergence-only: poison diverge_me.py's manifest to hash A. Disk and + // server both hold the original, so CONVERGED → skip. Divergence: BASE + // = A != REMOTE = original. + poisonManifestHash(t, dir, "diverge_me.py", poisonedA) + + var logged string + + var execErr error + + e := engineFor(t, dir, Options{Verify: true, Yes: true}, fake, catalogID, versionID) + + logged = captureWarnLog(t, func() { + _, execErr = e.Run() + }) + + require.NoError(t, execErr, "the maximal mixed plan must succeed (exit 0)") + + // --- Every plan field is populated --- + + plan := e.plan + require.NotNil(t, plan) + assert.NotEmpty(t, plan.Uploads, "upload field must be populated") + assert.NotEmpty(t, plan.Downloads, "download field must be populated") + assert.NotEmpty(t, plan.Deletes, "delete field must be populated") + assert.NotEmpty(t, plan.Conflicts, "conflict field must be populated") + + // Verify each action type is present. + uploadPaths := uploadPathsOf(plan) + assert.Contains(t, uploadPaths, "upload_me.py", "upload_me.py must be in uploads") + + downloadPaths := make([]string, 0, len(plan.Downloads)) + for _, fa := range plan.Downloads { + downloadPaths = append(downloadPaths, fa.Path) + } + + assert.Contains(t, downloadPaths, "download_me.py", "download_me.py must be in downloads") + + deletePaths := make([]string, 0, len(plan.Deletes)) + for _, fa := range plan.Deletes { + deletePaths = append(deletePaths, fa.Path) + } + + assert.Contains(t, deletePaths, "delete_me.py", "delete_me.py must be in deletes") + + conflictPaths := plan.ConflictPaths() + assert.Contains(t, conflictPaths, "conflict_me.py", "conflict_me.py must be in conflicts") + + // --- Every notice reaches stderr (warn log) --- + + // Symlink notices. + assert.Contains(t, logged, "link_to_file.py", "file symlink notice must reach stderr") + assert.Contains(t, logged, "link_to_dir", "directory symlink notice must reach stderr") + assert.Contains(t, logged, "was not uploaded", "file symlink wording must be present") + assert.Contains(t, logged, "was not synced", "directory symlink wording must be present") + + // Divergence notices. + assert.Contains(t, logged, "divergence", "divergence notice must reach stderr") + assert.Contains(t, logged, "download_me.py", "divergence must name download_me.py") + assert.Contains(t, logged, "conflict_me.py", "divergence must name conflict_me.py") + assert.Contains(t, logged, "diverge_me.py", "divergence must name diverge_me.py") + + // --- Both symlink kinds are reported --- + + skipped := e.SkippedSymlinks() + assert.Contains(t, symlinkPaths(skipped), "link_to_file.py") + assert.False(t, symlinkIsDir(skipped, "link_to_file.py")) + assert.Contains(t, symlinkPaths(skipped), "link_to_dir") + assert.True(t, symlinkIsDir(skipped, "link_to_dir")) + + // --- The manifest matches the server for every path --- + + manifest := manifestHashes(t, dir) + server := serverHashes(t, fake, catalogID, newVer) + + // The uploaded file has the streamed hash. + assert.Equal(t, server["upload_me.py"], manifest["upload_me.py"], + "uploaded file: manifest must match server") + + // The downloaded file has the remote hash (server's original). + assert.Equal(t, server["download_me.py"], manifest["download_me.py"], + "downloaded file: manifest must match server") + + // The conflict file has the remote hash (remote wins). + assert.Equal(t, server["conflict_me.py"], manifest["conflict_me.py"], + "conflict file: manifest must match server (remote wins)") + + // The divergent-only file has the real remote hash (repaired). + assert.Equal(t, server["diverge_me.py"], manifest["diverge_me.py"], + "divergent file: manifest must match server (repaired)") + + // The deleted file is absent from both. + _, manifestHasDelete := manifest["delete_me.py"] + assert.False(t, manifestHasDelete, "deleted file must not be in the manifest") + + _, serverHasDelete := server["delete_me.py"] + assert.False(t, serverHasDelete, "deleted file must not be on the server") + + // Neither symlink is in the manifest or server. + _, manifestHasFileLink := manifest["link_to_file.py"] + assert.False(t, manifestHasFileLink, "file symlink must not be in the manifest") + + _, manifestHasDirLink := manifest["link_to_dir"] + assert.False(t, manifestHasDirLink, "directory symlink must not be in the manifest") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-009: All notices at once (engine level) +// --------------------------------------------------------------------------- + +// TestCombined_AllNoticesAtOnce proves that the .wapiignore shadow warning, +// symlink notices, and the divergence notice all fire in a single --verify +// --dry-run run without suppressing each other. +// +// The state-migration notice requires a non-preview run (phase0Preflight +// skips migration for preview modes), so it cannot coexist with --dry-run. +// The cmd-level test (TestCmd_Combined_AllNotices_JSONMode) uses the +// fakeEngine to verify the display layer renders all four notice types +// simultaneously, including the migration notice. +func TestCombined_AllNoticesAtOnce(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-x9" + versionID = "ver-x9-synced" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + // Synced project with app.py. + dir := syncedProject(t, map[string]string{"app.py": contentA}, catalogID, versionID) + + // Add a file symlink and a directory symlink. + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + addDirSymlink(t, dir, "realdir", "inner.py", "link_to_dir") + + // Add a .wapiignore file alongside .drignore to trigger the shadow warning. + require.NoError(t, os.WriteFile( + filepath.Join(dir, ignore.LegacyFileName), []byte("*.old\n"), 0o644)) + + // Poison the manifest so --verify detects divergence. + poisonedHash := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + poisonManifestHash(t, dir, "app.py", poisonedHash) + + // Seed the server with B (different from the poisoned BASE). + seed := seededServerContents(t, dir, map[string]string{"app.py": contentA}, map[string][]byte{ + "app.py": []byte(contentB), + }) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + var logged string + + e := engineFor(t, dir, Options{Verify: true, DryRun: true, Yes: true}, fake, catalogID, versionID) + + logged = captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + // No notice suppresses another: all four are present in the warn log. + + // Shadow warning. + assert.Contains(t, logged, wantShadowWarning, + "the .wapiignore shadow warning must fire alongside the other notices") + + // Symlink notices. + assert.Contains(t, logged, "link_to_file.py", + "the file symlink notice must fire alongside the other notices") + assert.Contains(t, logged, "link_to_dir", + "the directory symlink notice must fire alongside the other notices") + + // Divergence notice. + assert.Contains(t, logged, "divergence", + "the divergence notice must fire alongside the other notices") + assert.Contains(t, logged, "app.py", + "the divergence notice must name the divergent path") + + // All four notice types are independently present (none suppresses + // another). Verify each is a distinct substring. + assert.Contains(t, logged, wantShadowWarning, + "shadow warning must be present") + assert.Contains(t, logged, "link_to_file.py", + "file symlink notice must be present") + assert.Contains(t, logged, "link_to_dir", + "directory symlink notice must be present") + assert.Contains(t, logged, "divergence", + "divergence notice must be present") +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-010: Default no-verify user experience +// --------------------------------------------------------------------------- + +// TestCombined_DefaultNoVerifyUX proves that a plain run without --verify +// across a sequence of operations: records correct hashes, emits no divergence +// notice, makes no extra AllFiles round-trip on a non-drifted artifact, and +// only prints "Up to date." (empty plan) when disk and server genuinely agree. +// +// Each sub-test uses its own project directory to avoid sync-lock contention +// between engines, since the lock is held until Close and the cleanup runs at +// test end, not between sections. +func TestCombined_DefaultNoVerifyUX(t *testing.T) { + t.Run("first_sync_correct_hashes_no_divergence", func(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "util.py": "def u(): pass\n", + "realfile.py": "target\n", + }) + + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + + fake := &fakeFilesClient{ + catalogID: "cid-fs", stageID: "stage-1", versionID: "ver-1", + } + + var logged string + + e := engineFor(t, dir, Options{Yes: true}, fake, "", "") + + logged = captureWarnLog(t, func() { + _, err := e.Run() + require.NoError(t, err) + }) + + // The symlink notice fires on first sync. + assert.Contains(t, logged, "link_to_file.py", "symlink notice must fire on first sync") + // No divergence notice without --verify. + assert.NotContains(t, logged, "divergence", "no divergence notice without --verify") + + // Manifest hashes are correct (match disk). + manifest := manifestHashes(t, dir) + for _, rel := range []string{"app.py", "util.py", ignore.FileName} { + h, _, err := hashLocal(t, dir, rel) + require.NoError(t, err) + assert.Equal(t, h, manifest[rel], "manifest hash for %s must match disk", rel) + } + + // The symlink is not in the manifest. + _, hasLink := manifest["link_to_file.py"] + assert.False(t, hasLink, "symlink must not be in the manifest") + }) + + t.Run("no_change_resync_up_to_date_no_allfiles", func(t *testing.T) { + const ( + catalogID = "cid-nc" + versionID = "ver-nc" + ) + + files := map[string]string{"app.py": "print('hi')\n"} + dir := syncedProject(t, files, catalogID, versionID) + + seed := seededServerContents(t, dir, files, nil) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, versionID) + + plan, err := e.Plan() + require.NoError(t, err) + assert.True(t, plan.IsEmpty(), "no-change re-sync must produce an empty plan (Up to date.)") + assert.Zero(t, fake.AllFilesCalls(), "no extra AllFiles round-trip on a non-drifted artifact") + }) + + t.Run("mid_sync_edit_records_streamed_hash", func(t *testing.T) { + const ( + catalogID = "cid-me" + versionID = "ver-me" + newVer = "ver-me-new" + ) + + original := "content AAAA\n" + planContent := "content BBBB\n" // same length, different from original + streamContent := "content CCCC\n" // same length, different from planContent + + require.Len(t, planContent, len(original), "fixture: same-size rewrite") + require.Len(t, streamContent, len(original), "fixture: same-size rewrite") + + dir := syncedProject(t, map[string]string{"app.py": original}, catalogID, versionID) + + // Seed server with original content (before disk edit). + seed := map[string][]byte{ + ignore.FileName: diskBytes(t, dir, ignore.FileName), + "app.py": []byte(original), + } + + fake := (&fakeFilesClient{ + catalogID: catalogID, stageID: "stage-me", versionID: newVer, + }).withVersionContent(catalogID, versionID, seed) + + // Modify app.py BEFORE Plan so the plan has an upload row. + modifyFile(t, dir, "app.py", planContent) + + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, versionID) + + plan, err := e.Plan() + require.NoError(t, err) + require.False(t, plan.IsEmpty(), "the modification must produce a plan") + + // Rewrite app.py to different bytes of the same size between Plan and + // Execute. The streamed hash will differ from the Phase-2 planned hash. + modifyFile(t, dir, "app.py", streamContent) + + _, err = e.Execute(plan) + require.NoError(t, err) + + // The manifest holds the streamed hash, not the Phase-2 planned hash. + manifest := manifestHashes(t, dir) + assert.Equal(t, sha256HexOf(streamContent), manifest["app.py"], + "the manifest must hold the streamed hash after a mid-sync edit") + assert.NotEqual(t, sha256HexOf(planContent), manifest["app.py"], + "the Phase-2 planned hash must not survive the run") + }) + + t.Run("symlink_replacement_no_divergence_notice", func(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-sr" + versionID = "ver-sr" + newVer = "ver-sr-new" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "util.py": "def u(): pass\n", + }, catalogID, versionID) + + // Replace util.py with a symlink. + require.NoError(t, os.Remove(filepath.Join(dir, "util.py"))) + require.NoError(t, os.Symlink( + filepath.Join(dir, "app.py"), + filepath.Join(dir, "util.py"))) + + // Seed server with original content (before the replacement). + seed := map[string][]byte{ + ignore.FileName: diskBytes(t, dir, ignore.FileName), + "app.py": []byte("print('hi')\n"), + "util.py": []byte("def u(): pass\n"), + } + + fake := (&fakeFilesClient{ + catalogID: catalogID, stageID: "stage-sr", versionID: newVer, + }).withVersionContent(catalogID, versionID, seed) + + var logged string + + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, versionID) + + logged = captureWarnLog(t, func() { + _, err := e.Run() + require.NoError(t, err) + }) + + // The symlink notice fires for the replaced util.py. + assert.Contains(t, logged, "util.py", "the replaced path must produce a symlink notice") + // No divergence notice without --verify. + assert.NotContains(t, logged, "divergence", "no divergence notice without --verify") + + // The manifest no longer has util.py (it was deleted from the server). + manifest := manifestHashes(t, dir) + _, hasUtil := manifest["util.py"] + assert.False(t, hasUtil, "the deleted path must not be in the manifest") + }) + + t.Run("mixed_plan_no_divergence_notice", func(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-mx" + versionID = "ver-mx-1" + remoteVer = "ver-mx-remote" + newVer = "ver-mx-2" + ) + + dir := syncedProject(t, map[string]string{ + "keep.py": "keep content\n", + "modify.py": "original\n", + "remove.py": "to be deleted\n", + }, catalogID, versionID) + + // Build the seed for the REMOTE version (which the artifact's codeRef + // points at). The remote version has the original files plus a new + // file (remote-added → download). Using a different version than the + // config's LastSyncedVersionID creates drift, so the engine fetches + // AllFiles and discovers the new file — without --verify. + seed := map[string][]byte{ + ignore.FileName: diskBytes(t, dir, ignore.FileName), + "keep.py": []byte("keep content\n"), + "modify.py": []byte("original\n"), + "remove.py": []byte("to be deleted\n"), + "new_remote.py": []byte("remote added\n"), + } + + // Add a file symlink. + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + + // Modify modify.py locally. + modifyFile(t, dir, "modify.py", "modified content\n") + + // Remove remove.py locally. + require.NoError(t, os.Remove(filepath.Join(dir, "remove.py"))) + + fake := (&fakeFilesClient{ + catalogID: catalogID, stageID: "stage-mx", versionID: newVer, + }).withVersionContent(catalogID, remoteVer, seed) + + var logged string + + // The artifact points at remoteVer (drifted from versionID in config). + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, remoteVer) + + logged = captureWarnLog(t, func() { + _, err := e.Run() + require.NoError(t, err) + }) + + // No divergence notice without --verify. + assert.NotContains(t, logged, "divergence", "no divergence notice without --verify in a mixed plan") + + // The symlink notice fires. + assert.Contains(t, logged, "link_to_file.py", "symlink notice must fire in a mixed plan") + + // Manifest hashes are correct. + manifest := manifestHashes(t, dir) + h, _, err := hashLocal(t, dir, "modify.py") + require.NoError(t, err) + assert.Equal(t, h, manifest["modify.py"], "modified file: manifest must match disk") + assert.Equal(t, sha256HexOf("remote added\n"), manifest["new_remote.py"], + "downloaded file: manifest must match server") + _, hasRemove := manifest["remove.py"] + assert.False(t, hasRemove, "deleted file must not be in the manifest") + }) +} + +// --------------------------------------------------------------------------- +// VAL-CROSS-012: Exit-code coherence +// --------------------------------------------------------------------------- + +// TestCombined_ExitCodeCoherence proves that exit code 0 means success or +// diagnostics-only findings, and non-zero means a genuine failure. At the +// engine level, "exit code" is whether Run() returns an error. +func TestCombined_ExitCodeCoherence(t *testing.T) { + t.Run("symlinks_only_exit0", func(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + + fake := &fakeFilesClient{ + catalogID: "cid-ec1", stageID: "stage-1", versionID: "ver-1", + } + + e := engineFor(t, dir, Options{Yes: true}, fake, "", "") + + _, err := e.Run() + assert.NoError(t, err, "symlinks only: exit 0 (diagnostic, not failure)") + }) + + t.Run("divergence_only_exit0", func(t *testing.T) { + const ( + catalogID = "cid-ec2" + versionID = "ver-ec2" + ) + + dir := syncedProject(t, map[string]string{"app.py": "print('A')\n"}, catalogID, versionID) + + poisonedHash := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + poisonManifestHash(t, dir, "app.py", poisonedHash) + + seed := seededServerContents(t, dir, map[string]string{"app.py": "print('A')\n"}, + map[string][]byte{"app.py": []byte("print('B')\n")}) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + e := engineFor(t, dir, Options{Verify: true, Yes: true}, fake, catalogID, versionID) + + _, err := e.Run() + assert.NoError(t, err, "divergence only: exit 0 (diagnostic, not failure)") + }) + + t.Run("symlinks_and_divergence_exit0", func(t *testing.T) { + skipNonWindowsSymlink(t) + + const ( + catalogID = "cid-ec3" + versionID = "ver-ec3" + newVer = "ver-ec3-new" + ) + + dir := syncedProject(t, map[string]string{"app.py": "print('A')\n"}, catalogID, versionID) + addFileSymlink(t, dir, "realfile.py", "link_to_file.py") + + poisonedHash := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + poisonManifestHash(t, dir, "app.py", poisonedHash) + + // The server holds B; disk holds A; BASE is poisoned. Classify + // sees all three different → CONFLICT → remote wins → download. + // The symlink target realfile.py is a new file → LOCAL_ADDED → upload. + // So the fake needs stageID/versionID for the upload path. + seed := map[string][]byte{ + ignore.FileName: diskBytes(t, dir, ignore.FileName), + "app.py": []byte("print('B')\n"), + } + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-ec3", + versionID: newVer, + }).withVersionContent(catalogID, versionID, seed) + + e := engineFor(t, dir, Options{Verify: true, Yes: true}, fake, catalogID, versionID) + + _, err := e.Run() + assert.NoError(t, err, "symlinks + divergence: exit 0 (both are diagnostics)") + }) + + t.Run("healthy_up_to_date_exit0", func(t *testing.T) { + const ( + catalogID = "cid-ec4" + versionID = "ver-ec4" + ) + + dir := syncedProject(t, map[string]string{"app.py": "print('hi')\n"}, catalogID, versionID) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, + seededServerContents(t, dir, map[string]string{"app.py": "print('hi')\n"}, nil)) + + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, versionID) + + _, err := e.Run() + assert.NoError(t, err, "healthy up-to-date: exit 0") + }) + + t.Run("successful_sync_exit0", func(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + fake := &fakeFilesClient{ + catalogID: "cid-ec5", stageID: "stage-1", versionID: "ver-1", + } + + e := engineFor(t, dir, Options{Yes: true}, fake, "", "") + + _, err := e.Run() + assert.NoError(t, err, "successful sync: exit 0") + }) + + t.Run("upload_failure_nonzero", func(t *testing.T) { + const ( + catalogID = "cid-ec6" + versionID = "ver-ec6" + newVer = "ver-ec6-new" + ) + + dir := syncedProject(t, map[string]string{"app.py": "print('A')\n"}, catalogID, versionID) + modifyFile(t, dir, "app.py", "print('B')\n") + + seed := seededServerContents(t, dir, map[string]string{"app.py": "print('A')\n"}, nil) + + fake := (&fakeFilesClient{ + catalogID: catalogID, stageID: "stage-1", versionID: newVer, + }).withVersionContent(catalogID, versionID, seed).withFailNthUpload(1) + + e := engineFor(t, dir, Options{Yes: true}, fake, catalogID, versionID) + + _, err := e.Run() + require.Error(t, err, "upload failure: non-zero exit") + assert.Contains(t, err.Error(), "upload") + }) + + t.Run("post_apply_verification_mismatch_nonzero", func(t *testing.T) { + const ( + catalogID = "cid-ec7" + versionID = "ver-ec7" + newVer = "ver-ec7-new" + ) + + appPy := "print('A')\n" + untouched := "print('u')\n" + + dir := syncedProject(t, map[string]string{ + "app.py": appPy, + "untouched.py": untouched, + }, catalogID, versionID) + + // Build the seed from explicit original bytes BEFORE modifying disk, + // so the server holds the pre-change content (matching BASE) and the + // plan has exactly one upload row for app.py. + seed := map[string][]byte{ + ignore.FileName: diskBytes(t, dir, ignore.FileName), + "app.py": []byte(appPy), + "untouched.py": []byte(untouched), + } + + newBody := "print('B')\n" + modifyFile(t, dir, "app.py", newBody) + + // The post-apply AllFiles (for the NEW version) returns a wrong + // checksum for app.py, so post-apply verification catches it. + fake := (&fakeFilesClient{ + catalogID: catalogID, stageID: "stage-1", versionID: newVer, + }).withVersionContent(catalogID, versionID, seed). + withWrongChecksumForVersion(newVer, "app.py", "deadbeef") + + e := engineFor(t, dir, Options{Verify: true, Yes: true}, fake, catalogID, versionID) + + _, err := e.Run() + require.Error(t, err, "post-apply verification mismatch: non-zero exit") + assert.Contains(t, err.Error(), "post-apply verification") + }) + + t.Run("case_collision_nonzero", func(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "x"}) + + // The collision must exist ON DISK for Run() to see it, and a + // case-insensitive filesystem (macOS APFS and Windows NTFS by + // default) collapses APP.py into app.py instead of creating a + // second entry — so the expectation is filesystem-aware, not + // universal. The detector itself is covered platform-independently + // by TestCaseCollision_FailsBeforeUpload, and the end-to-end shape + // by TestCaseCollision_EndToEnd_FailsBeforeUpload. + if fsIsCaseInsensitive(t, dir) { + t.Skip("case-insensitive filesystem: a case-only colliding tree cannot exist here, so no non-zero exit is reachable") + } + + // Create a file whose name differs only in case from app.py. + // On a case-insensitive filesystem this is the same file, so the + // test skips. On Linux CI (case-sensitive) it produces a collision. + require.NoError(t, os.WriteFile(filepath.Join(dir, "APP.py"), []byte("y"), 0o644)) + + fake := &fakeFilesClient{ + catalogID: "cid-ec8", stageID: "stage-1", versionID: "ver-1", + } + + e := engineFor(t, dir, Options{Yes: true}, fake, "", "") + + _, err := e.Run() + require.Error(t, err, "case collision: non-zero exit") + assert.Contains(t, err.Error(), "case") + }) + + t.Run("locked_non_dry_run_nonzero", func(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + e := lockedEngine(t, dir, Options{Yes: true}) + + _, err := e.Run() + require.Error(t, err, "locked non-dry-run sync: non-zero exit") + assert.Contains(t, err.Error(), "locked") + }) +} diff --git a/internal/workload/sync/detection_regression_test.go b/internal/workload/sync/detection_regression_test.go new file mode 100644 index 000000000..39fea4adb --- /dev/null +++ b/internal/workload/sync/detection_regression_test.go @@ -0,0 +1,361 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests lock in the fact that change detection is purely content-hash +// based. The ticket (RAPTOR-19525) alleged a size+mtime fast path that silently +// skips files whose content changed but whose size and mtime are preserved. +// That mechanism does not exist in this codebase: Diff receives only hashes, +// and hashEntries rehashes every file every run. These tests would fail loudly +// if anyone ever introduced such a shortcut as a "performance improvement". + +// regressionEngine builds an engine over a syncedProject directory wired to a +// draft artifact whose codeRef matches the manifest's catalogID and versionID. +// The returned fakeFilesClient lets tests inspect call counters; the directory +// lets tests modify files on disk between Plan calls. +func regressionEngine(t *testing.T, files map[string]string) (*Engine, *fakeFilesClient, string) { + t.Helper() + + const ( + catalogID = "cid-regression" + versionID = "ver-regression" + ) + + dir := syncedProject(t, files, catalogID, versionID) + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-regression", + versionID: "ver-regression-next", + } + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e, fake, dir +} + +// uploadPathSet returns the set of paths in the upload portion of the plan. +func uploadPathSet(plan *SyncPlan) map[string]struct{} { + out := make(map[string]struct{}, len(plan.Uploads)) + for _, fa := range plan.Uploads { + out[fa.Path] = struct{}{} + } + + return out +} + +// TestSameSizeSameMtimeDetected is the codified refutation of RAPTOR-19525's +// stated mechanism. A file is rewritten to different bytes of identical length, +// and its mtime is restored to the pre-change value via os.Chtimes. If change +// detection were size+mtime based (as the ticket alleged), this file would be +// invisible. Because detection is content-hash based, it appears in the plan +// as a modification to upload. +// +// Fulfills VAL-REGRESSION-001. +func TestSameSizeSameMtimeDetected(t *testing.T) { + const ( + original = "print('A')\n" // 12 bytes + modified = "print('B')\n" // 12 bytes, different content + ) + + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": original, + }) + + abs := filepath.Join(dir, "app.py") + + // Capture the original mtime before modification. + info, err := os.Stat(abs) + require.NoError(t, err) + + origMtime := info.ModTime() + + // Rewrite to different content of the same byte length. + require.NoError(t, os.WriteFile(abs, []byte(modified), 0o644)) + + // Restore the original mtime so both size and mtime match the pre-change + // state. This is the exact scenario the ticket alleged would be skipped. + require.NoError(t, os.Chtimes(abs, origMtime, origMtime)) + + // Confirm the mtime was actually restored (some filesystems have coarse + // mtime resolution, so verify rather than assume). + postInfo, err := os.Stat(abs) + require.NoError(t, err) + + assert.True(t, postInfo.ModTime().Equal(origMtime), + "mtime must be restored for the test to be meaningful") + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.True(t, present, + "same-size same-mtime content change must be detected as a modification; "+ + "if this fails, a size+mtime fast path was introduced") +} + +// TestDetectionIsContentOnly is a 2x2 matrix over {size matches manifest, size +// differs} x {content matches manifest, content differs}. Detection must fire +// if and only if content differs, in every cell. This proves detection is +// purely content-hash based and never consults size or mtime. +// +// The "size differs, content matches" cell is synthetic: the manifest's size +// field is manually corrupted while the hash stays correct. A size-based fast +// path would flag this file as changed; the hash-based mechanism correctly +// leaves it alone. +// +// Fulfills VAL-REGRESSION-005. +func TestDetectionIsContentOnly(t *testing.T) { + const ( + original = "print('A')\n" // 12 bytes + modified = "print('B')\n" // 12 bytes, same size, different content + grown = "print('BB')\n" // 13 bytes, different size and content + ) + + cases := []struct { + name string + setup func(t *testing.T, dir string) + wantDetected bool + }{ + { + name: "size_matches_content_matches", + // File unchanged: size and content both match the manifest. + // Detection must NOT fire. + setup: func(_ *testing.T, _ string) {}, + wantDetected: false, + }, + { + name: "size_matches_content_differs", + // Same-size content change: the ticket's exact scenario. + // Detection MUST fire. + setup: func(t *testing.T, dir string) { + t.Helper() + + modifyFile(t, dir, "app.py", modified) + }, + wantDetected: true, + }, + { + name: "size_differs_content_matches", + // Manifest's size field is corrupted while the hash stays correct. + // A size-based check would flag this; hash-based detection must NOT. + setup: func(t *testing.T, dir string) { + t.Helper() + + corruptManifestSize(t, dir, "app.py", 9999) + }, + wantDetected: false, + }, + { + name: "size_differs_content_differs", + // Normal modification: both size and content change. + // Detection MUST fire. + setup: func(t *testing.T, dir string) { + t.Helper() + + modifyFile(t, dir, "app.py", grown) + }, + wantDetected: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": original, + }) + + tc.setup(t, dir) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.Equal(t, tc.wantDetected, present, + "detection must fire iff content differs, regardless of size") + }) + } +} + +// TestDetectionControls provides true-positive and false-positive controls so +// the detection assertions above cannot be satisfied trivially. +// +// True positives: +// - size_and_content_change: both size and content change → detected. +// - large_file_content_change: a 4 MiB file whose content changes → detected, +// guarding against a future size-threshold shortcut for large files. +// +// False positives: +// - untouched_file: a file identical to its manifest entry → absent from plan. +// - mtime_only_advance: content unchanged, mtime advanced → not detected. +// +// Fulfills VAL-REGRESSION-003 and VAL-REGRESSION-004. +func TestDetectionControls(t *testing.T) { + t.Run("size_and_content_change", func(t *testing.T) { + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": "print('A')\n", + }) + + modifyFile(t, dir, "app.py", "print('hello world')\n") + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.True(t, present, "a size-and-content change must be detected") + }) + + t.Run("large_file_content_change", func(t *testing.T) { + // A 4 MiB file guards against a future size-threshold shortcut that + // skips hashing for "large" files under the assumption they are + // unlikely to change without a size change. + const largeSize = 4 * 1024 * 1024 // 4 MiB + + original := strings.Repeat("A", largeSize) + modified := strings.Repeat("B", largeSize) + + e, _, dir := regressionEngine(t, map[string]string{ + "big.bin": original, + }) + + modifyFile(t, dir, "big.bin", modified) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["big.bin"] + assert.True(t, present, "a multi-megabyte content change must be detected") + }) + + t.Run("untouched_file", func(t *testing.T) { + e, _, _ := regressionEngine(t, map[string]string{ + "app.py": "print('A')\n", + }) + + // No modifications: the file is identical to its manifest entry. + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.False(t, present, + "an untouched file must not appear in the upload plan") + }) + + t.Run("mtime_only_advance", func(t *testing.T) { + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": "print('A')\n", + }) + + abs := filepath.Join(dir, "app.py") + + // Advance the mtime without changing the content. + future := time.Now().Add(2 * time.Hour) + require.NoError(t, os.Chtimes(abs, future, future)) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.False(t, present, + "an mtime-only advance with identical content must not be detected") + }) +} + +// TestPlanRowSizesMatchDisk asserts that each upload plan row's displayed byte +// size (fa.LocalSize) equals the real on-disk file size at plan time. This +// ensures the sizes a validator reads off the plan can be trusted as evidence. +// +// Fulfills VAL-REGRESSION-014. +func TestPlanRowSizesMatchDisk(t *testing.T) { + files := map[string]string{ + "app.py": "print('A')\n", + "config.yaml": "name: test\n", + "data.bin": "ABCDEF", + } + + e, _, dir := regressionEngine(t, files) + + // Modify files to different sizes so the plan is non-empty and the + // sizes on disk differ from the manifest's recorded sizes. + modifyFile(t, dir, "app.py", "print('hello world')\n") + modifyFile(t, dir, "data.bin", "ABCDEFGHIJKL") + + plan, err := e.Plan() + require.NoError(t, err) + + require.NotEmpty(t, plan.Uploads, "plan must have uploads for the size assertion to be meaningful") + + for _, fa := range plan.Uploads { + abs := filepath.Join(dir, filepath.FromSlash(fa.Path)) + info, err := os.Stat(abs) + require.NoError(t, err, "stat %s", fa.Path) + + assert.Equal(t, info.Size(), fa.LocalSize, + "upload plan row for %s: displayed size must equal real on-disk size", fa.Path) + } +} + +// corruptManifestSize loads the manifest, changes the size field for the given +// path to a wrong value while keeping the hash correct, and re-saves it. This +// creates a synthetic "size differs, content matches" state for the detection +// matrix: the file on disk is unchanged (same hash), but the manifest's size +// field no longer matches the real size. A size-based fast path would flag +// this file as changed; the hash-based mechanism correctly leaves it alone. +func corruptManifestSize(t *testing.T, dir, rel string, wrongSize int64) { + t.Helper() + + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + fm, ok := manifest.Files[rel] + require.True(t, ok, "manifest must contain %s", rel) + + // Keep the hash, corrupt only the size. + fm.Size = wrongSize + manifest.Files[rel] = fm + + require.NoError(t, wapi.SaveManifest(dir, manifest)) +} diff --git a/internal/workload/sync/display/json.go b/internal/workload/sync/display/json.go index 9ebdedec2..26322615a 100644 --- a/internal/workload/sync/display/json.go +++ b/internal/workload/sync/display/json.go @@ -37,6 +37,43 @@ type PlanJSON struct { // like, and a script reading only this document has nothing else to go on: // the human warning goes to stderr and the exit status is 0. Locked bool `json:"locked"` + + // Divergence lists the paths where manifest.json (BASE) disagrees with + // the server (REMOTE), as detected by a --verify run. Like Locked it is + // always emitted and explicitly empty when there is nothing to report, so + // a script never has to guess whether a missing key meant "checked and + // clean" or "never checked". Only a --verify run can populate it; every + // other run leaves it empty. The findings are diagnostics: the plan + // already reconciles them and the exit status is unchanged. + Divergence []DivergenceJSON `json:"divergence"` + + // SkippedSymlinks lists every symlink the walk did not follow, filtered + // through the ignore matcher. Like Locked and Divergence it is always + // emitted and explicitly empty when there are none, so a script never + // has to guess whether a missing key meant "no symlinks" or "never + // checked". The findings are diagnostics: the plan already excludes them + // and the exit status is unchanged. + SkippedSymlinks []SkippedSymlinkJSON `json:"skippedSymlinks"` +} + +// SkippedSymlinkJSON is one symlink the walk did not follow, in the plan +// document. IsDir distinguishes a single skipped file from an entire omitted +// subtree (a directory symlink prunes all of its children), which is the +// distinction that makes the notice actionable. +type SkippedSymlinkJSON struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` +} + +// DivergenceJSON is one BASE-vs-REMOTE disagreement in the plan document. +// Kind is the fixed vocabulary shared with the stderr prose (hash_mismatch / +// base_only / remote_only); the hash of the side a kind does not involve is +// omitted, so a script can identify each finding without re-fetching. +type DivergenceJSON struct { + Path string `json:"path"` + Kind sync.DivergenceKind `json:"kind"` + BaseHash string `json:"baseHash,omitempty"` + RemoteHash string `json:"remoteHash,omitempty"` } type FileActionJSON struct { @@ -58,21 +95,31 @@ type PlanStatsJSON struct { OldVersionShort string `json:"oldVersionShort,omitempty"` } -// RenderPlanJSON writes plan as pretty-printed JSON to w. -func RenderPlanJSON(w io.Writer, plan *sync.SyncPlan, locked bool) error { +// RenderPlanJSON writes plan as pretty-printed JSON to w. divergences are +// the BASE-vs-REMOTE findings of a --verify run; skippedSymlinks are the +// symlinks the walk did not follow. Both are rendered even when plan is nil, +// because they are detected in Phase 2 regardless of what the plan goes on +// to do with the paths. +func RenderPlanJSON(w io.Writer, plan *sync.SyncPlan, locked bool, divergences []sync.Divergence, skippedSymlinks []sync.SkippedSymlink) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") if plan == nil { - return enc.Encode(PlanJSON{Locked: locked}) + return enc.Encode(PlanJSON{ + Locked: locked, + Divergence: divergencesJSON(divergences), + SkippedSymlinks: skippedSymlinksJSON(skippedSymlinks), + }) } out := PlanJSON{ - Locked: locked, - Uploads: actionsJSON(plan.Uploads), - Downloads: actionsJSON(plan.Downloads), - Deletes: actionsJSON(plan.Deletes), - Conflicts: actionsJSON(plan.Conflicts), + Locked: locked, + Divergence: divergencesJSON(divergences), + SkippedSymlinks: skippedSymlinksJSON(skippedSymlinks), + Uploads: actionsJSON(plan.Uploads), + Downloads: actionsJSON(plan.Downloads), + Deletes: actionsJSON(plan.Deletes), + Conflicts: actionsJSON(plan.Conflicts), Stats: PlanStatsJSON{ UploadCount: len(plan.Uploads), DownloadCount: len(plan.Downloads), @@ -91,6 +138,41 @@ func RenderPlanJSON(w io.Writer, plan *sync.SyncPlan, locked bool) error { return nil } +// divergencesJSON converts the engine's findings to the wire shape. The +// result is never nil: an empty input yields an explicit empty slice so the +// rendered document says "no divergence" instead of null. +func divergencesJSON(in []sync.Divergence) []DivergenceJSON { + out := make([]DivergenceJSON, len(in)) + + for i, d := range in { + out[i] = DivergenceJSON{ + Path: d.Path, + Kind: d.Kind, + BaseHash: d.BaseHash, + RemoteHash: d.RemoteHash, + } + } + + return out +} + +// skippedSymlinksJSON converts the engine's skipped-symlink findings to the +// wire shape. The result is never nil: an empty input yields an explicit +// empty slice so the rendered document says "no skipped symlinks" instead +// of null, mirroring the divergence and locked precedents. +func skippedSymlinksJSON(in []sync.SkippedSymlink) []SkippedSymlinkJSON { + out := make([]SkippedSymlinkJSON, len(in)) + + for i, s := range in { + out[i] = SkippedSymlinkJSON{ + Path: s.Path, + IsDir: s.IsDir, + } + } + + return out +} + type ResultJSON struct { OldVersion string `json:"oldVersion,omitempty"` NewVersion string `json:"newVersion,omitempty"` diff --git a/internal/workload/sync/display/json_test.go b/internal/workload/sync/display/json_test.go new file mode 100644 index 000000000..70ebdd8cc --- /dev/null +++ b/internal/workload/sync/display/json_test.go @@ -0,0 +1,181 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package display + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/datarobot/cli/internal/workload/sync" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// planDocWithDivergence renders a plan with the given divergence findings and +// returns the decoded top-level document. +func planDocWithDivergence(t *testing.T, plan *sync.SyncPlan, divergences []sync.Divergence) map[string]any { + t.Helper() + + var buf bytes.Buffer + + require.NoError(t, RenderPlanJSON(&buf, plan, false, divergences, nil)) + + var doc map[string]any + + require.NoError(t, json.Unmarshal(buf.Bytes(), &doc), "the plan document must decode as JSON") + + return doc +} + +// The divergence field is part of the contract, not a bonus: always present +// and explicitly empty when there is no divergence, so a script reading the +// document never has to guess whether a missing key meant "checked, clean" +// or "never checked" (the locked:false precedent). +func TestRenderPlanJSON_DivergenceAlwaysPresent(t *testing.T) { + doc := planDocWithDivergence(t, &sync.SyncPlan{}, nil) + + div, ok := doc["divergence"] + require.True(t, ok, "the divergence key must always be emitted") + + require.IsType(t, []any{}, div) + assert.Empty(t, div, "no divergence must render as an explicit empty array, not null and not an omitted key") + assert.Contains(t, doc, "locked") +} + +func TestRenderPlanJSON_DivergenceEntries(t *testing.T) { + divergences := []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "h-base", RemoteHash: "h-remote"}, + {Path: "gone.py", Kind: sync.DivergenceBaseOnly, BaseHash: "h-gone"}, + {Path: "stray.py", Kind: sync.DivergenceRemoteOnly, RemoteHash: "h-stray"}, + } + + doc := planDocWithDivergence(t, &sync.SyncPlan{}, divergences) + + raw, err := json.Marshal(doc["divergence"]) + require.NoError(t, err) + + var entries []struct { + Path string `json:"path"` + Kind string `json:"kind"` + BaseHash string `json:"baseHash"` + RemoteHash string `json:"remoteHash"` + } + + require.NoError(t, json.Unmarshal(raw, &entries)) + require.Len(t, entries, 3) + + assert.Equal(t, "app.py", entries[0].Path) + assert.Equal(t, "hash_mismatch", entries[0].Kind) + assert.Equal(t, "h-base", entries[0].BaseHash) + assert.Equal(t, "h-remote", entries[0].RemoteHash) + + assert.Equal(t, "gone.py", entries[1].Path) + assert.Equal(t, "base_only", entries[1].Kind) + assert.Equal(t, "h-gone", entries[1].BaseHash) + assert.Empty(t, entries[1].RemoteHash) + + assert.Equal(t, "stray.py", entries[2].Path) + assert.Equal(t, "remote_only", entries[2].Kind) + assert.Equal(t, "h-stray", entries[2].RemoteHash) + assert.Empty(t, entries[2].BaseHash) +} + +// An empty plan (Up to date.) can still carry findings: divergence is +// detected in Phase 2 and reported regardless of what the plan does with the +// paths, so a nil plan must not drop the field. +func TestRenderPlanJSON_NilPlan_KeepsDivergence(t *testing.T) { + divergences := []sync.Divergence{ + {Path: "app.py", Kind: sync.DivergenceHashMismatch, BaseHash: "h-base", RemoteHash: "h-remote"}, + } + + doc := planDocWithDivergence(t, nil, divergences) + + div, ok := doc["divergence"] + require.True(t, ok, "the divergence key must be emitted even for a nil plan") + + require.IsType(t, []any{}, div) + assert.Len(t, div, 1) +} + +// planDocWithSymlinks renders a plan with the given skipped symlinks and +// returns the decoded top-level document. +func planDocWithSymlinks(t *testing.T, plan *sync.SyncPlan, symlinks []sync.SkippedSymlink) map[string]any { + t.Helper() + + var buf bytes.Buffer + + require.NoError(t, RenderPlanJSON(&buf, plan, false, nil, symlinks)) + + var doc map[string]any + + require.NoError(t, json.Unmarshal(buf.Bytes(), &doc), "the plan document must decode as JSON") + + return doc +} + +// The skippedSymlinks field is part of the contract: always present and +// explicitly empty when there are none (the locked:false precedent). +func TestRenderPlanJSON_SkippedSymlinksAlwaysPresent(t *testing.T) { + doc := planDocWithSymlinks(t, &sync.SyncPlan{}, nil) + + sym, ok := doc["skippedSymlinks"] + require.True(t, ok, "the skippedSymlinks key must always be emitted") + + require.IsType(t, []any{}, sym) + assert.Empty(t, sym, "no skipped symlinks must render as an explicit empty array, not null and not an omitted key") +} + +func TestRenderPlanJSON_SkippedSymlinkEntries(t *testing.T) { + symlinks := []sync.SkippedSymlink{ + {Path: "link_to_file.py", IsDir: false}, + {Path: "link_to_dir", IsDir: true}, + } + + doc := planDocWithSymlinks(t, &sync.SyncPlan{}, symlinks) + + raw, err := json.Marshal(doc["skippedSymlinks"]) + require.NoError(t, err) + + var entries []struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` + } + + require.NoError(t, json.Unmarshal(raw, &entries)) + require.Len(t, entries, 2) + + assert.Equal(t, "link_to_file.py", entries[0].Path) + assert.False(t, entries[0].IsDir, "file symlink must report isDir false") + + assert.Equal(t, "link_to_dir", entries[1].Path) + assert.True(t, entries[1].IsDir, "directory symlink must report isDir true") +} + +// A nil plan can still carry findings: skipped symlinks are detected in Phase 2 +// and reported regardless of what the plan does with the paths. +func TestRenderPlanJSON_NilPlan_KeepsSkippedSymlinks(t *testing.T) { + symlinks := []sync.SkippedSymlink{ + {Path: "link.py", IsDir: false}, + } + + doc := planDocWithSymlinks(t, nil, symlinks) + + sym, ok := doc["skippedSymlinks"] + require.True(t, ok, "the skippedSymlinks key must be emitted even for a nil plan") + + require.IsType(t, []any{}, sym) + assert.Len(t, sym, 1) +} diff --git a/internal/workload/sync/display/plan.go b/internal/workload/sync/display/plan.go index a12247bf2..c5d337464 100644 --- a/internal/workload/sync/display/plan.go +++ b/internal/workload/sync/display/plan.go @@ -23,7 +23,10 @@ import ( ) // PrintPlan writes the human-readable sync plan to w. Empty plans print -// "Up to date." and return. +// "Up to date." and return — with one exception an applying run routes +// through PrintEmptyPlanRepair instead, because a plan that is empty only +// because a --verify run is repairing the manifest must not read as a +// project with nothing wrong. func PrintPlan(w io.Writer, plan *sync.SyncPlan) error { if plan == nil || plan.IsEmpty() { _, err := fmt.Fprintln(w, "Up to date.") @@ -51,6 +54,18 @@ func PrintPlan(w io.Writer, plan *sync.SyncPlan) error { return nil } +// PrintEmptyPlanRepair writes the one-line stdout account of an applying +// run whose plan is empty but whose --verify findings force the manifest +// repair: the plan has no rows to apply, and the repair happens as the +// state write that rewrites manifest.json from the server's state. Printing +// the ordinary "Up to date." immediately before that rewrite would claim +// nothing is being fixed. +func PrintEmptyPlanRepair(w io.Writer) error { + _, err := fmt.Fprintln(w, "The plan is empty, but manifest.json is being rewritten from the server's state to repair the divergences found by --verify.") + + return err +} + func printGroup(w io.Writer, header string, files []sync.FileAction, marker func(sync.FileAction) string) error { if len(files) == 0 { return nil diff --git a/internal/workload/sync/divergence.go b/internal/workload/sync/divergence.go new file mode 100644 index 000000000..df0fc3682 --- /dev/null +++ b/internal/workload/sync/divergence.go @@ -0,0 +1,148 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "fmt" + "sort" + + "github.com/datarobot/cli/internal/log" +) + +// DivergenceKind names the way a single path diverges between BASE +// (manifest.json) and REMOTE (the server's actual file listing). The three +// values are a fixed vocabulary shared by the stderr prose and the JSON +// rendering, so a script can act on the same distinction the message draws. +type DivergenceKind string + +const ( + // DivergenceHashMismatch: the path exists on both sides and the hashes + // differ — the recorded state no longer describes the server's bytes. + DivergenceHashMismatch DivergenceKind = "hash_mismatch" + + // DivergenceBaseOnly: BASE records the path but REMOTE does not have it. + DivergenceBaseOnly DivergenceKind = "base_only" + + // DivergenceRemoteOnly: REMOTE has the path but BASE does not record it. + DivergenceRemoteOnly DivergenceKind = "remote_only" +) + +// Divergence is one path where BASE and REMOTE disagree. It is a diagnostic, +// never an error: the reconciling plan already follows from Diff once the +// real remote is known, and the exit status must not change. +type Divergence struct { + Path string + Kind DivergenceKind + + // BaseHash and RemoteHash carry the two sides of the disagreement so a + // script can identify the mismatch without re-fetching anything. The + // side a kind does not involve is empty (base_only leaves RemoteHash + // empty, remote_only leaves BaseHash empty). + BaseHash string + RemoteHash string +} + +// detectDivergence compares BASE against REMOTE per path and reports every +// disagreement, distinguishing a hash difference from the two one-sided +// cases. The result is sorted by path so notices listing several paths are +// deterministic across runs. +func detectDivergence(base, remote BaseManifest) []Divergence { + var out []Divergence + + for path, b := range base { + r, ok := remote[path] + + switch { + case !ok: + out = append(out, Divergence{Path: path, Kind: DivergenceBaseOnly, BaseHash: b.Hash}) + case b.Hash != r.Hash: + out = append(out, Divergence{ + Path: path, Kind: DivergenceHashMismatch, + BaseHash: b.Hash, RemoteHash: r.Hash, + }) + } + } + + for path, r := range remote { + if _, ok := base[path]; !ok { + out = append(out, Divergence{Path: path, Kind: DivergenceRemoteOnly, RemoteHash: r.Hash}) + } + } + + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + + return out +} + +// maybeDetectDivergence checks BASE's claim about the remote when --verify +// has just fetched the real listing for a non-drifted artifact: there — and +// only there — BASE claims to describe exactly the version fetched, so a +// mismatch is a lie worth reporting. On a drifted artifact the remote is a +// newer version by design, and BASE-vs-REMOTE differences are ordinary +// drift, not findings. +// +// Findings are stored on the engine for the display layer AND logged from +// the phase via log.Warn: the log writes stderr immediately, so the findings +// survive a later phase failing and never touch stdout in JSON mode. The +// prose is bounded at DivergenceNoticeBound entries with an "and N more" +// tail; the structured field on the engine carries every divergence +// regardless. +func maybeDetectDivergence(e *Engine) { + if e.drifted || !e.opts.Verify { + return + } + + e.divergences = detectDivergence(e.base, e.remote) + + for i, d := range e.divergences { + if i >= DivergenceNoticeBound { + break + } + + log.Warn(divergenceNotice(d)) + } + + if len(e.divergences) > DivergenceNoticeBound { + log.Warn(fmt.Sprintf( + "divergence: and %d more divergence(s) were found (see the plan JSON for the full list)", + len(e.divergences)-DivergenceNoticeBound)) + } +} + +// DivergenceNoticeBound is the maximum number of divergences the stderr prose +// lists individually before summarizing the remainder as a count. The plan +// JSON always lists every divergence. Fixed at 5 so workers and validators +// assert the same number rather than each choosing a bound. +const DivergenceNoticeBound = 5 + +// divergenceNotice renders one divergence as a self-contained stderr line. +// Each kind gets its own wording: a reader must be able to tell "the server +// holds different bytes" apart from "the path exists on one side only". +func divergenceNotice(d Divergence) string { + switch d.Kind { + case DivergenceHashMismatch: + return fmt.Sprintf( + "divergence: %s: BASE (manifest.json) hash %s differs from REMOTE (server) hash %s; the manifest no longer describes the server", + d.Path, d.BaseHash, d.RemoteHash) + case DivergenceBaseOnly: + return fmt.Sprintf( + "divergence: %s: present in BASE (manifest.json) but absent from REMOTE (server)", d.Path) + case DivergenceRemoteOnly: + return fmt.Sprintf( + "divergence: %s: present in REMOTE (server) but absent from BASE (manifest.json)", d.Path) + } + + return fmt.Sprintf("divergence: %s: unknown kind %q", d.Path, d.Kind) +} diff --git a/internal/workload/sync/divergence_test.go b/internal/workload/sync/divergence_test.go new file mode 100644 index 000000000..138328ae4 --- /dev/null +++ b/internal/workload/sync/divergence_test.go @@ -0,0 +1,488 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// diskBytes reads a file straight off the project tree so tests can seed the +// fake server with exactly the bytes the local side holds (or deliberately +// different ones) without re-deriving them. +func diskBytes(t *testing.T, dir, rel string) []byte { + t.Helper() + + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + + return b +} + +// seededServerContents builds a version payload whose entries match the given +// project files byte for byte, plus optional overrides for paths that must +// diverge. Including .drignore in the default set keeps the ignore template +// from showing up as an unrelated base-only divergence. +func seededServerContents(t *testing.T, dir string, files map[string]string, overrides map[string][]byte) map[string][]byte { + t.Helper() + + contents := make(map[string][]byte, len(files)+1) + + contents[ignore.FileName] = diskBytes(t, dir, ignore.FileName) + + for rel := range files { + contents[rel] = diskBytes(t, dir, rel) + } + + for rel, body := range overrides { + contents[rel] = body + } + + return contents +} + +func TestDetectDivergence(t *testing.T) { + base := BaseManifest{ + "same.py": {Hash: "h-same", Size: 1}, + "changed.py": {Hash: "h-old", Size: 2}, + "gone.py": {Hash: "h-gone", Size: 3}, + } + + remote := BaseManifest{ + "same.py": {Hash: "h-same", Size: 1}, + "changed.py": {Hash: "h-new", Size: 4}, + "stray.py": {Hash: "h-stray", Size: 5}, + } + + got := detectDivergence(base, remote) + + require.Len(t, got, 3, "every divergent path must be reported, not just the first") + + // Deterministic order: sorted by path, so a notice naming several paths + // is stable across runs. + assert.Equal(t, []Divergence{ + {Path: "changed.py", Kind: DivergenceHashMismatch, BaseHash: "h-old", RemoteHash: "h-new"}, + {Path: "gone.py", Kind: DivergenceBaseOnly, BaseHash: "h-gone"}, + {Path: "stray.py", Kind: DivergenceRemoteOnly, RemoteHash: "h-stray"}, + }, got) +} + +func TestDetectDivergence_NoneAndEmpty(t *testing.T) { + both := BaseManifest{"a.py": {Hash: "h1", Size: 1}} + + assert.Empty(t, detectDivergence(both, both), "identical manifests must not diverge") + assert.Empty(t, detectDivergence(BaseManifest{}, BaseManifest{}), "empty BASE and empty REMOTE (first sync) must not diverge") +} + +// The prose must let a reader tell the three kinds apart: a hash difference is +// not the same fact as a path existing on one side only. +func TestDivergenceNoticeText(t *testing.T) { + mismatch := divergenceNotice(Divergence{ + Path: "app.py", Kind: DivergenceHashMismatch, + BaseHash: "aaaa", RemoteHash: "bbbb", + }) + baseOnly := divergenceNotice(Divergence{Path: "gone.py", Kind: DivergenceBaseOnly, BaseHash: "cccc"}) + remoteOnly := divergenceNotice(Divergence{Path: "stray.py", Kind: DivergenceRemoteOnly, RemoteHash: "dddd"}) + + assert.Contains(t, mismatch, "app.py") + assert.Contains(t, mismatch, "aaaa") + assert.Contains(t, mismatch, "bbbb") + assert.Contains(t, mismatch, "differs", "a hash mismatch must say the hashes differ") + + assert.Contains(t, baseOnly, "gone.py") + assert.Contains(t, baseOnly, "BASE") + assert.NotContains(t, baseOnly, "differs", "base-only must not read as a hash comparison") + + assert.Contains(t, remoteOnly, "stray.py") + assert.Contains(t, remoteOnly, "REMOTE") + + assert.NotEqual(t, mismatch, baseOnly) + assert.NotEqual(t, baseOnly, remoteOnly) +} + +// TestMaybeDetectDivergence_WarnBoundedAtFive verifies that the per-path +// divergence warnings are bounded the same way the sibling symlink notices +// are: the first DivergenceNoticeBound paths in deterministic order, then a +// count of the remainder. The engine's divergences field still carries every +// path (the plan JSON lists the full set). +func TestMaybeDetectDivergence_WarnBoundedAtFive(t *testing.T) { + remote := BaseManifest{} + + for i := 0; i < 7; i++ { + name := fmt.Sprintf("%c.py", rune('a'+i)) + remote[name] = FileEntry{Hash: "h", Size: 1} + } + + e := &Engine{opts: Options{Verify: true}, base: BaseManifest{}, remote: remote} + + logged := captureWarnLog(t, func() { maybeDetectDivergence(e) }) + + // The structured field carries every divergence regardless of the prose + // bound. + require.Len(t, e.divergences, 7) + + // The first five appear in the prose. + for i := 0; i < 5; i++ { + name := fmt.Sprintf("%c.py", rune('a'+i)) + assert.Contains(t, logged, name, + "the first %d divergences must appear in the bounded prose", 5) + } + + // The sixth and seventh do not appear individually. + assert.NotContains(t, logged, "f.py", + "the 6th divergence must not appear in the bounded prose") + assert.NotContains(t, logged, "g.py", + "the 7th divergence must not appear in the bounded prose") + + // The count of the remainder appears. + assert.Contains(t, logged, "2 more divergence(s)", + "the count of the remainder must appear in the prose") +} + +// engineFor returns an engine bound to dir with the given options and a fake +// artifact store pointing at the given catalog/version — the common fixture +// shape of every Phase-2 divergence test. +func engineFor(t *testing.T, dir string, opts Options, fake *fakeFilesClient, catalogID, versionID string) *Engine { + t.Helper() + + e, err := newWithDeps(dir, opts, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e +} + +// The flagship poisoned-manifest scenario at the engine level: local == base, +// the server holds different bytes for the same version, and only --verify +// looks. Expect exactly one AllFiles call, the divergence recorded on the +// engine, and a reconciling download row in the plan instead of "Up to date.". +func TestPhase2_VerifyForcesAllFilesAndDetectsDivergence(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + serverAppPy := []byte("server holds different bytes\n") + localAppPy := "print('A')\n" + + dir := syncedProject(t, map[string]string{"app.py": localAppPy}, catalogID, versionID) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, map[string]string{ + "app.py": localAppPy, + }, map[string][]byte{"app.py": serverAppPy})) + + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + plan, err := e.Plan() + require.NoError(t, err) + + assert.Equal(t, 1, fake.AllFilesCalls(), "--verify must force exactly one AllFiles round-trip on a non-drifted artifact") + + divs := e.Divergences() + require.Len(t, divs, 1) + + assert.Equal(t, "app.py", divs[0].Path) + assert.Equal(t, DivergenceHashMismatch, divs[0].Kind) + assert.Equal(t, sha256Hex([]byte(localAppPy)), divs[0].BaseHash) + assert.Equal(t, sha256Hex(serverAppPy), divs[0].RemoteHash) + + // The real remote is now known, so Diff produces the reconciling row. + require.Len(t, plan.Downloads, 1, "the plan must reconcile BASE with the real REMOTE instead of skipping") + assert.Equal(t, "app.py", plan.Downloads[0].Path) + assert.Equal(t, ClsRemoteModified, plan.Downloads[0].Classification) +} + +// Hard rule 9: the default sync must stay exactly as it was — no AllFiles +// round-trip on a non-drifted artifact and no divergence findings. +func TestPhase2_DefaultPath_NoAllFiles_NoDivergence(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + serverAppPy := []byte("server holds different bytes\n") + localAppPy := "print('A')\n" + + dir := syncedProject(t, map[string]string{"app.py": localAppPy}, catalogID, versionID) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, map[string]string{ + "app.py": localAppPy, + }, map[string][]byte{"app.py": serverAppPy})) + + e := engineFor(t, dir, Options{}, fake, catalogID, versionID) + + plan, err := e.Plan() + require.NoError(t, err) + + assert.Zero(t, fake.AllFilesCalls(), "the fast path must not fetch AllFiles") + assert.Empty(t, e.Divergences(), "no divergence findings without --verify") + assert.True(t, plan.IsEmpty(), "without --verify the poisoned BASE stays invisible (the defect --verify exists to expose)") +} + +// On an already-drifted artifact the drift path fetches AllFiles anyway, so +// --verify must not add a second fetch — and BASE-vs-REMOTE differences are +// ordinary drift there, not findings. +func TestPhase2_VerifyOnDriftedArtifact_OneAllFilesCall_NoDivergence(t *testing.T) { + const ( + catalogID = "cid-1" + lastSynced = "ver-1" + currentVer = "ver-2" + ) + + files := map[string]string{"app.py": "print('A')\n"} + + dir := syncedProject(t, files, catalogID, lastSynced) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, currentVer, seededServerContents(t, dir, files, nil)) + + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, currentVer) + + _, err := e.Plan() + require.NoError(t, err) + + assert.Equal(t, 1, fake.AllFilesCalls(), "the drift fetch already happens; --verify must not add a second") + assert.Empty(t, e.Divergences(), "a remote that moved on is drift, not a BASE-vs-REMOTE finding") +} + +// A healthy project must stay silent under --verify. +func TestPhase2_Verify_HealthyProject_NoDivergence(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + files := map[string]string{"app.py": "print('A')\n", "util.py": "def u(): pass\n"} + + dir := syncedProject(t, files, catalogID, versionID) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, files, nil)) + + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + plan, err := e.Plan() + require.NoError(t, err) + + assert.True(t, plan.IsEmpty(), "a healthy project must stay Up to date under --verify") + assert.Empty(t, e.Divergences(), "no divergence when BASE describes the server") +} + +// VAL-VERIFY-006: a plain local modification is not a divergence, and the +// upload plan must be identical with and without --verify. +func TestPhase2_Verify_LocalOnlyModification_NoDivergence_PlanUnchanged(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + files := map[string]string{"app.py": "print('A')\n", "util.py": "def u(): pass\n"} + + dir := syncedProject(t, files, catalogID, versionID) + + // Seed the server with the ORIGINAL bytes (== BASE) before touching disk, + // so remote == base and only the disk moves. + seed := seededServerContents(t, dir, files, nil) + + modifyFile(t, dir, "app.py", "print('A2')\n") + + // Each engine gets its own fake over the same server state, so neither + // Plan() call can influence the other's call counts. The project lock is + // held until Close, so the first engine must be closed before the second + // one plans. + newFake := func() *fakeFilesClient { + return (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + } + + ePlain := engineFor(t, dir, Options{}, newFake(), catalogID, versionID) + + planPlain, err := ePlain.Plan() + require.NoError(t, err) + require.NoError(t, ePlain.Close(), "release the project lock before the second engine plans") + + eVerify := engineFor(t, dir, Options{Verify: true}, newFake(), catalogID, versionID) + + planVerify, err := eVerify.Plan() + require.NoError(t, err) + + assert.Empty(t, eVerify.Divergences(), "a local modification is not a BASE-vs-REMOTE divergence") + assert.Equal(t, planPlain, planVerify, "the upload plan must be identical with and without --verify") + require.Len(t, planVerify.Uploads, 1) + assert.Equal(t, "app.py", planVerify.Uploads[0].Path) +} + +// First sync against an empty artifact: there is no BASE and no REMOTE to +// diverge, so --verify must stay silent. +func TestPhase2_Verify_FirstSyncEmptyArtifact_NoDivergence(t *testing.T) { + dir := initProject(t, map[string]string{"agent.py": "print('hi')\n"}) + + fake := &fakeFilesClient{catalogID: "cid-new", stageID: "stage-1", versionID: "ver-1"} + + e := engineFor(t, dir, Options{Verify: true}, fake, "", "") + + plan, err := e.Plan() + require.NoError(t, err) + + assert.Empty(t, e.Divergences(), "a first sync has nothing to diverge") + assert.NotEmpty(t, plan.Uploads, "the first sync still plans the uploads") +} + +// VAL-VERIFY-007 at the engine level: both one-sided directions are detected +// with distinct kinds, and the reconciling plan repairs the manifest to match +// the server exactly. +func TestPhase2_Verify_OneSidedDivergence_RepairMatchesServer(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + files := map[string]string{"app.py": "print('A')\n", "util.py": "def u(): pass\n"} + + stray := []byte("only on the server\n") + remoteAppPy := []byte("print('B')\n") + + dir := syncedProject(t, files, catalogID, versionID) + + // The server version drops util.py (BASE-only), rewrites app.py (hash + // mismatch) and carries a path BASE never recorded (REMOTE-only). + seed := seededServerContents(t, dir, files, map[string][]byte{ + "app.py": remoteAppPy, + "stray.py": stray, + }) + delete(seed, "util.py") + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + plan, err := e.Plan() + require.NoError(t, err) + + // Both one-sided directions, each under its own kind. + kinds := make(map[string]DivergenceKind, len(e.Divergences())) + for _, d := range e.Divergences() { + kinds[d.Path] = d.Kind + } + + assert.Equal(t, DivergenceBaseOnly, kinds["util.py"], "a path in BASE but not on the server is base-only") + assert.Equal(t, DivergenceRemoteOnly, kinds["stray.py"], "a path on the server but not in BASE is remote-only") + + // The reconciling plan: download the REMOTE-only and modified paths, + // remove the BASE-only path locally (remote wins). + downloaded := make([]string, 0, len(plan.Downloads)) + + for _, fa := range plan.Downloads { + downloaded = append(downloaded, fa.Path) + } + + deleted := make([]string, 0, len(plan.Deletes)) + + for _, fa := range plan.Deletes { + deleted = append(deleted, fa.Path) + } + + assert.ElementsMatch(t, []string{"app.py", "stray.py"}, downloaded) + assert.ElementsMatch(t, []string{"util.py"}, deleted) + + _, err = e.Execute(plan) + require.NoError(t, err) + + // After apply the manifest must describe the true server state. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + want := map[string]string{ + "app.py": sha256Hex(remoteAppPy), + ignore.FileName: sha256Hex(diskBytes(t, dir, ignore.FileName)), + "stray.py": sha256Hex(stray), + } + + got := make(map[string]string, len(manifest.Files)) + for p, meta := range manifest.Files { + got[p] = meta.Hash + } + + assert.Equal(t, want, got, "the repaired manifest must match the server (util.py gone, stray.py present)") + + server, err := fake.AllFiles(catalogID, versionID) + require.NoError(t, err) + + assert.Len(t, server, len(manifest.Files), "manifest and server must hold the same path set") +} + +// VAL-VERIFY-017: the divergence warning is emitted from Phase 2 through the +// warn logger, so it must already be on stderr when a later phase fails — +// not rendered only after a successful plan render. +func TestPhase2_Verify_DivergenceWarningSurvivesPhase5Failure(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + files := map[string]string{"app.py": "print('A')\n"} + + dir := syncedProject(t, files, catalogID, versionID) + + // Poisoned app.py (BASE==local, server differs) plus a brand-new local + // file, so the plan both downloads and uploads — and the upload fails. + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, files, map[string][]byte{ + "app.py": []byte("server holds different bytes\n"), + })).withFailNthUpload(1) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "new.py"), []byte("brand new\n"), 0o644)) + + var execErr error + + out := captureWarnLog(t, func() { + e, err := newWithDeps(dir, Options{Verify: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, perr := e.Plan() + require.NoError(t, perr) + require.False(t, plan.IsEmpty(), "the scenario must produce a plan with rows") + + _, execErr = e.Execute(plan) + }) + + require.Error(t, execErr, "Phase 5 must fail for this test to mean anything") + assert.Contains(t, out, "app.py", "the divergence warning must name the affected path") + assert.Contains(t, out, "divergence", "the divergence warning must survive the Phase 5 failure") +} diff --git a/internal/workload/sync/download_path_test.go b/internal/workload/sync/download_path_test.go new file mode 100644 index 000000000..de8b54dd7 --- /dev/null +++ b/internal/workload/sync/download_path_test.go @@ -0,0 +1,219 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests exercise the download half of Execute through the fake's +// server model: a remote-modified file is pulled to disk, the bytes that +// land are asserted byte-for-byte and by checksum (not merely "no error"), +// and the two download fault hooks prove the failure paths trip the +// rollback instead of silently leaving a bad file behind. +// +// The download-plan shape recorded here is also the scaffolding later +// verify-milestone tests build on: a downloads-only plan whose rows carry +// the server's advertised hashes. + +// downloadScenario builds a synced project whose disk is untouched while +// the server holds a newer version of app.py, so Plan() produces exactly +// one download row for it. The remote version is seeded with recorded +// content (not just hashes), which is what makes it downloadable through +// the fake. The .drignore bytes are seeded as-is so the ignore file is +// unchanged on both sides and does not add plan rows. +type downloadScenario struct { + dir string + fake *fakeFilesClient + remoteContent string + remoteHash string +} + +func newDownloadScenario(t *testing.T, mutate func(*fakeFilesClient) *fakeFilesClient) downloadScenario { + t.Helper() + + const ( + catalogID = "cid-dl" + versionID = "ver-dl-synced" + remoteVerID = "ver-dl-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + remoteContent := "print('remote-newer')\n" + + // The server holds a newer app.py than both the manifest and the disk; + // the artifact's codeRef points at that version so the engine sees drift + // and fetches the real remote listing. + drignoreBytes, err := os.ReadFile(filepath.Join(dir, ignore.FileName)) + require.NoError(t, err) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-dl", + versionID: "ver-dl-next", + }).withVersionContent(catalogID, remoteVerID, map[string][]byte{ + "app.py": []byte(remoteContent), + ignore.FileName: drignoreBytes, + }) + + if mutate != nil { + fake = mutate(fake) + } + + return downloadScenario{ + dir: dir, + fake: fake, + remoteContent: remoteContent, + remoteHash: sha256Hex([]byte(remoteContent)), + } +} + +func (s downloadScenario) engine(t *testing.T) *Engine { + t.Helper() + + e, err := newWithDeps(s.dir, Options{Yes: true}, Deps{ + Files: s.fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "cid-dl", "ver-dl-remote"), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e +} + +// TestExecute_RemoteModifiedDownloadLandsOnDisk is the fault-off control +// for the download path and the assertion the fake previously could not +// support: the download writes the server's exact bytes to disk, those +// bytes hash to the advertised checksum, and the manifest records that +// hash — "the right bytes were written", not merely "no error returned". +func TestExecute_RemoteModifiedDownloadLandsOnDisk(t *testing.T) { + s := newDownloadScenario(t, nil) + e := s.engine(t) + + plan, err := e.Plan() + require.NoError(t, err) + + // The plan must be exactly one download row carrying the server's + // advertised hash and size — the structure later downloads-only + // coverage relies on. + require.Len(t, plan.Downloads, 1) + + dl := plan.Downloads[0] + assert.Equal(t, "app.py", dl.Path) + assert.Equal(t, ActDownloadModify, dl.Action) + assert.Equal(t, s.remoteHash, dl.RemoteHash, "plan must carry the advertised checksum") + assert.Equal(t, int64(len(s.remoteContent)), dl.RemoteSize) + assert.Empty(t, plan.Uploads) + assert.Empty(t, plan.Deletes) + assert.Empty(t, plan.Conflicts) + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, 1, result.DownloadedCount) + + // The remote bytes must be on disk, byte for byte, and hash to the + // advertised checksum. + onDisk, err := os.ReadFile(filepath.Join(s.dir, "app.py")) + require.NoError(t, err) + + assert.Equal(t, s.remoteContent, string(onDisk), "downloaded file must hold the server's exact bytes") + assert.Equal(t, s.remoteHash, sha256Hex(onDisk), "downloaded bytes must hash to the advertised checksum") + + // The new BASE must describe what the server holds for the downloaded + // path, which is only truthful if the write above really happened. + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + assert.Equal(t, s.remoteHash, manifest.Files["app.py"].Hash, "manifest must record the downloaded file's remote hash") + + assert.Equal(t, 1, s.fake.DownloadFileCalls(), "exactly one download must have been served") +} + +// TestExecute_FailedDownloadRollsBack proves a download that errors aborts +// the sync: Execute fails naming the path, and the rollback restores the +// original local file, so a failed download never leaves the tree half +// written. +func TestExecute_FailedDownloadRollsBack(t *testing.T) { + s := newDownloadScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + return f.withFailDownload("app.py") + }) + e := s.engine(t) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Downloads, 1) + + _, err = e.Execute(plan) + require.Error(t, err, "a failed download must fail the sync") + assert.Contains(t, err.Error(), "app.py", "the error must name the failed path") + assert.Contains(t, err.Error(), "injected failure", "the fake's fault must be visible in the chain") + + // The rollback must have restored the pre-sync local content: the + // remote bytes never landed despite the file having been opened for + // writing mid-download. + onDisk, readErr := os.ReadFile(filepath.Join(s.dir, "app.py")) + require.NoError(t, readErr) + assert.Equal(t, "print('hi')\n", string(onDisk), "failed download must leave the original local file in place") + + assert.Equal(t, 1, s.fake.DownloadFileCalls(), "the failed download still counts as a call") +} + +// TestExecute_CorruptDownloadChecksumMismatchRollsBack proves the +// post-download checksum verification has teeth: the fake serves bytes +// whose hash differs from the advertised checksum (same length, so the +// size check passes), and the client must catch the mismatch, fail the +// sync, and restore the original file rather than keep corrupt bytes. +func TestExecute_CorruptDownloadChecksumMismatchRollsBack(t *testing.T) { + s := newDownloadScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + return f.withCorruptDownload("app.py") + }) + e := s.engine(t) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Downloads, 1) + + _, err = e.Execute(plan) + require.Error(t, err, "a checksum mismatch must fail the sync") + assert.Contains(t, err.Error(), "checksum mismatch", "the checksum check must be what catches the corruption") + assert.Contains(t, err.Error(), "app.py", "the error must name the corrupted path") + + // The corrupt bytes were written and then removed by the download's + // own cleanup; the rollback must have brought the original file back. + onDisk, readErr := os.ReadFile(filepath.Join(s.dir, "app.py")) + require.NoError(t, readErr) + assert.Equal(t, "print('hi')\n", string(onDisk), "corrupt bytes must not survive a failed download") + + assert.Equal(t, 1, s.fake.DownloadFileCalls()) +} diff --git a/internal/workload/sync/dry_run_integrity_test.go b/internal/workload/sync/dry_run_integrity_test.go new file mode 100644 index 000000000..44c2510a1 --- /dev/null +++ b/internal/workload/sync/dry_run_integrity_test.go @@ -0,0 +1,156 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDryRun_LeavesStateUntouched verifies that a --dry-run run with pending +// changes leaves manifest.json (content AND mtime), config.json, and the +// fake's server state all unchanged. The plan is non-empty (a file was +// modified), but Run returns after Plan without calling Execute, so no +// upload-side calls are issued and no state files are written. +// +// Go-level coverage: +// - manifest.json content is byte-identical before and after. +// - manifest.json mtime is unchanged (SaveManifest is never called; the +// file is not touched at all, so even the mtime survives). +// - config.json content is byte-identical (SaveConfig is never called). +// - The fake's upload counters are all zero (no stage, upload, apply, or +// zip calls). +// - AllFiles is not called (the artifact is not drifted, so the fast path +// copies BASE to REMOTE without a round-trip). +// +// What remains for a staging validator to confirm through the real binary: +// - Real mtime preservation across a process invocation (a Go test shares +// a process and a filesystem cache; a separate `dr` process touching +// the file would reset the mtime even if the content is unchanged). +// - The server's AllFiles listing is unchanged after the dry-run (the Go +// test asserts zero upload calls, which implies this, but the staging +// validator confirms it against the real server). +// +// Fulfills VAL-UPLOAD-016. +func TestDryRun_LeavesStateUntouched(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change so the plan is non-empty. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: "ver-should-not-be-used", + } + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + // Capture pre-run state: manifest bytes, manifest mtime, config bytes. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + preManifest, err := os.ReadFile(mPath) + require.NoError(t, err) + + // Set a known mtime well in the past so any touch (even at the same + // content) is detectable regardless of filesystem mtime resolution. + knownTime := time.Now().Add(-1 * time.Hour) + require.NoError(t, os.Chtimes(mPath, knownTime, knownTime)) + + preInfo, err := os.Stat(mPath) + require.NoError(t, err) + + preManifestMtime := preInfo.ModTime() + + preConfig, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + // Run the dry-run. It must stop after Plan (DryRun is true). + result, err := e.Run() + require.NoError(t, err) + + require.NotNil(t, result, "dry-run must return a result") + + // The plan must be non-empty — the positive control that proves the + // zero-call assertions below are not vacuously true. Read the plan Run + // already computed (Plan stores it on the engine) rather than re-running + // Plan() after the fact: a second Plan is a full re-execution of phases + // 0-4 and could itself issue calls or touch state, which would silently + // couple the zero-call assertions to plan-phase purity. Reading the + // stored plan is passive and keeps the control independent of that + // assumption. + require.NotNil(t, e.plan, "Run must have computed a plan for the control to mean anything") + + assert.False(t, e.plan.IsEmpty(), + "plan must be non-empty (a file was modified) for the zero-call assertions to be meaningful") + + // manifest.json content must be byte-identical. + postManifest, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, preManifest, postManifest, + "manifest.json content must be unchanged after a dry-run") + + // manifest.json mtime must be unchanged — SaveManifest was never called. + postInfo, err := os.Stat(mPath) + require.NoError(t, err) + + assert.True(t, postInfo.ModTime().Equal(preManifestMtime), + "manifest.json mtime must be unchanged after a dry-run (SaveManifest not called)") + + // config.json content must be byte-identical. + postConfig, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, preConfig, postConfig, + "config.json content must be unchanged after a dry-run") + + // No upload-side calls: the fake's server state is untouched. + assert.Equal(t, 0, fake.CreateStageCalls(), + "CreateStage must not be called in a dry-run") + assert.Equal(t, 0, fake.UploadToStageCalls(), + "UploadToStage must not be called in a dry-run") + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called in a dry-run") + assert.Equal(t, 0, fake.UploadFromZipCalls(), + "UploadFromZip must not be called in a dry-run") + assert.Equal(t, 0, fake.AllFilesCalls(), + "AllFiles must not be called in a dry-run on a non-drifted artifact (fast path)") +} diff --git a/internal/workload/sync/engine.go b/internal/workload/sync/engine.go index 883dc77c8..4e3b9c18c 100644 --- a/internal/workload/sync/engine.go +++ b/internal/workload/sync/engine.go @@ -30,6 +30,14 @@ type Options struct { DryRun bool ShowDiffs bool Yes bool + + // Verify opts into the network-cost integrity checks: a remote + // round-trip even when the artifact is not drifted, and post-apply + // verification that the server holds what was uploaded. It changes how + // much a run checks, not whether the run applies its plan, so it must + // never be treated as a preview mode: previewOnly must not consider it, + // and a Verify run without DryRun/ShowDiffs still reaches Execute. + Verify bool } // Result is the outcome of a successful sync. @@ -101,10 +109,12 @@ type Engine struct { local LocalManifest remote RemoteManifest plan *SyncPlan + divergences []Divergence lock *SyncLock rollback *Rollback newCatalogID string newVersionID string + uploadOutcome *UploadOutcome conflictCopies []string result *Result startedAt time.Time @@ -113,6 +123,11 @@ type Engine struct { ignoreNotice string lockedNote string + // skippedSymlinks holds symlinks the walk did not follow, filtered + // through the ignore matcher so deliberately-ignored or system-excluded + // links are absent. Populated in Phase 2; exposed by the display layer. + skippedSymlinks []SkippedSymlink + lockfileFn LockfileRunner lockfileGenerated bool lockfileHint string @@ -197,13 +212,20 @@ func (e *Engine) Execute(plan *SyncPlan) (_ *Result, retErr error) { } // Run is Plan + Execute. With DryRun or ShowDiffs it stops after Plan. +// An empty plan normally short-circuits before Execute too, but a --verify +// run that recorded BASE-vs-REMOTE divergences must still run Phase 6: the +// plan has nothing to apply, yet the manifest on disk is a lie about the +// server, and Phase 6 is what rewrites it from the real remote now in hand. +// The sharpest shape — BASE poisoned to A while disk and server both hold B +// — classifies as CONVERGED and plans nothing, so without this the poison +// survives the very run that detected it. func (e *Engine) Run() (*Result, error) { plan, err := e.Plan() if err != nil { return nil, err } - if e.previewOnly() || plan.IsEmpty() { + if e.previewOnly() || (plan.IsEmpty() && len(e.divergences) == 0) { if relErr := e.releaseLock(); relErr != nil { return nil, fmt.Errorf("release lock: %w", relErr) } @@ -247,6 +269,25 @@ func (e *Engine) IgnoreFileNotice() string { return e.ignoreNotice } // would read as a sync that is going to work. func (e *Engine) LockedNotice() string { return e.lockedNote } +// Divergences reports the paths where BASE (manifest.json) and the real +// REMOTE listing disagree, as detected in Phase 2 of a --verify run. A +// non-drifted artifact without --verify never fetches the remote (the fast +// path copies BASE into REMOTE), so an empty result can mean "checked and +// clean", "nothing was checked", or "nothing can be checked" (first sync) — +// only a --verify run with a fetched remote populates the slice. +// +// The findings are diagnostics, not errors: the plan already reconciles them +// because the real remote is in hand, and the exit status must not change. +func (e *Engine) Divergences() []Divergence { return e.divergences } + +// SkippedSymlinks reports the symlinks the walk did not follow, filtered +// through the ignore matcher so deliberately-ignored or system-excluded +// links are absent. Each entry distinguishes a single skipped file from an +// entire omitted subtree (a directory symlink prunes all of its children). +// The findings are diagnostics: the plan already excludes them, and the +// exit status must not change. +func (e *Engine) SkippedSymlinks() []SkippedSymlink { return e.skippedSymlinks } + // previewOnly reports that this run stops after Plan and sends nothing to the // platform, which is what makes the artifact's own mutability beside the point // in phase 1. It is not a promise that the working tree is untouched: phase 0 diff --git a/internal/workload/sync/engine_test.go b/internal/workload/sync/engine_test.go index 17cdeadb1..613b87191 100644 --- a/internal/workload/sync/engine_test.go +++ b/internal/workload/sync/engine_test.go @@ -16,10 +16,8 @@ package sync import ( "errors" - "io" "os" "path/filepath" - stdsync "sync" "testing" "time" @@ -56,93 +54,6 @@ func (f *fakeArtifactStore) PatchCodeRef(artifactID, catalogID, catalogVersionID return f.PatchFn(artifactID, catalogID, catalogVersionID) } -// fakeFilesClient is the in-memory FilesAPI fake used by engine tests. -// Unexpected methods return errors so off-happy-path drift fails loudly. -type fakeFilesClient struct { - allFiles map[string]filesapi.FileMeta - catalogID string - versionID string - stageID string - uploadedFiles map[string][]byte - deletedPaths []string - mu stdsync.Mutex -} - -func (f *fakeFilesClient) CreateCatalog() (*filesapi.CatalogResp, error) { - if f.catalogID == "" { - return nil, errors.New("fakeFilesClient.CreateCatalog: no catalogID configured") - } - - return &filesapi.CatalogResp{CatalogID: f.catalogID, CatalogVersionID: ""}, nil -} - -func (f *fakeFilesClient) CreateStage(_ string) (*filesapi.StageResp, error) { - if f.stageID == "" { - return nil, errors.New("fakeFilesClient.CreateStage: no stageID configured") - } - - return &filesapi.StageResp{CatalogID: f.catalogID, StageID: f.stageID}, nil -} - -func (f *fakeFilesClient) UploadToStage(_, _, name string, _ int64, body io.Reader) error { - data, err := io.ReadAll(body) - if err != nil { - return err - } - - f.mu.Lock() - defer f.mu.Unlock() - - if f.uploadedFiles == nil { - f.uploadedFiles = map[string][]byte{} - } - - f.uploadedFiles[name] = data - - return nil -} - -func (f *fakeFilesClient) ApplyStage(_, _, _ string) (*filesapi.ApplyStageResp, error) { - if f.versionID == "" { - return nil, errors.New("fakeFilesClient.ApplyStage: no versionID configured") - } - - return &filesapi.ApplyStageResp{ - CatalogID: f.catalogID, - CatalogVersionID: f.versionID, - NumFiles: len(f.uploadedFiles), - }, nil -} - -func (f *fakeFilesClient) UploadFromZipNew(_ string, _ int64, _ io.Reader) (*filesapi.FromFileResp, error) { - return nil, errors.New("fakeFilesClient: UploadFromZipNew not expected") -} - -func (f *fakeFilesClient) UploadFromZipExisting(_, _, _ string, _ int64, _ io.Reader) (*filesapi.FromFileResp, error) { - return nil, errors.New("fakeFilesClient: UploadFromZipExisting not expected") -} - -func (f *fakeFilesClient) PollStatus(_ string) (*filesapi.StatusResp, error) { - return nil, errors.New("fakeFilesClient: PollStatus not expected") -} - -func (f *fakeFilesClient) AllFiles(_, _ string) (map[string]filesapi.FileMeta, error) { - return f.allFiles, nil -} - -func (f *fakeFilesClient) DownloadFile(_, _, _ string, _ io.Writer) (string, int64, error) { - return "", 0, errors.New("fakeFilesClient: DownloadFile not expected") -} - -func (f *fakeFilesClient) DeleteFiles(_ string, paths []string) (*filesapi.DeleteFilesResp, error) { - f.deletedPaths = append(f.deletedPaths, paths...) - return &filesapi.DeleteFilesResp{}, nil -} - -func (f *fakeFilesClient) ListVersions(_ string, _ int) ([]filesapi.CatalogVersion, error) { - return nil, errors.New("fakeFilesClient: ListVersions not expected") -} - func initProject(t *testing.T, files map[string]string) string { t.Helper() diff --git a/internal/workload/sync/fake_files_client_self_test.go b/internal/workload/sync/fake_files_client_self_test.go new file mode 100644 index 000000000..e12d1cb0d --- /dev/null +++ b/internal/workload/sync/fake_files_client_self_test.go @@ -0,0 +1,903 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sha256Hex computes the SHA-256 hex of content, matching what the fake +// records and what the real server returns. +func sha256Hex(data []byte) string { + h := sha256.Sum256(data) + + return hex.EncodeToString(h[:]) +} + +// stageUpload uploads files through the stage path and applies, returning the +// ApplyStage response. This is the helper for tests that exercise the fake's +// server-state model without going through the full engine. +func stageUpload(t *testing.T, fake *fakeFilesClient, files map[string][]byte) *filesapi.ApplyStageResp { + t.Helper() + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + for path, data := range files { + err := fake.UploadToStage(fake.catalogID, stage.StageID, path, int64(len(data)), bytes.NewReader(data)) + require.NoError(t, err) + } + + resp, err := fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.NoError(t, err) + + return resp +} + +// TestFakeServerState_StagePathRecordsContentAndChecksum verifies that after an +// upload+apply, AllFiles for the new version returns exactly the paths uploaded +// with fileChecksum equal to the SHA-256 hex of the recorded bytes. +func TestFakeServerState_StagePathRecordsContentAndChecksum(t *testing.T) { + files := map[string][]byte{ + "app.py": []byte("print('hello')\n"), + "utils/helper.py": []byte("def help(): pass\n"), + } + + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + resp := stageUpload(t, fake, files) + require.Equal(t, "ver-1", resp.CatalogVersionID) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + // AllFiles must return exactly the paths uploaded, with correct checksums. + assert.Len(t, all, len(files)) + + for path, data := range files { + fm, ok := all[path] + require.True(t, ok, "path %s missing from AllFiles", path) + assert.Equal(t, sha256Hex(data), fm.Hash, "checksum for %s", path) + assert.Equal(t, int64(len(data)), fm.Size, "size for %s", path) + } +} + +// TestFakeServerState_StagePathMergeSemantics verifies that ApplyStage merges +// staged files into the prior version (REPLACE merge), not replacing the +// whole catalog. Uploading 1 file over a 3-file version yields a 4-file +// version. +func TestFakeServerState_StagePathMergeSemantics(t *testing.T) { + // Pre-populate a version with 3 files. + priorFiles := map[string]filesapi.FileMeta{ + "a.py": {Hash: sha256Hex([]byte("a")), Size: 1}, + "b.py": {Hash: sha256Hex([]byte("b")), Size: 1}, + "c.py": {Hash: sha256Hex([]byte("c")), Size: 1}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersion("cid-1", "ver-1", priorFiles) + + // Upload 1 new file over the 3-file version. + resp := stageUpload(t, fake, map[string][]byte{ + "d.py": []byte("d"), + }) + + // numFiles must count ALL files in the resulting version (3 + 1 = 4), + // not just the ones uploaded (1). + assert.Equal(t, 4, resp.NumFiles, "numFiles must count all files in the resulting version, not just uploaded") + + all, err := fake.AllFiles(fake.catalogID, "ver-2") + require.NoError(t, err) + + // The resulting version must contain all 4 files: 3 from the prior + // version plus 1 newly uploaded. + assert.Len(t, all, 4) + + for path := range priorFiles { + assert.Contains(t, all, path, "prior file %s must survive the merge", path) + } + + assert.Contains(t, all, "d.py", "new file must be in the version") + assert.Equal(t, sha256Hex([]byte("d")), all["d.py"].Hash) +} + +// TestWithVersionCopiesTheSeed pins that withVersion copies the caller's map +// into server state: mutating the map after handing it over must not change +// what the fake serves. The fake used to alias the caller's map directly, so +// a reused or later-mutated seed silently rewrote an already-seeded version +// behind the test's back — the kind of cross-test state leak -shuffle=on is +// meant to surface. +func TestWithVersionCopiesTheSeed(t *testing.T) { + seed := map[string]filesapi.FileMeta{ + "a.py": {Hash: "hash-a", Size: 1}, + } + + fake := (&fakeFilesClient{}).withVersion("cid-1", "ver-1", seed) + + // Mutate the caller's map after handing it over; server state must not + // follow. + seed["a.py"] = filesapi.FileMeta{Hash: "hash-mutated", Size: 99} + seed["injected.py"] = filesapi.FileMeta{Hash: "hash-injected", Size: 5} + + all, err := fake.AllFiles("cid-1", "ver-1") + require.NoError(t, err) + + fm, ok := all["a.py"] + require.True(t, ok, "seeded path must remain present") + + assert.Equal(t, "hash-a", fm.Hash, "a later mutation of the caller's map must not reach server state") + assert.Equal(t, int64(1), fm.Size, "a later mutation of the caller's map must not reach server state") + + assert.NotContains(t, all, "injected.py", + "paths added to the caller's map after the call must not appear in server state") +} + +// TestFakeServerState_StagePathReplacesExisting verifies that uploading a file +// that already exists in the prior version replaces its content (REPLACE +// semantics for staged paths). +func TestFakeServerState_StagePathReplacesExisting(t *testing.T) { + priorFiles := map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("old")), Size: 3}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersion("cid-1", "ver-1", priorFiles) + + newContent := []byte("new content here") + + resp := stageUpload(t, fake, map[string][]byte{"app.py": newContent}) + assert.Equal(t, 1, resp.NumFiles, "version still has 1 file after replace") + + all, err := fake.AllFiles(fake.catalogID, "ver-2") + require.NoError(t, err) + + assert.Equal(t, sha256Hex(newContent), all["app.py"].Hash, "checksum must reflect new content") + assert.Equal(t, int64(len(newContent)), all["app.py"].Size) +} + +// TestFakeServerState_ZipPathRecordsContent verifies that the zip upload path +// extracts the archive and records per-file content with SHA-256 checksums. +func TestFakeServerState_ZipPathRecordsContent(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "main.py": "if __name__ == '__main__': pass\n", + }) + + // Build a real zip from the project files (same as the production ZipUploader). + plan := &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalSize: 11}, + {Path: "main.py", LocalSize: 34}, + }, + } + + zipPath, _, err := buildZip(dir, plan.Uploads) + require.NoError(t, err) + + zipData, err := os.ReadFile(zipPath) + require.NoError(t, err) + + fake := &fakeFilesClient{ + catalogID: "cid-zip", + versionID: "ver-zip", + } + + resp, err := fake.UploadFromZipNew("wapi-sync.zip", int64(len(zipData)), bytes.NewReader(zipData)) + require.NoError(t, err) + + assert.Equal(t, "cid-zip", resp.CatalogID) + assert.Equal(t, "ver-zip", resp.CatalogVersionID) + + all, err := fake.AllFiles(fake.catalogID, "ver-zip") + require.NoError(t, err) + + // The zip path must record the same content and checksums as the stage path. + assert.Len(t, all, 2) + + for _, fa := range plan.Uploads { + fm, ok := all[fa.Path] + require.True(t, ok, "path %s missing from zip AllFiles", fa.Path) + + // The checksum must match the SHA-256 of the file content on disk. + data, err := os.ReadFile(dir + "/" + fa.Path) + require.NoError(t, err) + + assert.Equal(t, sha256Hex(data), fm.Hash, "zip checksum for %s", fa.Path) + assert.Equal(t, int64(len(data)), fm.Size, "zip size for %s", fa.Path) + } +} + +// TestFakePagination_PageSizeOne verifies that AllFiles with page size 1 and +// two files returns both files (the fake follows the next link). If the fake +// failed to follow next links, only the first file would be returned. +func TestFakePagination_PageSizeOne(t *testing.T) { + files := map[string][]byte{ + "alpha.py": []byte("a"), + "beta.py": []byte("b"), + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withPageSize(1) + + stageUpload(t, fake, files) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + // Both files must be returned despite page size 1. + assert.Len(t, all, 2, "AllFiles must follow next links and return all files") + assert.Contains(t, all, "alpha.py") + assert.Contains(t, all, "beta.py") +} + +// TestFakePagination_SinglePage verifies that when all files fit in one page, +// AllFiles returns them all without needing to follow a next link. +func TestFakePagination_SinglePage(t *testing.T) { + files := map[string][]byte{ + "alpha.py": []byte("a"), + "beta.py": []byte("b"), + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withPageSize(10) // larger than the number of files + + stageUpload(t, fake, files) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Len(t, all, 2) +} + +// TestFakeNumFiles_DefaultsToAllFiles verifies that ApplyStage returns numFiles +// equal to the count of ALL files in the resulting version, not just the ones +// uploaded. +func TestFakeNumFiles_DefaultsToAllFiles(t *testing.T) { + priorFiles := map[string]filesapi.FileMeta{ + "a.py": {Hash: sha256Hex([]byte("a")), Size: 1}, + "b.py": {Hash: sha256Hex([]byte("b")), Size: 1}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersion("cid-1", "ver-1", priorFiles) + + // Upload 1 file over a 2-file version. + resp := stageUpload(t, fake, map[string][]byte{"c.py": []byte("c")}) + assert.Equal(t, 3, resp.NumFiles, "numFiles = all files in version (2+1=3), not uploaded count (1)") +} + +// TestFakeNumFiles_Override verifies that the numFilesOverride hook makes +// ApplyStage return a configured value instead of the real count. +func TestFakeNumFiles_Override(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withNumFilesOverride(99) + + resp := stageUpload(t, fake, map[string][]byte{"app.py": []byte("x")}) + assert.Equal(t, 99, resp.NumFiles, "numFilesOverride must override the real count") +} + +// TestFakeCallCounters verifies that per-method call counters are incremented +// correctly for stage-create, upload-to-stage, apply-stage, AllFiles, and +// DownloadFile. +func TestFakeCallCounters(t *testing.T) { + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + // No calls yet. + assert.Equal(t, 0, fake.CreateStageCalls()) + assert.Equal(t, 0, fake.UploadToStageCalls()) + assert.Equal(t, 0, fake.ApplyStageCalls()) + assert.Equal(t, 0, fake.AllFilesCalls()) + assert.Equal(t, 0, fake.UploadFromZipCalls()) + assert.Equal(t, 0, fake.DownloadFileCalls()) + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + assert.Equal(t, 1, fake.CreateStageCalls()) + + for i := 0; i < 3; i++ { + path := "file" + string(rune('a'+i)) + ".py" + err := fake.UploadToStage(fake.catalogID, stage.StageID, path, 1, strings.NewReader("x")) + require.NoError(t, err) + } + + assert.Equal(t, 3, fake.UploadToStageCalls()) + + _, err = fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.NoError(t, err) + + assert.Equal(t, 1, fake.ApplyStageCalls()) + + _, err = fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + _, err = fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Equal(t, 2, fake.AllFilesCalls()) + + var buf bytes.Buffer + + _, _, err = fake.DownloadFile(fake.catalogID, "ver-1", "filea.py", &buf) + require.NoError(t, err) + assert.Equal(t, "x", buf.String(), "the downloaded bytes must be what was staged") + + assert.Equal(t, 1, fake.DownloadFileCalls()) +} + +// TestFakeFaultInjection_DropPath verifies that the dropPathFromApply hook +// omits a path from the resulting version. Passes with the fault off, fails +// with it on (the path is missing from AllFiles). +func TestFakeFaultInjection_DropPath(t *testing.T) { + files := map[string][]byte{ + "app.py": []byte("app"), + "config.py": []byte("config"), + } + + // Without the fault: both files present. + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stageUpload(t, fake, files) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + assert.Len(t, all, 2, "without fault, both files present") + + // With the fault: app.py is dropped from the resulting version. + fake2 := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withDropPath("app.py") + + stageUpload(t, fake2, files) + + all2, err := fake2.AllFiles(fake2.catalogID, "ver-2") + require.NoError(t, err) + + assert.Len(t, all2, 1, "with fault, dropped path is missing") + assert.NotContains(t, all2, "app.py", "app.py must be dropped") + assert.Contains(t, all2, "config.py", "other files must survive") +} + +// TestFakeFaultInjection_WrongChecksum verifies that the wrongChecksum hook +// makes AllFiles return a wrong checksum for a chosen path. Passes with the +// fault off, fails with it on. +func TestFakeFaultInjection_WrongChecksum(t *testing.T) { + data := []byte("content here") + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stageUpload(t, fake, map[string][]byte{"app.py": data}) + + // Without the fault: correct checksum. + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + assert.Equal(t, sha256Hex(data), all["app.py"].Hash) + + // With the fault: wrong checksum. + wrong := strings.Repeat("z", 64) + fake2 := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withWrongChecksum("app.py", wrong) + + stageUpload(t, fake2, map[string][]byte{"app.py": data}) + + all2, err := fake2.AllFiles(fake2.catalogID, "ver-1") + require.NoError(t, err) + + assert.Equal(t, wrong, all2["app.py"].Hash, "wrong checksum must be returned") + assert.NotEqual(t, sha256Hex(data), all2["app.py"].Hash, "wrong checksum must differ from real") +} + +// TestFakeFaultInjection_FailNthUpload verifies that the failNthUpload hook +// fails the Nth UploadToStage call. Passes with the fault off, fails with it on. +func TestFakeFaultInjection_FailNthUpload(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withFailNthUpload(2) + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + // First upload succeeds. + err = fake.UploadToStage(fake.catalogID, stage.StageID, "a.py", 1, strings.NewReader("a")) + require.NoError(t, err, "first upload must succeed") + + // Second upload fails. + err = fake.UploadToStage(fake.catalogID, stage.StageID, "b.py", 1, strings.NewReader("b")) + require.Error(t, err, "second upload must fail with fault injection") + require.Contains(t, err.Error(), "injected failure") + + // Third upload succeeds (fault only fires on the Nth call). + err = fake.UploadToStage(fake.catalogID, stage.StageID, "c.py", 1, strings.NewReader("c")) + require.NoError(t, err, "third upload must succeed after the fault fired") + + assert.Equal(t, 3, fake.UploadToStageCalls()) +} + +// TestFakeFaultInjection_FailApplyStage verifies that the failApplyStage hook +// makes ApplyStage return an error after all files have been staged. +func TestFakeFaultInjection_FailApplyStage(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withFailApplyStage() + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + err = fake.UploadToStage(fake.catalogID, stage.StageID, "app.py", 3, strings.NewReader("app")) + require.NoError(t, err, "upload must succeed; fault is on ApplyStage only") + + _, err = fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.Error(t, err, "ApplyStage must fail with fault injection") + require.Contains(t, err.Error(), "injected failure") +} + +// TestFakeRaceFree_ConcurrentUploads verifies that the fake is race-free when +// the uploader runs with 4-way concurrency over 8+ files. Run with -race to +// detect data races. +func TestFakeRaceFree_ConcurrentUploads(t *testing.T) { + // 8 files, uploaded concurrently by 4 goroutines (matching UploadConcurrency). + numFiles := 8 + + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + var wg sync.WaitGroup + + for i := 0; i < numFiles; i++ { + path := "file" + string(rune('a'+i)) + ".py" + data := []byte(strings.Repeat("x", i+1)) // varying sizes including 0-byte-like + + wg.Add(1) + + go func() { + defer wg.Done() + + err := fake.UploadToStage(fake.catalogID, stage.StageID, path, int64(len(data)), bytes.NewReader(data)) + assert.NoError(t, err) + }() + } + + wg.Wait() + + resp, err := fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.NoError(t, err) + + assert.Equal(t, numFiles, resp.NumFiles) + assert.Equal(t, numFiles, fake.UploadToStageCalls()) + + // Verify all files are present with correct checksums. + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Len(t, all, numFiles, "all files must be recorded despite concurrent uploads") + + // Verify a few checksums deterministically. + paths := make([]string, 0, numFiles) + for i := 0; i < numFiles; i++ { + paths = append(paths, "file"+string(rune('a'+i))+".py") + } + + sort.Strings(paths) + + for i, p := range paths { + expected := sha256Hex([]byte(strings.Repeat("x", i+1))) + assert.Equal(t, expected, all[p].Hash, "checksum for %s", p) + } +} + +// --------------------------------------------------------------------------- +// Download modelling +// --------------------------------------------------------------------------- + +// TestFakeDownload_ServesRecordedBytes_StagePath verifies that DownloadFile +// returns byte-for-byte what the stage path recorded for a path at a +// version, so a download test can assert "the right bytes were served" +// rather than merely "no error was returned". +func TestFakeDownload_ServesRecordedBytes_StagePath(t *testing.T) { + files := map[string][]byte{ + "app.py": []byte("print('hello')\n"), + "utils/helper.py": []byte("def help(): pass\n"), + } + + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stageUpload(t, fake, files) + + for path, want := range files { + var buf bytes.Buffer + + _, n, err := fake.DownloadFile(fake.catalogID, "ver-1", path, &buf) + require.NoError(t, err, "download %s", path) + + assert.Equal(t, string(want), buf.String(), "downloaded bytes for %s", path) + assert.Equal(t, int64(len(want)), n, "downloaded byte count for %s", path) + } + + assert.Equal(t, len(files), fake.DownloadFileCalls()) +} + +// TestFakeDownload_ServesRecordedBytes_ZipPath verifies that the zip upload +// path also records content, so downloads work for zip-synced catalogs too. +func TestFakeDownload_ServesRecordedBytes_ZipPath(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "main.py": "if __name__ == '__main__': pass\n", + }) + + plan := &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalSize: 11}, + {Path: "main.py", LocalSize: 34}, + }, + } + + zipPath, _, err := buildZip(dir, plan.Uploads) + require.NoError(t, err) + + zipData, err := os.ReadFile(zipPath) + require.NoError(t, err) + + fake := &fakeFilesClient{ + catalogID: "cid-zip", + versionID: "ver-zip", + } + + _, err = fake.UploadFromZipNew("wapi-sync.zip", int64(len(zipData)), bytes.NewReader(zipData)) + require.NoError(t, err) + + for _, fa := range plan.Uploads { + want, err := os.ReadFile(filepath.Join(dir, fa.Path)) + require.NoError(t, err) + + var buf bytes.Buffer + + _, n, err := fake.DownloadFile(fake.catalogID, "ver-zip", fa.Path, &buf) + require.NoError(t, err, "download %s", fa.Path) + + assert.Equal(t, string(want), buf.String(), "zip-path downloaded bytes for %s", fa.Path) + assert.Equal(t, int64(len(want)), n, "zip-path downloaded byte count for %s", fa.Path) + } +} + +// TestFakeDownload_WithVersionContentConsistentWithAllFiles verifies the +// self-consistency the whole fake rests on: a version seeded through +// withVersionContent advertises via AllFiles exactly the checksums its +// recorded bytes hash to, and DownloadFile serves exactly those bytes. A +// fake whose advertised checksums disagree with its served bytes would make +// every checksum-verification test vacuous. +func TestFakeDownload_WithVersionContentConsistentWithAllFiles(t *testing.T) { + contents := map[string][]byte{ + "app.py": []byte("server holds this\n"), + "b.py": []byte("and this"), + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + versionID: "ver-1", + }).withVersionContent("cid-1", "ver-1", contents) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + assert.Len(t, all, len(contents)) + + for path, want := range contents { + fm, ok := all[path] + require.True(t, ok, "path %s missing from AllFiles", path) + + assert.Equal(t, sha256Hex(want), fm.Hash, "advertised checksum for %s must be the bytes' hash", path) + assert.Equal(t, int64(len(want)), fm.Size, "advertised size for %s", path) + + var buf bytes.Buffer + + _, _, err := fake.DownloadFile(fake.catalogID, "ver-1", path, &buf) + require.NoError(t, err, "download %s", path) + + assert.Equal(t, string(want), buf.String(), "served bytes for %s must match the advertised content", path) + } +} + +// TestFakeDownload_UnknownPathOrVersionErrors pins the faithful-404 +// behaviour: an unknown version, an unknown path in a known version, and a +// hash-only withVersion seed (metadata without recorded content) all error +// rather than serving empty bytes that would hash like the empty string. +func TestFakeDownload_UnknownPathOrVersionErrors(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + versionID: "ver-1", + }).withVersionContent("cid-1", "ver-1", map[string][]byte{"a.py": []byte("a")}) + + t.Run("unknown version", func(t *testing.T) { + var buf bytes.Buffer + + _, _, err := fake.DownloadFile(fake.catalogID, "ver-missing", "a.py", &buf) + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") + }) + + t.Run("unknown path in known version", func(t *testing.T) { + var buf bytes.Buffer + + _, _, err := fake.DownloadFile(fake.catalogID, "ver-1", "missing.py", &buf) + require.Error(t, err) + assert.Contains(t, err.Error(), "not in version") + }) + + t.Run("hash-only seed has no recorded content", func(t *testing.T) { + hashOnly := (&fakeFilesClient{ + catalogID: "cid-2", + versionID: "ver-2", + }).withVersion("cid-2", "ver-2", map[string]filesapi.FileMeta{ + "a.py": {Hash: sha256Hex([]byte("a")), Size: 1}, + }) + + var buf bytes.Buffer + + _, _, err := hashOnly.DownloadFile("cid-2", "ver-2", "a.py", &buf) + require.Error(t, err, "a version seeded with hashes only has no bytes to serve") + assert.Contains(t, err.Error(), "no recorded content") + }) +} + +// TestFakeDownload_ContentCarriedAcrossStageMerge verifies that applying a +// stage over an existing version carries the prior version's recorded +// content into the new version, so files the sync did not touch remain +// downloadable — matching the real API's REPLACE-merge semantics. +func TestFakeDownload_ContentCarriedAcrossStageMerge(t *testing.T) { + prior := map[string][]byte{ + "a.py": []byte("alpha"), + "b.py": []byte("beta"), + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersionContent("cid-1", "ver-1", prior) + + stageUpload(t, fake, map[string][]byte{"c.py": []byte("gamma")}) + + all, err := fake.AllFiles(fake.catalogID, "ver-2") + require.NoError(t, err) + assert.Len(t, all, 3, "merge must keep prior paths and add the new one") + + for path, want := range map[string][]byte{ + "a.py": prior["a.py"], + "b.py": prior["b.py"], + "c.py": []byte("gamma"), + } { + var buf bytes.Buffer + + _, _, err := fake.DownloadFile(fake.catalogID, "ver-2", path, &buf) + require.NoError(t, err, "download %s from merged version", path) + + assert.Equal(t, string(want), buf.String(), "carried content for %s", path) + } +} + +// TestFakeDownload_ContentCarriedAcrossDelete verifies that a remote delete +// produces a new version whose surviving paths stay downloadable and whose +// deleted path is gone. +func TestFakeDownload_ContentCarriedAcrossDelete(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + versionID: "ver-1", + }).withVersionContent("cid-1", "ver-1", map[string][]byte{ + "a.py": []byte("alpha"), + "b.py": []byte("beta"), + }) + + resp, err := fake.DeleteFiles("cid-1", []string{"a.py"}) + require.NoError(t, err) + + newVer := resp.CatalogVersionID + require.NotEmpty(t, newVer) + + var buf bytes.Buffer + + _, _, err = fake.DownloadFile("cid-1", newVer, "b.py", &buf) + require.NoError(t, err, "surviving path must stay downloadable after a delete") + assert.Equal(t, "beta", buf.String()) + + _, _, err = fake.DownloadFile("cid-1", newVer, "a.py", &buf) + require.Error(t, err, "deleted path must no longer be downloadable") +} + +// TestFakeFaultInjection_FailDownload verifies the fail-download hook: +// without it the same call succeeds (control), with it DownloadFile errors +// and the call is still counted. +func TestFakeFaultInjection_FailDownload(t *testing.T) { + seed := func() *fakeFilesClient { + return (&fakeFilesClient{ + catalogID: "cid-1", + versionID: "ver-1", + }).withVersionContent("cid-1", "ver-1", map[string][]byte{ + "app.py": []byte("content"), + }) + } + + // Control: no fault, download succeeds. + plain := seed() + + var buf bytes.Buffer + + _, _, err := plain.DownloadFile("cid-1", "ver-1", "app.py", &buf) + require.NoError(t, err, "without the fault the download must succeed") + assert.Equal(t, 1, plain.DownloadFileCalls()) + + // With the fault: the download fails and still counts as a call. + faulty := seed().withFailDownload("app.py") + + _, _, err = faulty.DownloadFile("cid-1", "ver-1", "app.py", &buf) + require.Error(t, err, "the faulted download must fail") + assert.Contains(t, err.Error(), "injected failure") + assert.Equal(t, 1, faulty.DownloadFileCalls(), "a faulted call still counts") +} + +// TestFakeFaultInjection_CorruptDownload verifies the corrupt-download +// hook: the served bytes keep their length (so a size check passes) but +// hash to something other than the advertised checksum, which is what lets +// a test target the client-side checksum verification specifically. The +// control pins that the unhooked fake serves bytes matching the advertised +// checksum. +func TestFakeFaultInjection_CorruptDownload(t *testing.T) { + content := []byte("genuine bytes") + + seed := func() *fakeFilesClient { + return (&fakeFilesClient{ + catalogID: "cid-1", + versionID: "ver-1", + }).withVersionContent("cid-1", "ver-1", map[string][]byte{ + "app.py": content, + }) + } + + // Control: unhooked fake serves bytes whose hash matches AllFiles'. + plain := seed() + + var plainBuf bytes.Buffer + + _, _, err := plain.DownloadFile("cid-1", "ver-1", "app.py", &plainBuf) + require.NoError(t, err) + assert.Equal(t, sha256Hex(content), sha256Hex(plainBuf.Bytes()), + "without the fault the served bytes must match the advertised checksum") + + // With the fault: same length, different bytes, different hash. + faulty := seed().withCorruptDownload("app.py") + + var corruptBuf bytes.Buffer + + _, n, err := faulty.DownloadFile("cid-1", "ver-1", "app.py", &corruptBuf) + require.NoError(t, err, "the corruption hook alters bytes, not the outcome") + + all, err := faulty.AllFiles("cid-1", "ver-1") + require.NoError(t, err) + + assert.Equal(t, int64(len(content)), n, "corrupt bytes must keep the advertised size") + assert.Equal(t, len(content), corruptBuf.Len(), "corrupt bytes must be the same length") + assert.NotEqual(t, content, corruptBuf.Bytes(), "corrupt bytes must differ from the recorded content") + assert.NotEqual(t, all["app.py"].Hash, sha256Hex(corruptBuf.Bytes()), + "corrupt bytes must hash to something other than the advertised checksum") +} + +// TestFakeRaceFree_ConcurrentDownloads verifies the download path is +// race-free under the same concurrency the real downloader uses +// (DownloadConcurrency = 4 workers over more files). Run with -race. +func TestFakeRaceFree_ConcurrentDownloads(t *testing.T) { + const numFiles = 8 + + contents := make(map[string][]byte, numFiles) + + for i := 0; i < numFiles; i++ { + contents["file"+string(rune('a'+i))+".py"] = []byte(strings.Repeat("x", i+1)) + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + versionID: "ver-1", + }).withVersionContent("cid-1", "ver-1", contents) + + paths := make([]string, 0, numFiles) + for p := range contents { + paths = append(paths, p) + } + + sort.Strings(paths) + + var wg sync.WaitGroup + + for _, path := range paths { + want := contents[path] + + wg.Add(1) + + go func() { + defer wg.Done() + + var buf bytes.Buffer + + _, _, err := fake.DownloadFile("cid-1", "ver-1", path, &buf) + assert.NoError(t, err) + assert.Equal(t, string(want), buf.String(), "concurrent download bytes for %s", path) + }() + } + + wg.Wait() + + assert.Equal(t, numFiles, fake.DownloadFileCalls()) +} diff --git a/internal/workload/sync/fake_files_client_test.go b/internal/workload/sync/fake_files_client_test.go new file mode 100644 index 000000000..fa2f1934a --- /dev/null +++ b/internal/workload/sync/fake_files_client_test.go @@ -0,0 +1,984 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + stdsync "sync" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload/fileops" +) + +// fakeFilesClient is a self-consistent in-memory model of the Files API +// server. Unlike a shallow stub, it records per-path content and its SHA-256 +// so tests can assert "what the server ended up holding" after an upload. +// +// Downloads are served from the same model: DownloadFile returns exactly the +// bytes recorded for the requested path at the requested version, so a test +// can assert the right bytes were pulled — not merely that no error was +// returned. Versions carry content only when it was actually recorded (via +// uploads or the withVersionContent builder); hash-only seeds honestly +// report nothing to download. +// +// Server state is modeled as versions: each ApplyStage (or zip upload) creates +// a new version whose files are the REPLACE-merge of the staged paths into the +// prior version. AllFiles serves exactly what was recorded, keyed by version, +// with configurable pagination via a next-link simulation. ApplyStage returns +// a numFiles that defaults to the count of ALL files in the resulting version +// (matching the real API semantics), not just the ones uploaded. +// +// Per-method call counters let tests assert that specific network calls were +// or were not issued. Fault-injection hooks are all opt-in: a test that does +// not ask for a fault gets a faithful server. +// +// Every shared map is guarded by a mutex so -race is clean when the uploader +// runs concurrently (UploadConcurrency = 4). +type fakeFilesClient struct { + mu stdsync.Mutex + + // Configured IDs returned by the fake server. CreateCatalog returns + // catalogID, CreateStage returns stageID, ApplyStage/zip uploads return + // versionID. + catalogID string + stageID string + versionID string + + // Server state: versionID → (path → FileMeta with SHA-256 hash + size). + versions map[string]map[string]filesapi.FileMeta + + // Per-catalog latest version ID, for REPLACE-merge semantics. + latestVersion map[string]string + + // Current staging area: path → content bytes (stage path only). + stagedFiles map[string][]byte + + // Content store: versionID → (path → the exact bytes recorded for that + // path in that version). DownloadFile serves from this map so a download + // returns byte-for-byte what an upload (or a withVersionContent seed) + // recorded, and AllFiles' checksums stay consistent with those bytes. + // Versions seeded through the hash-only withVersion builder carry no + // content, and DownloadFile reports them as not found rather than + // inventing bytes the server never held. + versionContents map[string]map[string][]byte + + // Backward-compatible fields: recorded uploaded content and deleted + // paths, kept so existing tests that inspect them still work. + uploadedFiles map[string][]byte + deletedPaths []string + + // Call counters (guarded by mu). + createCatalogCalls int + createStageCalls int + uploadToStageCalls int + applyStageCalls int + uploadFromZipCalls int + allFilesCalls int + downloadFileCalls int + + // Configurable AllFiles page size. 0 = return all at once (no pagination + // simulation). When > 0, AllFiles internally splits files into pages and + // follows next links, mirroring the real client's pagination behavior. + allFilesPageSize int + + // Fault-injection hooks (all opt-in; zero values are no-ops). + dropPathFromApply string // omit this path from the resulting version after apply + wrongChecksumPath string // return wrong checksum for this path in AllFiles + wrongChecksumValue string // the wrong checksum to substitute + failNthUpload int // fail the Nth UploadToStage call (1-indexed; 0 = disabled) + failApplyStage bool // ApplyStage returns error after staging succeeded + numFilesOverride *int // if non-nil, ApplyStage returns this numFiles; otherwise real count + + // Version-scoped checksum fault: served only when AllFiles is called + // for this version (see withWrongChecksumForVersion). + wrongChecksumForVersion *wrongChecksumFault + + // Download faults, keyed by path. failDownloadPath makes DownloadFile + // return an error for that path; corruptDownloadPath makes it serve + // bytes whose SHA-256 differs from the advertised checksum (same + // length, so the size check passes and the checksum check is what + // catches the corruption). + failDownloadPath string + corruptDownloadPath string +} + +// --- Builder methods for test setup (chainable, mutex-safe) --- + +// withVersion pre-populates a version in the fake's server state. Useful for +// testing REPLACE-merge semantics without going through the upload flow. +// +// The seed map is copied, not aliased: a caller that reuses or mutates its +// map after handing it over must not silently rewrite server state mid-test. +// (The fake used to store the caller's map directly, so a later mutation +// changed an already-seeded version behind the test's back. +// TestWithVersionCopiesTheSeed pins the copy.) +func (f *fakeFilesClient) withVersion(catalogID, versionID string, files map[string]filesapi.FileMeta) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + copied := make(map[string]filesapi.FileMeta, len(files)) + + for path, fm := range files { + copied[path] = fm + } + + f.versions[versionID] = copied + f.latestVersion[catalogID] = versionID + + // This builder seeds metadata only; drop any content a previous seed + // may have left under the same version ID so the store never disagrees + // with the advertised checksums. + delete(f.versionContents, versionID) + + return f +} + +// withVersionContent pre-populates a version from raw bytes, computing each +// path's SHA-256 hash and size the way the real server does. This is the +// builder for download tests: DownloadFile later serves exactly these bytes +// for this version, and AllFiles advertises the checksums those bytes hash +// to. Use withVersion instead when only metadata is needed. +func (f *fakeFilesClient) withVersionContent(catalogID, versionID string, contents map[string][]byte) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.versionContents == nil { + f.versionContents = make(map[string]map[string][]byte) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + metas := make(map[string]filesapi.FileMeta, len(contents)) + + for path, content := range contents { + metas[path] = fileMetaOf(content) + } + + f.versions[versionID] = metas + f.versionContents[versionID] = contents + f.latestVersion[catalogID] = versionID + + return f +} + +// withPageSize sets the AllFiles page size for pagination simulation. +func (f *fakeFilesClient) withPageSize(n int) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.allFilesPageSize = n + + return f +} + +// withDropPath configures the fake to omit the given path from the resulting +// version after ApplyStage, simulating a server-side dropped upload. +func (f *fakeFilesClient) withDropPath(path string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.dropPathFromApply = path + + return f +} + +// withWrongChecksum configures AllFiles to return the given checksum for the +// given path instead of the real SHA-256, simulating a server-side checksum +// mismatch. +func (f *fakeFilesClient) withWrongChecksum(path, checksum string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.wrongChecksumPath = path + f.wrongChecksumValue = checksum + + return f +} + +// withWrongChecksumForVersion is the version-scoped form of withWrongChecksum: +// the substitute checksum is served only when AllFiles is called for the given +// version. The engine's post-apply verification reads the NEW version, which +// Phase 2 never fetches, so a fault scoped to it models "the post-apply +// listing disagrees" without corrupting the Phase-2 fetch or the plan. +func (f *fakeFilesClient) withWrongChecksumForVersion(versionID, path, checksum string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.wrongChecksumForVersion = &wrongChecksumFault{ + versionID: versionID, + path: path, + checksum: checksum, + } + + return f +} + +// withFailDownload configures DownloadFile to return an error for the given +// path, simulating a download that fails server-side or mid-transfer. +func (f *fakeFilesClient) withFailDownload(path string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.failDownloadPath = path + + return f +} + +// withCorruptDownload configures DownloadFile to serve, for the given path, +// bytes of the same length as the recorded content but with every byte +// flipped. The served bytes therefore hash to something other than the +// checksum AllFiles advertises, which is what lets a test exercise the +// client-side post-download checksum verification rather than the size +// check. The hooked path must have non-empty recorded content: flipping an +// empty byte string yields empty again, and the fake refuses to serve a +// "corruption" indistinguishable from the real bytes. +func (f *fakeFilesClient) withCorruptDownload(path string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.corruptDownloadPath = path + + return f +} + +// withFailNthUpload configures the fake to fail the Nth UploadToStage call +// (1-indexed). 0 disables the hook. +func (f *fakeFilesClient) withFailNthUpload(n int) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.failNthUpload = n + + return f +} + +// withFailApplyStage configures ApplyStage to return an error after all files +// have been successfully staged, simulating a server-side apply failure. +func (f *fakeFilesClient) withFailApplyStage() *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.failApplyStage = true + + return f +} + +// withNumFilesOverride configures ApplyStage to return the given numFiles +// instead of the real count of all files in the resulting version. +func (f *fakeFilesClient) withNumFilesOverride(n int) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.numFilesOverride = &n + + return f +} + +// --- Counter accessors (mutex-safe for reading after concurrent work) --- + +func (f *fakeFilesClient) CreateCatalogCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.createCatalogCalls +} + +func (f *fakeFilesClient) CreateStageCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.createStageCalls +} + +func (f *fakeFilesClient) UploadToStageCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.uploadToStageCalls +} + +func (f *fakeFilesClient) ApplyStageCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.applyStageCalls +} + +func (f *fakeFilesClient) UploadFromZipCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.uploadFromZipCalls +} + +func (f *fakeFilesClient) AllFilesCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.allFilesCalls +} + +func (f *fakeFilesClient) DownloadFileCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.downloadFileCalls +} + +// --- filesapi.Client implementation --- + +func (f *fakeFilesClient) CreateCatalog() (*filesapi.CatalogResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.createCatalogCalls++ + + if f.catalogID == "" { + return nil, errors.New("fakeFilesClient.CreateCatalog: no catalogID configured") + } + + return &filesapi.CatalogResp{CatalogID: f.catalogID}, nil +} + +func (f *fakeFilesClient) CreateStage(_ string) (*filesapi.StageResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.createStageCalls++ + + if f.stageID == "" { + return nil, errors.New("fakeFilesClient.CreateStage: no stageID configured") + } + + // Initialize a fresh staging area for this stage. The fake models ONE + // active stage: the staging area is shared and wiped here, so anything + // staged but not yet applied is discarded when a test calls CreateStage + // again mid-flow. This matches production usage — exactly one + // CreateStage per sync, then the uploads, then ApplyStage — so a test + // that interleaves two stages exercises a flow the real server never + // sees. + f.stagedFiles = make(map[string][]byte) + + return &filesapi.StageResp{CatalogID: f.catalogID, StageID: f.stageID}, nil +} + +func (f *fakeFilesClient) UploadToStage(_, _, name string, _ int64, body io.Reader) error { + // Read the body outside the mutex: each call has its own reader and + // holding the mutex during I/O would serialize uploads unnecessarily. + data, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("read upload body: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploadToStageCalls++ + + // Fault injection: fail the Nth upload. + if f.failNthUpload > 0 && f.uploadToStageCalls == f.failNthUpload { + return fmt.Errorf("fakeFilesClient.UploadToStage: injected failure on upload %d", f.uploadToStageCalls) + } + + if f.stagedFiles == nil { + f.stagedFiles = make(map[string][]byte) + } + + f.stagedFiles[name] = data + + // Backward-compatible: record uploaded content for tests that inspect it. + if f.uploadedFiles == nil { + f.uploadedFiles = make(map[string][]byte) + } + + f.uploadedFiles[name] = data + + return nil +} + +// mergeStagedFiles merges staged files into a copy of the latest version's +// files, matching the real API's stage REPLACE semantics: staged paths +// replace existing entries, unmentioned paths from the prior version remain. +func (f *fakeFilesClient) mergeStagedFiles(catalogID string) map[string]filesapi.FileMeta { + newVersion := make(map[string]filesapi.FileMeta) + + if latest, ok := f.latestVersion[catalogID]; ok { + if files, ok := f.versions[latest]; ok { + for k, v := range files { + newVersion[k] = v + } + } + } + + for path, data := range f.stagedFiles { + h := sha256.Sum256(data) + newVersion[path] = filesapi.FileMeta{ + Hash: hex.EncodeToString(h[:]), + Size: int64(len(data)), + } + } + + return newVersion +} + +// mergeStagedContents is mergeStagedFiles' content-side twin: it starts from +// the latest version's recorded bytes and overlays the staged files, so the +// resulting version's content store matches the metadata merge exactly. +// Prior paths with no recorded content (hash-only withVersion seeds) carry +// no entry, keeping "no content recorded" truthful through the merge. +func (f *fakeFilesClient) mergeStagedContents(catalogID string) map[string][]byte { + newContents := make(map[string][]byte) + + if latest, ok := f.latestVersion[catalogID]; ok { + if contents, ok := f.versionContents[latest]; ok { + for k, v := range contents { + newContents[k] = v + } + } + } + + for path, data := range f.stagedFiles { + newContents[path] = data + } + + return newContents +} + +func (f *fakeFilesClient) ApplyStage(catalogID, _, _ string) (*filesapi.ApplyStageResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.applyStageCalls++ + + // Fault injection: fail ApplyStage after staging succeeded. + if f.failApplyStage { + return nil, errors.New("fakeFilesClient.ApplyStage: injected failure") + } + + if f.versionID == "" { + return nil, errors.New("fakeFilesClient.ApplyStage: no versionID configured") + } + + newVersion := f.mergeStagedFiles(catalogID) + + newContents := f.mergeStagedContents(catalogID) + + // Fault injection: drop a path from the resulting version. + if f.dropPathFromApply != "" { + delete(newVersion, f.dropPathFromApply) + delete(newContents, f.dropPathFromApply) + } + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.versionContents == nil { + f.versionContents = make(map[string]map[string][]byte) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[f.versionID] = newVersion + f.versionContents[f.versionID] = newContents + f.latestVersion[catalogID] = f.versionID + + // Clear the staging area for the next stage. + f.stagedFiles = make(map[string][]byte) + + // numFiles defaults to the count of ALL files in the resulting version, + // not just the ones uploaded. This matches the real API semantics. + numFiles := len(newVersion) + if f.numFilesOverride != nil { + numFiles = *f.numFilesOverride + } + + return &filesapi.ApplyStageResp{ + CatalogID: f.catalogID, + CatalogVersionID: f.versionID, + NumFiles: numFiles, + }, nil +} + +func (f *fakeFilesClient) UploadFromZipNew(_ string, _ int64, body io.Reader) (*filesapi.FromFileResp, error) { + data, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read zip body: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploadFromZipCalls++ + + if f.catalogID == "" { + return nil, errors.New("fakeFilesClient.UploadFromZipNew: no catalogID configured") + } + + zipContents, err := extractZipEntries(data) + if err != nil { + return nil, err + } + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.versionContents == nil { + f.versionContents = make(map[string]map[string][]byte) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[f.versionID] = zipFileMetas(zipContents) + f.versionContents[f.versionID] = zipContents + f.latestVersion[f.catalogID] = f.versionID + + // Inline completion (no StatusID), matching the real API for small archives. + return &filesapi.FromFileResp{ + CatalogID: f.catalogID, + CatalogVersionID: f.versionID, + }, nil +} + +func (f *fakeFilesClient) UploadFromZipExisting(catalogID, _, _ string, _ int64, body io.Reader) (*filesapi.FromFileResp, error) { + data, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read zip body: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploadFromZipCalls++ + + zipContents, err := extractZipEntries(data) + if err != nil { + return nil, err + } + + newVersion, newContents := f.mergeZipIntoExisting(catalogID, zipContents) + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.versionContents == nil { + f.versionContents = make(map[string]map[string][]byte) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[f.versionID] = newVersion + f.versionContents[f.versionID] = newContents + f.latestVersion[catalogID] = f.versionID + + return &filesapi.FromFileResp{ + CatalogID: catalogID, + CatalogVersionID: f.versionID, + }, nil +} + +// zipFileMetas derives the server's metadata map from extracted zip content, +// the only checksum derivation for the zip path. +func zipFileMetas(zipContents map[string][]byte) map[string]filesapi.FileMeta { + metas := make(map[string]filesapi.FileMeta, len(zipContents)) + + for path, content := range zipContents { + metas[path] = fileMetaOf(content) + } + + return metas +} + +// mergeZipIntoExisting builds the resulting version for an incremental zip +// upload: the latest version's metadata and recorded content with the zip's +// entries overlaid, matching the real API's REPLACE merge. Both maps stay in +// lockstep so AllFiles' checksums always describe the bytes DownloadFile +// serves. +func (f *fakeFilesClient) mergeZipIntoExisting(catalogID string, zipContents map[string][]byte) (map[string]filesapi.FileMeta, map[string][]byte) { + newVersion := make(map[string]filesapi.FileMeta) + + newContents := make(map[string][]byte) + + if latest, ok := f.latestVersion[catalogID]; ok { + if files, ok := f.versions[latest]; ok { + for k, v := range files { + newVersion[k] = v + } + } + + if contents, ok := f.versionContents[latest]; ok { + for k, v := range contents { + newContents[k] = v + } + } + } + + for k, v := range zipFileMetas(zipContents) { + newVersion[k] = v + } + + for k, v := range zipContents { + newContents[k] = v + } + + return newVersion, newContents +} + +func (f *fakeFilesClient) PollStatus(_ string) (*filesapi.StatusResp, error) { + return &filesapi.StatusResp{Status: filesapi.StatusCompleted}, nil +} + +// applyWrongChecksum substitutes the configured wrong checksum for the +// chosen path in the result map, simulating a server-side checksum mismatch. +func (f *fakeFilesClient) applyWrongChecksum(result map[string]filesapi.FileMeta) { + if f.wrongChecksumPath == "" { + return + } + + if fm, exists := result[f.wrongChecksumPath]; exists { + fm.Hash = f.wrongChecksumValue + result[f.wrongChecksumPath] = fm + } +} + +// wrongChecksumFault is the version-scoped checksum fault record (see +// withWrongChecksumForVersion). +type wrongChecksumFault struct { + versionID string + path string + checksum string +} + +// applyWrongChecksumForVersion applies the version-scoped checksum fault when +// AllFiles was called for the fault's version, leaving every other version's +// listing truthful. +func (f *fakeFilesClient) applyWrongChecksumForVersion(versionID string, result map[string]filesapi.FileMeta) { + flt := f.wrongChecksumForVersion + if flt == nil || flt.versionID != versionID { + return + } + + if fm, exists := result[flt.path]; exists { + fm.Hash = flt.checksum + result[flt.path] = fm + } +} + +// paginateFiles simulates the server's paginated response and the client's +// next-link following. With a page size of 1 and N files, the fake processes +// N pages; if it failed to follow next links, only the first file would be +// returned. A no-op when pageSize is 0 or when all files fit in one page. +func paginateFiles(files map[string]filesapi.FileMeta, pageSize int) map[string]filesapi.FileMeta { + if pageSize <= 0 || len(files) <= pageSize { + return files + } + + paths := make([]string, 0, len(files)) + for k := range files { + paths = append(paths, k) + } + + sort.Strings(paths) + + collected := make(map[string]filesapi.FileMeta, len(paths)) + + // Process page by page, following next links. + idx := 0 + for idx < len(paths) { + end := idx + pageSize + if end > len(paths) { + end = len(paths) + } + + for _, p := range paths[idx:end] { + collected[p] = files[p] + } + + // Advance to the next page (follow the next link). + idx = end + } + + return collected +} + +func (f *fakeFilesClient) AllFiles(_, versionID string) (map[string]filesapi.FileMeta, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.allFilesCalls++ + + // Get the files for the requested version from server state. + files, ok := f.versions[versionID] + if !ok { + return make(map[string]filesapi.FileMeta), nil + } + + // Copy so the caller cannot mutate our state. + result := make(map[string]filesapi.FileMeta, len(files)) + for k, v := range files { + result[k] = v + } + + // Fault injection: return a wrong checksum for a chosen path. + f.applyWrongChecksum(result) + f.applyWrongChecksumForVersion(versionID, result) + + // Simulate pagination: split into pages and follow next links, mirroring + // the real client. A bug here (not following next links) would cause + // AllFiles to return only the first page, which the pagination test catches. + result = paginateFiles(result, f.allFilesPageSize) + + return result, nil +} + +// DownloadFile serves the exact bytes recorded for path at versionID, +// streaming them to w the way the real client streams the response body. +// The string return is unused by the sync engine (the production client +// returns "" for inline downloads); n is the number of bytes written. +// +// The fault hooks apply before the content lookup, so a faulted path fails +// whether or not the version has recorded content, and every call is +// counted whether it succeeds or fails. +func (f *fakeFilesClient) DownloadFile(_, versionID, path string, w io.Writer) (string, int64, error) { + data, err := f.downloadPayload(versionID, path) + if err != nil { + return "", 0, err + } + + n, err := w.Write(data) + if err != nil { + return "", int64(n), fmt.Errorf("fakeFilesClient.DownloadFile: write %s: %w", path, err) + } + + return "", int64(n), nil +} + +// downloadPayload gathers the bytes to serve for one download call. It takes +// the mutex so the counter, fault decision, and content snapshot are atomic +// with respect to concurrent uploads creating new versions, and returns a +// copy so the caller's write happens without holding the lock (mirroring the +// read-body-outside-the-mutex convention in UploadToStage). +func (f *fakeFilesClient) downloadPayload(versionID, path string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.downloadFileCalls++ + + if path == f.failDownloadPath { + return nil, fmt.Errorf("fakeFilesClient.DownloadFile: injected failure for %s", path) + } + + contents, ok := f.versionContents[versionID] + if !ok { + // A version can exist with metadata only (hash-only withVersion + // seed); report that honestly instead of serving empty bytes, + // which would hash like the empty string and silently pass + // checksum assertions. + if _, versionExists := f.versions[versionID]; versionExists { + return nil, fmt.Errorf( + "fakeFilesClient.DownloadFile: version %s has no recorded content (seeded via withVersion; use withVersionContent to make it downloadable)", versionID) + } + + return nil, fmt.Errorf("fakeFilesClient.DownloadFile: version %s not found", versionID) + } + + data, ok := contents[path] + if !ok { + return nil, fmt.Errorf("fakeFilesClient.DownloadFile: path %s not in version %s", path, versionID) + } + + if path == f.corruptDownloadPath { + data = corruptBytes(data) + } + + // Copy under the lock: the recorded content could otherwise be replaced + // by a concurrent version write between the snapshot and the write. + snapshot := make([]byte, len(data)) + copy(snapshot, data) + + return snapshot, nil +} + +// corruptBytes returns a same-length copy of data with every byte flipped, +// guaranteeing a different SHA-256 while keeping the advertised size intact. +// Empty input is its own corruption (flipping nothing changes nothing), so +// corrupt-download tests must hook a path with non-empty recorded content. +func corruptBytes(data []byte) []byte { + out := make([]byte, len(data)) + + for i, b := range data { + out[i] = b ^ 0xFF + } + + return out +} + +func (f *fakeFilesClient) DeleteFiles(catalogID string, paths []string) (*filesapi.DeleteFilesResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.deletedPaths = append(f.deletedPaths, paths...) + + if len(paths) == 0 { + return &filesapi.DeleteFilesResp{}, nil + } + + // Create a new version with the deleted paths removed from the latest version. + latest, ok := f.latestVersion[catalogID] + if !ok { + return &filesapi.DeleteFilesResp{}, nil + } + + newVersion, newContents := f.versionAfterDeletes(latest, paths) + + newVerID := fmt.Sprintf("%s-del-%d", f.versionID, len(f.versions)) + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.versionContents == nil { + f.versionContents = make(map[string]map[string][]byte) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[newVerID] = newVersion + f.versionContents[newVerID] = newContents + f.latestVersion[catalogID] = newVerID + + return &filesapi.DeleteFilesResp{ + CatalogID: catalogID, + CatalogVersionID: newVerID, + NumFiles: len(newVersion), + }, nil +} + +// versionAfterDeletes builds the post-delete version's metadata and recorded +// content: the latest version's state minus the deleted paths, so surviving +// files keep their checksums and stay downloadable. +func (f *fakeFilesClient) versionAfterDeletes(latest string, paths []string) (map[string]filesapi.FileMeta, map[string][]byte) { + newVersion := make(map[string]filesapi.FileMeta) + + if files, ok := f.versions[latest]; ok { + for k, v := range files { + newVersion[k] = v + } + } + + newContents := make(map[string][]byte) + + if contents, ok := f.versionContents[latest]; ok { + for k, v := range contents { + newContents[k] = v + } + } + + for _, p := range paths { + delete(newVersion, p) + delete(newContents, p) + } + + return newVersion, newContents +} + +func (f *fakeFilesClient) ListVersions(_ string, _ int) ([]filesapi.CatalogVersion, error) { + return nil, errors.New("fakeFilesClient: ListVersions not expected") +} + +// extractZipEntries reads a zip archive from raw bytes and returns a map of +// path → the entry's uncompressed content. This models what the server does +// when it extracts an uploaded archive: the extracted bytes are what the +// server holds, so both the recorded checksums and later downloads must +// derive from exactly these bytes. +func extractZipEntries(data []byte) (map[string][]byte, error) { + zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("open zip: %w", err) + } + + files := make(map[string][]byte) + + for _, zf := range zipReader.File { + content, err := readZipEntry(zf) + if err != nil { + return nil, err + } + + files[fileops.NormalizePath(zf.Name)] = content + } + + return files, nil +} + +// fileMetaOf derives the server's FileMeta (SHA-256 hex + size) from file +// content, the single derivation every version-recording path goes through. +func fileMetaOf(content []byte) filesapi.FileMeta { + h := sha256.Sum256(content) + + return filesapi.FileMeta{ + Hash: hex.EncodeToString(h[:]), + Size: int64(len(content)), + } +} + +// readZipEntry reads and closes a single zip file entry, returning its +// uncompressed content. +func readZipEntry(zf *zip.File) ([]byte, error) { + rc, err := zf.Open() + if err != nil { + return nil, fmt.Errorf("open zip entry %s: %w", zf.Name, err) + } + + defer func() { _ = rc.Close() }() + + content, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("read zip entry %s: %w", zf.Name, err) + } + + return content, nil +} diff --git a/internal/workload/sync/hashstream.go b/internal/workload/sync/hashstream.go new file mode 100644 index 000000000..5c5acda26 --- /dev/null +++ b/internal/workload/sync/hashstream.go @@ -0,0 +1,44 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "crypto/sha256" + "encoding/hex" + "hash" +) + +// newStreamHasher returns a SHA-256 hasher for computing the hash of bytes +// as they are streamed to their destination, without a separate read pass. +// Both uploaders use this so the hash recorded in the manifest is of the +// bytes that actually crossed the wire, not the bytes hashed during planning. +// A file rewritten between plan and upload must leave BASE describing what +// the server received, and only a hash of the streamed bytes can guarantee +// that. +func newStreamHasher() hash.Hash { + return sha256.New() +} + +// streamedEntry finalizes a hasher that observed the streamed bytes into a +// FileEntry carrying the hex-encoded SHA-256 and the byte count. The size +// comes from the caller — f.Stat on the already-open handle for the stage +// path, io.Copy's return for the zip path — so it always describes the same +// bytes the hasher observed, never the Phase-2 planned size. +func streamedEntry(h hash.Hash, size int64) FileEntry { + return FileEntry{ + Hash: hex.EncodeToString(h.Sum(nil)), + Size: size, + } +} diff --git a/internal/workload/sync/interruption_test.go b/internal/workload/sync/interruption_test.go new file mode 100644 index 000000000..c11aa978a --- /dev/null +++ b/internal/workload/sync/interruption_test.go @@ -0,0 +1,249 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "testing" + "time" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestInterruption_MidUpload_NonZeroAndConverges simulates a sync interrupted +// mid-upload and verifies that (1) the interrupted sync exits non-zero without +// advancing manifest.json or config.json, (2) the rollback directory remains +// for stale-rollback recovery, and (3) a following sync converges to a manifest +// matching the fake's server state. +// +// The interruption is simulated by injecting an upload failure (withFailNthUpload) +// rather than by delivering a real SIGINT. The effect is the same from the +// engine's perspective: Phase 5 fails, Phase 6 never runs, and the rollback +// directory is left on disk. The second sync's Phase 0 restores the stale +// rollback via RestoreStaleIfPresent, then proceeds normally. +// +// The fake for the second sync is pre-populated with the prior version (the +// server state from the original successful sync) so that ApplyStage's REPLACE +// merge produces a version containing both the uploaded files and the +// unchanged files (like .drignore). Without this, the fake's server would +// only hold the uploaded files and the convergence check would fail for +// unchanged paths that the manifest correctly carries forward from BASE. +// +// Go-level coverage: +// - The first sync returns an error (non-zero) without advancing manifest +// or config. +// - The rollback directory exists after the failed sync (it is NOT cleaned +// up by Restore, only by Discard on success or by the next sync's Phase 0). +// - The second sync's Phase 0 restores the stale rollback +// (e.StaleRollbackRestored() == true) and removes the rollback directory. +// - The second sync converges: the manifest's hashes match the fake's +// AllFiles server state for every file. +// +// What remains for a staging validator to confirm through the real binary: +// - Real SIGINT delivery to a running `dr` process (a Go test cannot +// deliver a signal to itself in a meaningful way; the test simulates the +// interruption via a fault-injected upload failure, which has the same +// engine-level effect: Phase 5 fails, Phase 6 never runs, the rollback +// directory remains for the next sync's stale-rollback recovery). +// - The real binary exits with a non-zero code on SIGINT (the Go test +// asserts the engine returns an error, which maps to a non-zero exit in +// the CLI, but the signal-to-exit-code path through the CLI's signal +// handler is the CLI's responsibility, not the engine's). +// +// Fulfills VAL-CROSS-011. +func TestInterruption_MidUpload_NonZeroAndConverges(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + "c.py": "CCCC\n", + } + + dir := syncedProject(t, files, catalogID, versionID) + + // Compute the prior version's server state (old content hashes) before + // modifying any files. The second sync's fake must be pre-populated with + // this so ApplyStage's REPLACE merge yields a version containing both + // the uploaded files and the unchanged ones (like .drignore). + priorVersion := make(map[string]filesapi.FileMeta, len(files)+1) + + for rel := range files { + hash, size, err := hashLocal(t, dir, rel) + require.NoError(t, err) + + priorVersion[rel] = filesapi.FileMeta{Hash: hash, Size: size} + } + + // Include .drignore (created by Initialize) so it is in the server state. + drHash, drSize, err := hashLocal(t, dir, ignore.FileName) + require.NoError(t, err) + + priorVersion[ignore.FileName] = filesapi.FileMeta{Hash: drHash, Size: drSize} + + // Now modify the files to introduce pending changes. + for rel, content := range mods { + modifyFile(t, dir, rel, content) + } + + // Capture pre-sync state. + pre := captureState(t, dir, []string{"a.py", "b.py", "c.py"}) + + // --- First sync: simulate interruption mid-upload --- + + fake1 := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(2) + + e1, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake1, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + plan1, err := e1.Plan() + require.NoError(t, err) + + require.Len(t, plan1.Uploads, 3, "all three files should be pending uploads") + + _, err = e1.Execute(plan1) + + require.Error(t, err, "interrupted sync must exit non-zero (error)") + assert.Contains(t, err.Error(), "upload", + "error must come from the upload step") + + require.NoError(t, e1.Close()) + + // manifest.json and config.json must not be advanced. + assertStateUntouched(t, dir, pre) + + // The rollback directory must exist — it is the stale rollback that + // the next sync's Phase 0 will restore. + rollDir := wapi.RollbackDir(dir) + + _, err = os.Stat(rollDir) + require.NoError(t, err, + "rollback directory must exist after a failed sync (for stale-rollback recovery)") + + // ApplyStage must not have been called — the upload failure prevented it. + assert.Equal(t, 0, fake1.ApplyStageCalls(), + "ApplyStage must not be called when the upload was interrupted") + + // --- Second sync: converge --- + + // Pre-populate the fake with the prior version so ApplyStage's REPLACE + // merge produces a version containing both uploaded and unchanged files. + fake2 := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-2", + versionID: "ver-new", + }).withVersion(catalogID, versionID, priorVersion) + + e2, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake2, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + // Phase 0 must restore the stale rollback from the first sync. + plan2, err := e2.Plan() + require.NoError(t, err) + + assert.True(t, e2.StaleRollbackRestored(), + "Phase 0 must restore the stale rollback from the interrupted sync") + + require.Len(t, plan2.Uploads, 3, + "the same pending changes must still produce uploads") + + // Execute the second sync — it must converge. + result, err := e2.Execute(plan2) + require.NoError(t, err, "the second sync must succeed and converge") + + require.NotNil(t, result) + + // The rollback directory must be gone after a successful sync. + _, err = os.Stat(rollDir) + assert.True(t, os.IsNotExist(err), + "rollback directory must be removed after a successful sync") + + // The manifest must match the fake's server state for every file. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + all, err := fake2.AllFiles(catalogID, "ver-new") + require.NoError(t, err) + + // Set-size equality closes the reverse direction: the loop below proves + // every manifest path exists on the server, but without this check a + // path the server holds and the manifest forgot would fall out of the + // record silently. Map keys are unique, so equal sizes plus the + // per-path matches below mean the two sets are exactly equal. + assert.Len(t, all, len(manifest.Files), + "manifest and server must hold exactly the same path set after convergence") + + // Every file in the manifest must have a hash matching the server. + for path, fm := range manifest.Files { + serverFM, ok := all[path] + require.True(t, ok, "server must have %s", path) + + assert.Equal(t, fm.Hash, serverFM.Hash, + "manifest hash for %s must match the server's checksum (converged)", path) + assert.Equal(t, fm.Size, serverFM.Size, + "manifest size for %s must match the server's size", path) + } + + // The manifest's syncedVersionId must equal the new version. + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, "ver-new", *manifest.SyncedVersionID, + "manifest syncedVersionId must equal the new version after convergence") + + // config.json's LastSyncedVersionID must also equal the new version. + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, "ver-new", *cfg.LastSyncedVersionID, + "config LastSyncedVersionID must equal the new version after convergence") +} diff --git a/internal/workload/sync/log_capture_test.go b/internal/workload/sync/log_capture_test.go new file mode 100644 index 000000000..62752a06e --- /dev/null +++ b/internal/workload/sync/log_capture_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "bytes" + "os" + "testing" + + "github.com/datarobot/cli/internal/log" + "github.com/stretchr/testify/require" +) + +// captureWarnLog redirects os.Stderr to a pipe and reinitializes the log +// package's stderr logger, runs fn, and returns everything the logger wrote +// during fn. This is the seam for pinning phase warnings (the .wapiignore +// shadow warning, and later the --verify divergence notice) that are emitted +// only through log.Warn and therefore never reach an engine accessor. +// +// It mirrors the captureLog helper in internal/tools — the repo's existing +// log-capture pattern — rather than inventing a mechanism. The helper cannot +// be imported from another package's _test.go file, so it is copied here +// with the same semantics: the logger is built against the redirected +// os.Stderr, writes synchronously on each call, and StopStderr in a cleanup +// restores a nil logger so no later test writes through the stale pipe. +// Nothing in the log package changes; production behaviour is untouched. +// +// Tests using this helper must not call t.Parallel: the os.Stderr swap is +// process-global while fn runs. +func captureWarnLog(t *testing.T, fn func()) string { + t.Helper() + + r, w, err := os.Pipe() + require.NoError(t, err) + + origStderr := os.Stderr + os.Stderr = w + + log.StartStderr() + + // Register the restore before fn runs: a require.* inside fn that + // FailNows unwinds via runtime.Goexit, skipping every statement after + // fn() but still running registered cleanups. If os.Stderr stayed + // swapped (and w open), the process-global stderr would point at a + // closed pipe for the rest of the package run. t.Cleanup runs even after + // Goexit, so one closure restores stderr, closes the pipe, and tears down + // the stderr logger regardless of how fn exits. + t.Cleanup(func() { + os.Stderr = origStderr + + _ = w.Close() + + log.StopStderr() + }) + + fn() + + w.Close() + + var buf bytes.Buffer + + _, err = buf.ReadFrom(r) + require.NoError(t, err) + + r.Close() + + return buf.String() +} diff --git a/internal/workload/sync/path_safety_test.go b/internal/workload/sync/path_safety_test.go index 066c5993a..ee9845948 100644 --- a/internal/workload/sync/path_safety_test.go +++ b/internal/workload/sync/path_safety_test.go @@ -43,15 +43,15 @@ var unsafeServerPaths = []string{ func TestDownloadOne_RejectsUnsafeServerPath(t *testing.T) { for _, bad := range unsafeServerPaths { t.Run(bad, func(t *testing.T) { - // fakeFilesClient.DownloadFile returns "DownloadFile not - // expected" — if SafeRelPath fails first the error message - // is the unsafe-path wrapper, proving no remote call ran. + // The fake now serves recorded content for safe paths, so the + // assertion that the error is the unsafe-path wrapper proves + // SafeRelPath fired before any remote call ran — the fake was + // never reached. e := &Engine{projectDir: t.TempDir(), files: &fakeFilesClient{}} err := downloadOne(e, "cid", "vid", FileAction{Path: bad}) require.Error(t, err) assert.Contains(t, err.Error(), "server returned unsafe download path") - assert.NotContains(t, err.Error(), "DownloadFile not expected") }) } } diff --git a/internal/workload/sync/phase2_manifests.go b/internal/workload/sync/phase2_manifests.go index 339d3d875..82e084c75 100644 --- a/internal/workload/sync/phase2_manifests.go +++ b/internal/workload/sync/phase2_manifests.go @@ -16,6 +16,7 @@ package sync import ( "fmt" + "sort" "github.com/datarobot/cli/internal/log" "github.com/datarobot/cli/internal/workload/fileops" @@ -44,10 +45,29 @@ func phase2Manifests(e *Engine) error { warnIfLockfileIgnored(e, matcher) - var skippedSymlinks []string + // The walk's symlink arm returns before the ignore check (walk.go tests + // ModeSymlink before calling ignore), so filtering must happen here rather + // than in the walker. Without it, a symlink the user deliberately .drignore'd + // or that is system-excluded (e.g. named .git) would still be announced — the + // classic unfiltered-warning trap. matcher.Match applies both the user's + // .drignore patterns and the hardcoded system excludes, with the same + // case-folding rules used for regular files. + walkOnSymlink := func(rel, _ string, isDir, dangling bool) { + matched := matcher.Match(rel, isDir) + + // A dangling link's kind is unknowable, so a directory-only pattern + // ("node_modules/") cannot match through the isDir=false branch. + // Treat it as excluded when either spelling matches so a dangling + // node_modules link is still filtered rather than warned about. + if dangling { + matched = matcher.Match(rel, false) || matcher.Match(rel, true) + } + + if matched { + return + } - walkOnSymlink := func(rel, _ string) { - skippedSymlinks = append(skippedSymlinks, rel) + e.skippedSymlinks = append(e.skippedSymlinks, SkippedSymlink{Path: rel, IsDir: isDir}) } entries, err := fileops.Walk(e.projectDir, matcher.Match, walkOnSymlink) @@ -55,6 +75,31 @@ func phase2Manifests(e *Engine) error { return fmt.Errorf("walk project directory: %w", err) } + // Sort the skipped symlinks by path so notices and the structured field + // are deterministic across runs, then emit the warning from within the + // phase via log.Warn. Like the .wapiignore shadow warning, logging here + // (rather than returning a notice for the display layer to render) means + // the user hears it even when a later phase fails before anything gets a + // chance to render. The prose is bounded at SymlinkNoticeBound entries; + // the structured field on the engine carries every symlink regardless. + sort.Slice(e.skippedSymlinks, func(i, j int) bool { + return e.skippedSymlinks[i].Path < e.skippedSymlinks[j].Path + }) + + for i, s := range e.skippedSymlinks { + if i >= SymlinkNoticeBound { + break + } + + log.Warn(skippedSymlinkNotice(s)) + } + + if len(e.skippedSymlinks) > SymlinkNoticeBound { + log.Warn(fmt.Sprintf( + "skipped symlink: and %d more symlink(s) were not uploaded or synced (see the plan JSON for the full list)", + len(e.skippedSymlinks)-SymlinkNoticeBound)) + } + local, err := hashEntries(entries) if err != nil { return err @@ -66,9 +111,21 @@ func phase2Manifests(e *Engine) error { return fmt.Errorf("%s", fileops.FormatCaseCollisions(cs)) } - if !e.drifted { + return loadRemote(e) +} + +// loadRemote decides where REMOTE comes from: copied from BASE by the +// solo-developer fast path, empty on a first sync, or fetched from the +// FilesAPI. It is split out of the phase body because the fast-path guard +// and the divergence check each carry compound conditions, and the phase +// function is at the complexity ceiling. +func loadRemote(e *Engine) error { + if !e.drifted && !e.opts.Verify { // Nobody else changed the remote since our last sync; skip the - // allFiles round-trip and reuse BASE. + // allFiles round-trip and reuse BASE. --verify opts out of this + // trust: its whole point is to check that BASE still describes the + // server, which requires actually asking the server. The bypass is + // unconditional on dry-run — a non-dry-run verify run must fetch too. e.remote = copyManifest(e.base) return nil @@ -89,6 +146,13 @@ func phase2Manifests(e *Engine) error { e.remote = FromFilesAPI(remote) + // Only a verify-forced fetch on a non-drifted artifact checks BASE's + // claim: here — and only here — BASE claims to describe exactly the + // version just fetched, so a mismatch is a lie worth reporting. On a + // drifted artifact the remote is a newer version by design, and + // BASE-vs-REMOTE differences are ordinary drift, not findings. + maybeDetectDivergence(e) + return nil } diff --git a/internal/workload/sync/phase2_symlink_test.go b/internal/workload/sync/phase2_symlink_test.go new file mode 100644 index 000000000..ae88d1138 --- /dev/null +++ b/internal/workload/sync/phase2_symlink_test.go @@ -0,0 +1,383 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// skipNonWindowsSymlink skips on Windows following the existing +// runtime.GOOS == "windows" precedent. The skip reason is visible so CI shows +// why the test did not run. +func skipNonWindowsSymlink(t *testing.T) { + t.Helper() + + if runtime.GOOS == "windows" { + t.Skip("symlink tests skipped on Windows: os.Symlink needs Developer Mode; junctions are not equivalent") + } +} + +// symlinkPaths extracts the Path field from each SkippedSymlink on the engine. +func symlinkPaths(ss []SkippedSymlink) []string { + out := make([]string, len(ss)) + for i, s := range ss { + out[i] = s.Path + } + + return out +} + +// symlinkIsDir extracts the IsDir field for a given path from the engine's +// skippedSymlinks, returning false if the path is absent. +func symlinkIsDir(ss []SkippedSymlink, path string) bool { + for _, s := range ss { + if s.Path == path { + return s.IsDir + } + } + + return false +} + +// TestPhase2_Symlink_FilteredByDrignore verifies that a symlink matching a +// .drignore pattern is neither uploaded nor reported in the skipped-symlink +// list. The walk's symlink arm returns before the ignore check, so filtering +// must happen at the collection site in phase2_manifests.go via the same +// matcher used for regular files. +// +// The directory symlink's .drignore pattern is asserted in both spellings +// gitignore accepts — trailing slash ("link_to_dir/") and bare +// ("link_to_dir"). The two reach the matcher through different branches: +// Match tries the bare path first and retries with a trailing slash only +// when isDir is set, so pinning one spelling does not prove the other at +// the collection site. +// +// Fulfills VAL-SYMLINK-002 (go test portion). +func TestPhase2_Symlink_FilteredByDrignore(t *testing.T) { + skipNonWindowsSymlink(t) + + cases := []struct { + name string + // dirPattern is the .drignore line filtering the directory symlink; + // the file symlink's bare pattern is fixed for every case. + dirPattern string + }{ + {name: "directory_pattern_trailing_slash", dirPattern: "link_to_dir/"}, + {name: "directory_pattern_bare", dirPattern: "link_to_dir"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // realfile.py is a real file; link_to_file.py is a symlink to it. + require.NoError(t, os.WriteFile(filepath.Join(dir, "realfile.py"), []byte("x"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realfile.py"), + filepath.Join(dir, "link_to_file.py"))) + + // realdir is a real directory with a child; link_to_dir is a symlink to it. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "realdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "realdir", "inner.py"), []byte("y"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realdir"), + filepath.Join(dir, "link_to_dir"))) + + // .drignore excludes the file symlink and the directory symlink + // in this case's spelling. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), + []byte("link_to_file.py\n"+tc.dirPattern+"\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + // Neither symlink is in the upload plan. + uploadPaths := uploadPathsOf(plan) + assert.NotContains(t, uploadPaths, "link_to_file.py") + assert.NotContains(t, uploadPaths, "link_to_dir") + assert.NotContains(t, uploadPaths, "link_to_dir/inner.py") + + // Neither ignored symlink is in the skipped-symlink list. + skipped := symlinkPaths(e.skippedSymlinks) + assert.NotContains(t, skipped, "link_to_file.py", + "a .drignore-matched file symlink must not be reported") + assert.NotContains(t, skipped, "link_to_dir", + "a .drignore-matched directory symlink must not be reported") + + // The real files are still in the plan. + assert.Contains(t, uploadPaths, "app.py") + assert.Contains(t, uploadPaths, "realfile.py") + assert.Contains(t, uploadPaths, "realdir/inner.py") + }) + } +} + +// TestPhase2_Symlink_SystemExcludedNotReported verifies that a symlink whose +// relative path is a system-excluded path (e.g. named .git, or living under +// .datarobot/workload) is not reported in the skipped-symlink list. System +// excludes are hardcoded in the ignore matcher and fold case, so a symlink +// named .git is excluded the same way a real .git directory would be. +// +// Fulfills VAL-SYMLINK-003. +func TestPhase2_Symlink_SystemExcludedNotReported(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // A real directory to point the symlinks at. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "realdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "realdir", "inner.py"), []byte("y"), 0o644)) + + // .git symlink -> realdir (system-excluded name) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realdir"), + filepath.Join(dir, ".git"))) + + // .datarobot/workload/ is created by wapi.Initialize as a real directory. + // A symlink placed inside it is system-excluded by prefix (.datarobot/workload/...). + require.NoError(t, os.Symlink( + filepath.Join(dir, "realdir"), + filepath.Join(dir, ".datarobot", "workload", "link_to_dir"))) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + // Neither system-excluded symlink is in the upload plan. + uploadPaths := uploadPathsOf(plan) + assert.NotContains(t, uploadPaths, ".git") + assert.NotContains(t, uploadPaths, ".datarobot/workload/link_to_dir") + + // Neither system-excluded symlink is in the skipped-symlink list. + skipped := symlinkPaths(e.skippedSymlinks) + assert.NotContains(t, skipped, ".git", + "a symlink named .git must not be reported (system-excluded)") + assert.NotContains(t, skipped, ".datarobot/workload/link_to_dir", + "a symlink under .datarobot/workload must not be reported (system-excluded)") + + // The real files are still in the plan. + assert.Contains(t, uploadPaths, "app.py") + assert.Contains(t, uploadPaths, "realdir/inner.py") +} + +// TestPhase2_Symlink_InsideDrignorePrunedDirectory verifies that a symlink +// inside a .drignore-pruned real directory is not reported and none of its +// contents are uploaded. The walk prunes the directory before descending, so +// the symlink inside it is never visited and never produces a callback. +// +// Fulfills VAL-SYMLINK-002(c). +func TestPhase2_Symlink_InsideDrignorePrunedDirectory(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // A real directory with a symlink inside it. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "ignored", "sub"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "ignored", "real.py"), []byte("x"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "ignored", "real.py"), + filepath.Join(dir, "ignored", "link.py"))) + + // .drignore prunes the entire ignored/ directory. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), + []byte("ignored/\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + // Nothing under ignored/ is in the upload plan. + uploadPaths := uploadPathsOf(plan) + assert.NotContains(t, uploadPaths, "ignored/real.py") + assert.NotContains(t, uploadPaths, "ignored/link.py") + + // The symlink inside the pruned directory is not in the skipped-symlink + // list (the walk never visited it). + skipped := symlinkPaths(e.skippedSymlinks) + assert.NotContains(t, skipped, "ignored/link.py", + "a symlink inside a .drignore-pruned directory must not be reported") + + // The real file outside the pruned directory is still in the plan. + assert.Contains(t, uploadPaths, "app.py") +} + +// TestPhase2_Symlink_ReportedWithIsDir verifies that the engine's +// skippedSymlinks carries the correct isDir for each reported symlink: true +// for a directory symlink and false for a file symlink. This is the +// phase2-level complement to the walk-level isDir tests. +// +// Fulfills VAL-SYMLINK-011(b) at the collection site. +func TestPhase2_Symlink_ReportedWithIsDir(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // realfile.py and a file symlink to it. + require.NoError(t, os.WriteFile(filepath.Join(dir, "realfile.py"), []byte("x"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realfile.py"), + filepath.Join(dir, "link_to_file.py"))) + + // realdir with a child and a directory symlink to it. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "realdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "realdir", "inner.py"), []byte("y"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realdir"), + filepath.Join(dir, "link_to_dir"))) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + // Neither symlink is in the upload plan. + uploadPaths := uploadPathsOf(plan) + assert.NotContains(t, uploadPaths, "link_to_file.py") + assert.NotContains(t, uploadPaths, "link_to_dir") + + // Both symlinks are reported with the correct isDir. + skipped := e.skippedSymlinks + assert.Contains(t, symlinkPaths(skipped), "link_to_file.py") + assert.Contains(t, symlinkPaths(skipped), "link_to_dir") + assert.False(t, symlinkIsDir(skipped, "link_to_file.py"), + "file symlink must report isDir false") + assert.True(t, symlinkIsDir(skipped, "link_to_dir"), + "directory symlink must report isDir true") + + // The real files are still in the plan. + assert.Contains(t, uploadPaths, "app.py") + assert.Contains(t, uploadPaths, "realfile.py") + assert.Contains(t, uploadPaths, "realdir/inner.py") +} + +// TestPhase2_Symlink_DanglingReported verifies that a dangling symlink is +// reported in the skipped-symlink list with isDir false, and that the sync +// completes without error. The target is empty because os.Stat fails on a +// dangling link. +// +// Fulfills VAL-SYMLINK-004(a) at the go test level. +func TestPhase2_Symlink_DanglingReported(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // A symlink to a path that does not exist. + require.NoError(t, os.Symlink( + filepath.Join(dir, "nonexistent"), + filepath.Join(dir, "dangling.lnk"))) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err, "a dangling symlink must not fail the sync") + + // The dangling symlink is not in the upload plan. + assert.NotContains(t, uploadPathsOf(plan), "dangling.lnk") + + // The dangling symlink IS reported in the skipped-symlink list with + // isDir false. + skipped := e.skippedSymlinks + assert.Contains(t, symlinkPaths(skipped), "dangling.lnk") + assert.False(t, symlinkIsDir(skipped, "dangling.lnk"), + "dangling symlink must report isDir false") +} + +// TestPhase2_Symlink_DanglingDirectoryPatternFiltered verifies that a +// dangling symlink whose target has been deleted is still filtered by a +// directory-only ignore pattern ("node_modules/"). A dangling link's kind is +// unknowable, so the filter must check both the file and directory spellings +// of the pattern — otherwise "node_modules/" only matches a resolvable +// directory symlink and the dangling one leaks into the warning stream. +func TestPhase2_Symlink_DanglingDirectoryPatternFiltered(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // node_modules -> a target that does not exist (a dangling directory link). + require.NoError(t, os.Symlink( + filepath.Join(dir, "missing_target"), + filepath.Join(dir, "node_modules"))) + + // The directory-only spelling is what used to miss dangling links. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), + []byte("node_modules/\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + plan, err := e.Plan() + require.NoError(t, err) + + // Not in the upload plan. + assert.NotContains(t, uploadPathsOf(plan), "node_modules") + }) + + // Not reported, and therefore not warned about. + assert.NotContains(t, symlinkPaths(e.skippedSymlinks), "node_modules", + "a dangling node_modules link filtered by 'node_modules/' must not be reported") + assert.NotContains(t, logged, "node_modules", + "a dangling node_modules link filtered by 'node_modules/' must not be warned") +} + +// TestPhase2_Symlink_DanglingUncoveredStillWarned verifies that a dangling +// symlink covered by no ignore pattern is still reported and warned about, +// independent of the dangling-specific filter. +func TestPhase2_Symlink_DanglingUncoveredStillWarned(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // A dangling symlink no pattern covers. + require.NoError(t, os.Symlink( + filepath.Join(dir, "nonexistent"), + filepath.Join(dir, "dangling.lnk"))) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + assert.Contains(t, symlinkPaths(e.skippedSymlinks), "dangling.lnk", + "an uncovered dangling symlink must still be reported") + assert.Contains(t, logged, "dangling.lnk", + "an uncovered dangling symlink must still be warned") +} diff --git a/internal/workload/sync/phase5_execute.go b/internal/workload/sync/phase5_execute.go index 80c616dc6..557ce5a28 100644 --- a/internal/workload/sync/phase5_execute.go +++ b/internal/workload/sync/phase5_execute.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "github.com/datarobot/cli/internal/workload/fileops" ) @@ -68,8 +69,13 @@ func phase5Execute(e *Engine) error { return err } - // Phase 6 discards the rollback only after SaveConfig + SaveManifest - // succeed. + // Hand the rollback to Phase 6, which discards it at entry — before its + // first state write, not after its last. Once the plan has executed + // successfully the backup tree protects nothing, and discarding + // unconditionally keeps a Phase 6 write failure from stranding the dir + // for the next run's stale-restore to resurrect pre-sync bytes from. + // A Phase 5 failure above restores and returns, leaving the dir in place + // for exactly that stale-restore recovery. e.rollback = rb return nil @@ -98,6 +104,15 @@ func executePlan(e *Engine, rb *Rollback) error { e.newCatalogID = newCatalogID e.newVersionID = newVersionID + // --verify's post-apply check runs before this function returns so a + // mismatch propagates to phase5Execute, which restores the working tree + // and stops the pipeline before phase6State can persist anything. The + // check itself is a no-op unless the plan uploaded files and Options.Verify + // opted in. + if err := verifyPostApplyUploads(e, e.uploadOutcome); err != nil { + return err + } + return nil } @@ -276,13 +291,20 @@ func applyRemoteDeletesAndUploads(e *Engine, codeRef codeRefRef) (string, string if len(e.plan.Uploads) > 0 { uploader := ChooseUploader(e.plan) - cid, vid, err := uploader.ApplyUploads(e, e.plan.Uploads) + outcome, err := uploader.ApplyUploads(e, e.plan.Uploads) if err != nil { return "", "", err } - newCatalogID = cid - newVersionID = vid + // Store the outcome on the engine so Phase 6 can read + // outcome.Sent[path]. The phase pipeline runs each phase + // independently via runPhases with no per-phase return + // threading, so Engine state is the only channel between + // Phase 5 and Phase 6. + e.uploadOutcome = &outcome + + newCatalogID = outcome.CatalogID + newVersionID = outcome.VersionID } if newVersionID != "" && newVersionID != codeRef.CatalogVersionID { @@ -320,3 +342,66 @@ func applyDeletes(e *Engine, catalogID string) (string, error) { return resp.CatalogVersionID, nil } + +// verifyPostApplyUploads is --verify's second effect: after ApplyUploads, +// fetch the new version's file listing and confirm the server's checksum for +// every UPLOADED path equals the hash of the bytes this run streamed. The +// comparison is plain strings because the server's file checksum is the +// SHA-256 hex of the content, the same digest the uploader computed while +// streaming. +// +// It must run here in Phase 5, not in Phase 6: a mismatch has to fail the +// phase and trip the rollback while Phase 6 has written nothing — a Phase-6 +// check would run after SaveManifest/SaveConfig, and failing there cannot +// un-write either file. The check is placed after the codeRef PATCH: the +// remote version cannot be un-published, so the recoverable posture after a +// failed verification is the drifted one — config still names the old +// version, the next sync sees the difference, fetches the real remote, and +// reconciles. +// +// Only uploaded paths are compared. Stage REPLACE merges staged paths into +// the version in place, so the listing legitimately contains files this sync +// never touched, some carrying checksums from earlier runs; flagging one of +// those would fail honest runs. +// +// numFiles from ApplyStage counts every file in the resulting version, not +// the ones uploaded, so it has no arithmetic relationship to the plan and is +// never used to gate, skip, or abort this check. +// +// The check is skipped entirely when nothing was uploaded: a downloads-only +// or empty plan makes no AllFiles request here, and a nil outcome or an +// empty Sent map is not an error. +func verifyPostApplyUploads(e *Engine, outcome *UploadOutcome) error { + if !e.opts.Verify || outcome == nil || len(outcome.Sent) == 0 { + return nil + } + + listing, err := e.files.AllFiles(outcome.CatalogID, outcome.VersionID) + if err != nil { + return fmt.Errorf("post-apply verification: fetch files of version %s: %w", outcome.VersionID, err) + } + + // Sorted so the reported path is deterministic when several differ. + paths := make([]string, 0, len(outcome.Sent)) + + for path := range outcome.Sent { + paths = append(paths, path) + } + + sort.Strings(paths) + + for _, path := range paths { + sent := outcome.Sent[path] + + held, ok := listing[path] + if !ok { + return fmt.Errorf("post-apply verification: uploaded file %s is absent from server version %s", path, outcome.VersionID) + } + + if held.Hash != sent.Hash { + return fmt.Errorf("post-apply verification: server checksum for %s in version %s does not match the uploaded bytes (sent %s, server holds %s)", path, outcome.VersionID, sent.Hash, held.Hash) + } + } + + return nil +} diff --git a/internal/workload/sync/phase6_state.go b/internal/workload/sync/phase6_state.go index 26e1cf810..5ba7e27aa 100644 --- a/internal/workload/sync/phase6_state.go +++ b/internal/workload/sync/phase6_state.go @@ -21,16 +21,36 @@ import ( "github.com/datarobot/cli/internal/workload/wapi" ) -// phase6State writes the new BASE manifest, config, history entry, and -// discards the rollback. Failures here do NOT roll back Phase 5 since -// the remote has already advanced; the next sync will reconcile. +// phase6State writes the new BASE manifest, config, and history entry, and +// discards the rollback at entry. Failures here do NOT roll back Phase 5 +// since the remote has already advanced; the next sync will reconcile. A +// Discard failure is the one entry failure that leaves the rollback dir +// behind, and it aborts before any state write so that leftover dir pairs +// with un-advanced state — the safe mid-Phase-5 shape (see below). func phase6State(e *Engine) error { + // Discard the rollback BEFORE any state write, unconditionally. When this + // runs, Phase 5 executed the whole plan successfully (e.rollback is only + // assigned after executePlan returns nil; a Phase 5 failure restores and + // returns without reaching here), so the backup tree has no remaining + // purpose — and Phase 6 never restores on failure because the remote has + // already advanced. Discarding first makes the cleanup independent of + // write success: if SaveManifest or SaveConfig below fails, the early + // return must not strand the rollback dir, because the next run's + // stale-rollback recovery would blindly copy the pre-sync bytes back + // into the working tree. Against the manifest this run just wrote, those + // resurrected bytes look like local edits and are silently re-uploaded + // over the remote. + if err := discardRollback(e); err != nil { + return err + } + if e.plan == nil { return nil } now := e.nowFn().UTC() + // Shallow copy is safe only while wapi.Config has no reference-type field mutated in place; if one is added, switch to a deep copy. cfg := e.config if e.newCatalogID != "" { @@ -48,22 +68,46 @@ func phase6State(e *Engine) error { cfg.LastSyncedVersionID = &versionForState } - if err := wapi.SaveConfig(e.projectDir, cfg); err != nil { - return fmt.Errorf("save config: %w", err) + // Build and write the manifest BEFORE writing config. Both orders leave + // a one-file window on failure, and the safe direction is the one where + // the next sync detects drift and rebuilds from real remote data: + // + // - SaveManifest fails: config has not been advanced yet, so the next + // sync sees the old version in config, detects drift, fetches + // AllFiles, and rebuilds BASE from the remote — safe and + // self-healing. The manifest write is retried by that same sync. + // + // - SaveConfig fails: the manifest is already advanced while config + // still names the old version. The version mismatch makes the next + // sync detect drift and fetch the remote; BASE (the advanced + // manifest) truthfully describes that remote, so the plan is empty + // and the run merely converges config. The rollback dir is already + // gone by then — discarded at entry above — so no stale-restore can + // resurrect pre-sync bytes as false local edits. + // + // The converse (config advanced, manifest stale) is the poisonous + // direction: the next sync sees no drift, fast-paths, copies the stale + // BASE to REMOTE, and reports "Up to date." forever. + // + // This is data-safe because nothing between the two writes reads config + // from disk. buildNewBaseManifest reads only e.remote, e.plan, and + // e.uploadOutcome (all in-memory). e.config and populateResult are + // touched only after both writes complete. + manifest, err := buildNewBaseManifest(e, versionForState, now) + if err != nil { + return fmt.Errorf("build manifest: %w", err) } - manifest := buildNewBaseManifest(e, versionForState, now) if err := wapi.SaveManifest(e.projectDir, manifest); err != nil { return fmt.Errorf("save manifest: %w", err) } - if err := wapi.AppendHistory(e.projectDir, syncHistoryEntry(e, now)); err != nil { - return fmt.Errorf("append history: %w", err) + if err := wapi.SaveConfig(e.projectDir, cfg); err != nil { + return fmt.Errorf("save config: %w", err) } - if e.rollback != nil { - _ = e.rollback.Discard() - e.rollback = nil + if err := wapi.AppendHistory(e.projectDir, syncHistoryEntry(e, versionForState, now)); err != nil { + return fmt.Errorf("append history: %w", err) } e.config = cfg @@ -72,9 +116,36 @@ func phase6State(e *Engine) error { return nil } -// buildNewBaseManifest computes NEW_BASE = REMOTE + uploads (local hashes) -// - deletes, with conflicts resolved as remote-wins. -func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) wapi.Manifest { +// discardRollback removes the rollback tree at Phase 6 entry. A Discard +// failure must abort the phase here, before any state write: nothing is +// persisted yet, so returning an error leaves the next run with the rollback +// dir AND un-advanced state — the recoverable mid-Phase-5 outcome. The stale +// restore puts back bytes the un-advanced manifest still matches, and the +// next diff schedules downloads, not false uploads. Swallowing the error +// instead strands the rollback dir next to advanced state, where the same +// stale restore resurrects pre-sync bytes as phantom local edits which the +// next sync silently re-uploads over the remote. +func discardRollback(e *Engine) error { + if e.rollback == nil { + return nil + } + + if err := e.rollback.Discard(); err != nil { + return fmt.Errorf("discard rollback: %w", err) + } + + e.rollback = nil + + return nil +} + +// buildNewBaseManifest computes NEW_BASE = REMOTE + uploads (streamed hashes) +// - deletes, with conflicts resolved as remote-wins. Each uploaded path's +// hash and size come from the UploadOutcome recorded in Phase 5 — the bytes +// that actually crossed the wire, not the Phase-2 planned hash. A missing +// Sent entry is a hard error naming the path: a per-path fallback to the +// planned hash IS the original poisoning bug and must not exist here. +func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) (wapi.Manifest, error) { files := make(map[string]wapi.FileMeta, len(e.remote)) for path, fe := range e.remote { @@ -82,7 +153,20 @@ func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) } for _, fa := range e.plan.Uploads { - files[fa.Path] = wapi.FileMeta{Hash: fa.LocalHash, Size: fa.LocalSize} + if e.uploadOutcome == nil { + return wapi.Manifest{}, fmt.Errorf("internal: no upload outcome recorded for %s", fa.Path) + } + + sent, ok := e.uploadOutcome.Sent[fa.Path] + if !ok { + // Refuse rather than fall back: phase6 overwrites unconditionally, + // so a per-path fallback to fa.LocalHash silently reintroduces + // the poisoning. Either every upload has a Sent entry, or the + // sync fails. + return wapi.Manifest{}, fmt.Errorf("internal: no streamed hash recorded for %s", fa.Path) + } + + files[fa.Path] = wapi.FileMeta{Hash: sent.Hash, Size: sent.Size} } for _, fa := range e.plan.Deletes { @@ -105,15 +189,20 @@ func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) SyncedAt: &syncedAtCopy, SyncedVersionID: &versionCopy, Files: files, - } + }, nil } -// syncHistoryEntry assembles the JSONL line written to history.log. -func syncHistoryEntry(e *Engine, now time.Time) wapi.HistoryEntry { +// syncHistoryEntry assembles the JSONL line written to history.log. The +// destination version is the one the run ended on (resolved in phase6State), +// not the raw new-version ID: a sync that applies no uploads — a pull-only +// sync, or the empty-plan state repair a --verify run performs — ends at the +// version it started from, and an empty tail would read as a truncated +// entry rather than as "no new version". +func syncHistoryEntry(e *Engine, versionForState string, now time.Time) wapi.HistoryEntry { entry := wapi.HistoryEntry{ "ts": now.Format(time.RFC3339), "op": "sync", - "version": fmt.Sprintf("%s→%s", ShortVer(e.plan.OldVersionShort), ShortVer(e.newVersionID)), + "version": fmt.Sprintf("%s→%s", ShortVer(e.plan.OldVersionShort), ShortVer(versionForState)), "uploaded": len(e.plan.Uploads), "downloaded": len(e.plan.Downloads), "deleted": len(e.plan.Deletes), diff --git a/internal/workload/sync/phase6_state_test.go b/internal/workload/sync/phase6_state_test.go new file mode 100644 index 000000000..6f864c682 --- /dev/null +++ b/internal/workload/sync/phase6_state_test.go @@ -0,0 +1,385 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// emptyHash is the SHA-256 of the empty string, matching what a torn-to-empty +// file produces when streamed. +const emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +// TestExecuteRecordsStreamedHash is the deterministic form of the TOCTOU that +// poisoned the manifest 3/3 times through the real binary. Plan() computes a +// hash, then the file is rewritten, then Execute(plan) streams the new bytes. +// The manifest must record the streamed hash, not the Phase-2 planned hash. +func TestExecuteRecordsStreamedHash(t *testing.T) { + tests := []struct { + name string + planContent string // file content at Plan time + execContent string // file content at Execute time + }{ + { + name: "same_size_rewrite", + planContent: "print('ho')\n", // 11 bytes + execContent: "print('ok')\n", // 11 bytes, different content + }, + { + name: "grows", + planContent: "print('hi')\n", // 11 bytes + execContent: "print('hello')\n", // 14 bytes + }, + { + name: "shrinks", + planContent: "print('hello')\n", // 14 bytes + execContent: "print('hi')\n", // 11 bytes + }, + { + name: "torn_to_empty", + planContent: "print('hi')\n", // 11 bytes + execContent: "", // 0 bytes + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + // Start from a synced project so Plan sees exactly one upload. + dir := syncedProject(t, map[string]string{ + "app.py": "print('orig')\n", + }, catalogID, versionID) + + // Introduce a pending change so Plan sees app.py as an upload. + modifyFile(t, dir, "app.py", tc.planContent) + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 1, "exactly one upload expected") + require.Equal(t, "app.py", plan.Uploads[0].Path) + + // The Phase-2 planned hash is of the content present at Plan time. + plannedHash := plan.Uploads[0].LocalHash + plannedSize := plan.Uploads[0].LocalSize + + // The streamed hash is of the content present at Execute time. + streamedContent := []byte(tc.execContent) + streamedHash := sha256Hex(streamedContent) + streamedSize := int64(len(streamedContent)) + + // The test is only meaningful if the planned and streamed hashes differ. + require.NotEqual(t, plannedHash, streamedHash, + "planned and streamed hashes must differ for the test to be meaningful") + + // Between Plan and Execute, rewrite the file. This is the TOCTOU window. + modifyFile(t, dir, "app.py", tc.execContent) + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + + // Read the manifest and verify it records the streamed bytes. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + fm, ok := manifest.Files["app.py"] + require.True(t, ok, "app.py must be in the manifest") + + assert.Equal(t, streamedHash, fm.Hash, + "manifest must record the streamed hash, not the Phase-2 planned hash") + assert.NotEqual(t, plannedHash, fm.Hash, + "manifest must NOT record the Phase-2 planned hash") + assert.Equal(t, streamedSize, fm.Size, + "manifest must record the streamed size, not the Phase-2 planned size") + + // Size inequality only holds when the content actually changed size; + // the same_size case has identical planned and streamed sizes by + // construction, and that is correct. + if plannedSize != streamedSize { + assert.NotEqual(t, plannedSize, fm.Size, + "manifest must NOT record the Phase-2 planned size when sizes differ") + } + + // The torn-to-empty case has a specific expected hash. + if tc.execContent == "" { + assert.Equal(t, emptyHash, fm.Hash, + "torn-to-empty must record the SHA-256 of the empty string") + assert.Equal(t, int64(0), fm.Size, + "torn-to-empty must record size 0") + } + + // Every hash must be 64 lowercase hex characters and size >= 0. + assert.Len(t, fm.Hash, 64, "hash must be 64 hex characters") + assert.GreaterOrEqual(t, fm.Size, int64(0), "size must be >= 0") + }) + } +} + +// TestPhase6MissingSentHardFails verifies the hard rule: if Sent[path] is +// missing for an uploaded path, buildNewBaseManifest must return an error +// naming the path rather than falling back to the Phase-2 planned hash. A +// per-path fallback IS the original poisoning bug. +func TestPhase6MissingSentHardFails(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid", + VersionID: "ver", + Sent: map[string]FileEntry{}, // missing app.py + }, + } + + _, err := buildNewBaseManifest(e, "ver", time.Now()) + require.Error(t, err) + assert.Contains(t, err.Error(), "app.py", + "error must name the missing path") +} + +// TestPhase6NilOutcomeHardFails verifies that a nil uploadOutcome with a +// non-empty upload list also hard-fails naming the path. +func TestPhase6NilOutcomeHardFails(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: nil, + } + + _, err := buildNewBaseManifest(e, "ver", time.Now()) + require.Error(t, err) + assert.Contains(t, err.Error(), "app.py") +} + +// TestPhase6EmptyPlanStillWritesManifest verifies that a plan with zero +// uploads still produces a correct manifest. The upload loop does not execute, +// so the hard-fail does not misfire on an empty Sent map. A nil uploadOutcome +// is safe when there are no uploads. +func TestPhase6EmptyPlanStillWritesManifest(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{}, + Downloads: []FileAction{}, + Deletes: []FileAction{}, + Conflicts: []FileAction{}, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + }, + uploadOutcome: nil, // no uploads, so nil is fine + } + + manifest, err := buildNewBaseManifest(e, "ver-new", time.Now()) + require.NoError(t, err) + assert.Equal(t, wapi.ManifestVersion, manifest.Version) + assert.Len(t, manifest.Files, 1) + assert.Equal(t, sha256Hex([]byte("print('hi')\n")), manifest.Files["app.py"].Hash) +} + +// TestPhase6ConfigCopyIsNotMutatedInPlace pins the property that makes the +// `cfg := e.config` value copy in phase6State safe: Phase 6 must REASSIGN +// the copy's pointer fields, never write through the pointers it shares +// with the loaded config. Today wapi.Config holds only value types and +// reassigned pointers, so the shallow copy cannot corrupt e.config — but +// that safety invariant silently breaks if wapi.Config ever gains a +// reference-type field (a slice or map, or a pointer whose pointee is +// edited rather than replaced) that Phase 6 mutates in place: the mutation +// would reach e.config through the shared reference, with nothing at the +// type level to catch it. If this test fails, a field like that was added; +// deep-copy or rebuild that field inside Phase 6 rather than reverting the +// value copy. +func TestPhase6ConfigCopyIsNotMutatedInPlace(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + newVerID = "ver-new" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + // Retain the ORIGINAL pointers Phase 6's copy starts from, so the test + // can see a write-through mutation that comparing the (replaced) fields + // on the copy would miss. + origCatalogPtr := cfg.CatalogID + origVersionPtr := cfg.LastSyncedVersionID + + require.NotNil(t, origCatalogPtr) + require.NotNil(t, origVersionPtr) + + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + }, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: newVerID, + Sent: map[string]FileEntry{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + }, + }, + newCatalogID: "cid-new", + newVersionID: newVerID, + nowFn: time.Now, + } + + require.NoError(t, phase6State(e)) + + // The engine's config advanced to the new IDs. + require.NotNil(t, e.config.LastSyncedVersionID) + assert.Equal(t, newVerID, *e.config.LastSyncedVersionID) + + require.NotNil(t, e.config.CatalogID) + assert.Equal(t, "cid-new", *e.config.CatalogID) + + // The pre-existing pointer TARGETS must be untouched: Phase 6 reassigned + // its local copy's pointers instead of writing through the shared ones. + // A `*cfg.LastSyncedVersionID = ...` style in-place mutation would show + // up here as the new version leaking into the old pointer. + require.NotNil(t, origVersionPtr) + assert.Equal(t, versionID, *origVersionPtr, + "Phase 6 must not mutate the old LastSyncedVersionID pointer target in place") + + require.NotNil(t, origCatalogPtr) + assert.Equal(t, catalogID, *origCatalogPtr, + "Phase 6 must not mutate the old CatalogID pointer target in place") +} + +// TestManifestSchemaUnchanged verifies that the written manifest carries +// "version": 1 and that no new fields are added to manifest.json or config.json. +func TestManifestSchemaUnchanged(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // Verify manifest.json schema. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + assert.Equal(t, 1, manifest.Version, "manifest version must stay 1") + + // Verify the manifest has exactly the expected top-level keys by reading + // the raw JSON and checking no unexpected fields were added. + rawManifest, err := os.ReadFile(dir + "/.datarobot/workload/manifest.json") + require.NoError(t, err) + + // The manifest must contain "version": 1 and the standard fields only. + assert.Contains(t, string(rawManifest), `"version": 1`) + assert.Contains(t, string(rawManifest), `"files"`) + assert.Contains(t, string(rawManifest), `"syncedVersionId"`) + + // Verify config.json has no new fields. + rawConfig, err := os.ReadFile(dir + "/.datarobot/workload/config.json") + require.NoError(t, err) + + assert.Contains(t, string(rawConfig), `"artifactId"`) + assert.Contains(t, string(rawConfig), `"catalogId"`) + assert.Contains(t, string(rawConfig), `"lastSyncedVersionId"`) + + // syncedVersionId in manifest must equal the version in the result. + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, result.NewVersion, *manifest.SyncedVersionID, + "syncedVersionId must equal the version in the sync summary") + + // config.json's LastSyncedVersionID must equal manifest's syncedVersionId. + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, *manifest.SyncedVersionID, *cfg.LastSyncedVersionID, + "config LastSyncedVersionID must equal manifest syncedVersionId") +} diff --git a/internal/workload/sync/project_helper_test.go b/internal/workload/sync/project_helper_test.go new file mode 100644 index 000000000..dfbefc0fa --- /dev/null +++ b/internal/workload/sync/project_helper_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/require" +) + +// syncedProject creates a temp project tree with initialized workload state +// (config.json + manifest.json) where the manifest reflects the current file +// hashes, simulating a project that has already been synced. The config +// points at the given catalogID and versionID, and the manifest's file +// entries match the SHA-256 of every file on disk (user files + .drignore). +// +// This is the reusable helper for tests that need a project in a "synced" +// state: local == base, so a Plan with no disk changes produces an empty plan, +// and a Plan after modifying a file produces exactly that file as an upload. +func syncedProject(t *testing.T, files map[string]string, catalogID, versionID string) string { + t.Helper() + + dir := initProject(t, files) + + // Update config with catalog and version to simulate a prior sync. + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + cfg.CatalogID = &catalogID + cfg.LastSyncedVersionID = &versionID + require.NoError(t, wapi.SaveConfig(dir, cfg)) + + // Build manifest with hashes of ALL files on disk: user files plus the + // .drignore template that Initialize writes. Every manifest entry must + // match the real file hash so that local == base and the plan is empty. + manifestFiles := make(map[string]wapi.FileMeta) + + for rel := range files { + hash, size, err := hashLocal(t, dir, rel) + require.NoError(t, err) + + manifestFiles[rel] = wapi.FileMeta{Hash: hash, Size: size} + } + + // Include .drignore (created by Initialize) so it too is in sync. + hash, size, err := hashLocal(t, dir, ignore.FileName) + require.NoError(t, err) + + manifestFiles[ignore.FileName] = wapi.FileMeta{Hash: hash, Size: size} + + syncedAt := time.Now().UTC() + manifest := wapi.Manifest{ + Version: wapi.ManifestVersion, + SyncedAt: &syncedAt, + SyncedVersionID: &versionID, + Files: manifestFiles, + } + + require.NoError(t, wapi.SaveManifest(dir, manifest)) + + return dir +} + +// modifyFile rewrites a file in dir with new content, for tests that need +// pending changes after a syncedProject setup. +func modifyFile(t *testing.T, dir, rel, content string) { + t.Helper() + + abs := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.WriteFile(abs, []byte(content), 0o644)) +} diff --git a/internal/workload/sync/rollback_discard_test.go b/internal/workload/sync/rollback_discard_test.go new file mode 100644 index 000000000..f1a724917 --- /dev/null +++ b/internal/workload/sync/rollback_discard_test.go @@ -0,0 +1,455 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The tests in this file pin the Phase 6 discard-first ordering: the rollback +// directory must be discarded at Phase 6 ENTRY, before any state write, not +// after the last one succeeds. +// +// The hazard this guards: e.rollback is assigned only after Phase 5 executed +// the whole plan successfully, but a Phase 6 failure (SaveManifest or +// SaveConfig) returns early. If the discard sits below the state writes, a +// write failure strands the rollback dir. The next run's Phase 0 +// (RestoreStaleIfPresent) then blindly copies the pre-sync bytes back into +// the working tree, the advanced manifest makes the diff classify them as +// local edits, and the sync re-uploads the stale bytes over the remote — +// a self-perpetuating corruption chain with no prompt and no error. + +// rollbackDiscardScenario builds a synced project whose next sync carries one +// upload (a.py, locally modified) and one download (b.py, remotely modified). +// The download is the point: downloads are backed up into the rollback tree, +// so the rollback covers b.py's pre-sync bytes and a stranded rollback dir +// has real bytes to resurrect. An uploads-only plan would strand an empty +// rollback dir and could not demonstrate the corruption. +// +// Server timeline: ver-synced (what the project was synced at) → ver-remote +// (b.py changed remotely, drift) → ver-new (created by this sync's apply, +// which uploads the local a.py on top of ver-remote). +type rollbackDiscardScenario struct { + dir string + fake *fakeFilesClient + engine *Engine + plan *SyncPlan + origA string // a.py content at sync time + newA string // a.py content after the local edit + origB string // b.py content at sync time + remoteB string // b.py content after the remote edit + rollDir string + cfgBytes []byte // config.json bytes captured before the run +} + +// setupRollbackDiscardScenario wires the scenario and returns it after Plan(), +// so the caller can inject a Phase 6 fault and then Execute. +func setupRollbackDiscardScenario(t *testing.T) *rollbackDiscardScenario { + return setupRollbackDiscardScenarioWithPatchHook(t, nil) +} + +// setupRollbackDiscardScenarioWithPatchHook is the hook-aware variant. The +// hook, when non-nil, replaces the no-op PatchCodeRef fake: it runs at the +// very end of Phase 5 — after every backup, download, delete, and upload, +// with the rollback dir already on disk but Phase 6 not yet entered — which +// is exactly the window a Phase 6-entry fault must land in. +func setupRollbackDiscardScenarioWithPatchHook(t *testing.T, patchHook func(artifactID, catalogID, catalogVersionID string) error) *rollbackDiscardScenario { + t.Helper() + + const ( + catalogID = "cid-synced" + oldVersion = "ver-synced" + remoteVer = "ver-remote" + newVersion = "ver-new" + ) + + s := &rollbackDiscardScenario{ + origA: "aaa\n", + newA: "AAAA\n", + origB: "bbb\n", + remoteB: "BBBB-remote\n", + } + + s.dir = syncedProject(t, map[string]string{ + "a.py": s.origA, + "b.py": s.origB, + }, catalogID, oldVersion) + + // Pending local change: a.py becomes an upload. + modifyFile(t, s.dir, "a.py", s.newA) + + // The remote moved ahead: ver-remote holds the edited b.py. ApplyStage + // merges the staged a.py into this version, producing ver-new. + // withVersionContent seeds both the metadata and the downloadable + // bytes in one step, so the download of b.py serves the remote bytes. + drignoreBytes, err := os.ReadFile(filepath.Join(s.dir, ignore.FileName)) + require.NoError(t, err) + + s.fake = (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: newVersion, + }).withVersionContent(catalogID, remoteVer, map[string][]byte{ + "a.py": []byte(s.origA), + "b.py": []byte(s.remoteB), + ignore.FileName: drignoreBytes, + }) + + // The artifact's codeRef still points at ver-remote, so Phase 1 detects + // drift against the stale config and fetches real remote state. + e, err := newWithDeps(s.dir, Options{Yes: true}, Deps{ + Files: s.fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVer), nil + }, + // nil PatchFn is safe: the fake no-ops when the hook is unset. + PatchFn: patchHook, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + s.engine = e + s.rollDir = wapi.RollbackDir(s.dir) + + cfg, err := os.ReadFile(wapi.ConfigPath(s.dir)) + require.NoError(t, err) + + s.cfgBytes = cfg + + s.plan, err = e.Plan() + require.NoError(t, err) + + require.Len(t, s.plan.Uploads, 1, "a.py must be the one upload") + require.Equal(t, "a.py", s.plan.Uploads[0].Path) + require.Len(t, s.plan.Downloads, 1, "b.py must be the one download") + require.Equal(t, "b.py", s.plan.Downloads[0].Path) + + return s +} + +// replaceFileWithDir removes a state file and recreates it as a directory, so +// AtomicWriteFile's rename fails when the engine next writes it. Phase 1 has +// already loaded the real file, so the fault hits only Phase 6. +func replaceFileWithDir(t *testing.T, path string) { + t.Helper() + + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0o755)) + + t.Cleanup(func() { _ = os.RemoveAll(path) }) +} + +// assertNoRollbackDir asserts the rollback location is empty — the property +// a Phase 6 outcome must never violate. Non-fatal so a red run reports the +// full downstream corruption (stale restore, false uploads) alongside it. +func assertNoRollbackDir(t *testing.T, rollDir string) { + t.Helper() + + _, err := os.Stat(rollDir) + assert.True(t, os.IsNotExist(err), + "rollback directory must not survive the Phase 6 outcome at %s", rollDir) +} + +// TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads is the +// main regression: SaveConfig fails AFTER SaveManifest advanced the manifest. +// The rollback dir must already be gone (discarded at Phase 6 entry), and the +// next run must reconcile from the remote without resurrecting b.py's +// pre-sync bytes as a false LOCAL_MODIFIED upload. +// +// Fulfills VAL-ROLLBACK-001 (SaveConfig-failure leg) and VAL-ROLLBACK-002. +func TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads(t *testing.T) { + const ( + catalogID = "cid-synced" + oldVersion = "ver-synced" + newVersion = "ver-new" + ) + + s := setupRollbackDiscardScenario(t) + + // Fault: SaveConfig's atomic write fails (config.json is now a + // directory). SaveManifest has already run by the time the write is + // attempted, so the manifest advances while config stays stale. + replaceFileWithDir(t, wapi.ConfigPath(s.dir)) + + _, err := s.engine.Execute(s.plan) + + require.Error(t, err, "phase 6 must fail when SaveConfig fails") + assert.Contains(t, err.Error(), "save config", + "error must come from SaveConfig, not an earlier step") + + // The manifest DID advance (SaveManifest ran first): it records ver-new + // with the streamed upload hash for a.py and the remote hash for b.py. + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, newVersion, *manifest.SyncedVersionID, + "manifest must be advanced past the failed SaveConfig") + assert.Equal(t, sha256Hex([]byte(s.newA)), manifest.Files["a.py"].Hash, + "manifest must record the uploaded a.py hash") + assert.Equal(t, sha256Hex([]byte(s.remoteB)), manifest.Files["b.py"].Hash, + "manifest must record the downloaded b.py hash") + + // The rollback dir must be GONE. Discarding only after the state writes + // strands it here, and the stranded dir is what resurrects stale bytes + // on the next run. + assertNoRollbackDir(t, s.rollDir) + + // The working tree keeps the synced bytes: a.py holds the local edit, + // b.py holds the downloaded remote content. Nothing may restore over it. + bOnDisk, err := os.ReadFile(filepath.Join(s.dir, "b.py")) + require.NoError(t, err) + + assert.Equal(t, s.remoteB, string(bOnDisk), + "b.py must keep the downloaded remote content — no pre-sync restore") + + // Restore config.json with its pre-sync bytes: in the real world the + // atomic write never landed, so the file keeps its old content. + cPath := wapi.ConfigPath(s.dir) + + require.NoError(t, os.RemoveAll(cPath)) + + // gosec's taint analysis does not flag this write today, so no nolint + // directive is needed here; if a future gosec flags it again, add one + // back with the G703 rationale. + require.NoError(t, os.WriteFile(cPath, s.cfgBytes, 0o644)) + + staleCfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, staleCfg.LastSyncedVersionID) + assert.Equal(t, oldVersion, *staleCfg.LastSyncedVersionID, + "config.json must still hold the pre-sync version — the drift the next run must detect") + + // --- Next run: must reconcile from the remote, not from stale bytes --- + + all, err := s.fake.AllFiles(catalogID, newVersion) + require.NoError(t, err) + + fake2 := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-2", + versionID: newVersion, + }).withVersion(catalogID, newVersion, all) + + // The codeRef now points at ver-new: run 1's PatchCodeRef succeeded + // before Phase 6 failed, so the artifact genuinely moved ahead. + e2, err := newWithDeps(s.dir, Options{Yes: true}, Deps{ + Files: fake2, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVersion), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + plan2, err := e2.Plan() + require.NoError(t, err) + + // No stale restore: with the rollback discarded at entry, Phase 0 has + // nothing to resurrect. Pre-fix, this is true AND b.py has been rolled + // back to its pre-sync bytes. + assert.False(t, e2.StaleRollbackRestored(), + "the next run must not restore a stale rollback — the rollback dir was discarded at Phase 6 entry") + + // The plan must not contain false LOCAL_MODIFIED / LOCAL_ADDED uploads + // for the rollback-covered paths. b.py is the covered path here: its + // manifest entry (advanced) matches the remote, so there is nothing to + // upload. Pre-fix, the resurrected pre-sync bytes are classified as + // LOCAL_MODIFIED and silently re-uploaded over the remote. + assert.Empty(t, plan2.Uploads, + "the next run must not upload anything for rollback-covered paths — got %v", plan2.Uploads) + assert.True(t, plan2.IsEmpty(), + "the next run's plan must reconcile from the remote and be empty") + + // Executing the (empty) plan runs Phase 6, which converges config to the + // version the manifest already records — the self-healing direction. + _, err = e2.Execute(plan2) + require.NoError(t, err) + + convergedCfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, convergedCfg.LastSyncedVersionID) + assert.Equal(t, newVersion, *convergedCfg.LastSyncedVersionID, + "config must converge to the manifest's version on the next run") + + // Three-way consistency after convergence: manifest hash == server + // checksum for every file. + convergedManifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + + serverFiles, err := fake2.AllFiles(catalogID, newVersion) + require.NoError(t, err) + + for path, fm := range convergedManifest.Files { + serverFM, ok := serverFiles[path] + require.True(t, ok, "server must hold %s after convergence", path) + + assert.Equal(t, serverFM.Hash, fm.Hash, + "manifest hash for %s must match the server after convergence", path) + } +} + +// TestPhase6SaveManifestFailure_DiscardsRollback pins the other write-failure +// leg of VAL-ROLLBACK-001: SaveManifest itself fails. The discard-first +// ordering has already removed the rollback dir by then, so no Phase 6 +// outcome — not even the earliest write failing — strands it. +func TestPhase6SaveManifestFailure_DiscardsRollback(t *testing.T) { + s := setupRollbackDiscardScenario(t) + + // Fault: SaveManifest's atomic write fails (manifest.json is now a + // directory). Neither state file advances past this point. + replaceFileWithDir(t, filepath.Join(wapi.Dir(s.dir), "manifest.json")) + + _, err := s.engine.Execute(s.plan) + + require.Error(t, err, "phase 6 must fail when SaveManifest fails") + assert.Contains(t, err.Error(), "save manifest", + "error must come from SaveManifest, not an earlier step") + + // Discard-first means the rollback dir is already gone even though the + // very first state write failed. + assertNoRollbackDir(t, s.rollDir) + + // config.json must not have advanced — SaveConfig runs after the failed + // SaveManifest. + cfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *cfg.LastSyncedVersionID, + "config must not advance when SaveManifest fails") +} + +// TestPhase6CleanSuccessDiscardsRollback pins the no-failure leg of +// VAL-ROLLBACK-001 for completeness: a fully successful Phase 6 also leaves +// no rollback dir behind. The interruption tests already assert this for an +// uploads-only plan; this variant covers a plan that also downloads. +func TestPhase6CleanSuccessDiscardsRollback(t *testing.T) { + s := setupRollbackDiscardScenario(t) + + result, err := s.engine.Execute(s.plan) + require.NoError(t, err) + require.NotNil(t, result) + + assertNoRollbackDir(t, s.rollDir) + + // The synced state must reflect the applied plan. + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + + assert.Equal(t, sha256Hex([]byte(s.newA)), manifest.Files["a.py"].Hash) + assert.Equal(t, sha256Hex([]byte(s.remoteB)), manifest.Files["b.py"].Hash) +} + +// TestPhase6DiscardFailure_AbortsBeforeStateWrites covers the one Phase 6 +// failure mode the write-failure tests above cannot reach: Discard itself +// failing. The fault rides Phase 5's final step (PatchCodeRef), which fires +// after the rollback dir exists but before Phase 6 runs, and strips write +// permission from the rollback dir so Discard's os.RemoveAll fails. +// +// Phase 6 must abort with a wrapped error BEFORE SaveManifest/SaveConfig. +// At entry nothing has been persisted, so un-advanced state plus the +// surviving rollback dir is exactly the recoverable mid-Phase-5 outcome: +// the next run's stale-rollback restore puts back pre-sync bytes that the +// un-advanced manifest still matches, and the next diff schedules downloads, +// not false uploads. Swallowing the error instead (the pre-fix behavior) +// strands the rollback dir next to ADVANCED state, and that same stale +// restore resurrects pre-sync bytes as phantom local edits which the next +// sync silently re-uploads over the remote. +func TestPhase6DiscardFailure_AbortsBeforeStateWrites(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fault injection relies on POSIX directory permissions; windows ignores them") + } + + // Declare first so the hook closure can read s.rollDir: the hook fires + // during Execute, long after the assignment completes. + var s *rollbackDiscardScenario + + s = setupRollbackDiscardScenarioWithPatchHook(t, func(_, _, _ string) error { + // Phase 5's last step: make the rollback dir unremovable so the + // Phase 6 entry Discard fails. The hook itself must succeed so + // Phase 5 completes and Phase 6 is genuinely reached. + if err := os.Chmod(s.rollDir, 0o555); err != nil { + return fmt.Errorf("fault: chmod rollback dir: %w", err) + } + + return nil + }) + + // Restore permissions before t.TempDir cleanup so the read-only dir can + // be removed; cleanup runs LIFO, so this lands ahead of the TempDir one. + // Nil-guarded: if scenario setup itself fails, s (or s.rollDir) may never + // have been assigned and the cleanup must not dereference it. + t.Cleanup(func() { + if s == nil || s.rollDir == "" { + return + } + + _ = os.Chmod(s.rollDir, 0o755) + }) + + preManifest, err := os.ReadFile(filepath.Join(wapi.Dir(s.dir), "manifest.json")) + require.NoError(t, err) + + _, err = s.engine.Execute(s.plan) + + require.Error(t, err, "Phase 6 must abort when the rollback dir cannot be discarded") + assert.Contains(t, err.Error(), "discard rollback", + "error must be the wrapped Discard failure, not a later write error") + + // No state write may have happened: the manifest stays byte-identical to + // its pre-sync content, and config does not advance. + postManifest, err := os.ReadFile(filepath.Join(wapi.Dir(s.dir), "manifest.json")) + require.NoError(t, err) + + assert.Equal(t, preManifest, postManifest, + "manifest.json must be untouched when Discard fails at Phase 6 entry") + + cfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *cfg.LastSyncedVersionID, + "config.json must not advance when Discard fails at Phase 6 entry") + + // The rollback dir survives the failed Discard — that is the fault. Its + // presence is safe only because no state advanced (asserted above): the + // next run's stale-rollback recovery restores it against un-advanced + // state, which is the recoverable mid-Phase-5 outcome. + _, statErr := os.Stat(s.rollDir) + assert.NoError(t, statErr, "the unremovable rollback dir must still be present after the abort") +} diff --git a/internal/workload/sync/surroundings_regression_test.go b/internal/workload/sync/surroundings_regression_test.go new file mode 100644 index 000000000..ae6fb5f82 --- /dev/null +++ b/internal/workload/sync/surroundings_regression_test.go @@ -0,0 +1,1137 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/fileops" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests guard the behaviour immediately surrounding the upload-integrity +// fix so it cannot regress silently. They cover ignore rules, case-collision +// detection, path normalization, plan action mapping, the sync lock, +// stale-rollback recovery, and state migration — all through the engine's +// Plan/Execute seam so a regression in any of them is caught at the level +// that matters. +// +// Fulfills VAL-REGRESSION-007 and VAL-REGRESSION-009. + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(a): Ignore rules +// --------------------------------------------------------------------------- + +// TestDrignoreExcludesFromPlan verifies that files matching .drignore patterns +// are absent from the upload plan. A .drignore pattern that excludes *.tmp must +// keep scratch.tmp out of the uploads while letting app.py through. +func TestDrignoreExcludesFromPlan(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "scratch.tmp": "temporary\n", + }) + + // Overwrite the .drignore template with a pattern that excludes *.tmp. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), []byte("*.tmp\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + assert.Contains(t, paths, "app.py", "non-ignored file must be in the plan") + assert.NotContains(t, paths, "scratch.tmp", ".drignore-excluded file must be absent from the plan") +} + +// TestWapiignoreShadowWarning_LegacyOnly verifies that the deprecation notice +// fires when only the legacy .wapiignore filename is present (no .drignore). +// The notice tells the user the old name is deprecated and to rename it. +func TestWapiignoreShadowWarning_LegacyOnly(t *testing.T) { + dir := legacyProject(t, map[string]string{ + ignore.LegacyFileName: "*.tmp\n", + "scratch.tmp": "x", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + _, err := e.Plan() + require.NoError(t, err) + + notice := e.IgnoreFileNotice() + assert.Contains(t, notice, ignore.LegacyFileName, "notice must name the legacy file") + assert.Contains(t, notice, ignore.FileName, "notice must name the current file to rename to") +} + +// wantShadowWarning pins the exact ShadowWarning text emitted when both +// ignore filenames are present. The warning is the only signal a user gets +// that patterns they wrote are inert, so a wording change should be a +// deliberate, reviewed act — pinning the full sentence makes it one. The +// text lives in the ignore matcher; this constant is transcribed from it +// (not produced through the same Sprintf, which would make the pin +// self-fulfilling). +const wantShadowWarning = "Both .drignore and .wapiignore are present. .drignore is the one in effect, " + + "and the patterns in .wapiignore are not applied. Merge them into .drignore and delete .wapiignore." + +// TestWapiignoreShadowWarning_BothPresent verifies that the shadow warning +// fires when both .drignore and .wapiignore are present. The current name wins +// and the legacy patterns are inert, which the warning must say. The warning +// is emitted through log.Warn (never through IgnoreFileNotice), so the +// capture seam is what pins the actual text. +func TestWapiignoreShadowWarning_BothPresent(t *testing.T) { + dir := initProject(t, map[string]string{ + ignore.FileName: "*.new\n", + ignore.LegacyFileName: "*.old\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + // The shadow warning goes to log.Warn, not to IgnoreFileNotice. The + // notice is empty because the current name is in effect. + assert.Empty(t, e.IgnoreFileNotice(), "current name in effect has no deprecation notice") + + assert.Contains(t, logged, wantShadowWarning, + "the shadow warning must actually be emitted, with this exact text") +} + +// TestWapiignoreShadowWarning_NotFiredWithoutLegacyFile is the false-positive +// control for the pinned warning: with only the current filename present +// there is nothing being shadowed, and no shadow warning may reach the log. +// Without this control an always-warn implementation would pass the positive +// test above. +func TestWapiignoreShadowWarning_NotFiredWithoutLegacyFile(t *testing.T) { + dir := initProject(t, map[string]string{ + ignore.FileName: "*.tmp\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + assert.NotContains(t, logged, wantShadowWarning, + "no shadow warning when the legacy file is absent") +} + +// TestSystemExcludes_StillExcluded verifies that built-in system-excluded paths +// (.datarobot/workload, .wapi, .git, .gitignore, .datarobot.yaml) are absent +// from the upload plan even when no .drignore file exists. +func TestSystemExcludes_StillExcluded(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Create system-excluded paths inside the project. Use a file name that + // does not overwrite the real config.json created by initProject. + require.NoError(t, os.WriteFile(filepath.Join(dir, ".datarobot", "workload", "extra.json"), []byte("{}"), 0o644)) + + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0o644)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("*.tmp\n"), 0o644)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, ".datarobot.yaml"), []byte("name: test\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + assert.Contains(t, paths, "app.py", "regular file must be in the plan") + + for _, excluded := range []string{ + ".datarobot/workload/extra.json", + ".git/HEAD", + ".gitignore", + ".datarobot.yaml", + } { + assert.NotContains(t, paths, excluded, "system-excluded path must be absent from the plan: %s", excluded) + } +} + +// TestCaseFoldedSystemExcludes verifies that system excludes fold case: a +// .Datarobot/workload path is excluded just like .datarobot/workload. +func TestCaseFoldedSystemExcludes(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Create a differently-cased state directory. + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".Datarobot", "workload"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".Datarobot", "workload", "secret.json"), []byte("{}"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + assert.NotContains(t, paths, ".Datarobot/workload/secret.json", + "case-folded system exclude must still be excluded") +} + +// TestCaseFoldedIgnorePatterns verifies that case-folding of ignore patterns +// still matches paths differing only in case. A .drignore pattern "BUILD/" +// must exclude a directory named "build/" (case-folded match via system +// excludes is already tested above; this tests user-pattern case-folding +// through the gitignore library). +func TestCaseFoldedIgnorePatterns(t *testing.T) { + // The gitignore library is case-sensitive on Unix but the system excludes + // fold case. Here we test that a user pattern in .drignore matches a + // file of the same name with different case, which the system-exclude + // case-folding handles for system paths. For user patterns, the + // gitignore library is case-sensitive, so a pattern "*.TMP" does NOT + // match "scratch.tmp". We verify the actual behaviour: user patterns are + // case-sensitive (as documented), while system excludes fold case. + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "scratch.tmp": "x", + }) + + // Pattern with uppercase extension — gitignore is case-sensitive, + // so *.TMP does NOT match scratch.tmp. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), []byte("*.TMP\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + // User patterns are case-sensitive, so scratch.tmp is NOT excluded by *.TMP. + assert.Contains(t, paths, "scratch.tmp", + "user patterns are case-sensitive: *.TMP does not match scratch.tmp") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(b): Case-collision hard error before any upload +// --------------------------------------------------------------------------- + +// TestCaseCollision_FailsBeforeUpload verifies that two paths differing only +// in case cause the sync to fail with a case-collision error. On a +// case-insensitive filesystem (macOS, Windows) the walker cannot produce two +// such entries, so this test calls caseCollisionsFromManifest directly with a +// crafted manifest — the same function phase2Manifests calls after hashing. +func TestCaseCollision_FailsBeforeUpload(t *testing.T) { + // Craft a local manifest with two paths differing only in case. + local := LocalManifest{ + "Config.yaml": {Hash: "aaa", Size: 10}, + "config.yaml": {Hash: "bbb", Size: 20}, + } + + collisions := caseCollisionsFromManifest(local) + + require.Len(t, collisions, 1, "one case-collision group expected") + assert.Equal(t, "config.yaml", collisions[0].Lowered) + assert.ElementsMatch(t, []string{"Config.yaml", "config.yaml"}, collisions[0].Paths) + + msg := fileops.FormatCaseCollisions(collisions) + assert.Contains(t, msg, "case-only path collisions") + assert.Contains(t, msg, "Config.yaml vs config.yaml") +} + +// fsIsCaseInsensitive reports whether dir lives on a case-insensitive +// filesystem, using a throwaway probe pair inside dir: on a +// case-insensitive filesystem, os.Stat of the upper-case spelling resolves +// to the lower-case file that was just written. The probe directory is +// removed before the helper returns, so the probe never leaks into the +// caller's fixture: a test that probes for case sensitivity must not change +// the tree it is about to build its colliding pair in. +func fsIsCaseInsensitive(t *testing.T, dir string) bool { + t.Helper() + + probeDir := filepath.Join(dir, "case-probe") + require.NoError(t, os.MkdirAll(probeDir, 0o755)) + + lower := filepath.Join(probeDir, "probe.txt") + require.NoError(t, os.WriteFile(lower, []byte("x"), 0o644)) + + _, err := os.Stat(filepath.Join(probeDir, "PROBE.TXT")) + + // Synchronous removal rather than t.Cleanup: the caller builds its + // fixture immediately after this call, and the probe must be gone by + // then — on the non-skip path the project dir must hold exactly the + // files the caller creates afterwards. + require.NoError(t, os.RemoveAll(probeDir)) + + return err == nil +} + +// TestCaseCollision_EndToEnd_FailsBeforeUpload drives Plan() against a +// genuinely colliding tree — two real files whose paths differ only in case — +// and asserts the plan fails with the case-collision error before any +// upload-side call. This is the end-to-end complement to +// TestCaseCollision_FailsBeforeUpload, which must call +// caseCollisionsFromManifest directly because a colliding tree cannot exist +// on a case-insensitive filesystem: here the tree is real, so the exact +// walk → hash → collision-check seam the production code runs is exercised. +// The probe skips with a visible reason on macOS/Windows, where the two +// files would collapse into one; Linux CI exercises this test for real. +func TestCaseCollision_EndToEnd_FailsBeforeUpload(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + if fsIsCaseInsensitive(t, dir) { + t.Skip("case-insensitive filesystem: two paths differing only in case collapse into one file, so a genuinely colliding tree cannot be created here (TestCaseCollision_FailsBeforeUpload covers the collision check directly)") + } + + // The colliding pair. Neither name is ignored or system-excluded, so the + // walk collects both and the local manifest carries both. + require.NoError(t, os.WriteFile(filepath.Join(dir, "Greeting.txt"), []byte("hello\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "greeting.txt"), []byte("hi\n"), 0o644)) + + fake := &fakeFilesClient{} + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + _, err = e.Plan() + + require.Error(t, err, "a genuinely colliding tree must fail the plan") + assert.Contains(t, err.Error(), "case-only path collisions", + "the failure must be the case-collision error") + assert.Contains(t, err.Error(), "Greeting.txt", "the error must name the colliding paths") + assert.Contains(t, err.Error(), "greeting.txt", "the error must name the colliding paths") + + // The collision check sits in Phase 2, so the failure must precede every + // upload-side call: nothing may reach the network from a plan that never + // became executable. + assert.Equal(t, 0, fake.CreateStageCalls(), "no stage may be created when the plan fails") + assert.Equal(t, 0, fake.UploadToStageCalls(), "nothing may be uploaded when the plan fails") + assert.Equal(t, 0, fake.ApplyStageCalls(), "no stage may be applied when the plan fails") + assert.Equal(t, 0, fake.UploadFromZipCalls(), "no zip upload may be issued when the plan fails") +} + +// TestPlanWithoutCaseCollisions_MakesNoUploadCalls is the false-positive +// control for the case-collision tests: with no colliding tree, Plan must +// succeed with an empty plan and issue no upload-side calls. A colliding +// tree cannot be created on a case-insensitive filesystem (macOS), so this +// test observes no collision error at all — it does not verify where the +// error fires; that is TestCaseCollision_EndToEnd_FailsBeforeUpload's job on +// case-sensitive filesystems, and TestCaseCollision_FailsBeforeUpload's at +// function level everywhere else. +func TestPlanWithoutCaseCollisions_MakesNoUploadCalls(t *testing.T) { + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, "cid-cc", "ver-cc") + + fake := &fakeFilesClient{ + catalogID: "cid-cc", + stageID: "stage-cc", + versionID: "ver-cc-next", + } + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "cid-cc", "ver-cc"), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.True(t, plan.IsEmpty(), "synced project with no changes has an empty plan") + + // No case collision, no uploads — Plan does not call Execute. + assert.Equal(t, 0, fake.UploadToStageCalls(), "Plan must not upload") + assert.Equal(t, 0, fake.ApplyStageCalls(), "Plan must not apply") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(c): Path normalization +// --------------------------------------------------------------------------- + +// TestPathNormalization_ForwardSlash verifies that relative paths in the plan +// use forward slashes, not backslashes. On Unix this is trivially true; on +// Windows the walker must convert OS-native backslashes to forward slashes. +func TestPathNormalization_ForwardSlash(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "src/utils/helper.py": "def help(): pass\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + for _, fa := range plan.Uploads { + assert.NotContains(t, fa.Path, "\\", "path must use forward slashes: %s", fa.Path) + } +} + +// TestPathNormalization_NoLeadingDotSlash verifies that relative paths in the +// plan have no leading "./" prefix. +func TestPathNormalization_NoLeadingDotSlash(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + for _, fa := range plan.Uploads { + assert.False(t, strings.HasPrefix(fa.Path, "./"), + "path must not have leading ./: %s", fa.Path) + } +} + +// TestPathNormalization_NoTrailingSlash verifies that relative paths in the +// plan have no trailing slash. +func TestPathNormalization_NoTrailingSlash(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "src/utils/helper.py": "def help(): pass\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + for _, fa := range plan.Uploads { + assert.False(t, strings.HasSuffix(fa.Path, "/"), + "path must not have trailing slash: %s", fa.Path) + } +} + +// TestPathNormalization_NFC verifies that relative paths in the plan are +// NFC-normalized. On macOS the filesystem stores filenames in NFD, so a file +// named café.py (NFD: cafe + combining accent) must appear as café.py (NFC) +// in the plan. +func TestPathNormalization_NFC(t *testing.T) { + // NFC form of "café": precomposed é (U+00E9). + nfc := "café.py" + + // NFD form: e + combining acute (U+0301). macOS stores this on disk. + nfd := "cafe\u0301.py" + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Write the file with the NFD name — the filesystem will store it as-is. + require.NoError(t, os.WriteFile(filepath.Join(dir, nfd), []byte("x"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + // The plan must contain the NFC form, not the NFD form. + paths := uploadPathsOf(plan) + + assert.Contains(t, paths, nfc, "path must be NFC-normalized in the plan") + assert.NotContains(t, paths, nfd, "NFD form must not appear in the plan") + + // Verify the manifest key is also NFC by checking that NormalizePath + // produces NFC from the NFD input. + assert.Equal(t, nfc, fileops.NormalizePath(nfd)) +} + +// TestPathNormalization_ManifestAndComparison verifies that path normalization +// is consistent between the local manifest build and the base/remote +// comparison. When a synced project has a file with a non-ASCII name, the +// manifest key (base) and the plan path (local) must both be NFC-normalized +// so the diff does not falsely report a change. +func TestPathNormalization_ManifestAndComparison(t *testing.T) { + nfc := "café.py" + + nfd := "cafe\u0301.py" + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Write the file with the NFD name. + require.NoError(t, os.WriteFile(filepath.Join(dir, nfd), []byte("x"), 0o644)) + + // syncedProject builds the manifest from the file hashes, but we need + // to ensure the manifest key is NFC. Let's build it manually. + catalogID := "cid-nfc" + versionID := "ver-nfc" + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + cfg.CatalogID = &catalogID + cfg.LastSyncedVersionID = &versionID + require.NoError(t, wapi.SaveConfig(dir, cfg)) + + // Build manifest with NFC keys (what a correct sync would produce). + manifestFiles := make(map[string]wapi.FileMeta) + + for _, rel := range []string{"app.py", nfc, ignore.FileName} { + hash, size, err := hashLocal(t, dir, rel) + if err != nil { + // The file might be stored under NFD on disk; try the NFD name. + hash, size, err = hashLocal(t, dir, nfd) + require.NoError(t, err) + } + + manifestFiles[rel] = wapi.FileMeta{Hash: hash, Size: size} + } + + syncedAt := time.Now().UTC() + manifest := wapi.Manifest{ + Version: wapi.ManifestVersion, + SyncedAt: &syncedAt, + SyncedVersionID: &versionID, + Files: manifestFiles, + } + require.NoError(t, wapi.SaveManifest(dir, manifest)) + + // Now Plan: the local walk produces NFC paths, the base has NFC keys, + // so the diff should find no changes for café.py. + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-nfc", + versionID: "ver-nfc-next", + } + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // The plan must be empty: local == base (both NFC), and the fast path + // copies base to remote, so no diff. + assert.True(t, plan.IsEmpty(), + "NFC-normalized paths must match between local and base: plan should be empty, got uploads=%v deletes=%v", + uploadPathsOf(plan), deletePathsOf(plan)) +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(d): Plan action mapping +// --------------------------------------------------------------------------- + +// TestPlanAction_LocalDeleted verifies that a file deleted locally (present in +// base and remote, absent from local) produces an upload-delete action in the +// plan's Deletes list. +func TestPlanAction_LocalDeleted(t *testing.T) { + const ( + catalogID = "cid-del" + versionID = "ver-del" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "to-delete.py": "x\n", + }, catalogID, versionID) + + // Delete the file locally. + require.NoError(t, os.Remove(filepath.Join(dir, "to-delete.py"))) + + // Pre-populate the fake's server state with the synced version. + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-del", + versionID: "ver-del-next", + }).withVersion(catalogID, versionID, map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + "to-delete.py": {Hash: sha256Hex([]byte("x\n")), Size: 2}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + // The artifact must point at the synced version so the engine detects drift. + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // The deleted file must appear in the Deletes list with ActUploadDelete. + var deleteActions []FileAction + + for _, fa := range plan.Deletes { + if fa.Path == "to-delete.py" { + deleteActions = append(deleteActions, fa) + } + } + + require.Len(t, deleteActions, 1, "to-delete.py must be in the Deletes list") + assert.Equal(t, ActUploadDelete, deleteActions[0].Action, + "locally-deleted file must map to ActUploadDelete") + assert.Equal(t, ClsLocalDeleted, deleteActions[0].Classification) +} + +// TestPlanAction_RemoteModified verifies that a file modified on the remote +// only (base == local, remote differs) produces a download action. +func TestPlanAction_RemoteModified(t *testing.T) { + const ( + catalogID = "cid-rm" + versionID = "ver-rm" + remoteVerID = "ver-rm-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // The server has a different version of app.py. The artifact's codeRef + // points at remoteVerID (different from the synced versionID) so the + // engine detects drift and fetches AllFiles for the remote version. + remoteHash := sha256Hex([]byte("print('changed')\n")) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-rm", + versionID: "ver-rm-next", + }).withVersion(catalogID, remoteVerID, map[string]filesapi.FileMeta{ + "app.py": {Hash: remoteHash, Size: 16}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVerID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // app.py must appear in the Downloads list. + var downloads []FileAction + + for _, fa := range plan.Downloads { + if fa.Path == "app.py" { + downloads = append(downloads, fa) + } + } + + require.Len(t, downloads, 1, "app.py must be in the Downloads list") + assert.Equal(t, ActDownloadModify, downloads[0].Action, + "remote-only modification must map to ActDownloadModify") + assert.Equal(t, ClsRemoteModified, downloads[0].Classification) +} + +// TestPlanAction_BothSidesDifferent verifies that a file modified on both sides +// differently (local != base, remote != base, local != remote) produces a +// conflict. +func TestPlanAction_BothSidesDifferent(t *testing.T) { + const ( + catalogID = "cid-conf" + versionID = "ver-conf" + remoteVerID = "ver-conf-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Modify the file locally. + modifyFile(t, dir, "app.py", "print('local-change')\n") + + // The server has a different version of app.py. The artifact's codeRef + // points at remoteVerID (different from the synced versionID) so the + // engine detects drift and fetches AllFiles for the remote version. + remoteHash := sha256Hex([]byte("print('remote-change')\n")) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-conf", + versionID: "ver-conf-next", + }).withVersion(catalogID, remoteVerID, map[string]filesapi.FileMeta{ + "app.py": {Hash: remoteHash, Size: 22}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVerID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // app.py must appear in the Conflicts list. + var conflicts []FileAction + + for _, fa := range plan.Conflicts { + if fa.Path == "app.py" { + conflicts = append(conflicts, fa) + } + } + + require.Len(t, conflicts, 1, "app.py must be in the Conflicts list") + assert.Equal(t, ActConflictCopy, conflicts[0].Action, + "both-sides-different must map to ActConflictCopy") + assert.Equal(t, ClsConflict, conflicts[0].Classification) +} + +// TestPlanAction_RemoteWinsResolution verifies that the remote-wins +// resolution path downloads the remote version and that the remote bytes +// actually land on disk. The plan-structure half pins that the conflict +// row carries the server's hash (Phase 6's buildNewBaseManifest resolves +// conflicts through fa.RemoteHash); the Execute half pins that the +// resolution really wrote those bytes, kept the local side as a .LOCAL. +// copy, and recorded the remote hash in the new BASE. +func TestPlanAction_RemoteWinsResolution(t *testing.T) { + const ( + catalogID = "cid-rw" + versionID = "ver-rw" + remoteVerID = "ver-rw-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Modify the file locally. + localContent := "print('local-change')\n" + modifyFile(t, dir, "app.py", localContent) + + // The server has a different version. The artifact's codeRef points at + // remoteVerID (different from the synced versionID) so the engine detects + // drift and fetches AllFiles for the remote version. The remote version + // is seeded with recorded content so the fake can actually serve the + // conflict's download; .drignore is seeded with the disk's own bytes so + // it is unchanged on both sides and adds no plan rows. + remoteContent := "print('remote-change')\n" + remoteHash := sha256Hex([]byte(remoteContent)) + + drignoreBytes, err := os.ReadFile(filepath.Join(dir, ignore.FileName)) + require.NoError(t, err) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-rw", + versionID: "ver-rw-next", + }).withVersionContent(catalogID, remoteVerID, map[string][]byte{ + "app.py": []byte(remoteContent), + ignore.FileName: drignoreBytes, + }) + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVerID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + require.True(t, plan.HasConflicts(), "plan must have conflicts") + + var conflict FileAction + + for _, fa := range plan.Conflicts { + if fa.Path == "app.py" { + conflict = fa + } + } + + require.Equal(t, "app.py", conflict.Path) + assert.Equal(t, remoteHash, conflict.RemoteHash, + "conflict's RemoteHash must be the server's hash (remote-wins resolution uses this)") + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, 1, result.ConflictCount) + + // Remote wins: the server's exact bytes must now sit at the original + // path, hashing to the advertised checksum. + onDisk, readErr := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, readErr) + + assert.Equal(t, remoteContent, string(onDisk), "remote-wins resolution must write the remote bytes") + assert.Equal(t, remoteHash, sha256Hex(onDisk), "written bytes must hash to the server's checksum") + + // The local side of the conflict survives as a .LOCAL. copy. + entries, readErr := os.ReadDir(dir) + require.NoError(t, readErr) + + var localCopy string + + for _, ent := range entries { + if strings.HasPrefix(ent.Name(), "app.py.LOCAL.") { + localCopy = ent.Name() + } + } + + require.NotEmpty(t, localCopy, "local side of the conflict must be preserved as a .LOCAL. copy") + + localBytes, readErr := os.ReadFile(filepath.Join(dir, localCopy)) + require.NoError(t, readErr) + assert.Equal(t, localContent, string(localBytes), "the .LOCAL. copy must hold the pre-conflict local bytes") + + // The new BASE records the remote hash for the conflicted path — the + // remote-wins rule made real by Phase 6. + manifest, manifestErr := wapi.LoadManifest(dir) + require.NoError(t, manifestErr) + assert.Equal(t, remoteHash, manifest.Files["app.py"].Hash, + "manifest must record the remote hash for the conflict-resolved path") + + assert.Equal(t, 1, fake.DownloadFileCalls(), "conflict resolution must have pulled the remote bytes exactly once") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-009: Sync lock, stale-rollback recovery, state migration +// --------------------------------------------------------------------------- + +// TestSyncLock_SecondConcurrentSyncRejected verifies that a second concurrent +// sync against the same project is rejected by the sync lock. On Unix, the +// second Plan() must fail with a lock error; on Windows the lock is a no-op +// (tracked in RAPTOR-16928) so the test skips. +func TestSyncLock_SecondConcurrentSyncRejected(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("v1 sync lock is a no-op on windows; tracked in RAPTOR-16928") + } + + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + // First engine acquires the lock. + e1, err := newWithDeps(dir, Options{}, Deps{ + Files: &fakeFilesClient{}, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + plan, err := e1.Plan() + require.NoError(t, err) + require.False(t, plan.IsEmpty(), "fixture must produce a non-empty plan") + + t.Cleanup(func() { _ = e1.Close() }) + + // Second engine on the same project must fail to acquire the lock. + e2, err := newWithDeps(dir, Options{}, Deps{ + Files: &fakeFilesClient{}, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + _, err = e2.Plan() + require.Error(t, err, "second concurrent sync must be rejected by the lock") + assert.Contains(t, err.Error(), "another sync is already running") +} + +// TestSyncLock_StaleRollbackRecovery verifies that a stale rollback from a +// crashed prior run is restored so a new sync proceeds. Phase 0 calls +// RestoreStaleIfPresent before acquiring the lock, so a stale rollback +// directory must be cleaned up and the engine's StaleRollbackRestored() must +// report true. +func TestSyncLock_StaleRollbackRecovery(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + // Simulate a crashed prior run: create a stale rollback directory with + // a backup of app.py. + rollDir := wapi.RollbackDir(dir) + require.NoError(t, os.MkdirAll(rollDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(rollDir, "app.py"), []byte("intact\n"), 0o644)) + + // Clobber the working tree to simulate the crash mid-sync. + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.py"), []byte("BROKEN\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err, "stale rollback must be recovered so Plan proceeds") + require.NotNil(t, plan) + + assert.True(t, e.StaleRollbackRestored(), "engine must report that a stale rollback was restored") + + // The file must be restored to its pre-crash content. + got, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err) + assert.Equal(t, "intact\n", string(got), "stale rollback must restore the original file") + + // The rollback directory must be cleaned up. + _, err = os.Stat(rollDir) + assert.ErrorIs(t, err, os.ErrNotExist, "stale rollback directory must be removed after recovery") +} + +// TestStateMigration_ForwardMigration verifies that an older on-disk state +// format (legacy .wapi/ directory) is migrated forward to .datarobot/workload/ +// without error. Phase 0 calls EnsureMigrated, which moves the legacy +// directory. The engine's StateMigrationNotice() must report the move. +func TestStateMigration_ForwardMigration(t *testing.T) { + // Initialize a proper project (creates .datarobot/workload/ with a valid + // config.json that has createdAt and cliVersion), then move the state + // directory to the legacy .wapi/ location to simulate an older on-disk + // state format. + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + // Move the current state directory to the legacy location. + currentDir := filepath.Join(dir, wapi.RootDirName, wapi.StateDirName) + legacyDir := filepath.Join(dir, wapi.LegacyDirName) + require.NoError(t, os.Rename(currentDir, legacyDir)) + + // Remove the now-empty .datarobot/ parent so EnsureMigrated sees only the + // legacy directory. + require.NoError(t, os.RemoveAll(filepath.Join(dir, wapi.RootDirName))) + + // Verify the legacy directory exists before migration. + assert.DirExists(t, legacyDir) + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: &fakeFilesClient{}, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + _, err = e.Plan() + require.NoError(t, err, "migration must succeed so Plan proceeds") + + // The legacy directory must be gone. + assert.NoDirExists(t, legacyDir, "legacy state directory must be moved") + + // The current state directory must exist. + assert.DirExists(t, currentDir, "current state directory must exist after migration") + + // The engine must report the migration. + notice := e.StateMigrationNotice() + assert.Contains(t, notice, wapi.LegacyDirName, "notice must name the legacy directory") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-008: dr workload up code-change measurement +// --------------------------------------------------------------------------- + +// TestWorkloadUp_CodeChangeCount_ModifiedTree verifies that the sync engine's +// Plan (which `dr workload up`'s defaultCodeChange builds as a dry-run) reports +// the correct upload + delete count for a modified tree. defaultCodeChange +// computes `len(plan.Uploads) + len(plan.Deletes)`, so we verify that count +// matches the real number of changed files. +func TestWorkloadUp_CodeChangeCount_ModifiedTree(t *testing.T) { + const ( + catalogID = "cid-wup" + versionID = "ver-wup" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "utils.py": "def util(): pass\n", + "to-delete.py": "x\n", + }, catalogID, versionID) + + // Modify one file, add one new file, delete one file. + modifyFile(t, dir, "app.py", "print('changed')\n") + require.NoError(t, os.WriteFile(filepath.Join(dir, "new.py"), []byte("new\n"), 0o644)) + require.NoError(t, os.Remove(filepath.Join(dir, "to-delete.py"))) + + // Pre-populate the fake's server state with the synced version. + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-wup", + versionID: "ver-wup-next", + }).withVersion(catalogID, versionID, map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + "utils.py": {Hash: sha256Hex([]byte("def util(): pass\n")), Size: 17}, + "to-delete.py": {Hash: sha256Hex([]byte("x\n")), Size: 2}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // defaultCodeChange reports len(plan.Uploads) + len(plan.Deletes). + // Modified app.py + new new.py = 2 uploads. + // Deleted to-delete.py = 1 delete. + // Total = 3 changed files. + count := len(plan.Uploads) + len(plan.Deletes) + assert.Equal(t, 3, count, + "modified tree must report 3 changed files (2 uploads + 1 delete), got %d (uploads=%d, deletes=%d)", + count, len(plan.Uploads), len(plan.Deletes)) +} + +// TestWorkloadUp_CodeChangeCount_UnchangedTree verifies that the sync engine's +// Plan reports zero changed files for an unchanged tree (synced project with no +// modifications). defaultCodeChange reports `len(plan.Uploads) + len(plan.Deletes)`, +// which must be zero. +func TestWorkloadUp_CodeChangeCount_UnchangedTree(t *testing.T) { + const ( + catalogID = "cid-wup-uc" + versionID = "ver-wup-uc" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "utils.py": "def util(): pass\n", + }, catalogID, versionID) + + // No modifications — the tree is unchanged. + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-wup-uc", + versionID: "ver-wup-uc-next", + } + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + count := len(plan.Uploads) + len(plan.Deletes) + assert.Equal(t, 0, count, + "unchanged tree must report 0 changed files, got %d (uploads=%d, deletes=%d)", + count, len(plan.Uploads), len(plan.Deletes)) + assert.True(t, plan.IsEmpty(), "unchanged tree must produce an empty plan") +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// deletePathsOf returns the paths in the delete portion of the plan. +func deletePathsOf(plan *SyncPlan) []string { + paths := make([]string, 0, len(plan.Deletes)) + for _, fa := range plan.Deletes { + paths = append(paths, fa.Path) + } + + return paths +} diff --git a/internal/workload/sync/symlink.go b/internal/workload/sync/symlink.go new file mode 100644 index 000000000..258124e72 --- /dev/null +++ b/internal/workload/sync/symlink.go @@ -0,0 +1,50 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import "fmt" + +// SymlinkNoticeBound is the maximum number of skipped symlinks the stderr +// prose lists individually before summarizing the remainder as a count. +// The JSON field always lists every symlink. Fixed at 5 so workers and +// validators assert the same number rather than each choosing a bound. +const SymlinkNoticeBound = 5 + +// SkippedSymlink describes a symlink the walk did not follow, collected so +// the display layer can tell the user a file they expect on the remote is +// absent. IsDir distinguishes a single skipped file from an entire omitted +// subtree (a directory symlink prunes all of its children), which is the +// distinction that makes the notice actionable. +type SkippedSymlink struct { + Path string + IsDir bool +} + +// skippedSymlinkNotice renders one skipped symlink as a self-contained stderr +// line. The wording differs by kind so a reader can tell "lost one file" +// apart from "lost an entire subtree", and both say the symlink was NOT +// uploaded or synced — not merely that one was found — so the user +// understands their file is absent from the remote. +func skippedSymlinkNotice(s SkippedSymlink) string { + if s.IsDir { + return fmt.Sprintf( + "skipped symlink: %s was not synced (it is a directory symlink; the entire subtree under it is omitted from the sync)", + s.Path) + } + + return fmt.Sprintf( + "skipped symlink: %s was not uploaded (it is a symlink, not a regular file)", + s.Path) +} diff --git a/internal/workload/sync/symlink_notice_test.go b/internal/workload/sync/symlink_notice_test.go new file mode 100644 index 000000000..ed679e631 --- /dev/null +++ b/internal/workload/sync/symlink_notice_test.go @@ -0,0 +1,228 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPhase2_SymlinkNotice_EmittedFromPhase2 verifies that the skipped-symlink +// warning is emitted via log.Warn from within Phase 2, so it survives a later +// phase failing. The warning must name the symlink and say it was NOT +// uploaded or synced. +// +// Fulfills VAL-SYMLINK-001(d,e) and VAL-SYMLINK-011(f) at the go test level. +func TestPhase2_SymlinkNotice_EmittedFromPhase2(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // A file symlink and a directory symlink. + require.NoError(t, os.WriteFile(filepath.Join(dir, "realfile.py"), []byte("x"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realfile.py"), + filepath.Join(dir, "link_to_file.py"))) + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "realdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "realdir", "inner.py"), []byte("y"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realdir"), + filepath.Join(dir, "link_to_dir"))) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + // Both symlinks are named in the warning. + assert.Contains(t, logged, "link_to_file.py") + assert.Contains(t, logged, "link_to_dir") + + // The file-symlink wording says "not uploaded". + assert.Contains(t, logged, "was not uploaded", + "the file-symlink notice must say the symlink was not uploaded") + + // The directory-symlink wording says "not synced" and conveys that a + // whole subtree is omitted. + assert.Contains(t, logged, "was not synced", + "the directory-symlink notice must say the symlink was not synced") + assert.Contains(t, logged, "subtree", + "the directory-symlink wording must convey that a whole subtree is omitted") + + // The file and directory wordings differ. + assert.NotEqual(t, + strings.Index(logged, "was not uploaded"), + strings.Index(logged, "was not synced"), + "file-symlink wording must differ from directory-symlink wording") +} + +// TestPhase2_SymlinkNotice_SurvivesLaterPhaseFailure verifies that the +// skipped-symlink warning reaches stderr even when a later phase fails, so +// the user hears it exactly when they need it most. The warning is emitted +// from within Phase 2 via log.Warn, not after a successful plan. +// +// Fulfills VAL-SYMLINK-011(f). +func TestPhase2_SymlinkNotice_SurvivesLaterPhaseFailure(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "realfile.py"), []byte("x"), 0o644)) + require.NoError(t, os.Symlink( + filepath.Join(dir, "realfile.py"), + filepath.Join(dir, "link_to_file.py"))) + + // Make a file unreadable so hashEntries fails after the walk and the + // log.Warn emission. The walk (filepath.WalkDir) only needs directory + // execute permission to enumerate entries; it does not open individual + // files. hashEntries then opens each file to hash it, and an unreadable + // file produces a "permission denied" error that fails Phase 2 — but + // the symlink warning has already been emitted to stderr by then. + require.NoError(t, os.WriteFile(filepath.Join(dir, "unreadable.py"), []byte("x"), 0o644)) + require.NoError(t, os.Chmod(filepath.Join(dir, "unreadable.py"), 0o000)) + + t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "unreadable.py"), 0o644) }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.Error(t, err, "hashEntries must fail on an unreadable file") + }) + + // The symlink warning still appears despite the phase failure. + assert.Contains(t, logged, "link_to_file.py", + "the symlink warning must survive a later phase failing") + assert.Contains(t, logged, "was not uploaded", + "the warning must say the symlink was not uploaded") +} + +// TestPhase2_SymlinkNotice_BoundedAtFive verifies that when more than +// SymlinkNoticeBound symlinks are skipped, the stderr prose lists the first +// SymlinkNoticeBound in deterministic order followed by a count of the +// remainder, while the engine's skippedSymlinks field carries every one. +// +// Fulfills VAL-SYMLINK-009(a). +func TestPhase2_SymlinkNotice_BoundedAtFive(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Create a real file to point the symlinks at. + require.NoError(t, os.WriteFile(filepath.Join(dir, "real.py"), []byte("x"), 0o644)) + + // Create 7 file symlinks (exceeding the bound of 5). + for i := 0; i < 7; i++ { + name := "link" + string(rune('a'+i)) + ".py" + require.NoError(t, os.Symlink( + filepath.Join(dir, "real.py"), + filepath.Join(dir, name))) + } + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + // The first 5 symlinks appear in the log. + for i := 0; i < 5; i++ { + name := "link" + string(rune('a'+i)) + ".py" + assert.Contains(t, logged, name, + "the first 5 symlinks must appear in the bounded prose") + } + + // The 6th and 7th do not appear individually. + assert.NotContains(t, logged, "linkf.py", + "the 6th symlink must not appear in the bounded prose") + assert.NotContains(t, logged, "linkg.py", + "the 7th symlink must not appear in the bounded prose") + + // The count of the remainder appears. + assert.Contains(t, logged, "2 more", + "the count of the remainder must appear in the prose") + + // The engine's skippedSymlinks field carries every symlink. + require.Len(t, e.skippedSymlinks, 7, + "the structured field must list every symlink even when prose is bounded") +} + +// TestPhase2_SymlinkNotice_NoSymlinksNoNotice verifies that a project with no +// symlinks emits no symlink-related warning. +// +// Fulfills VAL-SYMLINK-005(a) at the go test level. +func TestPhase2_SymlinkNotice_NoSymlinksNoNotice(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err) + }) + + assert.NotContains(t, logged, "symlink", + "a project with no symlinks must emit no symlink-related warning") + assert.Empty(t, e.skippedSymlinks, + "the skippedSymlinks field must be empty when there are no symlinks") +} + +// TestPhase2_SymlinkNotice_DeterministicOrder verifies that the skipped +// symlinks are sorted by path on the engine, so notices and the structured +// field are deterministic across runs. +func TestPhase2_SymlinkNotice_DeterministicOrder(t *testing.T) { + skipNonWindowsSymlink(t) + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "real.py"), []byte("x"), 0o644)) + + // Create symlinks in non-sorted order of creation. + for _, name := range []string{"z_link.py", "a_link.py", "m_link.py"} { + require.NoError(t, os.Symlink( + filepath.Join(dir, "real.py"), + filepath.Join(dir, name))) + } + + e := lockfileEngine(t, dir, noLockfileRunner) + + _, err := e.Plan() + require.NoError(t, err) + + // The skippedSymlinks on the engine are sorted by path. + require.Len(t, e.skippedSymlinks, 3) + assert.Equal(t, "a_link.py", e.skippedSymlinks[0].Path) + assert.Equal(t, "m_link.py", e.skippedSymlinks[1].Path) + assert.Equal(t, "z_link.py", e.skippedSymlinks[2].Path) +} diff --git a/internal/workload/sync/temp_cleanup_test.go b/internal/workload/sync/temp_cleanup_test.go new file mode 100644 index 000000000..2195bdfa7 --- /dev/null +++ b/internal/workload/sync/temp_cleanup_test.go @@ -0,0 +1,288 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This file pins the "no litter" contract of a sync run: after a sync +// finishes — successfully or not — every temporary artifact it created must +// be gone. Two real temp producers exist: +// +// - buildZip creates wapi-sync-*.zip in the SYSTEM temp dir, removed by +// ApplyUploads' defer on the happy path and by buildZip's error paths. +// - AtomicWriteFile creates .tmp.* siblings inside the state dir, +// removed on any failure before the rename. +// +// The rollback tree is deliberately NOT asserted to vanish on failure: a +// failed sync leaves .datarobot/workload/.rollback in place on purpose, as +// the only copy of the user's overwritten files until the next run's Phase 0 +// restores-and-removes it (Restore is best-effort, so the tree outlives a +// partial restore). That behaviour is pinned by the interruption tests; here +// it is asserted only where the design says it must hold (gone on success, +// present for recovery on failure), never treated as litter. + +// syncTempPrefix is the CreateTemp pattern buildZip uses for its archive. +const syncTempPrefix = "wapi-sync-" + +// snapshotSyncTemps lists the sync-pattern entries currently in the system +// temp dir. The before/after diff form matters: os.TempDir is shared with +// every other process on the machine, so "the set did not grow" is the only +// honest cleanup assertion — a pre-existing stray from a crashed old build +// must not make this test fail, and a concurrent creator must not make it +// pass vacuously. +func snapshotSyncTemps(t *testing.T) []string { + t.Helper() + + entries, err := os.ReadDir(os.TempDir()) + require.NoError(t, err) + + var found []string + + for _, e := range entries { + if strings.HasPrefix(e.Name(), syncTempPrefix) { + found = append(found, e.Name()) + } + } + + return found +} + +// stateDirTmpLitter returns every file under the project's state dir whose +// name matches AtomicWriteFile's temp pattern (.tmp.*). Persistent +// state (config.json, manifest.json, history.log, sync.lock) never matches. +func stateDirTmpLitter(t *testing.T, dir string) []string { + t.Helper() + + var litter []string + + err := filepath.WalkDir(wapi.Dir(dir), func(p string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if !d.IsDir() && strings.Contains(d.Name(), ".tmp.") { + litter = append(litter, p) + } + + return nil + }) + require.NoError(t, err) + + return litter +} + +// assertNoNewSyncTemps diffs the system-temp snapshot against the current +// set and fails when the sync added entries. +func assertNoNewSyncTemps(t *testing.T, before []string) { + t.Helper() + + after := snapshotSyncTemps(t) + + extra := setMinus(after, before) + + assert.Empty(t, extra, "sync must leave no wapi-sync-* files in the system temp dir") +} + +// setMinus returns the elements of a not present in b, sorted. +func setMinus(a, b []string) []string { + inB := make(map[string]bool, len(b)) + for _, s := range b { + inB[s] = true + } + + var out []string + + for _, s := range a { + if !inB[s] { + out = append(out, s) + } + } + + return out +} + +// newCleanupEngine wires an engine against a fake server for the cleanup +// tests, mirroring the seam the rest of the suite uses. The fake is built by +// the caller because the stage/zip fault hooks chain on the constructor. +func newCleanupEngine(t *testing.T, dir string, files *fakeFilesClient) *Engine { + t.Helper() + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: files, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e +} + +// TestSyncCleanup_StagePathSuccess proves a successful small sync (stage +// path, well under the zip threshold) leaves nothing behind: no zip temp in +// the system temp dir, no AtomicWriteFile litter in the state dir, and no +// rollback tree (Discard removes it once Phase 6 persisted). +func TestSyncCleanup_StagePathSuccess(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('ok')\n", + "utils/h.py": "def h(): pass\n", + }) + + files := &fakeFilesClient{catalogID: "cid-new", stageID: "stage-1", versionID: "ver-1"} + + before := snapshotSyncTemps(t) + + e := newCleanupEngine(t, dir, files) + + plan, err := e.Plan() + require.NoError(t, err) + + require.NotEmpty(t, plan.Uploads) + + _, err = e.Execute(plan) + require.NoError(t, err) + + assert.Positive(t, files.UploadToStageCalls(), "precondition: the stage path ran") + + assertNoNewSyncTemps(t, before) + assert.Empty(t, stateDirTmpLitter(t, dir), "no AtomicWriteFile temps may survive a successful sync") + assert.NoDirExists(t, wapi.RollbackDir(dir), "the rollback tree is discarded once the sync persisted") +} + +// TestSyncCleanup_StagePathFailure proves a failed stage sync is equally +// litter-free in both temp locations. The rollback tree remains by design — +// the next run's Phase 0 restores and removes it — so its presence is +// asserted here as the documented exception, not as a pass for litter. +func TestSyncCleanup_StagePathFailure(t *testing.T) { + dir := initProject(t, map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + }) + + files := (&fakeFilesClient{catalogID: "cid-new", stageID: "stage-1", versionID: "ver-1"}).withFailNthUpload(2) + + before := snapshotSyncTemps(t) + + e := newCleanupEngine(t, dir, files) + + plan, err := e.Plan() + require.NoError(t, err) + + _, err = e.Execute(plan) + require.Error(t, err, "the injected upload failure must fail the sync") + + assertNoNewSyncTemps(t, before) + assert.Empty(t, stateDirTmpLitter(t, dir), "Phase 6 never ran, but no AtomicWriteFile temp may survive either") + + // Deliberate design, pinned by the interruption tests: the rollback tree + // is the recovery copy for a run that died before it could restore. It is + // consumed (restored + removed) by Phase 0 of the next sync. + assert.DirExists(t, wapi.RollbackDir(dir), + "rollback tree must remain after a failed sync for stale-rollback recovery") +} + +// zipThresholdProject builds a project whose upload count crosses the +// ChooseUploader zip threshold, so Execute exercises buildZip's temp file. +func zipThresholdProject(t *testing.T) string { + t.Helper() + + files := make(map[string]string, 21) + + for i := 1; i <= 21; i++ { + name := fmt.Sprintf("file%02d.txt", i) + files[name] = fmt.Sprintf("content %d\n", i) + } + + return initProject(t, files) +} + +// TestSyncCleanup_ZipPathSuccess proves a successful zip-path sync removes +// its archive temp from the system temp dir and leaves no other litter. +func TestSyncCleanup_ZipPathSuccess(t *testing.T) { + dir := zipThresholdProject(t) + + files := &fakeFilesClient{catalogID: "cid-new", stageID: "stage-1", versionID: "ver-1"} + + before := snapshotSyncTemps(t) + + e := newCleanupEngine(t, dir, files) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Greater(t, len(plan.Uploads), 20, "precondition: the plan must cross the zip threshold") + + _, err = e.Execute(plan) + require.NoError(t, err) + + assert.Positive(t, files.UploadFromZipCalls(), "precondition: the zip path ran") + + assertNoNewSyncTemps(t, before) + assert.Empty(t, stateDirTmpLitter(t, dir)) + assert.NoDirExists(t, wapi.RollbackDir(dir)) +} + +// TestSyncCleanup_ZipPathFailure proves the failure path of buildZip also +// cleans its temp: the plan is built, then one upload's source file is +// deleted, so addToZip fails on an open error while the archive temp +// already exists on disk. buildZip must remove it before propagating. +func TestSyncCleanup_ZipPathFailure(t *testing.T) { + dir := zipThresholdProject(t) + + files := &fakeFilesClient{catalogID: "cid-new", stageID: "stage-1", versionID: "ver-1"} + + before := snapshotSyncTemps(t) + + e := newCleanupEngine(t, dir, files) + + plan, err := e.Plan() + require.NoError(t, err) + + // Remove one file that the plan uploads, after planning: the zip build + // then fails mid-archive with the temp file already on disk. + require.NoError(t, os.Remove(filepath.Join(dir, "file01.txt"))) + + _, err = e.Execute(plan) + require.Error(t, err, "the deleted source must fail the zip build") + + // Pin the failure to buildZip's per-file open ("... for zip: ..."), the + // only error text in the zip path carrying that phrase: if Execute ever + // failed earlier (rollback journal, a delete, an uploader swap), the + // litter assertions below would otherwise pass vacuously on a run that + // never reached the archive build. + require.ErrorContains(t, err, "for zip") + + assertNoNewSyncTemps(t, before) + assert.Empty(t, stateDirTmpLitter(t, dir)) +} diff --git a/internal/workload/sync/upload_concurrency_test.go b/internal/workload/sync/upload_concurrency_test.go new file mode 100644 index 000000000..16fe3fbcf --- /dev/null +++ b/internal/workload/sync/upload_concurrency_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// chunkBoundarySize is the default io.Copy buffer size (32 KiB). A file whose +// size is an exact multiple of this value exercises the chunk-boundary edge +// case: the last io.Copy chunk is exactly one buffer, not a partial. +const chunkBoundarySize = 32 * 1024 + +// TestConcurrentUpload_RaceFree_NoDroppedResults drives the stage uploader +// with 4-way concurrency (UploadConcurrency) over 13 files of mixed sizes — +// including 0-byte, small, and chunk-boundary sizes — and asserts: +// - No data race (automatic under `go test -race`). +// - Exactly one Sent entry per planned upload (no dropped result from a +// blocked channel send or an early worker return). +// - No file is uploaded twice (UploadToStageCalls == len(files)). +// - Every Sent hash equals the SHA-256 of the file content on disk. +// +// The result channel is buffered to len(files) so a send never blocks, and +// the orchestrator drains it after wg.Wait. A bug that closed resCh early, +// used an unbuffered channel, or returned before sending would drop a result +// and the Sent-count assertion would catch it. +// Fulfills VAL-UPLOAD-012. +func TestConcurrentUpload_RaceFree_NoDroppedResults(t *testing.T) { + // 13 files of mixed sizes: 0-byte, small, chunk-boundary, and larger. + fileSpecs := []struct { + path string + size int + }{ + {"f00.dat", 0}, + {"f01.dat", 0}, + {"f02.dat", 1}, + {"f03.dat", 10}, + {"f04.dat", 100}, + {"f05.dat", 500}, + {"f06.dat", 1000}, + {"f07.dat", chunkBoundarySize}, + {"f08.dat", chunkBoundarySize}, + {"f09.dat", chunkBoundarySize + 1}, + {"f10.dat", chunkBoundarySize * 2}, + {"f11.dat", 50000}, + {"f12.dat", 0}, + } + + dir := initProject(t, nil) + + // Write the files and build FileActions with known content. + actions := make([]FileAction, 0, len(fileSpecs)) + + expectedHashes := make(map[string]string, len(fileSpecs)) + + for _, spec := range fileSpecs { + content := generateContent(spec.path, spec.size) + + abs := filepath.Join(dir, spec.path) + require.NoError(t, os.WriteFile(abs, content, 0o644)) + + actions = append(actions, FileAction{ + Path: spec.path, + LocalHash: sha256Hex(content), + LocalSize: int64(len(content)), + }) + + expectedHashes[spec.path] = sha256Hex(content) + } + + const catalogID = "cid-conc" + + cid := catalogID + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-conc", + } + + engine := &Engine{ + projectDir: dir, + files: fake, + config: wapi.Config{CatalogID: &cid}, + } + + outcome, err := StageUploader{}.ApplyUploads(engine, actions) + + require.NoError(t, err, "all uploads must succeed with no fault injection") + + // Exactly one Sent entry per planned upload — no dropped result. + require.Len(t, outcome.Sent, len(fileSpecs), + "Sent must have exactly one entry per planned file (no dropped results)") + + // No file uploaded twice: UploadToStageCalls must equal the file count. + assert.Equal(t, len(fileSpecs), fake.UploadToStageCalls(), + "UploadToStage must be called exactly once per file (no duplicate uploads)") + + // ApplyStage called exactly once (the stage path applies in a single call). + assert.Equal(t, 1, fake.ApplyStageCalls(), + "ApplyStage must be called exactly once") + + // Every Sent entry must have the correct hash and size. + for _, fa := range actions { + entry, ok := outcome.Sent[fa.Path] + require.True(t, ok, "Sent must have an entry for %s", fa.Path) + + assert.Equal(t, expectedHashes[fa.Path], entry.Hash, + "Sent hash for %s must equal SHA-256 of file content", fa.Path) + assert.Equal(t, fa.LocalSize, entry.Size, + "Sent size for %s must equal file byte count", fa.Path) + } + + // The fake's server state must reflect every uploaded file with the + // correct checksum. This is the self-consistency check: AllFiles returns + // exactly what was uploaded. + all, err := fake.AllFiles(catalogID, "ver-conc") + require.NoError(t, err) + + assert.Len(t, all, len(fileSpecs), + "server must hold exactly one entry per uploaded file") + + for _, fa := range actions { + fm, ok := all[fa.Path] + require.True(t, ok, "server must have %s", fa.Path) + + assert.Equal(t, expectedHashes[fa.Path], fm.Hash, + "server checksum for %s must equal SHA-256 of content", fa.Path) + assert.Equal(t, fa.LocalSize, fm.Size, + "server size for %s must equal file byte count", fa.Path) + } +} + +// generateContent produces deterministic content of the given size for a +// given path. The content is a repeating pattern of the path name so each +// file has distinct bytes (avoiding hash collisions) while being fully +// deterministic across runs. +func generateContent(path string, size int) []byte { + if size == 0 { + return []byte{} + } + + pattern := path + ":" + + return []byte(strings.Repeat(pattern, size/len(pattern)+1))[:size] +} diff --git a/internal/workload/sync/upload_failure_test.go b/internal/workload/sync/upload_failure_test.go new file mode 100644 index 000000000..56be07c14 --- /dev/null +++ b/internal/workload/sync/upload_failure_test.go @@ -0,0 +1,840 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// preSyncState captures the on-disk state of manifest.json, config.json, and +// the project files so a test can assert nothing was advanced after a failed +// sync. The manifest and config are captured as raw bytes for byte-identical +// comparison; the file contents are captured per-path so a rollback failure +// that leaves a modified file is caught. +type preSyncState struct { + manifestBytes []byte + configBytes []byte + fileContents map[string][]byte +} + +// captureState reads manifest.json, config.json, and the given project files +// as raw bytes. The caller lists the relative paths whose content matters +// for the rollback assertion. +func captureState(t *testing.T, dir string, files []string) preSyncState { + t.Helper() + + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + contents := make(map[string][]byte, len(files)) + + for _, rel := range files { + content, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + + contents[rel] = content + } + + return preSyncState{ + manifestBytes: mBytes, + configBytes: cBytes, + fileContents: contents, + } +} + +// assertStateUntouched asserts manifest.json and config.json are byte-identical +// to their pre-sync content and every captured project file is unchanged. +// This is the core integrity assertion for every failure mode: a failed sync +// must never advance persisted state. +func assertStateUntouched(t *testing.T, dir string, pre preSyncState) { + t.Helper() + + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, pre.manifestBytes, mBytes, + "manifest.json must be byte-identical to its pre-sync content") + + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content") + + for rel, expected := range pre.fileContents { + content, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + + assert.Equal(t, expected, content, + "project file %s must be unchanged after a failed sync (rollback)", rel) + } +} + +// assertErrNamesAPlannedPath asserts the error message names at least one of +// the plan's upload paths. Which path loses the race is nondeterministic +// under 4-way concurrency — withFailNthUpload fails the Nth call received, +// and call order follows goroutine scheduling — so a hardcoded path would +// be flaky. Set membership over the planned paths is the strongest +// deterministic claim: production wraps every upload failure with its path +// ("upload : ..."), so an error that names none of them means the +// wrapping was lost. +func assertErrNamesAPlannedPath(t *testing.T, err error, plan *SyncPlan) { + t.Helper() + + require.Error(t, err) + + msg := err.Error() + + for _, p := range uploadPathsOf(plan) { + if strings.Contains(msg, p) { + return + } + } + + t.Errorf("error %q must name at least one planned upload path (planned: %v)", msg, uploadPathsOf(plan)) +} + +// newSyncedEngine builds an engine over a synced project with the given files +// modified to introduce pending changes. The fake and artifact store are +// returned so the caller can inspect counters and configure fault injection +// before running the engine. +func newSyncedEngine(t *testing.T, files map[string]string, mods map[string]string, fake *fakeFilesClient) (*Engine, string) { + t.Helper() + + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, files, catalogID, versionID) + + for rel, content := range mods { + modifyFile(t, dir, rel, content) + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e, dir +} + +// --- VAL-UPLOAD-011: Missing streamed hash hard-fails and persists nothing --- + +// TestMissingSentEntry_BuildNewBaseManifestHardFails verifies that +// buildNewBaseManifest returns an error naming the missing path when the +// Sent map lacks an entry for an uploaded file. The error must NOT fall back +// to the Phase-2 planned hash — a per-path fallback IS the original poisoning +// bug. This is the unit-level guard; the engine-level guard +// (TestMissingSentEntry_Phase6DoesNotAdvanceManifest) verifies the on-disk +// consequence. +func TestMissingSentEntry_BuildNewBaseManifestHardFails(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid", + VersionID: "ver", + Sent: map[string]FileEntry{}, // missing app.py + }, + } + + _, err := buildNewBaseManifest(e, "ver", time.Now()) + + require.Error(t, err, "missing Sent entry must hard-fail, not fall back") + assert.Contains(t, err.Error(), "app.py", + "error must name the missing path") +} + +// TestMissingSentEntry_Phase6DoesNotAdvanceManifest verifies that when +// buildNewBaseManifest fails inside phase6State (because Sent is missing an +// entry), neither manifest.json nor config.json is written — both retain +// their exact pre-sync content. This is the on-disk consequence of the +// hard-fail combined with the manifest-before-config write ordering. +// +// The invariant: config never moves ahead of the manifest. Phase 6 now +// builds and writes the manifest first, then writes config. When +// buildNewBaseManifest fails, neither write has occurred, so config stays +// at the old version. The missing-Sent scenario is not reachable through +// normal operation (uploadFilesParallel returns an error if any upload +// fails, so a partial Sent never reaches Phase 6), but the ordering +// invariant is what protects the reachable hazard (SaveManifest I/O +// failure), so this test guards it directly. +func TestMissingSentEntry_Phase6DoesNotAdvanceManifest(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + pre := captureState(t, dir, []string{"app.py"}) + + // Construct an engine in the state Phase 6 would see after a successful + // Phase 5 where the Sent map is missing app.py. This is not reachable + // through the engine (uploadFilesParallel returns an error on any upload + // failure), so we test phase6State directly. + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: "ver-new", + Sent: map[string]FileEntry{}, // missing app.py + }, + newCatalogID: "cid-new", + newVersionID: "ver-new", + nowFn: time.Now, + } + + err = phase6State(e) + + require.Error(t, err, "phase6State must fail when Sent is missing") + assert.Contains(t, err.Error(), "app.py", + "error must name the missing path") + + // manifest.json must NOT be written — SaveManifest runs after + // buildNewBaseManifest, which failed. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, pre.manifestBytes, mBytes, + "manifest.json must be byte-identical to its pre-sync content") + + // config.json must NOT be advanced. Phase 6 now writes the manifest + // before config: buildNewBaseManifest runs first, and when it fails, + // neither SaveManifest nor SaveConfig has executed. Config stays at + // the old version, preserving the invariant that config never moves + // ahead of the manifest. + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content — config never moves ahead of the manifest") + + // Config must still point at the old version, proving the invariant. + unchangedCfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, unchangedCfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *unchangedCfg.LastSyncedVersionID, + "config LastSyncedVersionID must NOT be advanced when the manifest build fails") + + // The project file must be unchanged (Phase 6 does not modify the tree). + content, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err) + + assert.Equal(t, pre.fileContents["app.py"], content, + "app.py must be unchanged") +} + +// TestMissingSentEntry_DoesNotFallBackToPhase2Hash verifies that the error +// from buildNewBaseManifest does not silently produce a manifest with the +// Phase-2 hash. The manifest is not written at all, so there is no entry to +// check — but we also verify the returned manifest is the zero value so no +// caller can accidentally use it. +func TestMissingSentEntry_DoesNotFallBackToPhase2Hash(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid", + VersionID: "ver", + Sent: map[string]FileEntry{}, + }, + } + + manifest, err := buildNewBaseManifest(e, "ver", time.Now()) + + require.Error(t, err) + assert.Empty(t, manifest.Files, + "no manifest must be produced when Sent is missing — no fallback") +} + +// --- Phase 6 write-ordering invariants --- + +// TestSaveManifestFailure_DoesNotAdvanceConfig verifies that when SaveManifest +// fails inside phase6State (injected by making manifest.json a directory so +// the atomic rename fails), config.json is NOT advanced. Phase 6 now writes +// the manifest before config: SaveManifest runs first, and when it fails, +// SaveConfig has not executed. Config stays at the old version, preserving +// the invariant that config never moves ahead of the manifest. +// +// This is the reachable hazard (SaveManifest I/O failure) that the write +// reorder fixes. A stale config paired with an un-advanced manifest makes +// the next sync detect drift, fetch AllFiles, and rebuild BASE from real +// remote data — safe and self-healing. The converse (advanced config, +// stale manifest) silently poisons BASE. +func TestSaveManifestFailure_DoesNotAdvanceConfig(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + pre := captureState(t, dir, []string{"app.py"}) + + // Make manifest.json a directory so AtomicWriteFile's rename fails. + // SaveManifest creates a temp file in the parent dir, then renames it + // over the target — renaming a file over a directory fails with EISDIR. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + require.NoError(t, os.Remove(mPath)) + require.NoError(t, os.Mkdir(mPath, 0o755)) + + t.Cleanup(func() { _ = os.RemoveAll(mPath) }) + + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + }, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: "ver-new", + Sent: map[string]FileEntry{ + "app.py": {Hash: sha256Hex([]byte("print('changed')\n")), Size: 14}, + }, + }, + newCatalogID: "cid-new", + newVersionID: "ver-new", + nowFn: time.Now, + } + + err = phase6State(e) + + require.Error(t, err, "phase6State must fail when SaveManifest fails") + assert.Contains(t, err.Error(), "save manifest", + "error must come from SaveManifest, not SaveConfig") + + // config.json must NOT be advanced — SaveConfig runs after SaveManifest, + // which failed, so config was never written. + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content — config never moves ahead of the manifest") + + unchangedCfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, unchangedCfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *unchangedCfg.LastSyncedVersionID, + "config LastSyncedVersionID must NOT be advanced when SaveManifest fails") +} + +// TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs verifies the safe +// failure direction: when SaveManifest succeeds but SaveConfig fails (injected +// by making config.json a directory), the manifest IS advanced (new version, +// new hashes) while config stays stale (old version). The next sync detects +// drift (config old != artifact new), fetches AllFiles rather than +// fast-pathing, and converges — config is updated to match the manifest. +// +// This is the self-healing property that makes manifest-before-config the +// safe ordering: an advanced manifest paired with a stale config makes the +// next run re-sync, whereas an advanced config paired with a stale manifest +// silently poisons BASE and reports "Up to date." forever. +func TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + newVerID = "ver-new" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + pre := captureState(t, dir, []string{"app.py"}) + + // Make config.json a directory so SaveConfig's atomic rename fails. + // SaveManifest writes to manifest.json (a regular file), so it succeeds; + // SaveConfig writes to config.json (now a directory), so it fails. + cPath := wapi.ConfigPath(dir) + + require.NoError(t, os.Remove(cPath)) + require.NoError(t, os.Mkdir(cPath, 0o755)) + + t.Cleanup(func() { _ = os.RemoveAll(cPath) }) + + streamedContent := "print('changed')\n" + streamedHash := sha256Hex([]byte(streamedContent)) + streamedSize := int64(len(streamedContent)) + + // Modify the disk file to match what was "uploaded" so that after + // SaveManifest writes the streamed hash, local == base and the next + // sync's plan is empty (the point is drift detection, not finding work). + modifyFile(t, dir, "app.py", streamedContent) + + // Include .drignore in remote so the manifest carries it through — + // buildNewBaseManifest seeds from remote, then overrides uploaded paths. + ignoreHash, ignoreSize, err := hashLocal(t, dir, ignore.FileName) + require.NoError(t, err) + + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + ignore.FileName: {Hash: ignoreHash, Size: ignoreSize}, + }, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: newVerID, + Sent: map[string]FileEntry{ + "app.py": {Hash: streamedHash, Size: streamedSize}, + }, + }, + newCatalogID: "cid-new", + newVersionID: newVerID, + nowFn: time.Now, + } + + err = phase6State(e) + + require.Error(t, err, "phase6State must fail when SaveConfig fails") + assert.Contains(t, err.Error(), "save config", + "error must come from SaveConfig, not SaveManifest") + + // manifest.json IS advanced — SaveManifest ran before SaveConfig and + // succeeded. The manifest now carries the new version and the streamed hash. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + assert.Equal(t, wapi.ManifestVersion, manifest.Version) + + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, newVerID, *manifest.SyncedVersionID, + "manifest syncedVersionId must be advanced") + + fm, ok := manifest.Files["app.py"] + require.True(t, ok) + + assert.Equal(t, streamedHash, fm.Hash, + "manifest must carry the streamed hash") + assert.Equal(t, streamedSize, fm.Size, + "manifest must carry the streamed size") + + // Restore config.json with the old content so the next sync can load it. + // This simulates the real-world state after a SaveConfig failure: the + // file retains its pre-sync content because the atomic write never landed. + require.NoError(t, os.RemoveAll(cPath)) + require.NoError(t, os.WriteFile(cPath, pre.configBytes, 0o644)) + + // Build the fake's server state from the manifest that SaveManifest wrote, + // so AllFiles returns exactly what BASE describes. This ensures the next + // sync sees base == remote and the plan is empty — the point is that the + // sync runs the full pipeline (drift detection, AllFiles fetch) rather + // than fast-pathing, not that it finds work to do. + serverFiles := make(map[string]filesapi.FileMeta, len(manifest.Files)) + + for path, fm := range manifest.Files { + serverFiles[path] = filesapi.FileMeta{Hash: fm.Hash, Size: fm.Size} + } + + // The next sync must detect drift (config old != artifact new), fetch + // AllFiles rather than fast-pathing, and converge. + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: newVerID, + }).withVersion(catalogID, newVerID, serverFiles) + + e2, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVerID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + plan, err := e2.Plan() + require.NoError(t, err) + + // Drift was detected: AllFiles was called (not fast-pathed). + assert.Equal(t, 1, fake.AllFilesCalls(), + "drift must trigger an AllFiles round-trip, not the fast path") + + // The plan should be empty: manifest (advanced) == remote (AllFiles), + // and local == base (no disk changes since the failed sync). But the + // sync ran the full pipeline — it did not silently fast-path. + assert.True(t, plan.IsEmpty(), + "plan should be empty — manifest matches remote, no disk changes") + + // Execute to run Phase 6, which updates config to the new version. + result, err := e2.Execute(plan) + require.NoError(t, err) + + require.NotNil(t, result) + + // Config must now converge to the new version — self-healing. + convergedCfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, convergedCfg.LastSyncedVersionID) + assert.Equal(t, newVerID, *convergedCfg.LastSyncedVersionID, + "config must converge to the new version after the self-healing sync") + + // Manifest and config must agree. + convergedManifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + require.NotNil(t, convergedManifest.SyncedVersionID) + assert.Equal(t, *convergedCfg.LastSyncedVersionID, *convergedManifest.SyncedVersionID, + "config and manifest must agree on the version after convergence") +} + +// --- VAL-UPLOAD-011b / VAL-UPLOAD-013: Failure modes through the engine --- + +// TestPartialSent_WorkerError_FailsCleanly verifies that when some workers +// succeed and one errors (producing a partial Sent internally), the sync +// fails, Phase 6 never runs, and manifest.json and config.json retain their +// pre-sync content. The partial Sent is never merged with a fallback because +// uploadFilesParallel returns an error, not a partial map. +// Fulfills VAL-UPLOAD-011 (partial Sent) and VAL-UPLOAD-013. +func TestPartialSent_WorkerError_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + } + + mods := map[string]string{ + "a.py": "AAA\n", + "b.py": "BBB\n", + "c.py": "CCC\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(2) + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 3, "all three files should be pending uploads") + + pre := captureState(t, dir, []string{"a.py", "b.py", "c.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when a worker errors") + assertErrNamesAPlannedPath(t, err, plan) + + assertStateUntouched(t, dir, pre) + + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called when an upload fails — no partial apply") +} + +// TestUploadFailure_OneFileFailsLate_FailsCleanly verifies that when one file +// fails after others have been uploaded to the staging area, the sync fails, +// Phase 6 never runs, and all persisted state is untouched. Files staged but +// never applied must not appear in the manifest. +// Fulfills VAL-UPLOAD-013. +func TestUploadFailure_OneFileFailsLate_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + "d.py": "ddd\n", + "e.py": "eee\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + "c.py": "CCCC\n", + "d.py": "DDDD\n", + "e.py": "EEEE\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(4) + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 5, "all five files should be pending uploads") + + pre := captureState(t, dir, []string{"a.py", "b.py", "c.py", "d.py", "e.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when one file fails late") + assertErrNamesAPlannedPath(t, err, plan) + + assertStateUntouched(t, dir, pre) + + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called — staged-but-unapplied files must not reach the manifest") +} + +// TestUploadFailure_AllFilesFail_FailsCleanly verifies that when every upload +// fails, the sync fails immediately, Phase 6 never runs, and all persisted +// state is untouched. +// Fulfills VAL-UPLOAD-013. +func TestUploadFailure_AllFilesFail_FailsCleanly(t *testing.T) { + files := map[string]string{ + "app.py": "print('orig')\n", + } + + mods := map[string]string{ + "app.py": "print('changed')\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(1) + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 1, "one file should be pending upload") + + pre := captureState(t, dir, []string{"app.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when all uploads fail") + assert.Contains(t, err.Error(), "app.py", + "error must name the failing path") + + assertStateUntouched(t, dir, pre) + + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called when all uploads fail") +} + +// TestUploadFailure_ApplyStageFails_FailsCleanly verifies that when all files +// are staged successfully but ApplyStage fails, the sync fails, Phase 6 +// never runs, and all persisted state is untouched. Files staged but never +// applied must not appear in the manifest. +// Fulfills VAL-UPLOAD-013. +func TestUploadFailure_ApplyStageFails_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailApplyStage() + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 2, "both files should be pending uploads") + + pre := captureState(t, dir, []string{"a.py", "b.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when ApplyStage fails") + assert.Contains(t, err.Error(), "apply stage", + "error must come from the apply-stage step") + + assertStateUntouched(t, dir, pre) + + // ApplyStage WAS called (it failed), but the resulting version must not + // be persisted. The manifest must not record hashes for staged-but- + // unapplied files. + assert.Equal(t, 1, fake.ApplyStageCalls(), + "ApplyStage must be called (it failed)") +} + +// --- VAL-UPLOAD-021: File deleted between Plan and Execute --- + +// TestDeletedFileBetweenPlanAndExecute_FailsCleanly verifies that a planned +// upload file deleted from disk between Plan() and Execute() produces an +// error naming that path (no panic), with manifest and config unchanged and +// rollback performed. VAL-UPLOAD-021's letter is about the upload call +// itself, so it is asserted directly on the fake's UploadToStageCalls +// counter: the deleted file must never reach UploadToStage (its open fails +// before the call is issued), so the count can never reach the full plan +// size. The remaining planned files MAY complete in-flight uploads before +// the first error's cancel lands — uploadFilesParallel documents that +// in-flight workers finish their current upload — so a specific count would +// pin goroutine scheduling, and only the upper bound is deterministic. +// Fulfills VAL-UPLOAD-021. +func TestDeletedFileBetweenPlanAndExecute_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + "c.py": "CCCC\n", + } + + fake := &fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + } + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 3, "all three files should be pending uploads") + + // Capture the state of the files that will NOT be deleted. + pre := captureState(t, dir, []string{"a.py", "c.py"}) + + // Between Plan and Execute, delete b.py from disk. os.Open will return + // os.ErrNotExist when the uploader tries to read it. + require.NoError(t, os.Remove(filepath.Join(dir, "b.py"))) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when a planned file is missing") + assert.Contains(t, err.Error(), "b.py", + "error must name the deleted path") + assert.NotContains(t, err.Error(), "panic", + "error must not be a panic") + + // manifest.json and config.json must be byte-identical to pre-sync. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, pre.manifestBytes, mBytes, + "manifest.json must be byte-identical to its pre-sync content") + + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content") + + // The remaining files (not deleted by the test) must be unchanged — + // the rollback restored the working tree to its pre-Execute state. + for _, rel := range []string{"a.py", "c.py"} { + content, err := os.ReadFile(filepath.Join(dir, rel)) + require.NoError(t, err) + + assert.Equal(t, pre.fileContents[rel], content, + "file %s must be unchanged after the failed sync", rel) + } + + // VAL-UPLOAD-021, asserted directly: the deleted file never reaches + // UploadToStage, so the stage-call count stays below the full plan size + // no matter how the workers were scheduled. + assert.Less(t, fake.UploadToStageCalls(), len(plan.Uploads), + "the deleted file must never reach UploadToStage; at most the remaining planned files may have been uploaded in-flight before the error stopped the pipeline") +} diff --git a/internal/workload/sync/upload_scaffolding_test.go b/internal/workload/sync/upload_scaffolding_test.go new file mode 100644 index 000000000..cf8a1a369 --- /dev/null +++ b/internal/workload/sync/upload_scaffolding_test.go @@ -0,0 +1,174 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// assertZeroUploadCalls asserts that no upload-side network calls were issued +// on the fake: stage create, upload-to-stage, apply-stage, and zip upload all +// zero. This is the core assertion for VAL-UPLOAD-015 and VAL-UPLOAD-017. +func assertZeroUploadCalls(t *testing.T, fake *fakeFilesClient) { + t.Helper() + + assert.Equal(t, 0, fake.CreateStageCalls(), "CreateStage must not be called") + assert.Equal(t, 0, fake.UploadToStageCalls(), "UploadToStage must not be called") + assert.Equal(t, 0, fake.ApplyStageCalls(), "ApplyStage must not be called") + assert.Equal(t, 0, fake.UploadFromZipCalls(), "UploadFromZip must not be called") +} + +// TestEmptyPlan_ZeroUploadCalls verifies VAL-UPLOAD-015: when the plan has +// zero uploads (all files unchanged), no upload API calls are issued. The +// per-method counters on the fake prove this at the Go level. +func TestEmptyPlan_ZeroUploadCalls(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "main.py": "def main(): pass\n", + }, catalogID, versionID) + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: "ver-should-not-be-used", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // An empty plan (all files unchanged) issues zero upload calls. + assertZeroUploadCalls(t, fake) +} + +// TestDryRun_ZeroUploadCalls verifies VAL-UPLOAD-017: a --dry-run run with +// pending changes issues zero upload-side calls on the fake. The plan is +// non-empty (a file was modified), but Run returns after Plan without calling +// Execute, so no upload methods are invoked. +func TestDryRun_ZeroUploadCalls(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change so the plan is non-empty. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: "ver-should-not-be-used", + } + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + // Run stops after Plan because DryRun is true. The plan is non-empty + // (a file was modified), but Execute is never called, so no upload + // methods are invoked. The positive control (TestNonEmptyPlan_UploadCallsAreNonZero) + // proves the same setup with DryRun=false DOES issue upload calls, so + // the zero-count here is not vacuously true. + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // No upload-side calls despite a non-empty plan. + assertZeroUploadCalls(t, fake) +} + +// TestNonEmptyPlan_UploadCallsAreNonZero is the positive control for +// VAL-UPLOAD-015 and VAL-UPLOAD-017: a non-empty plan in a non-dry-run sync +// DOES issue upload-side calls. This proves the counters work and the +// zero-call assertions above are not vacuously true. +func TestNonEmptyPlan_UploadCallsAreNonZero(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // A non-empty plan in a non-dry-run sync must issue upload calls. + assert.Equal(t, 1, fake.CreateStageCalls(), "CreateStage must be called for a non-empty plan") + assert.Positive(t, fake.UploadToStageCalls(), "UploadToStage must be called for a non-empty plan") + assert.Equal(t, 1, fake.ApplyStageCalls(), "ApplyStage must be called for a non-empty plan") + assert.Equal(t, 0, fake.UploadFromZipCalls(), "zip path must not be used for a small upload") +} diff --git a/internal/workload/sync/upload_stage.go b/internal/workload/sync/upload_stage.go index 8949f092b..6efa14b67 100644 --- a/internal/workload/sync/upload_stage.go +++ b/internal/workload/sync/upload_stage.go @@ -16,6 +16,7 @@ package sync import ( "fmt" + "io" "os" "path/filepath" "sync" @@ -27,28 +28,34 @@ import ( // missing, create stage, upload each file, apply stage. type StageUploader struct{} -// ApplyUploads pushes files via the stage workflow. -func (StageUploader) ApplyUploads(e *Engine, files []FileAction) (string, string, error) { +// ApplyUploads pushes files via the stage workflow and returns the per-path +// streamed hashes so Phase 6 can record what the server actually received. +func (StageUploader) ApplyUploads(e *Engine, files []FileAction) (UploadOutcome, error) { catalogID, err := ensureCatalog(e) if err != nil { - return "", "", err + return UploadOutcome{}, err } stage, err := e.files.CreateStage(catalogID) if err != nil { - return "", "", fmt.Errorf("create stage: %w", err) + return UploadOutcome{}, fmt.Errorf("create stage: %w", err) } - if err := uploadFilesParallel(e, catalogID, stage.StageID, files); err != nil { - return "", "", err + sent, err := uploadFilesParallel(e, catalogID, stage.StageID, files) + if err != nil { + return UploadOutcome{}, err } apply, err := e.files.ApplyStage(catalogID, stage.StageID, filesapi.OverwriteReplace) if err != nil { - return "", "", fmt.Errorf("apply stage: %w", err) + return UploadOutcome{}, fmt.Errorf("apply stage: %w", err) } - return catalogID, apply.CatalogVersionID, nil + return UploadOutcome{ + CatalogID: catalogID, + VersionID: apply.CatalogVersionID, + Sent: sent, + }, nil } // ensureCatalog returns the catalog ID, creating a new one when neither @@ -66,12 +73,23 @@ func ensureCatalog(e *Engine) (string, error) { return cat.CatalogID, nil } -// uploadFilesParallel uploads files up to UploadConcurrency. The first -// error closes done to stop other workers from starting; in-flight -// workers still finish their current upload before the function returns. -func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileAction) error { +// uploadResult carries one file's streamed hash and size from a worker +// goroutine to the orchestrator via a buffered result channel, matching the +// existing errCh convention rather than adding a mutex-guarded map. +type uploadResult struct { + path string + entry FileEntry +} + +// uploadFilesParallel uploads files up to UploadConcurrency and collects +// per-path streamed hashes. The first error closes done to stop other workers +// from starting; in-flight workers still finish their current upload before +// the function returns. Both errCh and resCh are buffered to len(files) so a +// send never blocks or drops. resCh is closed by the orchestrator only, after +// wg.Wait, following the same convention as errCh. +func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileAction) (map[string]FileEntry, error) { if len(files) == 0 { - return nil + return nil, nil } done := make(chan struct{}) @@ -83,6 +101,7 @@ func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileActio sem := make(chan struct{}, UploadConcurrency) errCh := make(chan error, len(files)) + resCh := make(chan uploadResult, len(files)) var wg sync.WaitGroup @@ -103,39 +122,68 @@ func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileActio defer func() { <-sem }() - if err := uploadOneToStage(e, catalogID, stageID, fa); err != nil { + entry, err := uploadOneToStage(e, catalogID, stageID, fa) + if err != nil { select { case errCh <- err: cancel() default: } + + return } + + resCh <- uploadResult{path: fa.Path, entry: entry} }() } wg.Wait() close(errCh) + close(resCh) if err := <-errCh; err != nil { - return err + return nil, err + } + + sent := make(map[string]FileEntry, len(files)) + + for r := range resCh { + sent[r.path] = r.entry } - return nil + return sent, nil } -func uploadOneToStage(e *Engine, catalogID, stageID string, fa FileAction) error { +func uploadOneToStage(e *Engine, catalogID, stageID string, fa FileAction) (FileEntry, error) { abs := filepath.Join(e.projectDir, filepath.FromSlash(fa.Path)) f, err := os.Open(abs) if err != nil { - return fmt.Errorf("open %s: %w", fa.Path, err) + return FileEntry{}, fmt.Errorf("open %s: %w", fa.Path, err) } defer func() { _ = f.Close() }() - if err := e.files.UploadToStage(catalogID, stageID, fa.Path, fa.LocalSize, f); err != nil { - return fmt.Errorf("upload %s: %w", fa.Path, err) + // Content-length from the already-open handle, not a fresh os.Stat: a + // fresh stat reopens the TOCTOU window and can follow a symlink swapped + // in since the plan phase. Do not abort merely because this differs from + // fa.LocalSize — a mid-run edit is legitimate; upload the current bytes + // and let the recorded hash be the truth. + stat, err := f.Stat() + if err != nil { + return FileEntry{}, fmt.Errorf("stat %s: %w", fa.Path, err) + } + + size := stat.Size() + + // Hash the exact bytes streamed, not the Phase-2 planned hash. TeeReader + // is the right primitive: UploadToStage pipes the body through io.Copy, + // so every byte that reaches the wire passes through the hasher. + h := newStreamHasher() + + if err := e.files.UploadToStage(catalogID, stageID, fa.Path, size, io.TeeReader(f, h)); err != nil { + return FileEntry{}, fmt.Errorf("upload %s: %w", fa.Path, err) } - return nil + return streamedEntry(h, size), nil } diff --git a/internal/workload/sync/upload_stage_test.go b/internal/workload/sync/upload_stage_test.go new file mode 100644 index 000000000..5a2407c4c --- /dev/null +++ b/internal/workload/sync/upload_stage_test.go @@ -0,0 +1,166 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestStageUploadHashesStreamedBytes verifies that the per-path hash in +// UploadOutcome.Sent is computed from the bytes the server actually received, +// not from a re-hash of the file on disk. The fake records the bytes it +// received, so the assertion is "the recorded hash matches the received +// bytes' SHA-256" rather than "it matches the disk file.". +func TestStageUploadHashesStreamedBytes(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('orig')\n", + }, catalogID, versionID) + + // Introduce a pending change so Plan sees an upload. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + + // The outcome must carry a Sent entry for app.py. + require.NotNil(t, e.uploadOutcome, "Phase 5 must store the upload outcome on the engine") + + sent, ok := e.uploadOutcome.Sent["app.py"] + require.True(t, ok, "Sent must have an entry for app.py") + + // The streamed hash must equal the SHA-256 of the bytes the fake + // received, proving the hash is of the streamed bytes, not a re-hash + // of the disk file. + receivedBytes, ok := fake.uploadedFiles["app.py"] + require.True(t, ok, "fake must have recorded the uploaded bytes") + assert.Equal(t, sha256Hex(receivedBytes), sent.Hash, + "Sent hash must match the SHA-256 of the bytes the server received") + assert.Equal(t, int64(len(receivedBytes)), sent.Size, + "Sent size must match the byte count the server received") +} + +// TestStageUploadContentLengthFromOpenHandle verifies that content-length is +// derived from Stat() on the already-open handle, not from the Phase-2 planned +// size. A file that changes size between Plan and Execute must have the +// streamed size match the new size, not the planned size. This is only possible +// if the size comes from the open handle at upload time. +func TestStageUploadContentLengthFromOpenHandle(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('orig')\n", + }, catalogID, versionID) + + // Introduce a pending change so Plan sees an upload. + modifyFile(t, dir, "app.py", "print('ho')\n") // 11 bytes + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + plannedSize := plan.Uploads[0].LocalSize + + // Between Plan and Execute, grow the file. The streamed size must + // match the new size, not the Phase-2 planned size. + newContent := "print('hello world')\n" // 19 bytes + modifyFile(t, dir, "app.py", newContent) + + require.NotEqual(t, plannedSize, int64(len(newContent)), + "planned and streamed sizes must differ for the test to be meaningful") + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + + require.NotNil(t, e.uploadOutcome) + + sent, ok := e.uploadOutcome.Sent["app.py"] + require.True(t, ok) + + // The streamed size must match the bytes at Execute time, not the + // Phase-2 planned size. This proves content-length came from the open + // handle's Stat, not from fa.LocalSize. + assert.Equal(t, int64(len(newContent)), sent.Size, + "streamed size must match the file at Execute time, not the Phase-2 planned size") + assert.NotEqual(t, plannedSize, sent.Size, + "streamed size must differ from the Phase-2 planned size") + + // The hash must also match the new content. + assert.Equal(t, sha256Hex([]byte(newContent)), sent.Hash, + "streamed hash must match the content at Execute time") + + // The fake must have received exactly the new bytes. + receivedBytes := fake.uploadedFiles["app.py"] + assert.Equal(t, []byte(newContent), receivedBytes, + "server must have received the bytes present at Execute time") +} diff --git a/internal/workload/sync/upload_zip.go b/internal/workload/sync/upload_zip.go index 444e4a203..6727d849f 100644 --- a/internal/workload/sync/upload_zip.go +++ b/internal/workload/sync/upload_zip.go @@ -24,84 +24,125 @@ import ( "time" "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/log" ) // ZipUploader implements the async zip workflow: build a zip locally, // POST it to FilesAPI, poll until terminal. type ZipUploader struct{} -// ApplyUploads zips the files, POSTs, and polls until done. -func (ZipUploader) ApplyUploads(e *Engine, files []FileAction) (string, string, error) { - zipPath, err := buildZip(e.projectDir, files) +// ApplyUploads zips the files, POSTs, polls until done, and returns the +// per-path streamed hashes so Phase 6 can record what entered the archive. +func (ZipUploader) ApplyUploads(e *Engine, files []FileAction) (UploadOutcome, error) { + zipPath, sent, err := buildZip(e.projectDir, files) if err != nil { - return "", "", err + return UploadOutcome{}, err } defer func() { _ = os.Remove(zipPath) }() zipFile, err := os.Open(zipPath) if err != nil { - return "", "", fmt.Errorf("open built zip: %w", err) + return UploadOutcome{}, fmt.Errorf("open built zip: %w", err) } defer func() { _ = zipFile.Close() }() stat, err := zipFile.Stat() if err != nil { - return "", "", fmt.Errorf("stat built zip: %w", err) + return UploadOutcome{}, fmt.Errorf("stat built zip: %w", err) } resp, err := postZip(e, zipFile, stat.Size()) if err != nil { - return "", "", err + return UploadOutcome{}, err } // Small archives complete inline (201, no statusId); larger ones // come back 202 with a statusId we then poll. if resp.StatusID != "" { if err := waitForCompletion(e, resp.StatusID); err != nil { - return "", "", err + return UploadOutcome{}, err } } - return resp.CatalogID, resp.CatalogVersionID, nil + return UploadOutcome{ + CatalogID: resp.CatalogID, + VersionID: resp.CatalogVersionID, + Sent: sent, + }, nil } -// buildZip writes a zip archive to a temp file. Buffering on disk -// keeps very large zips from pinning a multi-GiB allocation. -func buildZip(projectDir string, files []FileAction) (string, error) { +// buildZip writes a zip archive to a temp file and returns the per-path +// streamed hashes. Buffering on disk keeps very large zips from pinning a +// multi-GiB allocation. +// +// The temp file is closed before any failure-path removal: on Windows an +// open handle blocks os.Remove (Go opens files without FILE_SHARE_DELETE, +// so the remove fails with a sharing violation), and the leak-prone order — +// remove-then-close via defers — would strand the archive in the system +// temp dir on every build failure. POSIX unlinks an open file freely, which +// is exactly why the leak only ever showed on Windows. +func buildZip(projectDir string, files []FileAction) (string, map[string]FileEntry, error) { tmp, err := os.CreateTemp("", "wapi-sync-*.zip") if err != nil { - return "", fmt.Errorf("create zip tempfile: %w", err) + return "", nil, fmt.Errorf("create zip tempfile: %w", err) + } + + sent, err := writeZip(tmp, projectDir, files) + + // A close error must not mask the real failure, so it is adopted only + // when the archive built cleanly. + if closeErr := tmp.Close(); err == nil { + err = closeErr + } + + if err != nil { + // The remove itself can fail — a Windows sharing violation was the + // original leak — so a discarded error here would hide exactly the + // recurrence this cleanup exists to prevent. Log it instead. + if rmErr := os.Remove(tmp.Name()); rmErr != nil { + log.Warn("zip temp cleanup failed; the archive may be stranded in the system temp dir", + "path", tmp.Name(), "err", rmErr) + } + + return "", nil, err } - defer func() { _ = tmp.Close() }() + return tmp.Name(), sent, nil +} +// writeZip streams every planned file into tmp and returns the per-path +// streamed hashes. It owns no lifecycle: the caller closes the file and +// removes it on failure. +func writeZip(tmp *os.File, projectDir string, files []FileAction) (map[string]FileEntry, error) { zw := zip.NewWriter(tmp) - defer func() { _ = zw.Close() }() + sent := make(map[string]FileEntry, len(files)) for _, fa := range files { abs := filepath.Join(projectDir, filepath.FromSlash(fa.Path)) - if err := addToZip(zw, abs, fa.Path); err != nil { - _ = os.Remove(tmp.Name()) - return "", err + entry, err := addToZip(zw, abs, fa.Path) + if err != nil { + // Lifecycle contract: zw is deliberately left unclosed here — the caller closes the temp file and removes the dead archive, and closing would only flush a doomed central directory that risks masking the real error. + return nil, err } + + sent[fa.Path] = entry } if err := zw.Close(); err != nil { - _ = os.Remove(tmp.Name()) - return "", fmt.Errorf("close zip writer: %w", err) + return nil, fmt.Errorf("close zip writer: %w", err) } - return tmp.Name(), nil + return sent, nil } -func addToZip(zw *zip.Writer, src, archivePath string) error { +func addToZip(zw *zip.Writer, src, archivePath string) (FileEntry, error) { in, err := os.Open(src) if err != nil { - return fmt.Errorf("open %s for zip: %w", src, err) + return FileEntry{}, fmt.Errorf("open %s for zip: %w", src, err) } defer func() { _ = in.Close() }() @@ -110,14 +151,22 @@ func addToZip(zw *zip.Writer, src, archivePath string) error { w, err := zw.CreateHeader(hdr) if err != nil { - return fmt.Errorf("zip header for %s: %w", archivePath, err) + return FileEntry{}, fmt.Errorf("zip header for %s: %w", archivePath, err) } - if _, err := io.Copy(w, in); err != nil { - return fmt.Errorf("copy %s into zip: %w", archivePath, err) + // Hash the bytes entering the archive, not the Phase-2 planned hash. + // MultiWriter mirrors the download verification at download.go: a file + // rewritten between plan and zip-build must leave BASE describing what + // the server extracted. The size comes from io.Copy's return, not from + // fa.LocalSize, so it always describes the bytes that entered the archive. + h := newStreamHasher() + + n, err := io.Copy(io.MultiWriter(w, h), in) + if err != nil { + return FileEntry{}, fmt.Errorf("copy %s into zip: %w", archivePath, err) } - return nil + return streamedEntry(h, n), nil } // postZip dispatches to UploadFromZipNew (first-sync, no catalog) or diff --git a/internal/workload/sync/upload_zip_test.go b/internal/workload/sync/upload_zip_test.go new file mode 100644 index 000000000..15d3bb3ca --- /dev/null +++ b/internal/workload/sync/upload_zip_test.go @@ -0,0 +1,392 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "archive/zip" + "bytes" + "io" + "os" + "sort" + "testing" + + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// readZipEntries opens a zip archive at zipPath, extracts every entry, and +// returns a map of archive-path → uncompressed content bytes. This is the +// genuine read-back assertion: we open the archive that buildZip produced +// and hash the extracted entries, rather than re-hashing the source files. +// The assertion is "the archive contains what we claimed," rather than a +// bare claim that something was hashed. +func readZipEntries(t *testing.T, zipPath string) map[string][]byte { + t.Helper() + + data, err := os.ReadFile(zipPath) + require.NoError(t, err) + + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + + entries := make(map[string][]byte) + + for _, zf := range zr.File { + rc, err := zf.Open() + require.NoError(t, err) + + content, err := io.ReadAll(rc) + + _ = rc.Close() + + require.NoError(t, err) + + entries[zf.Name] = content + } + + return entries +} + +// fileActionsFrom builds a sorted slice of FileActions from a map of +// relative paths to content. LocalHash and LocalSize are set from the +// content so the actions are realistic, but addToZip and uploadOneToStage +// do not read them — they hash the bytes that actually enter the archive. +func fileActionsFrom(files map[string]string) []FileAction { + actions := make([]FileAction, 0, len(files)) + + for path, content := range files { + actions = append(actions, FileAction{ + Path: path, + LocalHash: sha256Hex([]byte(content)), + LocalSize: int64(len(content)), + }) + } + + sort.Slice(actions, func(i, j int) bool { return actions[i].Path < actions[j].Path }) + + return actions +} + +// TestZipBuild_HashesStreamedBytes verifies that buildZip records per-path +// Sent hashes equal to the SHA-256 of each file's content, and that the +// bytes read back out of the produced archive hash to the same values. This +// is the baseline correctness test for the zip path's hash-while-streaming +// behaviour: a normal multi-file archive with no mid-build changes. +// +// Read-back approach: we call buildZip (the production function), open the +// archive it returns the path to, extract each entry, and hash the extracted +// bytes. This proves the archive contains what the Sent map claims, rather +// than just that we hashed the source file. +// Fulfills VAL-UPLOAD-009 (sub-condition e: zip path produces hashes matching +// shasum of disk content). +func TestZipBuild_HashesStreamedBytes(t *testing.T) { + files := map[string]string{ + "app.py": "print('hello world')\n", + "utils/helper.py": "def help(): pass\n", + "config.yaml": "key: value\n", + "README.md": "# Project\n\nA test project.\n", + } + + dir := initProject(t, files) + + actions := fileActionsFrom(files) + + zipPath, sent, err := buildZip(dir, actions) + require.NoError(t, err) + + t.Cleanup(func() { _ = os.Remove(zipPath) }) + + // Read back the produced archive and extract entries. + archived := readZipEntries(t, zipPath) + + // Assert each file's Sent hash equals SHA-256 of its content, and + // the bytes read back from the archive hash to the same value. + for path, content := range files { + entry, ok := sent[path] + require.True(t, ok, "Sent must have an entry for %s", path) + + expectedHash := sha256Hex([]byte(content)) + expectedSize := int64(len(content)) + + assert.Equal(t, expectedHash, entry.Hash, + "Sent hash for %s must equal SHA-256 of file content", path) + assert.Equal(t, expectedSize, entry.Size, + "Sent size for %s must equal file byte count", path) + + // Read-back: the archive must contain the same bytes, and the + // extracted bytes must hash to the same value recorded in Sent. + archivedContent, ok := archived[path] + require.True(t, ok, "archive must contain entry for %s", path) + + assert.Equal(t, []byte(content), archivedContent, + "archived bytes for %s must match file content", path) + assert.Equal(t, expectedHash, sha256Hex(archivedContent), + "SHA-256 of extracted bytes for %s must match Sent hash", path) + } +} + +// TestZipBuild_SameSizeContentChange verifies that a file rewritten to +// different bytes of the SAME size between plan and addToZip records a Sent +// hash equal to the archived bytes' hash, NOT the Phase-2 planned hash. This +// is the zip-path analogue of the silent-poison TOCTOU case: the same-size +// rewrite produces no error and the hash must describe what entered the +// archive, not what was on disk at plan time. +// Fulfills VAL-UPLOAD-010. +func TestZipBuild_SameSizeContentChange(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('orig')\n", + }) + + // Phase-2 "planned" content: what was on disk at plan time. + planContent := "print('aaaa')\n" // 12 bytes + + modifyFile(t, dir, "app.py", planContent) + + plannedHash := sha256Hex([]byte(planContent)) + plannedSize := int64(len(planContent)) + + // Between plan and zip-build, rewrite to different bytes of the + // SAME size. This is the silent-poison window. + buildContent := "print('bbbb')\n" // 12 bytes, different content + + modifyFile(t, dir, "app.py", buildContent) + + require.Equal(t, plannedSize, int64(len(buildContent)), + "planned and build content must be the same size for this test") + + // The FileAction carries the Phase-2 planned hash and size. addToZip + // must NOT use these — it must hash the bytes that enter the archive. + actions := []FileAction{{ + Path: "app.py", + LocalHash: plannedHash, + LocalSize: plannedSize, + }} + + zipPath, sent, err := buildZip(dir, actions) + require.NoError(t, err) + + t.Cleanup(func() { _ = os.Remove(zipPath) }) + + entry, ok := sent["app.py"] + require.True(t, ok, "Sent must have an entry for app.py") + + buildHash := sha256Hex([]byte(buildContent)) + + // The Sent hash must be the archived bytes' hash, not the Phase-2 + // planned hash. + assert.Equal(t, buildHash, entry.Hash, + "Sent hash must equal SHA-256 of the archived bytes, not the Phase-2 planned hash") + assert.NotEqual(t, plannedHash, entry.Hash, + "Sent hash must differ from the Phase-2 planned hash") + + // Read-back: confirm the archive contains the build-time bytes, and + // the extracted bytes hash to the recorded Sent hash. + archived := readZipEntries(t, zipPath) + + archivedContent := archived["app.py"] + require.NotNil(t, archivedContent, "archive must contain app.py") + + assert.Equal(t, []byte(buildContent), archivedContent, + "archive must contain the build-time bytes, not the plan-time bytes") + assert.Equal(t, buildHash, sha256Hex(archivedContent), + "SHA-256 of extracted bytes must match the recorded Sent hash") + assert.NotEqual(t, plannedHash, sha256Hex(archivedContent), + "extracted bytes must NOT hash to the Phase-2 planned hash") +} + +// TestZipBuild_SizeChange verifies that a file that grows or shrinks between +// plan and addToZip records a Sent size equal to the byte count that actually +// entered the archive (from io.Copy's return), not fa.LocalSize (the Phase-2 +// planned size). The hash and size must be self-consistent — both describe +// the same streamed bytes — and the upload must not fail. +// Fulfills VAL-UPLOAD-020. +func TestZipBuild_SizeChange(t *testing.T) { + tests := []struct { + name string + planContent string + buildContent string + }{ + { + name: "grows", + planContent: "print('hi')\n", // 11 bytes + buildContent: "print('hello')\n", // 14 bytes + }, + { + name: "shrinks", + planContent: "print('hello')\n", // 14 bytes + buildContent: "print('hi')\n", // 11 bytes + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('orig')\n", + }) + + // Phase-2 planned content. + modifyFile(t, dir, "app.py", tc.planContent) + + plannedHash := sha256Hex([]byte(tc.planContent)) + plannedSize := int64(len(tc.planContent)) + + // Between plan and zip-build, change the file size. + modifyFile(t, dir, "app.py", tc.buildContent) + + buildHash := sha256Hex([]byte(tc.buildContent)) + buildSize := int64(len(tc.buildContent)) + + require.NotEqual(t, plannedSize, buildSize, + "planned and build sizes must differ for this test to be meaningful") + + actions := []FileAction{{ + Path: "app.py", + LocalHash: plannedHash, + LocalSize: plannedSize, + }} + + zipPath, sent, err := buildZip(dir, actions) + require.NoError(t, err, "upload must not fail due to a size change") + + t.Cleanup(func() { _ = os.Remove(zipPath) }) + + entry, ok := sent["app.py"] + require.True(t, ok, "Sent must have an entry for app.py") + + // Sent size must equal the bytes that entered the archive, + // not the Phase-2 planned size (fa.LocalSize). + assert.Equal(t, buildSize, entry.Size, + "Sent size must equal the bytes that entered the archive, not fa.LocalSize") + assert.NotEqual(t, plannedSize, entry.Size, + "Sent size must differ from the Phase-2 planned size") + + // Hash and size must be self-consistent: both describe the + // same streamed bytes. + assert.Equal(t, buildHash, entry.Hash, + "Sent hash must equal SHA-256 of the build-time bytes") + assert.Equal(t, buildSize, entry.Size, + "Sent size must equal the build-time byte count") + + // Read-back: confirm the archive contains the build-time + // bytes, and the extracted bytes match the recorded Sent. + archived := readZipEntries(t, zipPath) + + archivedContent := archived["app.py"] + require.NotNil(t, archivedContent, "archive must contain app.py") + + assert.Equal(t, []byte(tc.buildContent), archivedContent, + "archive must contain the build-time bytes") + assert.Equal(t, buildHash, sha256Hex(archivedContent), + "SHA-256 of extracted bytes must match the recorded Sent hash") + assert.Equal(t, buildSize, int64(len(archivedContent)), + "extracted byte count must match the recorded Sent size") + }) + } +} + +// TestStageAndZipProduceIdenticalSent verifies that given the same project +// tree, the stage path and the zip path produce identical Sent maps (same +// hashes and sizes for every file). Since Phase 6 seeds the manifest from +// Sent, identical Sent maps mean identical manifests. Both uploaders are +// driven directly through their exported ApplyUploads methods against +// self-consistent fakes, so the comparison exercises the real production +// code paths on both sides. +// Fulfills VAL-UPLOAD-009 (sub-condition e: stage and zip paths both produce +// manifests where every hash matches shasum of disk content). +func TestStageAndZipProduceIdenticalSent(t *testing.T) { + files := map[string]string{ + "app.py": "print('hello world')\n", + "utils/helper.py": "def help(): pass\n", + "config.yaml": "key: value\n", + "README.md": "# Project\n", + } + + dir := initProject(t, files) + + actions := fileActionsFrom(files) + + const catalogID = "cid-equiv" + + cid := catalogID + + // Stage path: drive StageUploader.ApplyUploads directly. The engine + // only needs projectDir, files, and config.CatalogID for the upload + // code path — no phases run. + stageFake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-stage", + } + + stageEngine := &Engine{ + projectDir: dir, + files: stageFake, + config: wapi.Config{CatalogID: &cid}, + } + + stageOutcome, err := StageUploader{}.ApplyUploads(stageEngine, actions) + require.NoError(t, err) + + // Zip path: drive ZipUploader.ApplyUploads directly with a separate + // fake so the two do not share server state. + zipFake := &fakeFilesClient{ + catalogID: catalogID, + versionID: "ver-zip", + } + + zipEngine := &Engine{ + projectDir: dir, + files: zipFake, + config: wapi.Config{CatalogID: &cid}, + } + + zipOutcome, err := ZipUploader{}.ApplyUploads(zipEngine, actions) + require.NoError(t, err) + + // Both paths must produce Sent maps with the same number of entries. + require.Len(t, stageOutcome.Sent, len(actions), + "stage Sent must have one entry per file") + require.Len(t, zipOutcome.Sent, len(actions), + "zip Sent must have one entry per file") + + // Both must produce identical hashes and sizes for every file, and + // both must match the SHA-256 of the file content on disk. + for _, fa := range actions { + stageEntry, ok := stageOutcome.Sent[fa.Path] + require.True(t, ok, "stage Sent must have entry for %s", fa.Path) + + zipEntry, ok := zipOutcome.Sent[fa.Path] + require.True(t, ok, "zip Sent must have entry for %s", fa.Path) + + assert.Equal(t, stageEntry.Hash, zipEntry.Hash, + "stage and zip must produce the same hash for %s", fa.Path) + assert.Equal(t, stageEntry.Size, zipEntry.Size, + "stage and zip must produce the same size for %s", fa.Path) + + // Both must match the SHA-256 and byte count of the file content. + expectedHash := sha256Hex([]byte(files[fa.Path])) + expectedSize := int64(len(files[fa.Path])) + + assert.Equal(t, expectedHash, stageEntry.Hash, + "stage hash for %s must equal SHA-256 of content", fa.Path) + assert.Equal(t, expectedHash, zipEntry.Hash, + "zip hash for %s must equal SHA-256 of content", fa.Path) + assert.Equal(t, expectedSize, stageEntry.Size, + "stage size for %s must equal byte count", fa.Path) + assert.Equal(t, expectedSize, zipEntry.Size, + "zip size for %s must equal byte count", fa.Path) + } +} diff --git a/internal/workload/sync/uploader.go b/internal/workload/sync/uploader.go index b5e2e13b5..7ff454990 100644 --- a/internal/workload/sync/uploader.go +++ b/internal/workload/sync/uploader.go @@ -14,11 +14,23 @@ package sync -// Uploader pushes a SyncPlan's Uploads and returns the resulting -// (catalogID, newVersionID). When catalogID is empty (first-sync against -// an empty artifact) the implementation creates a new catalog. +// UploadOutcome is what an Uploader actually accomplished. Sent carries the +// hash and size of the bytes that really crossed the wire, keyed by the same +// forward-slash relative path used everywhere else in the manifest. Phase 6 +// seeds each uploaded file's manifest entry from Sent, never from the Phase-2 +// planned hash — a per-path fallback to the planned hash is the original +// poisoning bug and must not exist anywhere in the code. +type UploadOutcome struct { + CatalogID string + VersionID string + Sent map[string]FileEntry +} + +// Uploader pushes a SyncPlan's Uploads and returns the resulting outcome. +// When the artifact has no catalog (first-sync against an empty artifact) +// the implementation creates a new one. type Uploader interface { - ApplyUploads(e *Engine, files []FileAction) (catalogID, versionID string, err error) + ApplyUploads(e *Engine, files []FileAction) (UploadOutcome, error) } // ChooseUploader picks stage for small change sets (tight error semantics) diff --git a/internal/workload/sync/verify_options_test.go b/internal/workload/sync/verify_options_test.go new file mode 100644 index 000000000..d7ab20af4 --- /dev/null +++ b/internal/workload/sync/verify_options_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// previewOnly answers exactly one question: does this run stop after Plan? +// Verify changes how much a run checks, not whether it applies its plan, so +// folding it into this predicate would silently turn every verify user's +// sync into a preview. These cases pin the boundary. +func TestPreviewOnly_DoesNotConsiderVerify(t *testing.T) { + for _, tc := range []struct { + name string + opts Options + want bool + }{ + {"no flags", Options{}, false}, + {"verify only", Options{Verify: true}, false}, + {"dry run", Options{DryRun: true}, true}, + {"diff", Options{ShowDiffs: true}, true}, + {"verify alongside dry run is still a preview", Options{Verify: true, DryRun: true}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + e := Engine{opts: tc.opts} + + assert.Equal(t, tc.want, e.previewOnly()) + }) + } +} + +// TestEngine_Run_VerifyStillExecutes: Run() under Options{Verify: true} +// must still apply the plan and create a version, because previewOnly does +// not consider Verify. This is the behavioural form of the boundary above — +// the moment someone folds Verify into previewOnly, Run returns the empty +// pre-plan result and this fails. +func TestEngine_Run_VerifyStillExecutes(t *testing.T) { + dir := initProject(t, map[string]string{"agent.py": "print('hi')\n"}) + + fake := &fakeFilesClient{catalogID: "cid-new", stageID: "stage-1", versionID: "ver-1"} + + e, err := newWithDeps(dir, Options{Verify: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + assert.Equal(t, "ver-1", result.NewVersion, "a verify-only run must still create a version") + assert.Equal(t, 2, result.UploadedCount, "expect agent.py + .drignore") +} diff --git a/internal/workload/sync/verify_post_apply_test.go b/internal/workload/sync/verify_post_apply_test.go new file mode 100644 index 000000000..06c4d7a14 --- /dev/null +++ b/internal/workload/sync/verify_post_apply_test.go @@ -0,0 +1,496 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/config" + "github.com/datarobot/cli/internal/config/viperx" + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests cover --verify's second effect: after ApplyUploads, Phase 5 +// asks the server what the new version actually holds and fails before any +// state is persisted when an uploaded path's server checksum differs from +// the streamed hash. The check lives in Phase 5 so a mismatch trips the +// existing rollback and Phase 6 never runs — asserted here at the state-file +// level: manifest.json and config.json must be byte-identical to their +// pre-sync content after a failed verification. + +const ( + postApplyCatalogID = "cid-verify" + postApplyOldVer = "ver-old" + postApplyNewVer = "ver-new" +) + +// postApplyScenario builds a synced project (BASE == LOCAL == the server's +// pre-apply version) with one modified file, so Plan yields exactly one +// upload row for it. Mutate chains fake fault hooks. +type postApplyScenario struct { + dir string + fake *fakeFilesClient + newBody string +} + +func newPostApplyScenario(t *testing.T, mutate func(*fakeFilesClient) *fakeFilesClient) postApplyScenario { + t.Helper() + + appPy := "print('A')\n" + untouched := "print('untouched')\n" + + // The manifest matches disk, and the server's pre-apply version matches + // both, so Phase 2's verify-forced fetch finds no divergence and the + // plan is exactly one upload. + dir := syncedProject(t, map[string]string{ + "app.py": appPy, + "untouched.py": untouched, + }, postApplyCatalogID, postApplyOldVer) + + // The server seed is built from the pre-change bytes explicitly, NOT by + // reading the tree after the modification below: a seed captured after + // the edit would make the server agree with disk, classify app.py as + // CONVERGED, and yield an empty plan — the divergence scenario, not the + // plain-upload one these tests need. + drignoreBytes, err := os.ReadFile(filepath.Join(dir, ignore.FileName)) + require.NoError(t, err) + + seed := map[string][]byte{ + ignore.FileName: drignoreBytes, + "app.py": []byte(appPy), + "untouched.py": []byte(untouched), + } + + newBody := "print('B')\n" + modifyFile(t, dir, "app.py", newBody) + + fake := (&fakeFilesClient{ + catalogID: postApplyCatalogID, + stageID: "stage-verify", + versionID: postApplyNewVer, + }).withVersionContent(postApplyCatalogID, postApplyOldVer, seed) + + if mutate != nil { + fake = mutate(fake) + } + + return postApplyScenario{dir: dir, fake: fake, newBody: newBody} +} + +func (s postApplyScenario) engine(t *testing.T, opts Options) *Engine { + t.Helper() + + e, err := newWithDeps(s.dir, opts, Deps{ + Files: s.fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, postApplyCatalogID, postApplyOldVer), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e +} + +// persistedStateSnapshot reads both state files' raw bytes so a test can +// assert Phase 6 never ran: byte-identical after a failed verification means +// neither SaveManifest nor SaveConfig happened. +func persistedStateSnapshot(t *testing.T, dir string) (manifest, config []byte) { + t.Helper() + + manifestPath, configPath := statePaths(dir) + + manifest, err := os.ReadFile(manifestPath) + require.NoError(t, err) + + config, err = os.ReadFile(configPath) + require.NoError(t, err) + + return manifest, config +} + +// TestEngine_VerifyPostApplyChecksumMismatchFailsAndPersistsNothing is +// VAL-VERIFY-009: a server checksum that differs from the streamed hash for +// an uploaded path fails the sync with an error naming that path, and both +// persisted state files are byte-identical to their pre-sync content — +// Phase 6 must never run on a failed verification. +func TestEngine_VerifyPostApplyChecksumMismatchFailsAndPersistsNothing(t *testing.T) { + t.Run("wrong checksum on the uploaded path", func(t *testing.T) { + sentHash := sha256Hex([]byte("print('B')\n")) + serverHash := sha256Hex([]byte("the server holds something else\n")) + + s := newPostApplyScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + // Scoped to the NEW version so Phase 2's fetch of the old + // version stays truthful: the corruption must be visible only + // to the post-apply check. + return f.withWrongChecksumForVersion(postApplyNewVer, "app.py", serverHash) + }) + e := s.engine(t, Options{Yes: true, Verify: true}) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + manifestBefore, configBefore := persistedStateSnapshot(t, s.dir) + + _, err = e.Execute(plan) + require.Error(t, err, "a post-apply checksum mismatch must fail the sync") + assert.Contains(t, err.Error(), "app.py", "the error must name the path whose hash differs") + + manifestAfter, configAfter := persistedStateSnapshot(t, s.dir) + + assert.Equal(t, string(manifestBefore), string(manifestAfter), + "manifest.json must be byte-identical to its pre-sync content when verification fails") + assert.Equal(t, string(configBefore), string(configAfter), + "config.json must be byte-identical to its pre-sync content when verification fails") + + // The failure is the point: the streamed hash the manifest WOULD + // have recorded must be the one the server refused to confirm. + assert.Contains(t, err.Error(), sentHash, "the error must carry the hash that was sent") + assert.Contains(t, err.Error(), serverHash, "the error must carry the hash the server holds") + }) + + t.Run("uploaded path dropped from the resulting version", func(t *testing.T) { + s := newPostApplyScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + // The server accepted the upload but dropped the path from + // the resulting version: the post-apply listing must not + // contain it, and verification must catch the absence before + // Phase 6 records a hash for bytes the server does not hold. + return f.withDropPath("app.py") + }) + e := s.engine(t, Options{Yes: true, Verify: true}) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + manifestBefore, configBefore := persistedStateSnapshot(t, s.dir) + + _, err = e.Execute(plan) + require.Error(t, err, "an uploaded path missing from the server version must fail the sync") + assert.Contains(t, err.Error(), "app.py", "the error must name the dropped path") + + manifestAfter, configAfter := persistedStateSnapshot(t, s.dir) + + assert.Equal(t, string(manifestBefore), string(manifestAfter), + "manifest.json must not gain an entry the server does not hold") + assert.Equal(t, string(configBefore), string(configAfter), + "config.json must not advance past the manifest") + }) + + t.Run("without verify no post-apply check exists", func(t *testing.T) { + // The negative control for the whole feature: the same server-side + // checksum fault, but without --verify, the run must succeed — the + // default path gains no verification and no extra round-trip. + s := newPostApplyScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + return f.withWrongChecksumForVersion(postApplyNewVer, "app.py", sha256Hex([]byte("wrong\n"))) + }) + e := s.engine(t, Options{Yes: true}) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + result, err := e.Execute(plan) + require.NoError(t, err, "without --verify the post-apply check must not exist") + require.NotNil(t, result) + + assert.Equal(t, 0, s.fake.AllFilesCalls(), + "a non-drifted run without --verify must make no AllFiles call at all") + }) +} + +// TestEngine_VerifyPostApplyOnlyComparesUploadedPaths is VAL-VERIFY-010(a): +// stage REPLACE merges in place, so the post-apply listing legitimately holds +// files this sync never touched. A wrong checksum on such a path must not +// fail the run — only uploaded paths are compared. +func TestEngine_VerifyPostApplyOnlyComparesUploadedPaths(t *testing.T) { + s := newPostApplyScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + return f.withWrongChecksumForVersion(postApplyNewVer, "untouched.py", sha256Hex([]byte("stale\n"))) + }) + e := s.engine(t, Options{Yes: true, Verify: true}) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + assert.Equal(t, "app.py", plan.Uploads[0].Path, "untouched.py must not be in the upload plan") + + result, err := e.Execute(plan) + require.NoError(t, err, "a wrong checksum on a path this sync did not upload must not fail the run") + require.NotNil(t, result) + + // Exactly two AllFiles calls: Phase 2's verify-forced fetch of the old + // version, then the post-apply check of the new one. This is the + // network cost --verify adds beyond the single Phase-2 fetch. + assert.Equal(t, 2, s.fake.AllFilesCalls(), + "a non-drifted verify run with uploads makes exactly two AllFiles calls") +} + +// TestEngine_VerifyPostApplyNumFilesDoesNotGate is VAL-VERIFY-010(c): +// ApplyStage's numFiles counts every file in the resulting version, not the +// ones uploaded, so it cannot relate to the plan's size. A nonsense value +// must neither gate, skip, nor abort verification — the sync completes +// successfully. +func TestEngine_VerifyPostApplyNumFilesDoesNotGate(t *testing.T) { + s := newPostApplyScenario(t, func(f *fakeFilesClient) *fakeFilesClient { + return f.withNumFilesOverride(99) + }) + e := s.engine(t, Options{Yes: true, Verify: true}) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1, "the fixture uploads one file; numFiles=99 must not matter") + + result, err := e.Execute(plan) + require.NoError(t, err, "a numFiles that does not match the upload count must not affect the run") + require.NotNil(t, result) + assert.Equal(t, 1, result.UploadedCount) + + assert.Equal(t, 2, s.fake.AllFilesCalls(), + "verification must still run (Phase-2 fetch + post-apply check) with a nonsense numFiles") + + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + assert.Equal(t, sha256Hex([]byte(s.newBody)), manifest.Files["app.py"].Hash, + "the manifest must record the streamed hash") +} + +// TestEngine_VerifyPostApplySkippedWhenNoUploads is VAL-VERIFY-023 plus the +// empty-plan corner: with nothing uploaded there is nothing to verify, and +// the post-apply step must make no AllFiles call at all rather than erroring +// on a nil outcome or an empty Sent map. +func TestEngine_VerifyPostApplySkippedWhenNoUploads(t *testing.T) { + t.Run("downloads-only plan", func(t *testing.T) { + s := newDownloadScenario(t, nil) + + e, err := newWithDeps(s.dir, Options{Yes: true, Verify: true}, Deps{ + Files: s.fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "cid-dl", "ver-dl-remote"), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Downloads, 1) + assert.Empty(t, plan.Uploads) + + // The drifted artifact made Phase 2 fetch the remote; that call is + // Phase 2's, not verification's. + callsAfterPlan := s.fake.AllFilesCalls() + require.Equal(t, 1, callsAfterPlan, "the fixture expects exactly the Phase-2 fetch before Execute") + + result, err := e.Execute(plan) + require.NoError(t, err, "a downloads-only plan must not error on a nil upload outcome") + require.NotNil(t, result) + assert.Equal(t, 1, result.DownloadedCount) + + assert.Equal(t, callsAfterPlan, s.fake.AllFilesCalls(), + "post-apply verification must make zero AllFiles calls when nothing was uploaded") + + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + assert.Equal(t, s.remoteHash, manifest.Files["app.py"].Hash, + "the manifest must record the downloaded file's remote hash") + }) + + t.Run("empty plan that reaches phase 5 through a divergence", func(t *testing.T) { + // BASE poisoned while disk and server agree: the plan is empty but + // the divergence drives the run into Phase 5 (a no-op there) and + // Phase 6's repair. No uploads, so no post-apply call. + contentA := "print('A')\n" + contentB := "print('B')\n" + + dir := syncedProject(t, map[string]string{"app.py": contentB}, postApplyCatalogID, postApplyOldVer) + + poisonManifestHash(t, dir, "app.py", sha256Hex([]byte(contentA))) + + fake := (&fakeFilesClient{}).withVersionContent(postApplyCatalogID, postApplyOldVer, seededServerContents(t, dir, map[string]string{ + "app.py": contentB, + }, nil)) + + var runErr error + + out := captureWarnLog(t, func() { + e, err := newWithDeps(dir, Options{Yes: true, Verify: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, postApplyCatalogID, postApplyOldVer), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + _, runErr = e.Run() + }) + + require.NoError(t, runErr) + assert.Contains(t, out, "divergence", "the repair run must still report the stale BASE") + + assert.Equal(t, 1, fake.AllFilesCalls(), + "the only AllFiles call is Phase 2's fetch; the empty plan must add none") + }) + + t.Run("empty plan short-circuits before phase 5", func(t *testing.T) { + dir := syncedProject(t, map[string]string{"app.py": "print('hi')\n"}, postApplyCatalogID, postApplyOldVer) + + fake := (&fakeFilesClient{}).withVersionContent(postApplyCatalogID, postApplyOldVer, seededServerContents(t, dir, map[string]string{ + "app.py": "print('hi')\n", + }, nil)) + + e, err := newWithDeps(dir, Options{Yes: true, Verify: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, postApplyCatalogID, postApplyOldVer), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + assert.Equal(t, 1, fake.AllFilesCalls(), + "Phase 2's verify-forced fetch is the only AllFiles call on an up-to-date project") + }) +} + +// TestVerifyPostApplyUploads_FollowsPaginationViaRealClient drives the +// post-apply check at the httptest level against the REAL httpClient. The +// next-link loop lives below the filesapi.Client interface (inside +// httpClient.AllFiles), so the fake's page-size hook can never force +// production down a multi-page path — an engine test with the fake would +// assert nothing about pagination. Here the uploaded path's checksum exists +// only on page 2, behind a next link, with an unrelated decoy on page 1: a +// page-1-only implementation could neither find the path to verify nor +// mismatch it. +func TestVerifyPostApplyUploads_FollowsPaginationViaRealClient(t *testing.T) { + const ( + catalogID = "cid-pag" + versionID = "ver-pag" + ) + + targetBody := []byte("page two content\n") + decoyBody := []byte("page one decoy\n") + + targetHash := sha256Hex(targetBody) + decoyHash := sha256Hex(decoyBody) + + var srv *httptest.Server + + mux := http.NewServeMux() + + mux.HandleFunc("/api/v2/files/"+catalogID+"/versions/"+versionID+"/allFiles/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.URL.Query().Get("offset") == "" { + // Page 1: the decoy only, plus a same-host next link. The + // uploaded path is deliberately absent here. + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{ + {"fileName": "decoy.txt", "fileSize": len(decoyBody), "fileChecksum": decoyHash}, + }, + "next": srv.URL + "/api/v2/files/" + catalogID + "/versions/" + versionID + "/allFiles/?offset=1", + }) + + return + } + + // Page 2: only the uploaded path. + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{ + {"fileName": "app.py", "fileSize": len(targetBody), "fileChecksum": targetHash}, + }, + }) + }) + + srv = httptest.NewServer(mux) + t.Cleanup(srv.Close) + + // Point the production client at the test server and skip the token + // resolution round-trip; previous values are restored afterwards. + prevURL := viperx.GetString(config.DataRobotURL) + prevKey := viperx.GetString(config.DataRobotAPIKey) + prevSkipAuth := viperx.GetBool(config.SkipAuthKey) + + viperx.Set(config.DataRobotURL, srv.URL) + viperx.Set(config.DataRobotAPIKey, "pagination-test-token") + viperx.Set(config.SkipAuthKey, true) + + t.Cleanup(func() { + viperx.Set(config.DataRobotURL, prevURL) + viperx.Set(config.DataRobotAPIKey, prevKey) + viperx.Set(config.SkipAuthKey, prevSkipAuth) + }) + + e, err := newWithDeps(t.TempDir(), Options{Verify: true}, Deps{Files: filesapi.New()}) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + outcome := &UploadOutcome{ + CatalogID: catalogID, + VersionID: versionID, + Sent: map[string]FileEntry{ + "app.py": {Hash: targetHash, Size: int64(len(targetBody))}, + }, + } + + // Match: the page-2 checksum equals the streamed hash. + require.NoError(t, verifyPostApplyUploads(e, outcome), + "a checksum that only appears on page 2 must be found and compared") + + // Mismatch: the same page-2 path, a different streamed hash. + sentBody := []byte("what we actually sent\n") + outcome.Sent["app.py"] = FileEntry{Hash: sha256Hex(sentBody), Size: int64(len(sentBody))} + + err = verifyPostApplyUploads(e, outcome) + require.Error(t, err, "a page-2 checksum mismatch must fail verification") + assert.Contains(t, err.Error(), "app.py", "the error must name the path seen on page 2") +} diff --git a/internal/workload/sync/verify_repair_test.go b/internal/workload/sync/verify_repair_test.go new file mode 100644 index 000000000..ba08bdbd2 --- /dev/null +++ b/internal/workload/sync/verify_repair_test.go @@ -0,0 +1,612 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// poisonManifestHash overwrites the recorded hash of one path in the +// project's manifest.json, simulating a poisoned BASE: recorded state that +// describes bytes the server does not hold. This is the state --verify +// exists to detect and repair. +func poisonManifestHash(t *testing.T, dir, rel, hash string) { + t.Helper() + + m, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + fm, ok := m.Files[rel] + require.True(t, ok, "path %s must exist in the manifest to be poisoned", rel) + + fm.Hash = hash + m.Files[rel] = fm + + require.NoError(t, wapi.SaveManifest(dir, m)) +} + +// statePaths returns the two files whose bytes define the persisted sync +// state: manifest.json and config.json under the workload state directory. +func statePaths(dir string) (manifestPath, configPath string) { + return filepath.Join(dir, ".datarobot", "workload", "manifest.json"), + filepath.Join(dir, ".datarobot", "workload", "config.json") +} + +// readStateFile reads one persisted state file's raw bytes for byte-level +// comparison across runs. +func readStateFile(t *testing.T, path string) []byte { + t.Helper() + + b, err := os.ReadFile(path) + require.NoError(t, err) + + return b +} + +// TestEngine_Run_VerifyRepairsPoisonedManifestOnConvergedEmptyPlan is the +// sharpest repair case: BASE is poisoned to A while disk and server both hold +// B. Classify sees localChanged and remoteChanged with local == remote, so +// the classification is CONVERGED, every row is skipped, and the plan comes +// back empty. The divergence notice must still fire, the run must still exit +// cleanly, and Phase 6 must still rewrite the manifest from the real remote +// — an empty plan must not short-circuit the state write that repairs it. +func TestEngine_Run_VerifyRepairsPoisonedManifestOnConvergedEmptyPlan(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + // Disk holds B and the manifest agrees; poison only the recorded hash. + dir := syncedProject(t, map[string]string{"app.py": contentB}, catalogID, versionID) + + poisonManifestHash(t, dir, "app.py", sha256Hex([]byte(contentA))) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, map[string]string{ + "app.py": contentB, + }, nil)) + + manifestPath, _ := statePaths(dir) + before, err := os.ReadFile(manifestPath) + require.NoError(t, err) + + var ( + out string + runErr error + ) + + out = captureWarnLog(t, func() { + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + _, runErr = e.Run() + }) + + // The divergence is a diagnostic: exit 0, notice on the warn stream. + require.NoError(t, runErr, "a detected divergence must not change the exit status") + assert.Contains(t, out, "divergence", "the divergence notice must fire even though the plan is empty") + assert.Contains(t, out, "app.py", "the divergence notice must name the affected path") + assert.Contains(t, out, sha256Hex([]byte(contentB)), "the notice must carry the server's real hash") + + // Repair: the manifest hash for app.py is the server's checksum (B), + // not the poisoned A. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + assert.Equal(t, sha256Hex([]byte(contentB)), manifest.Files["app.py"].Hash, + "Phase 6 must seed the manifest from the real REMOTE, not from the poisoned BASE") + assert.NotEqual(t, sha256Hex([]byte(contentA)), manifest.Files["app.py"].Hash, + "the poisoned hash must not survive the run") + + server, err := fake.AllFiles(catalogID, versionID) + require.NoError(t, err) + assert.Equal(t, server["app.py"].Hash, manifest.Files["app.py"].Hash, + "the repaired manifest hash must equal the server's AllFiles checksum") + + // The repair is a local state write only: no upload-side network calls, + // no new version. + assert.Zero(t, fake.CreateStageCalls()) + assert.Zero(t, fake.UploadToStageCalls()) + assert.Zero(t, fake.ApplyStageCalls()) + assert.Zero(t, fake.UploadFromZipCalls()) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, versionID, *cfg.LastSyncedVersionID, "the repair must not advance the synced version") + + // Idempotent: a second --verify run finds no divergence and rewrites + // nothing. The short-circuit must stay in charge once the manifest is + // truthful, so the file is byte-identical afterwards. + afterRepair := readStateFile(t, manifestPath) + + var out2 string + + e2 := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + out2 = captureWarnLog(t, func() { + _, runErr = e2.Run() + }) + + require.NoError(t, runErr) + assert.Empty(t, e2.Divergences(), "the second verify run must find no divergence") + assert.NotContains(t, out2, "divergence", "the second verify run must be silent") + assert.Equal(t, afterRepair, readStateFile(t, manifestPath), + "the second verify run must not rewrite the manifest") + + // The manifest must not have been rewritten back to the poison, and the + // repair itself must have changed exactly the hash field's story: the + // pre-run bytes recorded A, the post-run bytes record B. + assert.NotEqual(t, before, afterRepair, "the repair run must rewrite the manifest") + + // A plain sync is now truthful: with the repaired manifest describing + // the server, the fast path yields an empty plan. + e3 := engineFor(t, dir, Options{}, fake, catalogID, versionID) + + plan, err := e3.Plan() + require.NoError(t, err) + assert.True(t, plan.IsEmpty(), "a plain sync must report Up to date. after the repair") +} + +// TestEngine_Run_VerifyRepairsWhenDivergentPathIsAlsoDeleted covers the +// delete-shaped empty plan: one path is a CONVERGED skip (poisoned hash, disk +// and server agree) and another is BOTH_DELETED (gone from disk and server, +// recorded only in the poisoned BASE). Both classify to skip, so the plan is +// empty — yet the run must still report both divergences and drop the +// phantom path from the manifest. +func TestEngine_Run_VerifyRepairsWhenDivergentPathIsAlsoDeleted(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + dir := syncedProject(t, map[string]string{ + "app.py": contentB, + "gone.py": "gone from both sides\n", + }, catalogID, versionID) + + // The server lost gone.py too, so its listing holds app.py and .drignore + // only. The seed is built while gone.py is still on disk (the helper + // reads the tree), then the path is dropped from it. + seed := seededServerContents(t, dir, map[string]string{ + "app.py": contentB, + "gone.py": "gone from both sides\n", + }, nil) + delete(seed, "gone.py") + + // gone.py is deleted locally as well. + require.NoError(t, os.Remove(filepath.Join(dir, "gone.py"))) + + // app.py's recorded hash is poisoned to A while disk and server hold B. + poisonManifestHash(t, dir, "app.py", sha256Hex([]byte(contentA))) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + var ( + out string + runErr error + ) + + out = captureWarnLog(t, func() { + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + _, runErr = e.Run() + }) + + require.NoError(t, runErr) + + // Both divergent paths are named: the hash mismatch and the phantom path. + assert.Contains(t, out, "app.py") + assert.Contains(t, out, "gone.py", "a BASE-only divergence must be reported even when deletion is already agreed") + + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + assert.Equal(t, sha256Hex([]byte(contentB)), manifest.Files["app.py"].Hash) + + _, ok := manifest.Files["gone.py"] + assert.False(t, ok, "a path absent from the server must be dropped from the manifest, not kept because the plan was empty") + + assert.Zero(t, fake.UploadToStageCalls(), "the repair makes no upload-side calls") +} + +// TestEngine_Run_VerifyRepairsWhenServerEditedDeletedPath: the divergent +// path is one the user deleted locally while the server edited it (BASE +// records the old hash, so the divergence is visible under --verify). The +// plan carries the download-over-delete row, remote wins, and after the run +// the restored file, the manifest, and the server all agree. +func TestEngine_Run_VerifyRepairsWhenServerEditedDeletedPath(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + dir := syncedProject(t, map[string]string{"app.py": contentA}, catalogID, versionID) + + // The server seed is captured while the file is still on disk (the + // helper reads the tree), with B overriding A; then the local deletion. + seed := seededServerContents(t, dir, map[string]string{ + "app.py": contentA, + }, map[string][]byte{"app.py": []byte(contentB)}) + + // The user deleted the file locally; the server moved on to B. + require.NoError(t, os.Remove(filepath.Join(dir, "app.py"))) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seed) + + var ( + out string + runErr error + ) + + out = captureWarnLog(t, func() { + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + _, runErr = e.Run() + }) + + require.NoError(t, runErr) + + assert.Contains(t, out, "divergence", "the stale BASE hash must be reported even though the path is locally deleted") + assert.Contains(t, out, "app.py") + + b, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err, "remote wins over the local deletion, so the file is restored from the server") + assert.Equal(t, contentB, string(b)) + + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + assert.Equal(t, sha256Hex([]byte(contentB)), manifest.Files["app.py"].Hash, + "the manifest must record the server's bytes after the repair") + + assert.Zero(t, fake.UploadToStageCalls(), "the repair makes no upload-side calls") +} + +// TestEngine_Run_VerifyRepairsFlagshipDivergence is the VAL-VERIFY-004 shape +// at the engine level: disk and manifest both hold A, the server holds B, so +// the plan carries a download row. After the run the disk and the manifest +// both describe the server, a second --verify finds nothing, and a plain +// sync reports Up to date. +func TestEngine_Run_VerifyRepairsFlagshipDivergence(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + dir := syncedProject(t, map[string]string{"app.py": contentA}, catalogID, versionID) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, map[string]string{ + "app.py": contentA, + }, map[string][]byte{"app.py": []byte(contentB)})) + + manifestPath, _ := statePaths(dir) + + e := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + assert.Equal(t, 1, fake.DownloadFileCalls(), "the reconciling row is a download") + + b, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err) + assert.Equal(t, contentB, string(b), "the download must land the server's bytes") + + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + assert.Equal(t, sha256Hex([]byte(contentB)), manifest.Files["app.py"].Hash, + "the manifest must record the server's checksum after the repair") + + afterRepair := readStateFile(t, manifestPath) + + e2 := engineFor(t, dir, Options{Verify: true}, fake, catalogID, versionID) + + _, err = e2.Run() + require.NoError(t, err) + assert.Empty(t, e2.Divergences(), "the second verify run must find no divergence") + assert.Equal(t, afterRepair, readStateFile(t, manifestPath), + "the second verify run must not rewrite the manifest") + + e3 := engineFor(t, dir, Options{}, fake, catalogID, versionID) + + plan, err := e3.Plan() + require.NoError(t, err) + assert.True(t, plan.IsEmpty(), "a plain sync must report Up to date. after the repair") +} + +// TestEngine_Plan_VerifyConflictDivergencePreviewsWithoutWriting: a +// both-sides conflict on a path whose BASE is also stale. The preview must +// surface the divergence and the conflict row while writing nothing — no +// manifest rewrite, no .LOCAL copy, disk untouched. +func TestEngine_Plan_VerifyConflictDivergencePreviewsWithoutWriting(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + contentC := "print('C')\n" + + dir := syncedProject(t, map[string]string{"app.py": contentA}, catalogID, versionID) + + // Local moves to C while the server moved to B: three-way conflict, and + // BASE (A) diverges from the server (B) too. + modifyFile(t, dir, "app.py", contentC) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, map[string]string{ + "app.py": contentA, + }, map[string][]byte{"app.py": []byte(contentB)})) + + manifestPath, _ := statePaths(dir) + manifestBefore := readStateFile(t, manifestPath) + + var ( + out string + planErr error + ) + + out = captureWarnLog(t, func() { + e := engineFor(t, dir, Options{Verify: true, DryRun: true}, fake, catalogID, versionID) + + var plan *SyncPlan + + plan, planErr = e.Plan() + require.NoError(t, planErr) + + require.Len(t, plan.Conflicts, 1, "the conflict row must be in the plan") + assert.Equal(t, "app.py", plan.Conflicts[0].Path) + }) + + require.NoError(t, planErr) + assert.Contains(t, out, "divergence", "the preview must surface the stale BASE") + assert.Contains(t, out, "app.py") + + assert.Equal(t, manifestBefore, readStateFile(t, manifestPath), + "a preview must not rewrite the manifest") + + b, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err) + assert.Equal(t, contentC, string(b), "a preview must not touch the working tree") + + copies, err := filepath.Glob(filepath.Join(dir, "*.LOCAL.*")) + require.NoError(t, err) + assert.Empty(t, copies, "a preview must not create conflict copies") +} + +// TestEngine_Run_VerifyOnLockedArtifactFails: --verify must not bypass the +// locked-artifact check. A non-dry-run verify fails with the same locked +// error a plain sync produces. +func TestEngine_Run_VerifyOnLockedArtifactFails(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + e := lockedEngine(t, dir, Options{Verify: true}) + + _, err := e.Run() + require.Error(t, err) + assert.Contains(t, err.Error(), "locked") +} + +// TestEngine_Plan_VerifyDryRunOnLockedArtifactIsPreview: --verify --dry-run +// on a locked artifact still plans, with the locked notice set — the +// preview exemption must survive --verify. +func TestEngine_Plan_VerifyDryRunOnLockedArtifactIsPreview(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + e := lockedEngine(t, dir, Options{Verify: true, DryRun: true}) + + plan, err := e.Plan() + require.NoError(t, err) + require.NotNil(t, plan) + + assert.Contains(t, e.LockedNotice(), "locked", + "a verify preview against a locked artifact must still carry the locked notice") +} + +// TestEngine_Run_DefaultPathDoesNotRepairPoison pins the documented residual +// risk: without --verify the poisoned manifest stays invisible and untouched +// — the default fast path copies BASE into REMOTE, sees no divergence, plans +// nothing, and never repairs. Only --verify looks. +func TestEngine_Run_DefaultPathDoesNotRepairPoison(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + ) + + contentA := "print('A')\n" + contentB := "print('B')\n" + + // Flagship poison: disk and manifest both hold A, the server holds B. + dir := syncedProject(t, map[string]string{"app.py": contentA}, catalogID, versionID) + + fake := (&fakeFilesClient{}).withVersionContent(catalogID, versionID, seededServerContents(t, dir, map[string]string{ + "app.py": contentA, + }, map[string][]byte{"app.py": []byte(contentB)})) + + manifestPath, _ := statePaths(dir) + before := readStateFile(t, manifestPath) + + e := engineFor(t, dir, Options{}, fake, catalogID, versionID) + + _, err := e.Run() + require.NoError(t, err) + + assert.Zero(t, fake.AllFilesCalls(), "the default path must not fetch the remote") + assert.Equal(t, before, readStateFile(t, manifestPath), + "without --verify the poison survives the run (the documented residual risk)") +} + +// TestEngine_Run_VerifyPersistsNoVerifyState: a --verify run must leave +// exactly the state a plain sync would leave. Two identical projects, one +// synced plainly and one synced with --verify first, end with byte-identical +// manifest.json and config.json — no verify-specific field anywhere, and no +// verify-dependent residue in the next plain sync's state. +func TestEngine_Run_VerifyPersistsNoVerifyState(t *testing.T) { + const ( + catalogID = "cid-1" + versionID = "ver-1" + newVersion = "ver-2" + ) + + files := map[string]string{"app.py": "print('A')\n"} + + // A fixed clock makes syncedAt deterministic, so byte-equality of the + // two projects' state files means the content is identical, not merely + // shaped the same. + fixedNow := time.Date(2025, 3, 1, 12, 0, 0, 0, time.UTC) + + // Plain sync only. The server seed is captured while the tree still + // holds the pre-change content, then disk moves on. + dirPlain := syncedProject(t, files, catalogID, versionID) + + plainSeed := seededServerContents(t, dirPlain, files, nil) + modifyFile(t, dirPlain, "app.py", "print('B')\n") + + plainFake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: newVersion, + }).withVersionContent(catalogID, versionID, plainSeed) + + ePlain, err := newWithDeps(dirPlain, Options{Yes: true}, Deps{ + Files: plainFake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: func() time.Time { return fixedNow }, + }) + require.NoError(t, err) + + _, err = ePlain.Run() + require.NoError(t, err) + require.NoError(t, ePlain.Close()) + + // --verify first, then a plain sync on top. Same seed discipline: the + // server holds the pre-change content, so BASE still describes it and + // the only divergence in play is the ordinary local modification. + dirVerify := syncedProject(t, files, catalogID, versionID) + + verifySeed := seededServerContents(t, dirVerify, files, nil) + modifyFile(t, dirVerify, "app.py", "print('B')\n") + + verifyFake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: newVersion, + }).withVersionContent(catalogID, versionID, verifySeed) + + eVerify, err := newWithDeps(dirVerify, Options{Yes: true, Verify: true}, Deps{ + Files: verifyFake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: func() time.Time { return fixedNow }, + }) + require.NoError(t, err) + + _, err = eVerify.Run() + require.NoError(t, err) + require.NoError(t, eVerify.Close()) + + assert.Empty(t, eVerify.Divergences(), "the server holds the pre-change content, so the verify run must find no divergence") + + ePlain2, err := newWithDeps(dirVerify, Options{Yes: true}, Deps{ + Files: verifyFake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVersion), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: func() time.Time { return fixedNow }, + }) + require.NoError(t, err) + + _, err = ePlain2.Run() + require.NoError(t, err) + require.NoError(t, ePlain2.Close()) + + manifestPlainPath, configPlainPath := statePaths(dirPlain) + manifestVerifyPath, configVerifyPath := statePaths(dirVerify) + + assert.Equal(t, string(readStateFile(t, manifestPlainPath)), string(readStateFile(t, manifestVerifyPath)), + "the verify run's persisted manifest must be byte-identical to a plain sync's") + + // createdAt is written at init from the real clock, so it legitimately + // differs between the two projects; compare everything else. + stripCreatedAt := func(t *testing.T, path string) string { + t.Helper() + + var cfg map[string]json.RawMessage + require.NoError(t, json.Unmarshal(readStateFile(t, path), &cfg)) + + delete(cfg, "createdAt") + + b, err := json.Marshal(cfg) + require.NoError(t, err) + + return string(b) + } + + assert.Equal(t, stripCreatedAt(t, configPlainPath), stripCreatedAt(t, configVerifyPath), + "the verify run's persisted config must match a plain sync's (modulo init-time createdAt)") + + // Schema discipline: the manifest carries only the four documented keys. + rawManifest := readStateFile(t, manifestVerifyPath) + + var manifestKeys map[string]json.RawMessage + require.NoError(t, json.Unmarshal(rawManifest, &manifestKeys)) + + wantKeys := []string{"version", "syncedAt", "syncedVersionId", "files"} + require.Len(t, manifestKeys, len(wantKeys)) + + for _, k := range wantKeys { + assert.Contains(t, manifestKeys, k) + } + + // config.json must have grown no verify-related field. + var configKeys map[string]json.RawMessage + require.NoError(t, json.Unmarshal(readStateFile(t, configVerifyPath), &configKeys)) + + for _, forbidden := range []string{"verified", "lastVerifiedVersionId", "lastVerifyAt"} { + assert.NotContains(t, configKeys, forbidden) + } +} diff --git a/internal/workload/sync/windows_crossplatform_test.go b/internal/workload/sync/windows_crossplatform_test.go new file mode 100644 index 000000000..1210bfecb --- /dev/null +++ b/internal/workload/sync/windows_crossplatform_test.go @@ -0,0 +1,361 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/fileops" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests provide Windows unit coverage requested on the ticket, scoped +// to unit tests only. They run on ALL platforms (including Windows) for the +// platform-independent behaviours, and explicitly skip with a visible +// reason for platform-inappropriate checks. +// +// The Windows CI leg has no -race and is continue-on-error (CFX-7059), so +// Linux remains the only race gate. These tests are written to be +// meaningful even on a best-effort leg: every skip carries a visible reason +// so a CI summary distinguishes "skipped on Windows" from "passed on +// Windows". +// +// Fulfills VAL-SYMLINK-012 and VAL-REGRESSION-013. + +// --------------------------------------------------------------------------- +// SHA-256 parity (VAL-REGRESSION-013a) +// --------------------------------------------------------------------------- + +// TestSHA256Parity_CRLFContent asserts that a file's computed SHA-256 hash +// is the same regardless of platform, including content containing CRLF +// line endings. Go's os.Open opens files in binary mode by default, so +// \r\n must not be translated to \n. The test guards against any future +// regression that introduces text-mode handling (e.g., a bufio.Scanner +// that strips \r, or a file open with O_TEXT on Windows). +// +// The expected hashes are hardcoded SHA-256 hex digests of the raw bytes, +// computed once with an independent tool (shasum -a 256 / python hashlib) +// rather than with crypto/sha256 in this test, so every expected value is +// externally eyeball-verifiable: the assertion cannot pass merely because +// the reference computation shares a translation bug with the code under +// test. A transcription error fails loudly on the first run, because +// fileops.HashFile hashes the bytes actually written to disk. +// +// Fulfills VAL-REGRESSION-013(a). +func TestSHA256Parity_CRLFContent(t *testing.T) { + cases := []struct { + name string + content []byte + wantHash string + }{ + // CRLF line endings — the primary case. If \r\n is translated to + // \n by any layer, the hash changes. + {name: "CRLFOnly", content: []byte("line1\r\nline2\r\nline3\r\n"), wantHash: "96e4d66c5a42d171e8aa107059eacaf52d3c2c095cbba1316b590a7392497718"}, + // Mixed line endings — CRLF and LF in the same file. + {name: "MixedLineEndings", content: []byte("unix\nwindows\r\nmixed\r\nunix\n"), wantHash: "c29fb97484f737e847c2efdf0ffad706c15398f9c38ccf85c677eb02e12c45ee"}, + // LF-only — the control case. Should always match. + {name: "LFOnly", content: []byte("line1\nline2\nline3\n"), wantHash: "66663af9c7aa341431a8ee2ff27b72abd06c9218f517bb6fef948e4803c19e03"}, + // Empty file — the SHA-256 of the empty string is a known constant. + {name: "EmptyFile", content: []byte{}, wantHash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + // Binary data containing \r without \n — must not be stripped. + {name: "BinaryWithCR", content: []byte("data\r\x00more\rdata"), wantHash: "fc440a4aa557bdf06c9e41f133e8380d371d23e4f564ff8f93c74d46329e1984"}, + // A lone \r at end of file — must not be stripped or translated. + {name: "TrailingCR", content: []byte("content\r"), wantHash: "98720b6e22244e28a7ac817c78fe102d2deb7902a607de513c5b38e81c178973"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Write the exact bytes to a temp file. + dir := t.TempDir() + p := filepath.Join(dir, "testfile.bin") + require.NoError(t, os.WriteFile(p, tc.content, 0o644)) + + gotHash, gotSize, err := fileops.HashFile(p) + require.NoError(t, err) + + assert.Equal(t, tc.wantHash, gotHash, + "SHA-256 must match the raw bytes exactly — no text-mode translation of \\r\\n") + assert.Equal(t, int64(len(tc.content)), gotSize, + "byte size must match the raw content length — no byte added or removed") + }) + } +} + +// TestSHA256Parity_StreamedMatchesFileHash asserts that the streaming hash +// used during uploads (newStreamHasher + streamedEntry) produces the same +// SHA-256 as fileops.HashFile for the same bytes, including CRLF content. +// The manifest records the streamed hash; the plan compares against the +// file hash. If they diverged, every file would appear modified on every +// sync — a silent correctness bug. +// +// Both pipelines are asserted against the same hardcoded digests the CRLF +// test pins: parity between the two alone would still pass if both shared +// a systematic translation bug, so the constants keep the parity claim +// externally verifiable. +// +// Fulfills VAL-REGRESSION-013(a) for the streaming hash path. +func TestSHA256Parity_StreamedMatchesFileHash(t *testing.T) { + cases := []struct { + name string + content []byte + wantHash string + }{ + {name: "crlf", content: []byte("line1\r\nline2\r\nline3\r\n"), wantHash: "96e4d66c5a42d171e8aa107059eacaf52d3c2c095cbba1316b590a7392497718"}, + {name: "mixed", content: []byte("unix\nwindows\r\nmixed\r\n"), wantHash: "3a36c87e075f1dce28dab20be3eb9fbac27d7c7f0b834647bf05b087a8501b5b"}, + {name: "binary_cr", content: []byte("binary\r\x00data\rmore"), wantHash: "3127d488be8b9d5a70e68996a339f4068898eeba2aaaae86382a44eb8942fd8f"}, + {name: "empty", content: []byte{}, wantHash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // File hash via fileops.HashFile. + dir := t.TempDir() + p := filepath.Join(dir, "stream.bin") + require.NoError(t, os.WriteFile(p, tc.content, 0o644)) + + fileHash, fileSize, err := fileops.HashFile(p) + require.NoError(t, err) + + // Streamed hash via the same pipeline the uploaders use. + h := newStreamHasher() + _, err = h.Write(tc.content) + require.NoError(t, err) + + streamed := streamedEntry(h, int64(len(tc.content))) + + assert.Equal(t, tc.wantHash, fileHash, + "file SHA-256 must match the hardcoded digest for the raw bytes") + assert.Equal(t, tc.wantHash, streamed.Hash, + "streamed SHA-256 must match the hardcoded digest for the raw bytes") + assert.Equal(t, fileSize, streamed.Size, + "streamed size must match file size for the same bytes") + }) + } +} + +// --------------------------------------------------------------------------- +// Path formatting (VAL-REGRESSION-013b) +// --------------------------------------------------------------------------- + +// TestPathFormatting_PlanUsesForwardSlashes asserts that all paths in the +// sync plan use forward slashes, identical to POSIX. On Windows the +// OS-native walker produces backslash-separated paths; NormalizePath +// converts them to forward slashes. This test runs on all platforms — +// trivially passing on POSIX where the separator is already '/', and +// meaningfully on Windows where the conversion is load-bearing. +// +// Fulfills VAL-REGRESSION-013(b) for plan output paths. +func TestPathFormatting_PlanUsesForwardSlashes(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "sub/dir/helper.py": "def help(): pass\n", + "deep/nested/path/f.py": "x\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + require.NotEmpty(t, plan.Uploads, "plan must have uploads to assert path formatting") + + checkForwardSlash := func(t *testing.T, path string, label string) { + t.Helper() + + assert.NotContains(t, path, "\\", "%s path must use forward slashes, got: %s", label, path) + } + + for _, fa := range plan.Uploads { + checkForwardSlash(t, fa.Path, "upload") + } + + for _, fa := range plan.Downloads { + checkForwardSlash(t, fa.Path, "download") + } + + for _, fa := range plan.Deletes { + checkForwardSlash(t, fa.Path, "delete") + } + + for _, fa := range plan.Conflicts { + checkForwardSlash(t, fa.Path, "conflict") + } +} + +// TestPathFormatting_ManifestKeysForwardSlashes asserts that manifest.json +// keys use forward slashes after a sync, identical to POSIX. The manifest +// is written from the plan paths, which are already normalized. This test +// runs on all platforms. +// +// Fulfills VAL-REGRESSION-013(b) for manifest keys. +func TestPathFormatting_ManifestKeysForwardSlashes(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "sub/dir/helper.py": "def help(): pass\n", + "deep/nested/path/f.py": "x\n", + }) + + // The fake must have IDs configured so CreateCatalog and ApplyStage + // succeed during Execute — a first-sync scenario against an empty + // artifact. + fake := &fakeFilesClient{ + catalogID: "cid-paths", + stageID: "stage-paths", + versionID: "ver-paths", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + require.NotEmpty(t, plan.Uploads, "plan must have uploads to produce a manifest") + + _, err = e.Execute(plan) + require.NoError(t, err) + + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + require.NotEmpty(t, manifest.Files, "manifest must have file entries after sync") + + for key := range manifest.Files { + assert.NotContains(t, key, "\\", + "manifest key must use forward slashes, got: %s", key) + } +} + +// --------------------------------------------------------------------------- +// No-symlink project on Windows (VAL-SYMLINK-012) +// --------------------------------------------------------------------------- + +// TestNoSymlinkProject_SyncsWithNoNotice asserts that a normal project with +// no symlinks produces no symlink notice and syncs normally. The no-symlink +// code path is platform-independent: the walk simply finds no symlinks to +// report, so skippedSymlinks stays empty and no warning is emitted. This +// test runs on all platforms, including Windows where symlink creation +// needs Developer Mode — a project without symlinks is the common case. +// +// Fulfills VAL-SYMLINK-012. +func TestNoSymlinkProject_SyncsWithNoNotice(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "sub/helper.py": "def help(): pass\n", + "requirements.txt": "flask\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + // Capture the warn log to verify no symlink notice is emitted. + logged := captureWarnLog(t, func() { + _, err := e.Plan() + require.NoError(t, err, "a no-symlink project must sync without error") + }) + + // No symlink notice is emitted. + assert.NotContains(t, logged, "symlink", + "a project with no symlinks must emit no symlink-related warning") + + // The skippedSymlinks field is empty. + assert.Empty(t, e.skippedSymlinks, + "the skippedSymlinks field must be empty when there are no symlinks") + + // The plan contains the real files (the sync works normally). + uploadPaths := uploadPathsOf(e.plan) + assert.Contains(t, uploadPaths, "app.py") + assert.Contains(t, uploadPaths, "sub/helper.py") + assert.Contains(t, uploadPaths, "requirements.txt") +} + +// --------------------------------------------------------------------------- +// Platform-inappropriate check skips +// --------------------------------------------------------------------------- + +// TestFileModePreservation_SkipOnWindows documents that file-mode +// assertions are skipped on Windows with a visible reason. Windows +// collapses POSIX mode bits: a file created with 0o644 and one created +// with 0o600 are indistinguishable through os.Stat on NTFS. Any test that +// asserts a specific file mode after a sync would silently pass on +// Windows even if the mode were wrong, so it must skip rather than report +// a vacuous pass. +// +// This test is the explicit skip: it creates a file with a specific mode, +// verifies the mode on POSIX, and skips on Windows with a visible reason. +// It serves as the documented precedent for future file-mode tests. +// +// Fulfills VAL-REGRESSION-013(c) for file-mode assertions. +func TestFileModePreservation_SkipOnWindows(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("file-mode assertions skipped on Windows: Windows collapses POSIX mode bits (0o644 vs 0o600 are indistinguishable on NTFS)") + } + + dir := t.TempDir() + p := filepath.Join(dir, "mode_test.py") + require.NoError(t, os.WriteFile(p, []byte("x"), 0o644)) + + info, err := os.Stat(p) + require.NoError(t, err) + + // On POSIX, the mode is preserved exactly. + assert.Equal(t, os.FileMode(0o644), info.Mode()&os.ModePerm, + "file mode must be preserved on POSIX") +} + +// TestSyncLock_NoCrossProcessExclusionOnWindows documents that the sync +// lock is a no-op on Windows (synclock_windows.go), so no test may assume +// cross-process mutual exclusion there. On POSIX, a second acquire fails; +// on Windows it succeeds silently. This test asserts the POSIX behaviour +// and skips on Windows with a visible reason, mirroring the existing +// TestSyncLock_DoubleAcquireFailsOnUnix precedent. +// +// Fulfills VAL-REGRESSION-013(c) for the sync lock. +func TestSyncLock_NoCrossProcessExclusionOnWindows(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sync lock is a no-op on Windows (synclock_windows.go); cross-process mutual exclusion is not available — tracked in RAPTOR-16928") + } + + dir := setupProject(t) + + lock1, err := AcquireSyncLock(dir) + require.NoError(t, err) + + t.Cleanup(func() { _ = lock1.Release() }) + + // A second acquire must fail on POSIX — the lock provides mutual + // exclusion. On Windows this would succeed (no-op lock), so the + // test skips rather than asserting a vacuous pass. + _, err = AcquireSyncLock(dir) + assert.Error(t, err, + "second acquire must fail on POSIX while the first is held") +} diff --git a/internal/workload/up/codechange_regression_test.go b/internal/workload/up/codechange_regression_test.go new file mode 100644 index 000000000..5d9663e18 --- /dev/null +++ b/internal/workload/up/codechange_regression_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package up + +import ( + "testing" + + "github.com/datarobot/cli/internal/workload/manifest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests guard `dr workload up`'s code-change measurement so the +// upload-integrity fix cannot regress it silently. defaultCodeChange builds +// a dry-run sync engine and counts plan upload + delete rows; the first-deploy +// flag is set when the project is not linked. +// +// The modified-tree and unchanged-tree count tests live in the sync package +// (TestWorkloadUp_CodeChangeCount_*) because defaultCodeChange uses sync.New +// with production deps for linked projects, which cannot be tested without +// staging. The sync engine's Plan() is the measurement's core, and testing it +// with injected fakes verifies the same logic. +// +// Fulfills VAL-REGRESSION-008. + +// dockerfileManifestYAML is a minimal manifest with a Dockerfile build mode, +// which is what defaultCodeChange requires to measure code changes. +const dockerfileManifestYAML = `name: my-app +artifact: + name: my-app-artifact + spec: + type: service + containerGroups: + - name: default + containers: + - name: primary + primary: true + port: 8080 + imageBuildConfig: + dockerfile: + source: provided +runtime: + containerGroups: + - name: default + replicaCount: 1 + containers: + - name: primary + resources: + cpu: 0.5 + memory: 1Gi +` + +// TestDefaultCodeChange_FirstDeploy verifies that defaultCodeChange reports +// the first-deploy flag for an unlinked project. When wapi.Exists returns +// false (no .datarobot/workload or .wapi directory), the measurement returns +// CodeChange{Applies: true, FirstDeploy: true} without calling the sync +// engine — there is nothing to compare against, so every file is new. +func TestDefaultCodeChange_FirstDeploy(t *testing.T) { + dir := t.TempDir() + + // The project is not linked: no .datarobot/workload or .wapi directory. + m, err := manifest.Parse([]byte(dockerfileManifestYAML), "") + require.NoError(t, err) + + require.Equal(t, manifest.BuildModeDockerfile, m.BuildMode(), + "fixture must have a Dockerfile build mode for code change to apply") + + loaded := Loaded{ + ProjectDir: dir, + Manifest: m, + } + + change, err := defaultCodeChange(loaded, Live{}) + require.NoError(t, err) + + assert.True(t, change.Applies, "Dockerfile build mode means code change applies") + assert.True(t, change.FirstDeploy, + "unlinked project must report FirstDeploy=true (nothing to compare against)") + assert.Equal(t, 0, change.Files, + "first deploy has no diff count (every file is new, not changed)") +} + +// TestDefaultCodeChange_ImageModeDoesNotApply verifies that a manifest with +// an image build mode (not Dockerfile or Generated) does not report a code +// change. A published image has no code to sync. +func TestDefaultCodeChange_ImageModeDoesNotApply(t *testing.T) { + dir := t.TempDir() + + const imageManifestYAML = `name: my-app +artifact: + name: my-app-artifact + spec: + type: service + containerGroups: + - name: default + containers: + - name: primary + primary: true + port: 8080 + imageUri: registry.example.com/my-app:v1 +runtime: + containerGroups: + - name: default + replicaCount: 1 + containers: + - name: primary + resources: + cpu: 0.5 + memory: 1Gi +` + + m, err := manifest.Parse([]byte(imageManifestYAML), "") + require.NoError(t, err) + + assert.Equal(t, manifest.BuildModeImage, m.BuildMode(), + "fixture must have an image build mode") + + loaded := Loaded{ + ProjectDir: dir, + Manifest: m, + } + + change, err := defaultCodeChange(loaded, Live{}) + require.NoError(t, err) + + assert.False(t, change.Applies, "image build mode means no code change to measure") + assert.False(t, change.FirstDeploy, "non-Dockerfile mode must not set FirstDeploy") +} diff --git a/internal/workload/up/run_test.go b/internal/workload/up/run_test.go index 904880d68..8971e6a65 100644 --- a/internal/workload/up/run_test.go +++ b/internal/workload/up/run_test.go @@ -432,7 +432,7 @@ func TestRun_WizardRedirectIsFollowed(t *testing.T) { return wizard.Result{Path: manifest.Path(app)}, nil }, create: func(any) (*workload.Workload, error) { return running("wl-new"), nil }, - wait: func(string, time.Duration, time.Duration, func(*workload.Workload)) (*workload.Workload, error) { + wait: func(string, workload.Serving, time.Duration, time.Duration, func(*workload.Workload)) (*workload.Workload, error) { return running("wl-new"), nil }, writeID: func(path, _ string) error {