Add --git-fetch-base-branch to fetch the base branch at checkout

agent/agent_configuration.go

@@ -36,6 +36,7 @@ type AgentConfiguration struct {
 	GitSubmoduleCloneConfig         []string
 	SkipCheckout                    bool
 	GitSkipFetchExistingCommits     bool
+	GitFetchBaseBranch              bool
 	CheckoutOverrideMode            env.CheckoutOverrideMode
 	CheckoutAttempts                int
 	AllowedRepositories             []*regexp.Regexp

agent/job_runner.go

@@ -615,6 +615,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 		setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
 		setEnv("BUILDKITE_SKIP_CHECKOUT", fmt.Sprint(r.conf.AgentConfiguration.SkipCheckout))
 		setEnv("BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS", fmt.Sprint(r.conf.AgentConfiguration.GitSkipFetchExistingCommits))
+		setEnv("BUILDKITE_GIT_FETCH_BASE_BRANCH", fmt.Sprint(r.conf.AgentConfiguration.GitFetchBaseBranch))
 		// A zero timeout means no checkout timeout; emit it anyway under strict so
 		// a job-supplied value can't reintroduce one past the agent config.
 		setEnv("BUILDKITE_GIT_CHECKOUT_TIMEOUT", strconv.Itoa(r.conf.AgentConfiguration.GitCheckoutTimeout))
@@ -633,6 +634,9 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 		if r.conf.AgentConfiguration.GitSkipFetchExistingCommits {
 			setCheckoutEnv("BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS", "true")
 		}
+		if r.conf.AgentConfiguration.GitFetchBaseBranch {
+			setCheckoutEnv("BUILDKITE_GIT_FETCH_BASE_BRANCH", "true")
+		}
 		if r.conf.AgentConfiguration.GitCheckoutTimeout > 0 {
 			setCheckoutEnv("BUILDKITE_GIT_CHECKOUT_TIMEOUT", strconv.Itoa(r.conf.AgentConfiguration.GitCheckoutTimeout))
 		}

clicommand/agent_start.go

@@ -174,6 +174,7 @@ type AgentStartConfig struct {
 	GitSubmoduleCloneConfig     []string `cli:"git-submodule-clone-config"`
 	SkipCheckout                bool     `cli:"skip-checkout"`
 	GitSkipFetchExistingCommits bool     `cli:"git-skip-fetch-existing-commits"`
+	GitFetchBaseBranch          bool     `cli:"git-fetch-base-branch"`
 	CheckoutOverrideMode        string   `cli:"checkout-override-mode"`
 	CheckoutAttempts            int      `cli:"checkout-attempts"`
 
@@ -546,6 +547,7 @@ var AgentStartCommand = &cli.Command{
 		GitCheckoutTimeoutFlag,
 		GitSubmoduleCloneConfigFlag,
 		GitSkipFetchExistingCommitsFlag,
+		GitFetchBaseBranchFlag,
 		CheckoutAttemptsFlag,
 
 		&cli.StringFlag{
@@ -1006,6 +1008,7 @@ var AgentStartCommand = &cli.Command{
 			GitSubmoduleCloneConfig:         cfg.GitSubmoduleCloneConfig,
 			SkipCheckout:                    cfg.SkipCheckout,
 			GitSkipFetchExistingCommits:     cfg.GitSkipFetchExistingCommits,
+			GitFetchBaseBranch:              cfg.GitFetchBaseBranch,
 			CheckoutOverrideMode:            checkoutMode,
 			CheckoutAttempts:                cfg.CheckoutAttempts,
 			SSHKeyscan:                      !cfg.NoSSHKeyscan,

clicommand/bootstrap.go

@@ -75,6 +75,7 @@ type BootstrapConfig struct {
 	SkipCheckout                 bool          `cli:"skip-checkout"`
 	GitCheckoutTimeout           int           `cli:"git-checkout-timeout"`
 	GitSkipFetchExistingCommits  bool          `cli:"git-skip-fetch-existing-commits"`
+	GitFetchBaseBranch           bool          `cli:"git-fetch-base-branch"`
 	GitCheckoutFlags             string        `cli:"git-checkout-flags"`
 	GitCloneFlags                string        `cli:"git-clone-flags"`
 	GitFetchFlags                string        `cli:"git-fetch-flags"`
@@ -284,6 +285,7 @@ var BootstrapCommand = &cli.Command{
 		GitSubmoduleCloneConfigFlag,
 		GitCheckoutTimeoutFlag,
 		GitSkipFetchExistingCommitsFlag,
+		GitFetchBaseBranchFlag,
 		CheckoutAttemptsFlag,
 
 		&cli.StringFlag{
@@ -487,6 +489,7 @@ var BootstrapCommand = &cli.Command{
 			SkipCheckout:                 cfg.SkipCheckout,
 			GitCheckoutTimeout:           cfg.GitCheckoutTimeout,
 			GitSkipFetchExistingCommits:  cfg.GitSkipFetchExistingCommits,
+			GitFetchBaseBranch:           cfg.GitFetchBaseBranch,
 			CheckoutOverrideMode:         checkoutMode,
 			Command:                      cfg.Command,
 			CommandEval:                  cfg.CommandEval,

clicommand/global.go

@@ -334,6 +334,12 @@ var (
 		Sources: cli.EnvVars("BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS"),
 	}
 
+	GitFetchBaseBranchFlag = &cli.BoolFlag{
+		Name:    "git-fetch-base-branch",
+		Usage:   "Also fetch the base branch during checkout, so every command in the job can diff against its current tip. The base branch is the first non-empty value of {$BUILDKITE_PULL_REQUEST_BASE_BRANCH, $BUILDKITE_PIPELINE_DEFAULT_BRANCH}, and nothing is fetched when it is the branch being built (default: false)",
+		Sources: cli.EnvVars("BUILDKITE_GIT_FETCH_BASE_BRANCH"),
+	}
+
 	CheckoutAttemptsFlag = &cli.IntFlag{
 		Name:    "checkout-attempts",
 		Value:   6,

docs/remote-git-mirrors.md

@@ -176,7 +176,9 @@ point a reader would otherwise file a bug. §10 lists each one and where it goes
 4. `git clean`, optional `git lfs install --local`.
 5. `fetchSource` → `git fetch <flags> -- origin <refspec>`, where the refspec is
    the custom refspec, or `refs/pull/N/{head,merge}`, or the branch (when
-   `BUILDKITE_COMMIT=HEAD`), or — the common case — the commit SHA.
+   `BUILDKITE_COMMIT=HEAD`), or — the common case — the commit SHA. With
+   `--git-fetch-base-branch`, an additional fetch runs first and updates
+   `refs/remotes/origin/<base>`.
 6. Commit verification, sparse setup, `git checkout`, submodules, LFS, clean.
 
 Four things are easy to get wrong, and the plan depends on all of them:

env/protected.go

@@ -115,6 +115,7 @@ var checkoutOverrideScope = map[string]struct{}{
 	"BUILDKITE_GIT_CLEAN_FLAGS":                 {},
 	"BUILDKITE_GIT_CLONE_FLAGS":                 {},
 	"BUILDKITE_GIT_COMMIT_VERIFICATION":         {},
+	"BUILDKITE_GIT_FETCH_BASE_BRANCH":           {},
 	"BUILDKITE_GIT_FETCH_FLAGS":                 {},
 	"BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS": {},
 	"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE":        {},

internal/job/checkout_fetch.go

@@ -86,18 +86,27 @@ func (e *Executor) fetchSource(ctx context.Context, addBloblessFilter bool, atte
 	skipFetch := (e.GitSkipFetchExistingCommits || mirrorHit) && e.Commit != "HEAD" &&
 		hasGitCommit(ctx, e.shell, ".git", e.Commit)
 
-	span.SetAttributes(attribute.Bool("git.skipped", skipFetch))
-
-	if skipFetch {
-		e.shell.Commentf("Commit %q already exists locally, skipping fetch", e.Commit)
-		return nil
-	}
+	span.SetAttributes(
+		attribute.Bool("git.skipped", skipFetch),
+		attribute.Bool("git.fetch_base_branch", e.GitFetchBaseBranch),
+	)
 
 	gitFetchFlags := e.GitFetchFlags
 	if addBloblessFilter {
 		gitFetchFlags = "--filter=blob:none " + gitFetchFlags
 	}
 
+	// A job that asked for the base branch wants it whether or not its own commit
+	// still needs fetching, so this precedes the skip below.
+	if e.GitFetchBaseBranch {
+		e.fetchBaseBranch(ctx, gitFetchFlags)
+	}
+
+	if skipFetch {
+		e.shell.Commentf("Commit %q already exists locally, skipping fetch", e.Commit)
+		return nil
+	}
+
 	switch kind {
 	case refspecCustom:
 		// If a refspec is provided then use it instead.
@@ -265,6 +274,59 @@ func prMergeRefspecHint(isMergeRefspec bool) string {
 	return "\nThis is possibly due to a merge conflict, or GitHub being unable to create the merge ref automatically"
 }
 
+// baseBranchToFetch returns the branch a diff in this job would be taken against.
+//
+// The precedence is the one `pipeline upload` already uses for --git-diff-base,
+// minus its final literal "main": guessing a name would mean a failed fetch on
+// every repository whose default branch is called something else.
+func (e *Executor) baseBranchToFetch() string {
+	built := strings.TrimPrefix(e.Branch, "refs/heads/")
+	for _, name := range []string{"BUILDKITE_PULL_REQUEST_BASE_BRANCH", "BUILDKITE_PIPELINE_DEFAULT_BRANCH"} {
+		branch, _ := e.shell.Env.Get(name)
+		branch = strings.TrimPrefix(branch, "refs/heads/")
+		if branch == "" {
+			continue
+		}
+		if branch == built {
+			// The job's own fetch already brought the branch under test.
+			return ""
+		}
+		return branch
+	}
+	return ""
+}
+
+// fetchBaseBranch updates refs/remotes/origin/<base>, which the job's own fetch
+// does not: it asks for refs/pull/N/head and the commit, and checkout directories
+// are reused, so that ref otherwise holds whatever an earlier build left there.
+//
+// It is separate from that fetch because FETCH_HEAD resolves to the first ref
+// fetched and the checkout depends on it. The refspec is explicit and forced so the
+// remote-tracking ref lands whatever remote.origin.fetch says, and follows a
+// force-pushed base branch instead of stopping at a non-fast-forward rejection.
+//
+// Neither fatal nor retried: the checkout is complete without it, and the retry on
+// the job's own fetch waits out asynchronous creation of the ref being built, which
+// a deleted base branch never gets.
+func (e *Executor) fetchBaseBranch(ctx context.Context, gitFetchFlags string) {
+	base := e.baseBranchToFetch()
+	if base == "" {
+		return
+	}
+
+	e.shell.Commentf("Fetch base branch %q", base)
+
+	refspec := fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", base, base)
+	if err := gitFetch(ctx, gitFetchArgs{
+		Shell:         e.shell,
+		GitFetchFlags: gitFetchFlags,
+		Repository:    "origin",
+		RefSpecs:      []string{refspec},
+	}); err != nil {
+		e.shell.Warningf("Couldn't fetch base branch %q, continuing: %v", base, err)
+	}
+}
+
 func isExistingCheckoutRemoteMirrorAttempt(attempt *remoteMirrorAttempt) bool {
 	return attempt != nil &&
 		attempt.site == remoteMirrorSiteExistingCheckout &&