git: add fetch-depth and fetch-tags support

client/llb/git_test.go

@@ -69,6 +69,28 @@ func TestGit(t *testing.T) {
 				"git.fullurl":          "https://github.com/foo/bar.git",
 			},
 		},
+		{
+			name:       "fetch depth",
+			st:         Git("github.com/foo/bar.git", "ref", GitFetchDepth(0)),
+			identifier: "git://github.com/foo/bar.git#ref",
+			attrs: map[string]string{
+				"git.authheadersecret": "GIT_AUTH_HEADER",
+				"git.authtokensecret":  "GIT_AUTH_TOKEN",
+				"git.fetchdepth":       "0",
+				"git.fullurl":          "https://github.com/foo/bar.git",
+			},
+		},
+		{
+			name:       "fetch tags",
+			st:         Git("github.com/foo/bar.git", "ref", GitFetchTags()),
+			identifier: "git://github.com/foo/bar.git#ref",
+			attrs: map[string]string{
+				"git.authheadersecret": "GIT_AUTH_HEADER",
+				"git.authtokensecret":  "GIT_AUTH_TOKEN",
+				"git.fetchtags":        "true",
+				"git.fullurl":          "https://github.com/foo/bar.git",
+			},
+		},
 	}
 
 	for _, tc := range tcases {

client/llb/source.go

@@ -470,6 +470,14 @@ func Git(url, fragment string, opts ...GitOption) State {
 		attrs[pb.AttrGitChecksum] = checksum
 		addCap(&gi.Constraints, pb.CapSourceGitChecksum)
 	}
+	if gi.FetchDepth != nil {
+		attrs[pb.AttrGitFetchDepth] = strconv.Itoa(*gi.FetchDepth)
+		addCap(&gi.Constraints, pb.CapSourceGitFetchDepth)
+	}
+	if gi.FetchTags {
+		attrs[pb.AttrGitFetchTags] = "true"
+		addCap(&gi.Constraints, pb.CapSourceGitFetchTags)
+	}
 
 	if gi.SkipSubmodules {
 		attrs[pb.AttrGitSkipSubmodules] = "true"
@@ -500,6 +508,8 @@ type GitInfo struct {
 	KnownSSHHosts    string
 	MountSSHSock     string
 	Checksum         string
+	FetchDepth       *int
+	FetchTags        bool
 	Ref              string
 	SubDir           string
 	SkipSubmodules   bool
@@ -517,6 +527,21 @@ func GitSubDir(v string) GitOption {
 	})
 }
 
+func GitFetchDepth(v int) GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		if v < 0 {
+			return
+		}
+		gi.FetchDepth = &v
+	})
+}
+
+func GitFetchTags() GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.FetchTags = true
+	})
+}
+
 func GitSkipSubmodules() GitOption {
 	return gitOptionFunc(func(gi *GitInfo) {
 		gi.SkipSubmodules = true

frontend/dockerfile/dfgitutil/git_ref.go

@@ -56,6 +56,12 @@ type GitRef struct {
 
 	// Submodules is true for URL that controls whether to fetch git submodules.
 	Submodules *bool
+
+	// FetchDepth controls how much history to fetch.
+	FetchDepth *int
+
+	// FetchTags controls whether to fetch tag refs for shallow clones.
+	FetchTags *bool
 }
 
 // ParseGitRef parses a git ref.
@@ -131,7 +137,7 @@ func (gf *GitRef) loadQuery(query url.Values) error {
 		case 0, 1:
 			if len(v) == 0 || v[0] == "" {
 				switch k {
-				case "submodules", "keep-git-dir":
+				case "fetch-tags", "submodules", "keep-git-dir":
 					v = nil
 				default:
 					return errors.Errorf("query %q has no value", k)
@@ -158,6 +164,24 @@ func (gf *GitRef) loadQuery(query url.Values) error {
 			gf.SubDir = v[0]
 		case "checksum", "commit":
 			gf.Checksum = v[0]
+		case "fetch-depth":
+			vv, err := strconv.Atoi(v[0])
+			if err != nil || vv < 0 {
+				return errors.Errorf("invalid fetch-depth value: %q", v[0])
+			}
+			gf.FetchDepth = &vv
+		case "fetch-tags":
+			var vv bool
+			if len(v) == 0 {
+				vv = true
+			} else {
+				var err error
+				vv, err = strconv.ParseBool(v[0])
+				if err != nil {
+					return errors.Errorf("invalid fetch-tags value: %q", v[0])
+				}
+			}
+			gf.FetchTags = &vv
 		case "keep-git-dir":
 			var vv bool
 			if len(v) == 0 {

frontend/dockerfile/dfgitutil/git_ref_test.go

@@ -192,6 +192,24 @@ func TestParseGitRef(t *testing.T) {
 				Ref:       "refs/heads/v1.0",
 			},
 		},
+		{
+			ref: "https://github.com/moby/buildkit.git?fetch-depth=0#v1.2.3",
+			expected: &GitRef{
+				Remote:     "https://github.com/moby/buildkit.git",
+				ShortName:  "buildkit",
+				Ref:        "v1.2.3",
+				FetchDepth: ptrInt(0),
+			},
+		},
+		{
+			ref: "https://github.com/moby/buildkit.git?fetch-tags#v1.2.3",
+			expected: &GitRef{
+				Remote:    "https://github.com/moby/buildkit.git",
+				ShortName: "buildkit",
+				Ref:       "v1.2.3",
+				FetchTags: ptrBool(true),
+			},
+		},
 		{
 			ref: "https://github.com/moby/buildkit.git?ref=v1.0.0#v1.2.3",
 			err: "ref conflicts",
@@ -218,6 +236,14 @@ func TestParseGitRef(t *testing.T) {
 			ref: "https://github.com/moby/buildkit.git?invalid=123",
 			err: "unexpected query \"invalid\"",
 		},
+		{
+			ref: "https://github.com/moby/buildkit.git?fetch-depth=-1",
+			err: "invalid fetch-depth value",
+		},
+		{
+			ref: "https://github.com/moby/buildkit.git?fetch-tags=wat",
+			err: "invalid fetch-tags value",
+		},
 	}
 	for i, tt := range cases {
 		t.Run(fmt.Sprintf("case%d", i+1), func(t *testing.T) {
@@ -236,6 +262,14 @@ func TestParseGitRef(t *testing.T) {
 	}
 }
 
+func ptrInt(v int) *int {
+	return &v
+}
+
+func ptrBool(v bool) *bool {
+	return &v
+}
+
 func TestFragmentFormat(t *testing.T) {
 	cases := []struct {
 		ref      string

frontend/dockerfile/dockerfile2llb/convert_copy.go

@@ -164,6 +164,12 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 			if gitRef.SubDir != "" {
 				gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir))
 			}
+			if gitRef.FetchDepth != nil {
+				gitOptions = append(gitOptions, llb.GitFetchDepth(*gitRef.FetchDepth))
+			}
+			if gitRef.FetchTags != nil && *gitRef.FetchTags {
+				gitOptions = append(gitOptions, llb.GitFetchTags())
+			}
 			if gitRef.Submodules != nil && !*gitRef.Submodules {
 				gitOptions = append(gitOptions, llb.GitSkipSubmodules())
 			}

frontend/dockerui/build_test.go

@@ -1,10 +1,14 @@
 package dockerui
 
 import (
+	"context"
 	"testing"
 
 	"github.com/containerd/platforms"
+	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/solver/pb"
+	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/require"
 )
@@ -98,3 +102,64 @@ func TestNormalizePlatform(t *testing.T) {
 		require.Equal(t, platforms.FormatAll(platforms.Normalize(tc.p)), tc.expected.ID)
 	}
 }
+
+func TestDetectGitContextForwardsFetchDepth(t *testing.T) {
+	t.Parallel()
+
+	st, ok, err := DetectGitContext("https://github.com/crazy-max/diun.git?ref=refs/pull/1544/merge&subdir=.&fetch-depth=0", nil)
+	require.True(t, ok)
+	require.NoError(t, err)
+
+	g := marshalGitContext(t, st)
+	require.Equal(t, "git://github.com/crazy-max/diun.git#refs/pull/1544/merge:.", g.Identifier)
+	require.Equal(t, map[string]string{
+		"git.authheadersecret": "GIT_AUTH_HEADER",
+		"git.authtokensecret":  "GIT_AUTH_TOKEN",
+		"git.fetchdepth":       "0",
+		"git.fullurl":          "https://github.com/crazy-max/diun.git",
+	}, g.Attrs)
+}
+
+func TestDetectGitContextForwardsFetchTags(t *testing.T) {
+	t.Parallel()
+
+	st, ok, err := DetectGitContext("https://github.com/crazy-max/diun.git?ref=refs/pull/1544/merge&subdir=.&fetch-tags=true", nil)
+	require.True(t, ok)
+	require.NoError(t, err)
+
+	g := marshalGitContext(t, st)
+	require.Equal(t, "git://github.com/crazy-max/diun.git#refs/pull/1544/merge:.", g.Identifier)
+	require.Equal(t, map[string]string{
+		"git.authheadersecret": "GIT_AUTH_HEADER",
+		"git.authtokensecret":  "GIT_AUTH_TOKEN",
+		"git.fetchtags":        "true",
+		"git.fullurl":          "https://github.com/crazy-max/diun.git",
+	}, g.Attrs)
+}
+
+func marshalGitContext(t *testing.T, st *llb.State) *pb.SourceOp {
+	t.Helper()
+
+	def, err := st.Marshal(context.TODO())
+	require.NoError(t, err)
+
+	m := map[string]*pb.Op{}
+	arr := make([]*pb.Op, 0, len(def.Def))
+	for _, dt := range def.Def {
+		var op pb.Op
+		err := op.Unmarshal(dt)
+		require.NoError(t, err)
+		dgst := digest.FromBytes(dt)
+		m[string(dgst)] = &op
+		arr = append(arr, &op)
+	}
+
+	require.Equal(t, 2, len(arr))
+
+	last := arr[len(arr)-1]
+	require.Equal(t, 1, len(last.Inputs))
+	require.Equal(t, 0, int(last.Inputs[0].Index))
+	require.Equal(t, m[last.Inputs[0].Digest], arr[0])
+
+	return arr[0].Op.(*pb.Op_Source).Source
+}

frontend/dockerui/context.go

@@ -161,6 +161,12 @@ func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) {
 	if g.SubDir != "" {
 		gitOpts = append(gitOpts, llb.GitSubDir(g.SubDir))
 	}
+	if g.FetchDepth != nil {
+		gitOpts = append(gitOpts, llb.GitFetchDepth(*g.FetchDepth))
+	}
+	if g.FetchTags != nil && *g.FetchTags {
+		gitOpts = append(gitOpts, llb.GitFetchTags())
+	}
 	if g.Checksum != "" {
 		gitOpts = append(gitOpts, llb.GitChecksum(g.Checksum))
 	}

solver/pb/attr.go

@@ -7,6 +7,8 @@ const AttrAuthTokenSecret = "git.authtokensecret"
 const AttrKnownSSHHosts = "git.knownsshhosts"
 const AttrMountSSHSock = "git.mountsshsock"
 const AttrGitChecksum = "git.checksum"
+const AttrGitFetchDepth = "git.fetchdepth"
+const AttrGitFetchTags = "git.fetchtags"
 const AttrGitSkipSubmodules = "git.skipsubmodules"
 
 const AttrGitSignatureVerifyPubKey = "git.sig.pubkey"

solver/pb/caps.go

@@ -32,6 +32,8 @@ const (
 	CapSourceGitMountSSHSock    apicaps.CapID = "source.git.mountsshsock"
 	CapSourceGitSubdir          apicaps.CapID = "source.git.subdir"
 	CapSourceGitChecksum        apicaps.CapID = "source.git.checksum"
+	CapSourceGitFetchDepth      apicaps.CapID = "source.git.fetchdepth"
+	CapSourceGitFetchTags       apicaps.CapID = "source.git.fetchtags"
 	CapSourceGitSkipSubmodules  apicaps.CapID = "source.git.skipsubmodules"
 	CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify"
 
@@ -243,6 +245,18 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceGitFetchDepth,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceGitFetchTags,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceGitSkipSubmodules,
 		Enabled: true,

source/git/identifier.go

@@ -2,18 +2,22 @@ package git
 
 import (
 	"path"
+	"strconv"
 
 	"github.com/moby/buildkit/solver/llbsolver/provenance"
 	provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
 	"github.com/moby/buildkit/source"
 	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/gitutil"
+	"github.com/pkg/errors"
 )
 
 type GitIdentifier struct {
 	Remote           string
 	Ref              string
 	Checksum         string
+	FetchDepth       *int
+	FetchTags        bool
 	Subdir           string
 	KeepGitDir       bool
 	AuthTokenSecret  string
@@ -46,6 +50,23 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {
 		repo.Ref = u.Opts.Ref
 		repo.Subdir = u.Opts.Subdir
 	}
+	if v := u.Query.Get("fetch-depth"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 {
+			return nil, errors.Errorf("invalid fetch-depth value: %q", v)
+		}
+		repo.FetchDepth = &n
+	}
+	if vals, ok := u.Query["fetch-tags"]; ok {
+		repo.FetchTags = true
+		if len(vals) > 0 && vals[0] != "" {
+			v, err := strconv.ParseBool(vals[0])
+			if err != nil {
+				return nil, errors.Errorf("invalid fetch-tags value: %q", vals[0])
+			}
+			repo.FetchTags = v
+		}
+	}
 	if sd := path.Clean(repo.Subdir); sd == "/" || sd == "." {
 		repo.Subdir = ""
 	}

source/git/identifier_test.go

@@ -72,6 +72,23 @@ func TestNewGitIdentifier(t *testing.T) {
 				Subdir: "mydir/mysubdir/",
 			},
 		},
+		{
+			url: "https://github.com/user/repo.git?fetch-depth=0#mybranch:mydir/mysubdir/",
+			expected: GitIdentifier{
+				Remote:     "https://github.com/user/repo.git",
+				Ref:        "mybranch",
+				Subdir:     "mydir/mysubdir/",
+				FetchDepth: ptrInt(0),
+			},
+		},
+		{
+			url: "https://github.com/user/repo.git?fetch-tags#mybranch",
+			expected: GitIdentifier{
+				Remote:    "https://github.com/user/repo.git",
+				Ref:       "mybranch",
+				FetchTags: true,
+			},
+		},
 		{
 			url: "[email protected]:user/repo.git",
 			expected: GitIdentifier{
@@ -117,3 +134,7 @@ func TestNewGitIdentifier(t *testing.T) {
 		})
 	}
 }
+
+func ptrInt(v int) *int {
+	return &v
+}