Improve git cache performance by using reftable

docs/configuration.md

@@ -951,6 +951,13 @@ A transfer that receives nothing for 10 minutes is abandoned. Git's automatic
 garbage collection runs as part of fetches, and files over 8MB are kept
 without delta compression, so one large file does not take a fetch's memory.
 
+With `git` 2.56 or newer, repositories are cached with reftable reference
+storage, which writes a fetch's reference updates to one table rather than
+a file per reference, so repositories with many branches clone and refresh
+faster, most of all on network file systems. Repositories cached in the
+files format are converted at startup; one that fails to convert is served
+as it is. With an older `git`, repositories keep the files format.
+
 Packs for clients are built by at most two `upload-pack` processes per CPU
 the proxy may use, each with one delta-search thread and 64MB of delta
 window, and further clones wait for one to finish; listing refs does not

internal/gitcache/cache.go

@@ -98,6 +98,9 @@ type Cache struct {
 	// foldCase is set where the cache directory ignores case, so keys that
 	// differ only in case, which name one directory, are one repository.
 	foldCase bool
+	// reftable is set when git is new enough for the cache to keep
+	// references in the reftable format.
+	reftable bool
 	// stateReads lets one RepositoryState read run git at a time.
 	stateReads chan struct{}
 
@@ -245,6 +248,12 @@ func New(opts Options) (*Cache, error) {
 	if opts.maxRequestBody > 0 {
 		c.maxRequestBody = opts.maxRequestBody
 	}
+	out, err := c.command(context.Background(), nil, "version").Output()
+	c.reftable = err == nil && reftableReady(string(out))
+	if !c.reftable {
+		logger.Info("git cache: repositories keep the files reference format, as reftable needs git 2.56 or newer",
+			"version", strings.TrimSpace(string(out)))
+	}
 	if err := c.load(); err != nil {
 		_ = lock.Close()
 		return nil, err
@@ -289,7 +298,8 @@ func (c *Cache) tmpDir() string {
 }
 
 // load clears what an earlier run left in the temporary directory and
-// registers the repositories already cached.
+// registers the repositories already cached, converting those in the files
+// reference format to reftable when git supports it.
 func (c *Cache) load() error {
 	if err := os.RemoveAll(c.tmpDir()); err != nil {
 		return fmt.Errorf("clearing %s: %w", c.tmpDir(), err)
@@ -306,6 +316,9 @@ func (c *Cache) load() error {
 			}
 			rel, _ := filepath.Rel(c.dir, path)
 			rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path)
+			if c.reftable && !hasReftable(path) {
+				c.migrate(rp)
+			}
 			rp.size.Store(dirSize(path))
 			rp.lastChecked.Store(stampTime(path, checkedStamp))
 			rp.lastUsed.Store(stampTime(path, usedStamp))
@@ -320,6 +333,40 @@ func (c *Cache) load() error {
 	return nil
 }
 
+// reftableReady reports whether out, the output of "git version", names git
+// 2.56 or newer, the first with the reftable fixes the cache relies on.
+func reftableReady(out string) bool {
+	var major, minor int
+	_, err := fmt.Sscanf(out, "git version %d.%d", &major, &minor)
+	return err == nil && (major > 2 || major == 2 && minor >= 56)
+}
+
+// hasReftable reports whether the repository at dir keeps its references in
+// the reftable format. It looks for the table list rather than running git,
+// so a startup costs no process per repository once all are converted. Only
+// a list known to be missing counts as the files format.
+func hasReftable(dir string) bool {
+	_, err := os.Stat(filepath.Join(dir, "reftable", "tables.list"))
+	return !errors.Is(err, fs.ErrNotExist)
+}
+
+// migrate converts the references of the repository rp to the reftable
+// format. load calls it before any request can write to the repository,
+// which git refs migrate does not guard against. One that fails to convert
+// is served as it is.
+func (c *Cache) migrate(rp *repo) {
+	// GIT_DIR rather than -C, which would have git look for a repository in
+	// the directories above one that is not.
+	out, err := c.command(context.Background(), []string{"GIT_DIR=" + rp.dir},
+		"refs", "migrate", "--ref-format=reftable").CombinedOutput()
+	if err != nil {
+		c.logger.Warn("git cache: failed to convert a repository to the reftable format; serving it as it is",
+			"repository", rp.key, "error", err, "output", strings.TrimSpace(string(out)))
+		return
+	}
+	c.logger.Info("git cache: converted a repository to the reftable format", "repository", rp.key)
+}
+
 // upstreamDirPattern matches a directory UpstreamDir names, capturing the
 // upstream's name.
 var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`)

internal/gitcache/cache_test.go

@@ -692,6 +692,52 @@ func TestNewRemovesRepointedUpstreams(t *testing.T) {
 	}
 }
 
+// TestNewMigratesFilesFormatRepositories starts a cache whose directory
+// holds a repository in the files reference format: it is converted to
+// reftable with its refs, and served without being cloned again.
+func TestNewMigratesFilesFormatRepositories(t *testing.T) {
+	// Before the fixture, as git older than 2.45 has no --ref-format.
+	if !reftableReady(runGit(t, "", "version")) {
+		t.Skip("git is older than 2.56")
+	}
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	dir := t.TempDir()
+	bare := filepath.Join(dir, "up", "repo.git")
+	runGit(t, "", "clone", "--quiet", "--bare", "--ref-format=files", upstream.bare(), bare)
+	refs := runGit(t, bare, "for-each-ref", "--format=%(objectname) %(refname)")
+
+	tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir })
+	if !hasReftable(bare) {
+		t.Fatal("the repository was not converted to reftable")
+	}
+	if got := tc.Repositories(); len(got) != 1 {
+		t.Fatalf("startup registered %+v, want the converted repository", got)
+	}
+	if got := tc.cachedRefs(t); got != refs {
+		t.Errorf("refs after conversion:\n%s\nwant:\n%s", got, refs)
+	}
+	tc.clone(t, "repo")
+}
+
+// TestReftableReady checks which versions turn reftable on, across the ways
+// git prints its version.
+func TestReftableReady(t *testing.T) {
+	for out, want := range map[string]bool{
+		"git version 2.56.1\n":                 true,
+		"git version 2.56.0.windows.1\n":       true,
+		"git version 2.57.0-rc0\n":             true,
+		"git version 3.0.0\n":                  true,
+		"git version 2.55.2\n":                 false,
+		"git version 2.39.1 (Apple Git-146)\n": false,
+		"git version 2.x\n":                    false,
+	} {
+		if got := reftableReady(out); got != want {
+			t.Errorf("reftableReady(%q) = %v, want %v", out, got, want)
+		}
+	}
+}
+
 // TestCacheDirSharedWithOtherData points the cache at a directory that also
 // holds an artifact store and a database, as git.cache_dir: ./cache would.
 func TestCacheDirSharedWithOtherData(t *testing.T) {

internal/gitcache/refresh.go

@@ -221,7 +221,11 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo
 	}
 	defer func() { _ = os.RemoveAll(tmp) }()
 
-	if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil {
+	args := []string{"init", "--quiet", "--bare"}
+	if c.reftable {
+		args = append(args, "--ref-format=reftable")
+	}
+	if _, err := c.run(ctx, tmp, args...); err != nil {
 		return err
 	}
 	if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil {

internal/gitcache/stats_test.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 )
@@ -233,18 +232,17 @@ func TestRepositoryStateRetriesAFailedRead(t *testing.T) {
 	tc.clone(t, "repo")
 	waitIdle(t, tc.Cache)
 	rp := tc.repo("up/repo.git")
-	ref := filepath.Join(rp.dir, "refs", "heads", "main")
 	head := runGit(t, rp.dir, "rev-parse", "main")
-	if err := os.WriteFile(ref, []byte(strings.Repeat("1", len(head))+"\n"), 0o600); err != nil {
+	missing := runGit(t, rp.dir, "commit-tree", "-p", head, "-m", "missing", head+"^{tree}")
+	runGit(t, rp.dir, "update-ref", "refs/heads/main", missing)
+	if err := os.Remove(filepath.Join(rp.dir, "objects", missing[:2], missing[2:])); err != nil {
 		t.Fatal(err)
 	}
 
 	if st, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil {
 		t.Fatalf("state %+v of a branch with no commit, want an error", st)
 	}
-	if err := os.WriteFile(ref, []byte(head+"\n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
+	runGit(t, rp.dir, "update-ref", "refs/heads/main", head)
 	if _, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil {
 		t.Error("failed read not kept for a while")
 	}