Check cached Git repositories in the background

config.example.yaml

@@ -423,6 +423,7 @@ upstream:
 #   max_repo_size: 5GB       # refuse a repository that grows past this
 #   refresh_wait: 5s         # then serve the cached copy if the upstream check is slow
 #   stale_max_age: 24h       # never serve a copy not checked for this long
+#   warm_max_idle: 30d       # keep checking repositories used within this
 
 # Gradle HttpBuildCache configuration
 gradle:

docs/configuration.md

@@ -842,6 +842,7 @@ git:
   max_repo_size: 5GB     # refuse a repository that grows past this
   refresh_wait: 5s       # then serve the cached copy if the upstream check is slow
   stale_max_age: 24h     # never serve a copy not checked for this long
+  warm_max_idle: 30d     # keep checking repositories used within this
 ```
 
 Clients reach the proxy through Git's `insteadOf` setting, which rewrites the
@@ -920,6 +921,24 @@ location. A `git-upload-pack` request checks the upstream only when the last
 check refused the repository or the copy was last checked more than a minute
 past `stale_max_age` ago, so a clone checks once, at its ref listing.
 
+The proxy also checks cached repositories in the background, so a copy stays
+servable through an outage even when nobody cloned it for a day. A repository
+that a client used within `warm_max_idle` (default `30d`) is checked again
+once half of `stale_max_age` has passed since its last successful check,
+and fetched if its branches or tags moved. It is checked at the URL its last
+successful check used, so a repository cached by an older version waits for
+a client to refresh it once. After a failed check it waits a quarter of that
+interval, at least 10 minutes, so a few retries fit before the copy goes
+stale. Checks run one at a time and do not count as use, so they do not keep
+a repository from expiring out of `warm_max_idle`. With the cache at
+`max_size`, a background fetch makes room as any other fetch does, by
+evicting the least recently used idle repositories, which can be ones used
+more recently than the repository being checked. Repositories the upstream
+refused, and those the configuration no longer serves, are skipped. `warm_max_idle: 0`, or a
+`stale_max_age` under `20m` (which includes `0`), turns the background
+checks off. They run in `proxy serve` only, each process for its own
+`cache_dir`.
+
 The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it).
 The `git` processes never reach the network themselves: their fetches go
 through a loopback endpoint in the proxy that forwards them with the same
@@ -1026,8 +1045,8 @@ upstream, until retention removes it or you delete it through the
 Pushing, LFS uploads and locking, and the dumb HTTP protocol are not
 supported. The settings
 under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`,
-`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and
-`PROXY_GIT_STALE_MAX_AGE`.
+`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT`,
+`PROXY_GIT_STALE_MAX_AGE` and `PROXY_GIT_WARM_MAX_IDLE`.
 
 ### Maven repositories
 

internal/config/config.go

@@ -1618,6 +1618,7 @@ func (c *Config) LoadFromEnv() {
 	setEnvString(&c.Git.MaxRepoSize, "PROXY_GIT_MAX_REPO_SIZE")
 	setEnvString(&c.Git.RefreshWait, "PROXY_GIT_REFRESH_WAIT")
 	setEnvString(&c.Git.StaleMaxAge, "PROXY_GIT_STALE_MAX_AGE")
+	setEnvString(&c.Git.WarmMaxIdle, "PROXY_GIT_WARM_MAX_IDLE")
 }
 
 // validateAbsoluteURL returns an error if value is not a parseable URL with

internal/config/git.go

@@ -15,6 +15,7 @@ const (
 	defaultGitMaxRepoSize = "5GB"
 	defaultGitRefreshWait = "5s"
 	defaultGitStaleMaxAge = "24h"
+	defaultGitWarmMaxIdle = "30d"
 )
 
 // GitUpstream is an upstream.git entry: a Git server's base URL and,
@@ -104,10 +105,17 @@ type GitConfig struct {
 	// Default: 24h.
 	StaleMaxAge string `json:"stale_max_age" yaml:"stale_max_age"`
 
+	// WarmMaxIdle is how long after a client last used a repository the
+	// proxy keeps checking it in the background, so the cached copy stays
+	// within StaleMaxAge through an upstream outage. Accepts a "d" suffix.
+	// "0" turns the background checks off. Default: 30d.
+	WarmMaxIdle string `json:"warm_max_idle" yaml:"warm_max_idle"`
+
 	maxSize     int64
 	maxRepoSize int64
 	refreshWait time.Duration
 	staleMaxAge time.Duration
+	warmMaxIdle time.Duration
 }
 
 // MaxSizeBytes returns the parsed MaxSize, 0 for no limit. Validate must run first.
@@ -122,6 +130,9 @@ func (g *GitConfig) RefreshWaitDuration() time.Duration { return g.refreshWait }
 // StaleMaxAgeDuration returns the parsed StaleMaxAge. Validate must run first.
 func (g *GitConfig) StaleMaxAgeDuration() time.Duration { return g.staleMaxAge }
 
+// WarmMaxIdleDuration returns the parsed WarmMaxIdle, 0 for off. Validate must run first.
+func (g *GitConfig) WarmMaxIdleDuration() time.Duration { return g.warmMaxIdle }
+
 // Validate checks the Git cache settings, applying defaults where unset.
 func (g *GitConfig) Validate() error {
 	if g.CacheDir == "" {
@@ -136,6 +147,9 @@ func (g *GitConfig) Validate() error {
 	if g.StaleMaxAge == "" {
 		g.StaleMaxAge = defaultGitStaleMaxAge
 	}
+	if g.WarmMaxIdle == "" {
+		g.WarmMaxIdle = defaultGitWarmMaxIdle
+	}
 
 	var err error
 	if g.maxSize, err = parseNonNegativeSize("git.max_size", g.MaxSize); err != nil {
@@ -150,6 +164,9 @@ func (g *GitConfig) Validate() error {
 	if g.staleMaxAge, err = parseNonNegativeDuration("git.stale_max_age", g.StaleMaxAge); err != nil {
 		return err
 	}
+	if g.warmMaxIdle, err = parseRetentionDuration("git.warm_max_idle", g.WarmMaxIdle); err != nil {
+		return err
+	}
 	return nil
 }
 

internal/config/git_test.go

@@ -132,11 +132,19 @@ func TestGitConfigDefaults(t *testing.T) {
 		t.Fatalf("Validate: %v", err)
 	}
 	if g.CacheDir != defaultGitCacheDir || g.MaxSizeBytes() != 0 || g.MaxRepoSizeBytes() != 5<<30 ||
-		g.RefreshWaitDuration() != 5*time.Second || g.StaleMaxAgeDuration() != 24*time.Hour {
+		g.RefreshWaitDuration() != 5*time.Second || g.StaleMaxAgeDuration() != 24*time.Hour ||
+		g.WarmMaxIdleDuration() != 30*24*time.Hour {
 		t.Errorf("defaults = %+v", g)
 	}
 
-	for _, bad := range []GitConfig{{MaxSize: "lots"}, {RefreshWait: "soon"}, {StaleMaxAge: "-1h"}} {
+	for value, want := range map[string]time.Duration{"90d": 90 * 24 * time.Hour, "12h": 12 * time.Hour, "0": 0} {
+		g := GitConfig{WarmMaxIdle: value}
+		if err := g.Validate(); err != nil || g.WarmMaxIdleDuration() != want {
+			t.Errorf("warm_max_idle %q = %v, %v; want %v", value, g.WarmMaxIdleDuration(), err, want)
+		}
+	}
+
+	for _, bad := range []GitConfig{{MaxSize: "lots"}, {RefreshWait: "soon"}, {StaleMaxAge: "-1h"}, {WarmMaxIdle: "-1d"}, {WarmMaxIdle: "often"}} {
 		if err := bad.Validate(); err == nil {
 			t.Errorf("Validate(%+v) accepted it", bad)
 		}

internal/gitcache/cache.go

@@ -46,6 +46,9 @@ const (
 	ecosystem           = "git"
 	upstreamHashLen     = 8
 	checkedStamp        = "proxy-checked"
+	// urlFile holds the upstream URL the last successful refresh used, for
+	// background refreshes.
+	urlFile             = "proxy-url"
 	usedStamp           = "proxy-used"
 	dirPermissions      = 0o750
 	lockFilePermissions = 0o600
@@ -62,17 +65,21 @@ type Options struct {
 	// processes the cache runs. Redirects are never followed.
 	Client *http.Client
 	Logger *slog.Logger
+	// WarmMaxIdle, with StaleMaxAge, lets Warm refresh the repositories
+	// clients used within it before StaleMaxAge runs out. 0 turns that off.
+	WarmMaxIdle time.Duration
 	// Upstreams maps each configured upstream's name to its directory in
 	// CacheDir, as UpstreamDir names it. Only those directories are loaded
 	// at startup. A directory UpstreamDir would name for one of these names
 	// at another URL, left by repointing the upstream, is removed. Nothing
 	// else in CacheDir is touched, so it may hold other data.
 	Upstreams map[string]string
 
-	// refCheckTimeout, lowSpeedTime, maxRequestBody, maxUploadPacks and
-	// maxRefListings override their defaults in tests, and refreshWaitOver,
-	// when set, replaces the RefreshWait timer.
+	// refCheckTimeout, lowSpeedTime, maxRequestBody, maxUploadPacks,
+	// maxRefListings and warmTick override their defaults in tests, and
+	// refreshWaitOver, when set, replaces the RefreshWait timer.
 	refCheckTimeout time.Duration
+	warmTick        time.Duration
 	refreshWaitOver <-chan time.Time
 	lowSpeedTime    int
 	maxRequestBody  int64
@@ -155,6 +162,27 @@ type repo struct {
 	// is current while the count is the one it was read at.
 	state       atomic.Pointer[cachedState]
 	refsWritten atomic.Int64
+	// url is the upstream URL the last successful refresh used.
+	url atomic.Pointer[string]
+}
+
+// repoURL returns the upstream URL the last successful refresh used, or ""
+// if none is recorded.
+func (rp *repo) repoURL() string {
+	if u := rp.url.Load(); u != nil {
+		return *u
+	}
+	return ""
+}
+
+// rememberURL records repoURL as the one a refresh succeeded with, on disk
+// too once the repository is cached.
+func (rp *repo) rememberURL(repoURL string) {
+	if rp.repoURL() == repoURL || !rp.exists() {
+		return
+	}
+	rp.url.Store(&repoURL)
+	_ = os.WriteFile(filepath.Join(rp.dir, urlFile), []byte(repoURL+"\n"), lockFilePermissions)
 }
 
 var errLocked = errors.New("locked by another process")
@@ -310,6 +338,11 @@ func (c *Cache) load() error {
 			rp.lastChecked.Store(stampTime(path, checkedStamp))
 			rp.lastUsed.Store(stampTime(path, usedStamp))
 			rp.usedWritten.Store(rp.lastUsed.Load())
+			if data, err := os.ReadFile(filepath.Join(path, urlFile)); err == nil {
+				if u := strings.TrimSpace(string(data)); u != "" {
+					rp.url.Store(&u)
+				}
+			}
 			c.repos[rp.key] = rp
 			return filepath.SkipDir
 		})
@@ -392,6 +425,7 @@ func (c *Cache) repoKey(key string) string {
 	return key
 }
 
+// release ends a use of the repository by a client, recording it as used.
 func (c *Cache) release(rp *repo) {
 	now := time.Now()
 	rp.lastUsed.Store(now.UnixNano())
@@ -400,7 +434,12 @@ func (c *Cache) release(rp *repo) {
 		rp.usedWritten.Store(now.UnixNano())
 		touch(rp.dir, usedStamp, now)
 	}
+	c.drop(rp, exists)
+}
 
+// drop ends a use of the repository without recording it as used. exists
+// is whether the repository is cloned.
+func (c *Cache) drop(rp *repo, exists bool) {
 	c.mu.Lock()
 	rp.users--
 	idle := rp.users == 0 && c.repos[rp.key] == rp

internal/gitcache/refresh.go

@@ -130,6 +130,12 @@ func servesStale(err error) bool {
 func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	return c.startRefreshLocked(rp, repoURL, false)
+}
+
+// startRefreshLocked is startRefresh with c.mu held. A refresh started in
+// the background does not count as a use of the repository when it ends.
+func (c *Cache) startRefreshLocked(rp *repo, repoURL string, background bool) *refreshCall {
 	if rp.refreshing != nil {
 		return rp.refreshing
 	}
@@ -154,6 +160,9 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall {
 		switch {
 		case call.err == nil:
 			rp.lastError.Store(nil)
+			// The upstream answered at this URL, so background refreshes
+			// use it.
+			rp.rememberURL(repoURL)
 		case c.ctx.Err() == nil:
 			metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err})
 			c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err)
@@ -165,7 +174,11 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall {
 			rp.refused = call.err
 		}
 		c.mu.Unlock()
-		c.release(rp)
+		if background {
+			c.drop(rp, rp.exists())
+		} else {
+			c.release(rp)
+		}
 		close(call.done)
 		if call.err == nil {
 			c.evict()