Record versions cooldown filters hold back

docs/configuration.md

@@ -1296,7 +1296,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma
 
 Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families.
 
-The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`.
+The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. Neither do a Maven named repository and routed npm and Composer packages, whose publish times are kept apart from the default registry's, nor a Go module with capitals in its path, whose times are stored under the escaped path. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`.
 
 Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below.
 

internal/database/database_test.go

@@ -1245,5 +1245,101 @@ func TestVersionPublishedAtIsUTC(t *testing.T) {
 			t.Fatal(err)
 		}
 		check("pkg:cargo/demo@2.0.0")
+
+		if err := db.RecordWithheldVersion(&Package{PURL: "pkg:cargo/demo", Ecosystem: "cargo", Name: "demo"},
+			"pkg:cargo/demo@3.0.0", written); err != nil {
+			t.Fatal(err)
+		}
+		check("pkg:cargo/demo@3.0.0")
+	})
+}
+
+// A version a cooldown filter holds back is recorded with its publish time,
+// and its package gets a row so the package page exists before anything is
+// downloaded. Rows already there keep what they have.
+func TestRecordWithheldVersion(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		const pkgPURL = "pkg:npm/leftpad"
+		published := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC)
+		pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"}
+
+		if err := db.RecordWithheldVersion(pkg, pkgPURL+"@2.0.0", published); err != nil {
+			t.Fatalf("RecordWithheldVersion: %v", err)
+		}
+		got, err := db.GetPackageByEcosystemName("npm", "leftpad")
+		if err != nil || got == nil {
+			t.Fatalf("package row = %v, %v", got, err)
+		}
+		if got.PURL != pkgPURL {
+			t.Errorf("package purl = %q, want %q", got.PURL, pkgPURL)
+		}
+		ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0")
+		if err != nil || ver == nil {
+			t.Fatalf("version row = %v, %v", ver, err)
+		}
+		if ver.PackagePURL != pkgPURL || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) {
+			t.Errorf("version = %+v, want package %s published %v", ver, pkgPURL, published)
+		}
+	})
+}
+
+// For an ecosystem whose release time is its newest file's, a later time
+// replaces the stored one and an earlier one doesn't.
+func TestRecordWithheldVersionNewest(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		const pkgPURL = "pkg:pypi/ruff"
+		first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC)
+		later := first.Add(time.Hour)
+		pkg := &Package{PURL: pkgPURL, Ecosystem: "pypi", Name: "ruff"}
+		for _, at := range []time.Time{first, later, first} {
+			if err := db.RecordWithheldVersionNewest(pkg, pkgPURL+"@2.0.0", at); err != nil {
+				t.Fatalf("RecordWithheldVersionNewest: %v", err)
+			}
+		}
+		ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0")
+		if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) {
+			t.Errorf("version = %+v, %v; want published %v", ver, err, later)
+		}
+		if got, err := db.GetPackageByEcosystemName("pypi", "ruff"); err != nil || got == nil {
+			t.Errorf("package row = %v, %v", got, err)
+		}
+	})
+}
+
+func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		const pkgPURL = "pkg:npm/leftpad"
+		stored := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC)
+		other := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC)
+		if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad",
+			Description: sql.NullString{String: "pads left", Valid: true}}); err != nil {
+			t.Fatalf("UpsertPackage: %v", err)
+		}
+		if err := db.SetVersionPublishedAt(pkgPURL+"@1.0.0", pkgPURL, stored); err != nil {
+			t.Fatalf("SetVersionPublishedAt: %v", err)
+		}
+		if err := db.UpsertVersion(&Version{PURL: pkgPURL + "@3.0.0", PackagePURL: pkgPURL}); err != nil {
+			t.Fatalf("UpsertVersion: %v", err)
+		}
+
+		pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"}
+		for _, v := range []string{"1.0.0", "3.0.0"} {
+			if err := db.RecordWithheldVersion(pkg, pkgPURL+"@"+v, other); err != nil {
+				t.Fatalf("RecordWithheldVersion %s: %v", v, err)
+			}
+		}
+
+		got, err := db.GetPackageByPURL(pkgPURL)
+		if err != nil || got == nil || got.Description.String != "pads left" {
+			t.Errorf("package = %+v, %v; want the description kept", got, err)
+		}
+		ver, err := db.GetVersionByPURL(pkgPURL + "@1.0.0")
+		if err != nil || ver == nil || !ver.PublishedAt.Time.Equal(stored) {
+			t.Errorf("1.0.0 = %+v, %v; want the stored time %v kept", ver, err, stored)
+		}
+		ver, err = db.GetVersionByPURL(pkgPURL + "@3.0.0")
+		if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(other) {
+			t.Errorf("3.0.0 = %+v, %v; want the empty time filled with %v", ver, err, other)
+		}
 	})
 }

internal/database/queries.go

@@ -225,6 +225,57 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt
 	return nil
 }
 
+// RecordWithheldVersion records a version a cooldown filter held back: its
+// publish time, and a row for its package so the package page exists before
+// anything is downloaded. It only adds what is missing. An existing package
+// row is left as it is, and a stored publish time is not overwritten, so the
+// filters can call it for every version they hold back without reading
+// first.
+func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time) error {
+	return db.recordWithheldVersion(pkg, versionPURL, publishedAt, "versions.published_at IS NULL")
+}
+
+// RecordWithheldVersionNewest is RecordWithheldVersion for an ecosystem whose
+// release time is its newest file's, such as PyPI: a later time replaces the
+// stored one, since a file can be added to a release after it was held back.
+func (db *DB) RecordWithheldVersionNewest(pkg *Package, versionPURL string, publishedAt time.Time) error {
+	return db.recordWithheldVersion(pkg, versionPURL, publishedAt,
+		"versions.published_at IS NULL OR versions.published_at < excluded.published_at")
+}
+
+// recordWithheldVersion records a held back version, updating the stored
+// publish time where replace, a condition on the existing row, holds.
+func (db *DB) recordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time, replace string) error {
+	now := time.Now()
+	tx, err := db.Beginx()
+	if err != nil {
+		return fmt.Errorf("recording withheld version: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	if _, err := tx.Exec(db.Rebind(`
+		INSERT INTO packages (purl, ecosystem, name, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?)
+		ON CONFLICT(purl) DO NOTHING
+	`), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil {
+		return fmt.Errorf("recording withheld version's package: %w", err)
+	}
+	if _, err := tx.Exec(db.Rebind(`
+		INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?)
+		ON CONFLICT(purl) DO UPDATE SET
+			published_at = excluded.published_at,
+			updated_at = excluded.updated_at
+		WHERE `+replace+`
+	`), versionPURL, pkg.PURL, publishedAt.UTC(), now, now); err != nil {
+		return fmt.Errorf("recording withheld version: %w", err)
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("recording withheld version: %w", err)
+	}
+	return nil
+}
+
 // GetFetchedVersionPURLs returns the versions of a package that have an
 // artifact row, cached now or evicted since. Versions without one were only
 // seen in metadata.

internal/handler/cargo.go

@@ -180,6 +180,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr
 			h.proxy.Logger.Info("cooldown: filtering cargo version",
 				"crate", crate.Name, "version", crate.Version,
 				"published", crate.PublishTime)
+			h.proxy.recordWithheld("cargo", crate.Name, crate.Version, publishedAt)
 			continue
 		}
 

internal/handler/composer.go

@@ -409,6 +409,11 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version
 	if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) {
 		h.proxy.Logger.Info("cooldown: filtering composer version",
 			"package", packageName, "version", version)
+		// A routed package's times are the route's, and aren't stored under
+		// the default registry's PURLs (see releaseTimeStored).
+		if _, routed := h.routes.match(packageName); !routed {
+			h.proxy.recordWithheld("composer", packageName, version, publishedAt)
+		}
 		return true
 	}
 

internal/handler/cooldown_times.go

@@ -138,16 +138,20 @@ func (d timedDocuments) rewriteDigest() string {
 }
 
 // withheld reports whether cooldown leaves version out, given the
-// package's publish times.
-func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool {
+// package's publish times, and records a version it leaves out.
+func (d timedDocuments) withheld(p *Proxy, ecosystem, name, version string) bool {
 	published, ok := d.published[version]
 	if !ok {
 		return d.withholdUnknown
 	}
 	if published.IsZero() {
 		return false
 	}
-	return !p.cooldown().IsAllowed(ecosystem, packagePURL, published)
+	if p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), published) {
+		return false
+	}
+	p.recordWithheld(ecosystem, name, version, published)
+	return true
 }
 
 // decodedMetadataBody returns res's body without its Content-Encoding. The

internal/handler/cooldown_withheld.go

@@ -0,0 +1,42 @@
+package handler
+
+import (
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/database"
+)
+
+// recordWithheld stores the publish time of a version a cooldown filter held
+// back, so the package page can list it with when it becomes available,
+// also for a package nobody has downloaded yet. name is the package name as
+// the ecosystem's downloads store it, so both land on the same package row.
+// A version without a package name, a version or a known time is skipped,
+// and a failed write is only logged: the metadata is served either way.
+func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) {
+	p.recordWithheldWith(ecosystem, name, version, publishedAt, false)
+}
+
+// recordWithheldNewest is recordWithheld for a release whose time is its
+// newest file's: a later time replaces the stored one.
+func (p *Proxy) recordWithheldNewest(ecosystem, name, version string, publishedAt time.Time) {
+	p.recordWithheldWith(ecosystem, name, version, publishedAt, true)
+}
+
+func (p *Proxy) recordWithheldWith(ecosystem, name, version string, publishedAt time.Time, newest bool) {
+	if p.DB == nil || name == "" || version == "" || publishedAt.IsZero() {
+		return
+	}
+	packagePURL, versionPURL, err := packagePURLStrings(ecosystem, name, version)
+	if err != nil {
+		return
+	}
+	pkg := &database.Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name}
+	record := p.DB.RecordWithheldVersion
+	if newest {
+		record = p.DB.RecordWithheldVersionNewest
+	}
+	if err := record(pkg, versionPURL, publishedAt); err != nil {
+		p.Logger.Warn("cooldown: failed to record withheld version",
+			"ecosystem", ecosystem, "name", name, "version", version, "error", err)
+	}
+}