diff --git a/docs/configuration.md b/docs/configuration.md index b75650fc..dab732b6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1296,7 +1296,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. Neither do a Maven named repository and routed npm and Composer packages, whose publish times are kept apart from the default registry's, nor a Go module with capitals in its path, whose times are stored under the escaped path. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 8a2e0574..df1a66fc 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1245,5 +1245,101 @@ func TestVersionPublishedAtIsUTC(t *testing.T) { t.Fatal(err) } check("pkg:cargo/demo@2.0.0") + + if err := db.RecordWithheldVersion(&Package{PURL: "pkg:cargo/demo", Ecosystem: "cargo", Name: "demo"}, + "pkg:cargo/demo@3.0.0", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@3.0.0") + }) +} + +// A version a cooldown filter holds back is recorded with its publish time, +// and its package gets a row so the package page exists before anything is +// downloaded. Rows already there keep what they have. +func TestRecordWithheldVersion(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:npm/leftpad" + published := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"} + + if err := db.RecordWithheldVersion(pkg, pkgPURL+"@2.0.0", published); err != nil { + t.Fatalf("RecordWithheldVersion: %v", err) + } + got, err := db.GetPackageByEcosystemName("npm", "leftpad") + if err != nil || got == nil { + t.Fatalf("package row = %v, %v", got, err) + } + if got.PURL != pkgPURL { + t.Errorf("package purl = %q, want %q", got.PURL, pkgPURL) + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil { + t.Fatalf("version row = %v, %v", ver, err) + } + if ver.PackagePURL != pkgPURL || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) { + t.Errorf("version = %+v, want package %s published %v", ver, pkgPURL, published) + } + }) +} + +// For an ecosystem whose release time is its newest file's, a later time +// replaces the stored one and an earlier one doesn't. +func TestRecordWithheldVersionNewest(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:pypi/ruff" + first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + later := first.Add(time.Hour) + pkg := &Package{PURL: pkgPURL, Ecosystem: "pypi", Name: "ruff"} + for _, at := range []time.Time{first, later, first} { + if err := db.RecordWithheldVersionNewest(pkg, pkgPURL+"@2.0.0", at); err != nil { + t.Fatalf("RecordWithheldVersionNewest: %v", err) + } + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) { + t.Errorf("version = %+v, %v; want published %v", ver, err, later) + } + if got, err := db.GetPackageByEcosystemName("pypi", "ruff"); err != nil || got == nil { + t.Errorf("package row = %v, %v", got, err) + } + }) +} + +func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:npm/leftpad" + stored := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + other := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad", + Description: sql.NullString{String: "pads left", Valid: true}}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.SetVersionPublishedAt(pkgPURL+"@1.0.0", pkgPURL, stored); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: pkgPURL + "@3.0.0", PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"} + for _, v := range []string{"1.0.0", "3.0.0"} { + if err := db.RecordWithheldVersion(pkg, pkgPURL+"@"+v, other); err != nil { + t.Fatalf("RecordWithheldVersion %s: %v", v, err) + } + } + + got, err := db.GetPackageByPURL(pkgPURL) + if err != nil || got == nil || got.Description.String != "pads left" { + t.Errorf("package = %+v, %v; want the description kept", got, err) + } + ver, err := db.GetVersionByPURL(pkgPURL + "@1.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Time.Equal(stored) { + t.Errorf("1.0.0 = %+v, %v; want the stored time %v kept", ver, err, stored) + } + ver, err = db.GetVersionByPURL(pkgPURL + "@3.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(other) { + t.Errorf("3.0.0 = %+v, %v; want the empty time filled with %v", ver, err, other) + } }) } diff --git a/internal/database/queries.go b/internal/database/queries.go index 6f693178..24d75189 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -225,6 +225,57 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// RecordWithheldVersion records a version a cooldown filter held back: its +// publish time, and a row for its package so the package page exists before +// anything is downloaded. It only adds what is missing. An existing package +// row is left as it is, and a stored publish time is not overwritten, so the +// filters can call it for every version they hold back without reading +// first. +func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time) error { + return db.recordWithheldVersion(pkg, versionPURL, publishedAt, "versions.published_at IS NULL") +} + +// RecordWithheldVersionNewest is RecordWithheldVersion for an ecosystem whose +// release time is its newest file's, such as PyPI: a later time replaces the +// stored one, since a file can be added to a release after it was held back. +func (db *DB) RecordWithheldVersionNewest(pkg *Package, versionPURL string, publishedAt time.Time) error { + return db.recordWithheldVersion(pkg, versionPURL, publishedAt, + "versions.published_at IS NULL OR versions.published_at < excluded.published_at") +} + +// recordWithheldVersion records a held back version, updating the stored +// publish time where replace, a condition on the existing row, holds. +func (db *DB) recordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time, replace string) error { + now := time.Now() + tx, err := db.Beginx() + if err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if _, err := tx.Exec(db.Rebind(` + INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO NOTHING + `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { + return fmt.Errorf("recording withheld version's package: %w", err) + } + if _, err := tx.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE `+replace+` + `), versionPURL, pkg.PURL, publishedAt.UTC(), now, now); err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + return nil +} + // GetFetchedVersionPURLs returns the versions of a package that have an // artifact row, cached now or evicted since. Versions without one were only // seen in metadata. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 0c742145..337c8031 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -180,6 +180,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) + h.proxy.recordWithheld("cargo", crate.Name, crate.Version, publishedAt) continue } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 6b82cfff..6e77e315 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -409,6 +409,11 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) + // A routed package's times are the route's, and aren't stored under + // the default registry's PURLs (see releaseTimeStored). + if _, routed := h.routes.match(packageName); !routed { + h.proxy.recordWithheld("composer", packageName, version, publishedAt) + } return true } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index b765f9a5..fa9120da 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -138,8 +138,8 @@ func (d timedDocuments) rewriteDigest() string { } // withheld reports whether cooldown leaves version out, given the -// package's publish times. -func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool { +// package's publish times, and records a version it leaves out. +func (d timedDocuments) withheld(p *Proxy, ecosystem, name, version string) bool { published, ok := d.published[version] if !ok { return d.withholdUnknown @@ -147,7 +147,11 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin if published.IsZero() { return false } - return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) + if p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), published) { + return false + } + p.recordWithheld(ecosystem, name, version, published) + return true } // decodedMetadataBody returns res's body without its Content-Encoding. The diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go new file mode 100644 index 00000000..cbf5fad8 --- /dev/null +++ b/internal/handler/cooldown_withheld.go @@ -0,0 +1,42 @@ +package handler + +import ( + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// recordWithheld stores the publish time of a version a cooldown filter held +// back, so the package page can list it with when it becomes available, +// also for a package nobody has downloaded yet. name is the package name as +// the ecosystem's downloads store it, so both land on the same package row. +// A version without a package name, a version or a known time is skipped, +// and a failed write is only logged: the metadata is served either way. +func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) { + p.recordWithheldWith(ecosystem, name, version, publishedAt, false) +} + +// recordWithheldNewest is recordWithheld for a release whose time is its +// newest file's: a later time replaces the stored one. +func (p *Proxy) recordWithheldNewest(ecosystem, name, version string, publishedAt time.Time) { + p.recordWithheldWith(ecosystem, name, version, publishedAt, true) +} + +func (p *Proxy) recordWithheldWith(ecosystem, name, version string, publishedAt time.Time, newest bool) { + if p.DB == nil || name == "" || version == "" || publishedAt.IsZero() { + return + } + packagePURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return + } + pkg := &database.Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name} + record := p.DB.RecordWithheldVersion + if newest { + record = p.DB.RecordWithheldVersionNewest + } + if err := record(pkg, versionPURL, publishedAt); err != nil { + p.Logger.Warn("cooldown: failed to record withheld version", + "ecosystem", ecosystem, "name", name, "version", version, "error", err) + } +} diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go new file mode 100644 index 00000000..8ddcacce --- /dev/null +++ b/internal/handler/cooldown_withheld_test.go @@ -0,0 +1,517 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/database" +) + +// withheldTestProxy returns a proxy with a database and a three day cooldown, +// and the times of a version old enough to pass it and one it holds back. +func withheldTestProxy(t *testing.T) (proxy *Proxy, db *database.DB, old, recent time.Time) { + t.Helper() + proxy, db, _, _ = setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + now := time.Now().UTC().Truncate(time.Second) + return proxy, db, now.Add(-10 * 24 * time.Hour), now.Add(-time.Hour) +} + +// assertWithheldRecorded checks that a version a filter held back is stored +// with its publish time, under a package row the package page finds by +// ecosystem and name. +func assertWithheldRecorded(t *testing.T, db *database.DB, ecosystem, name, version string, published time.Time) { + t.Helper() + pkg, err := db.GetPackageByEcosystemName(ecosystem, name) + if err != nil || pkg == nil { + t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err) + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version)) + if err != nil || ver == nil { + t.Fatalf("version %s = %v, %v; want a row", canonicalVersionPURL(ecosystem, name, version), ver, err) + } + if ver.PackagePURL != pkg.PURL { + t.Errorf("version's package = %q, want %q", ver.PackagePURL, pkg.PURL) + } + if !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) { + t.Errorf("version %s published %v, want %v", version, ver.PublishedAt, published) + } +} + +// assertNotRecorded checks that a version the filter let through got no row: +// only held back versions are recorded. +func assertNotRecorded(t *testing.T, db *database.DB, ecosystem, name, version string) { + t.Helper() + ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version)) + if err != nil { + t.Fatalf("GetVersionByPURL: %v", err) + } + if ver != nil { + t.Errorf("version %s recorded although cooldown let it through", version) + } +} + +func TestRecordWithheldNeedsADatabase(t *testing.T) { + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Now()) +} + +func TestRecordWithheldSkipsUnknownTimes(t *testing.T) { + proxy, db, _, _ := withheldTestProxy(t) + proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Time{}) + if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a time", pkg, err) + } +} + +// Without a version the version PURL would be the package's own. +func TestRecordWithheldSkipsVersionsWithoutAName(t *testing.T) { + proxy, db, _, recent := withheldTestProxy(t) + proxy.recordWithheld("npm", "leftpad", "", recent) + if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a version", pkg, err) + } +} + +func TestNPMMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"name":"testpkg","dist-tags":{"latest":"2.0.0"}, + "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"}, + "versions":{ + "1.0.0":{"name":"testpkg","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-1.0.0.tgz"}}, + "2.0.0":{"name":"testpkg","version":"2.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-2.0.0.tgz"}}}}` + if _, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertWithheldRecorded(t, db, "npm", "testpkg", "2.0.0", recent) + assertNotRecorded(t, db, "npm", "testpkg", "1.0.0") +} + +func TestCargoIndexRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &CargoHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + line := func(version string, at time.Time) string { + return `{"name":"serde","vers":"` + version + `","cksum":"abcd","features":{},"yanked":false,"pubtime":"` + at.Format(time.RFC3339) + `"}` + "\n" + } + h.applyCooldownFiltering(httptest.NewRecorder(), []byte(line("1.0.0", old)+line("1.0.1", recent))) + assertWithheldRecorded(t, db, "cargo", "serde", "1.0.1", recent) + assertNotRecorded(t, db, "cargo", "serde", "1.0.0") +} + +func TestComposerMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"packages":{"symfony/console":[ + {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/5.0.0.zip","type":"zip"}}, + {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/6.0.0.zip","type":"zip"}}]}}` + if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertWithheldRecorded(t, db, "composer", "symfony/console", "6.0.0", recent) + assertNotRecorded(t, db, "composer", "symfony/console", "5.0.0") +} + +func TestPubMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"name":"flutter_bloc","latest":{"version":"2.0.0"},"versions":[ + {"version":"1.0.0","published":"` + old.Format(time.RFC3339) + `","archive_url":"https://pub.dev/1.0.0.tar.gz"}, + {"version":"2.0.0","published":"` + recent.Format(time.RFC3339) + `","archive_url":"https://pub.dev/2.0.0.tar.gz"}]}` + if _, err := h.rewriteMetadata("flutter_bloc", []byte(input)); err != nil { + t.Fatalf("rewriteMetadata: %v", err) + } + assertWithheldRecorded(t, db, "pub", "flutter_bloc", "2.0.0", recent) + assertNotRecorded(t, db, "pub", "flutter_bloc", "1.0.0") +} + +func TestTimedDocumentsRecordWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + docs := timedDocuments{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent, "3.0.0": {}}} + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0", "4.0.0"} { + docs.withheld(proxy, "hex", "testpkg", v) + } + assertWithheldRecorded(t, db, "hex", "testpkg", "2.0.0", recent) + for _, v := range []string{"1.0.0", "3.0.0", "4.0.0"} { + assertNotRecorded(t, db, "hex", "testpkg", v) + } +} + +func TestGemIndexRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + versionsJSON, _ := json.Marshal([]gemVersion{ + {Number: "1.0.0", Platform: "ruby", CreatedAt: old.Format(time.RFC3339)}, + {Number: "2.0.0", Platform: "ruby", CreatedAt: recent.Format(time.RFC3339)}, + }) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/info/"): + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("---\n1.0.0 dep1:>= 1.0|checksum:abc123\n2.0.0 dep1:>= 1.0|checksum:def456\n")) + case strings.HasPrefix(r.URL.Path, "/api/v1/versions/"): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(versionsJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer upstream.Close() + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + assertWithheldRecorded(t, db, "gem", "testgem", "2.0.0", recent) + assertNotRecorded(t, db, "gem", "testgem", "1.0.0") +} + +// Go stores the publish time of every version it reads, so what the filter +// adds is the package row. +func TestGoListRecordsWithheldVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + if w := goGet(h, "/example.com/mod/@v/list"); w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + assertPackageRow(t, proxy.DB, "golang", "example.com/mod") +} + +// A .info request for a version in cooldown is refused, and the version is +// recorded with the publish time the refusal was based on. +func TestGoRefusedInfoRecordsWithheldVersion(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", w.Code) + } + ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.1.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("version = %v, %v; want it stored with its publish time", ver, err) + } + assertPackageRow(t, proxy.DB, "golang", "example.com/mod") +} + +func assertPackageRow(t *testing.T, db *database.DB, ecosystem, name string) { + t.Helper() + if pkg, err := db.GetPackageByEcosystemName(ecosystem, name); err != nil || pkg == nil { + t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err) + } +} + +func TestMavenMetadataRecordsWithheldVersions(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + for _, v := range []string{"2.0", "2.1"} { + assertWithheldRecorded(t, proxy.DB, "maven", "com.example:lib", v, u.times[v].UTC().Truncate(time.Second)) + } +} + +func TestMavenNamedRepositoryRecordsNoWithheldVersions(t *testing.T) { + u := newMavenCooldownUpstream(t) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.server.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). + WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() + if w := getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if pkg, err := db.GetPackageByEcosystemName("maven", "com.example:lib"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want no row from a named repository", pkg, err) + } +} + +func TestPyPISimpleRecordsWithheldVersions(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old.Format(time.RFC3339) + `"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent.Format(time.RFC3339) + `"}]}` + return pypiHTTPResponse(r, pypiSimpleJSON, body), nil + case "/pypi/ruff/json": + body := `{"releases":{"1.0.0":[{"upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` + + `"2.0.0":[{"upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}` + return pypiHTTPResponse(r, "application/json", body), nil + default: + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + return nil, nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) + assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") +} + +// The JSON API filters releases on its own path, so a client that reads it +// rather than the simple index gets the held back version recorded too. +func TestPyPIJSONRecordsWithheldVersions(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + if r.URL.Path != "/pypi/ruff/json" { + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + } + body := `{"info":{"name":"ruff","version":"1.0.0"},"releases":{` + + `"1.0.0":[{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` + + `"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Add(-time.Minute).Format(time.RFC3339) + `"},` + + `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if strings.Contains(w.Body.String(), `"2.0.0"`) { + t.Fatalf("2.0.0 was served although it is in cooldown") + } + // A release's time is its newest file's. + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) + assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") +} + +// A release's time is its newest file's, and files can be added to a release +// after it was first held back, so the newer time replaces the stored one. +func TestPyPIWithheldReleaseKeepsItsNewestUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + first, added := now.Add(-2*time.Hour), now.Add(-time.Hour) + files := []string{`{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + first.Format(time.RFC3339) + `"}`} + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + body := `{"info":{"name":"ruff","version":"2.0.0"},"releases":{"2.0.0":[` + strings.Join(files, ",") + `]}}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + get := func() { + t.Helper() + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + } + + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first) + files = append(files, `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"`+added.Format(time.RFC3339)+`"}`) + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added) +} + +// The simple index and the version JSON also keep the newest upload time. +func TestPyPIWithheldReleaseKeepsItsNewestUploadTimeOnEveryPath(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + first, added := now.Add(-2*time.Hour), now.Add(-time.Hour) + for name, path := range map[string]string{"simple": "/simple/ruff/", "version json": "/pypi/ruff/2.0.0/json"} { + t.Run(name, func(t *testing.T) { + times := []time.Time{first} + files := func(key string) string { + var out []string + for i, at := range times { + out = append(out, fmt.Sprintf(`{"filename":"ruff-2.0.0-%d.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0-%d.tar.gz",%q:%q}`, i, i, key, at.Format(time.RFC3339))) + } + return strings.Join(out, ",") + } + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + return pypiHTTPResponse(r, pypiSimpleJSON, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[`+files("upload-time")+`]}`), nil + case "/pypi/ruff/json": + return pypiHTTPResponse(r, "application/json", `{"releases":{"2.0.0":[`+files("upload_time_iso_8601")+`]}}`), nil + default: + return pypiHTTPResponse(r, "application/json", `{"info":{"name":"ruff","version":"2.0.0"},"urls":[`+files("upload_time_iso_8601")+`]}`), nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + get := func() { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + } + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first) + times = append(times, added) + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added) + }) + } +} + +// The version JSON names one release, in urls, and records it when it is +// held back. +func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) { + recent := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + body := `{"info":{"name":"ruff","version":"2.0.0"},"urls":[` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/2.0.0/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if strings.Contains(w.Body.String(), "ruff-2.0.0.tar.gz") { + t.Fatalf("the files of 2.0.0 were served although it is in cooldown") + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) +} + +// The NuGet listing and the registration each find the held back version. +func TestNuGetMetadataRecordsWithheldVersions(t *testing.T) { + for _, path := range []string{ + "/nuget/v3-flatcontainer/testpkg/index.json", + "/nuget" + nugetRegistrationPath + "testpkg/index.json", + } { + t.Run(path, func(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, http.StripPrefix("/nuget", h.Routes()), path, http.StatusOK) + + assertPackageRow(t, db, "nuget", "testpkg") + ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err) + } + if age := time.Since(ver.PublishedAt.Time); age < 47*time.Hour || age > 49*time.Hour { + t.Errorf("2.0.0 published %v ago, want two days", age) + } + assertNotRecorded(t, db, "nuget", "testpkg", "1.0.0") + }) + } +} + +// Without a name the PURLs would name no package, such as pkg:cargo/. +func TestRecordWithheldSkipsPackagesWithoutAName(t *testing.T) { + proxy, db, _, recent := withheldTestProxy(t) + proxy.recordWithheld("cargo", "", "1.0.0", recent) + if pkg, err := db.GetPackageByEcosystemName("cargo", ""); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a name", pkg, err) + } +} + +// A routed package's times come from its route and are never stored under +// the default registry's PURLs, so the filter doesn't record them either. +func TestNPMMetadataOfRoutedPackageRecordsNothing(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080", + routes: newPackageRoutes(map[string]string{"@corp/*": "https://npm.corp.example"})} + input := `{"name":"@corp/lib","dist-tags":{"latest":"2.0.0"}, + "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"}, + "versions":{ + "1.0.0":{"name":"@corp/lib","version":"1.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-1.0.0.tgz"}}, + "2.0.0":{"name":"@corp/lib","version":"2.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-2.0.0.tgz"}}}}` + if _, _, err := h.rewriteMetadataKeeping("@corp/lib", []byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertNotRecorded(t, db, "npm", "@corp/lib", "2.0.0") +} + +func TestComposerMetadataOfRoutedPackageRecordsNothing(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080", + routes: newPackageRoutes(map[string]string{"corp/*": "https://composer.corp.example"})} + input := `{"packages":{"corp/lib":[ + {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/5.0.0.zip","type":"zip"}}, + {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/6.0.0.zip","type":"zip"}}]}}` + if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertNotRecorded(t, db, "composer", "corp/lib", "6.0.0") +} + +// Go keeps publish times under the escaped module path, which differs from +// the module path downloads are stored under when it has capitals. Recording +// such a module would give one release two version rows, so it is skipped. +func TestGoListOfModuleWithCapitalsRecordsNoSecondRow(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/example.com/!mod/@v/list": + _, _ = w.Write([]byte("v1.1.0\n")) + case "/example.com/!mod/@v/v1.1.0.info": + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.1.0", Time: now.Add(-time.Hour)}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes() + if w := goGet(h, "/example.com/!mod/@v/list"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/!mod", "v1.1.0")); err != nil || ver == nil { + t.Fatalf("time under the escaped path = %v, %v; want it stored", ver, err) + } + assertNotRecorded(t, db, "golang", "example.com/Mod", "v1.1.0") +} + +// A registration leaf carries no version of its own; it is the one its path +// names. +func TestNuGetRegistrationLeafRecordsWithheldVersion(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, http.StripPrefix("/nuget", h.Routes()), "/nuget"+nugetRegistrationPath+"testpkg/2.0.0.json", http.StatusNotFound) + + ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err) + } +} + +func TestNuGetRegistrationLeafVersion(t *testing.T) { + for path, want := range map[string]string{ + nugetRegistrationPath + "testpkg/2.0.0.json": "2.0.0", + nugetRegistrationPath + "testpkg/index.json": "", + nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json": "", + nugetRegistrationPath + "testpkg/2.0.0": "", + "/other/testpkg/2.0.0.json": "", + } { + if got := nugetRegistrationLeafVersion(path); got != want { + t.Errorf("nugetRegistrationLeafVersion(%q) = %q, want %q", path, got, want) + } + } +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index fa7c24ba..25983cdc 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -176,7 +176,6 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return } - packagePURL := canonicalPackagePURL("gem", name) m, err := h.proxy.cachedRewrite(r.Context(), "gem", h.proxyURL, "info/"+name, docs.doc.body, docs.rewriteDigest(), func([]byte) ([]byte, []string, error) { body, err := h.proxy.decodedMetadataBody(docs.doc) @@ -184,7 +183,7 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return nil, nil, err } out, kept := h.filterIndex(body, name, func(version string) bool { - return docs.withheld(h.proxy, "gem", packagePURL, version) + return docs.withheld(h.proxy, "gem", name, version) }) return out, kept, nil }) diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index efcc1e19..e788e688 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -168,6 +168,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s } keep := make([]bool, len(versions)) + times := make([]time.Time, len(versions)) g, gctx := errgroup.WithContext(ctx) g.SetLimit(goInfoFetchLimit) for i, v := range versions { @@ -177,6 +178,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s return fmt.Errorf("reading time of %s: %w", v, err) } keep[i] = h.allowedAt(module, published) + times[i] = published return nil }) } @@ -190,11 +192,23 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s allowed = append(allowed, v) } else { h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) + h.recordWithheld(module, v, times[i]) } } return allowed, nil } +// recordWithheld records a version the list filter held back. Publish times +// are stored under the escaped module path and downloads under the module +// path, so a module whose path has capitals, and the two differ, is skipped +// rather than given a second row for the same release. +func (h *GoHandler) recordWithheld(module, version string, published time.Time) { + if escaped, err := gomodule.EscapePath(module); err != nil || escaped != module { + return + } + h.proxy.recordWithheld("golang", module, version, published) +} + // serveCooldownList serves @v/list without versions in cooldown. func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) { allowed, err := h.allowedVersions(r.Context(), escapedModule, module) @@ -220,6 +234,7 @@ func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, es var info goInfo if err := json.Unmarshal(body, &info); err == nil && info.Version != "" { if !h.allowedAt(module, info.Time) { + h.recordWithheld(module, info.Version, info.Time) h.refuseCooldown(w, module, info.Version) return } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index d394c4df..17899f10 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -130,7 +130,6 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { return } - packagePURL := canonicalPackagePURL("hex", name) m, err := h.proxy.cachedRewrite(r.Context(), "hex", h.proxyURL, "packages/"+name, docs.doc.body, docs.rewriteDigest(), func([]byte) ([]byte, []string, error) { body, err := h.proxy.decodedMetadataBody(docs.doc) @@ -144,7 +143,7 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { filtered := make(map[string]bool) var kept []string for _, version := range versions { - if docs.withheld(h.proxy, "hex", packagePURL, version) { + if docs.withheld(h.proxy, "hex", name, version) { filtered[version] = true h.proxy.Logger.Info("cooldown: filtering hex version", "package", name, "version", version) continue diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 6155e0d7..beccb703 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -330,6 +330,11 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a } h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) removed[v] = true + // A named repository's times are its own, under PURLs that name it, + // and the package page shows the default repository's. + if source.name == "" { + h.proxy.recordWithheld("maven", group+":"+artifact, v, published) + } } if len(removed) == 0 { return body, nil diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 3cf3244d..95308b78 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -721,6 +721,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l // cooldown is evaluated at, and the pushed versions to list. The callbacks // log what the rewrite did. func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { + _, routed := h.routes.match(packageName) return npm.Options{ ProxyURL: h.proxyURL, Cooldown: h.proxy.cooldown(), @@ -736,6 +737,11 @@ func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry OnCooldownFilter: func(version, publishedAt string) { h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, "published", publishedAt) + // A routed package's times are the route's, and aren't stored + // under the default registry's PURLs (see versionInCooldown). + if at, err := time.Parse(time.RFC3339, publishedAt); err == nil && !routed { + h.proxy.recordWithheld("npm", packageName, version, at) + } }, } } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index 714f2192..1e6615a5 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -393,21 +393,27 @@ func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version str return false, err } _, published := leaf.release() - return h.nugetPublishedAllowed(id, published), nil + return h.nugetPublishedAllowed(id, version, published), nil } // Missing/invalid timestamps retain the existing permissive behavior, but -// fetch and JSON errors must not bypass the policy. -func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool { +// fetch and JSON errors must not bypass the policy. A version held back is +// recorded under the lowercase ID and version downloads are stored under. +func (h *NuGetHandler) nugetPublishedAllowed(id, version, published string) bool { when, err := time.Parse(time.RFC3339, published) if err != nil { return true } - return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) + name := strings.ToLower(id) + if h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", name), when) { + return true + } + h.proxy.recordWithheld("nuget", name, nugetVersionKey(version), when) + return false } func (h *NuGetHandler) collectNuGetBlockedVersions(document *nugetTimes, id string, blocked map[string]bool) { - if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, published) { + if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, version, published) { blocked[nugetVersionKey(version)] = true } for i := range document.Items { @@ -436,7 +442,11 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request return } if applyCooldown && document.Items == nil { - if _, published := document.release(); !h.nugetPublishedAllowed(id, published) { + version, published := document.release() + if version == "" { + version = nugetRegistrationLeafVersion(r.URL.Path) + } + if !h.nugetPublishedAllowed(id, version, published) { JSONError(w, http.StatusNotFound, "version not found") return } @@ -469,6 +479,25 @@ func nugetRegistrationID(path string) string { return "" } +// nugetRegistrationLeafVersion returns the version a registration leaf's +// path names ({prefix}{id}/{version}.json), or "" for any other path. A leaf +// as nuget.org serves it carries no version of its own. +func nugetRegistrationLeafVersion(path string) string { + for _, prefix := range nugetRegistrationPrefixes { + rest, ok := strings.CutPrefix(path, prefix) + if !ok { + continue + } + _, leaf, _ := strings.Cut(rest, "/") + version, isJSON := strings.CutSuffix(leaf, ".json") + if !isJSON || version == "" || version == "index" || strings.Contains(version, "/") { + return "" + } + return version + } + return "" +} + // expandNuGetPageTimes gives each page of a registration index that only // links to its leaves the leaves of the page it links to. With bodies not nil // the linked pages themselves are kept in it, by link. Only index pages are diff --git a/internal/handler/nuget_registration.go b/internal/handler/nuget_registration.go index 01da55de..e32273ff 100644 --- a/internal/handler/nuget_registration.go +++ b/internal/handler/nuget_registration.go @@ -389,7 +389,11 @@ func (rw *nugetRegistrationWriter) node(dec *jsontext.Decoder, depth int) (bool, if n.container { return n.kept > 0, release.version, nil } - return rw.h.nugetPublishedAllowed(rw.id, release.published), release.version, nil + var version string + if len(release.version) >= 2 && release.version[0] == '"' { + version = string(rw.unquote(release.version)) + } + return rw.h.nugetPublishedAllowed(rw.id, version, release.published), release.version, nil } // linksPage reports whether a node at depth is a page of an index whose diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 5a6d426c..1fe8f040 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -288,6 +288,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) + h.proxy.recordWithheld("pub", name, version, publishedAt) return true } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index c94e1515..ed8546a7 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -326,6 +326,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s for version, publishedAt := range times { if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true + h.proxy.recordWithheldNewest("pypi", name, version, publishedAt) } } @@ -625,9 +626,17 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } for version, files := range releases { - if h.proxy.versionDenied("pypi", packageName, version) || h.shouldFilterRelease(packagePURL, files) { + denied := h.proxy.versionDenied("pypi", packageName, version) + held, publishedAt := false, time.Time{} + if !denied { + held, publishedAt = h.shouldFilterRelease(packagePURL, files) + } + if denied || held { h.proxy.Logger.Info("policy: filtering pypi version", "package", packageName, "version", version) + if held { + h.proxy.recordWithheldNewest("pypi", packageName, version, publishedAt) + } delete(releases, version) continue } @@ -636,19 +645,20 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } } -// shouldFilterRelease returns true if a release should be excluded due to cooldown. -func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { +// shouldFilterRelease reports whether a release should be excluded due to +// cooldown, and the newest upload time that decided it. +func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) (bool, time.Time) { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() || packagePURL == "" { - return false + return false, time.Time{} } filesArr, ok := files.([]any) if !ok { - return false + return false, time.Time{} } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) + return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt), publishedAt } // versionInCooldown reports whether a version is still inside the cooldown @@ -744,7 +754,10 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL info, _ := metadata["info"].(map[string]any) name, _ := info["name"].(string) version, _ := info["version"].(string) - if h.proxy.versionDenied("pypi", name, version) || h.shouldFilterRelease(packagePURL, urls) { + if h.proxy.versionDenied("pypi", name, version) { + metadata["urls"] = []any{} + } else if held, publishedAt := h.shouldFilterRelease(packagePURL, urls); held { + h.proxy.recordWithheldNewest("pypi", name, version, publishedAt) metadata["urls"] = []any{} }