From 88e88607f3ae6f13d8ee86fb42ebb7dc79736ac5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 11 Nov 2026 21:19:36 +0100 Subject: [PATCH 1/7] List a container image's tags on its page The page of an image listed only its cached blobs by digest, so a new release such as pihole/pihole:2026.11.0 could not be told apart from the image before it. A pull by tag, with a GET or the HEAD Docker and containerd send, and a mirror run now record the digest the tag points at in a new image_tags table. The page lists the tags, the most recently seen first. Referrers and cosign tags are left out, and a copy served stale during an outage does not count as a sighting. --- README.md | 5 + internal/database/image_tags.go | 49 ++++++++ internal/database/image_tags_test.go | 72 ++++++++++++ internal/database/schema.go | 37 ++++++ internal/handler/container_image_tags.go | 29 +++++ internal/handler/container_image_tags_test.go | 107 ++++++++++++++++++ internal/handler/container_manifest.go | 9 +- internal/handler/container_mirror.go | 1 + internal/server/dashboard.go | 2 + internal/server/image_tags_ui_test.go | 74 ++++++++++++ internal/server/server.go | 8 ++ .../server/templates/pages/package_show.html | 20 ++++ 12 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 internal/database/image_tags.go create mode 100644 internal/database/image_tags_test.go create mode 100644 internal/handler/container_image_tags.go create mode 100644 internal/handler/container_image_tags_test.go create mode 100644 internal/server/image_tags_ui_test.go diff --git a/README.md b/README.md index 7ba3cb21..6c2ea70e 100644 --- a/README.md +++ b/README.md @@ -559,6 +559,11 @@ docker pull localhost:8080/library/nginx:latest The registry is pull-only. BuildKit cache exports (`--cache-to type=registry`) and other pushes need a separate registry. +An image's page in the web UI (`/ui/package/oci/{name}`) lists its tags, the +most recently seen first, with the digest each one pointed at when the +upstream last confirmed it, through a pull or a mirror run. Its versions are +the cached blobs, by digest. + SBOMs, signatures and attestations attached to an image are found through the OCI 1.1 referrers API (`GET /v2/{name}/referrers/{digest}`). `oras discover` and `notation verify` use it, and so does cosign for its newer signature diff --git a/internal/database/image_tags.go b/internal/database/image_tags.go new file mode 100644 index 00000000..b5ada1ed --- /dev/null +++ b/internal/database/image_tags.go @@ -0,0 +1,49 @@ +package database + +import ( + "time" +) + +// ImageTag is a tag of a container image repository as the proxy last saw +// it upstream. +type ImageTag struct { + Tag string `db:"tag"` + Digest string `db:"digest"` + // SeenAt is when the upstream last said the tag points at Digest. + SeenAt time.Time `db:"-"` +} + +type imageTagRow struct { + Tag string `db:"tag"` + Digest string `db:"digest"` + SeenAt int64 `db:"seen_at"` +} + +// RecordImageTag stores that a tag of a repository pointed at digest at the +// given time. An older sighting than the stored one changes nothing. +func (db *DB) RecordImageTag(packagePURL, tag, digest string, seenAt time.Time) error { + _, err := db.Exec(db.Rebind(` + INSERT INTO image_tags (package_purl, tag, digest, seen_at) VALUES (?, ?, ?, ?) + ON CONFLICT (package_purl, tag) DO UPDATE SET digest = excluded.digest, seen_at = excluded.seen_at + WHERE image_tags.seen_at < excluded.seen_at + `), packagePURL, tag, digest, seenAt.UnixMilli()) + return err +} + +// ImageTags returns up to limit tags of a repository, the most recently seen +// first. +func (db *DB) ImageTags(packagePURL string, limit int) ([]ImageTag, error) { + var rows []imageTagRow + err := db.Select(&rows, db.Rebind(` + SELECT tag, digest, seen_at FROM image_tags WHERE package_purl = ? + ORDER BY seen_at DESC, tag LIMIT ? + `), packagePURL, limit) + if err != nil { + return nil, err + } + tags := make([]ImageTag, len(rows)) + for i, row := range rows { + tags[i] = ImageTag{Tag: row.Tag, Digest: row.Digest, SeenAt: time.UnixMilli(row.SeenAt)} + } + return tags, nil +} diff --git a/internal/database/image_tags_test.go b/internal/database/image_tags_test.go new file mode 100644 index 00000000..ccc03f25 --- /dev/null +++ b/internal/database/image_tags_test.go @@ -0,0 +1,72 @@ +package database + +import ( + "testing" + "time" +) + +const testTagRepo = "pkg:oci/pihole/pihole" + +func recordTag(t *testing.T, db *DB, repo, tag, digest string, seen time.Time) { + t.Helper() + if err := db.RecordImageTag(repo, tag, digest, seen); err != nil { + t.Fatalf("RecordImageTag(%s): %v", tag, err) + } +} + +// The tags of a repository are listed most recently seen first, each with +// the digest it pointed at when last seen. +func TestImageTagsListsTagsMostRecentlySeenFirst(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().Truncate(time.Millisecond) + recordTag(t, db, testTagRepo, "2026.10.0", "sha256:old", now.Add(-48*time.Hour)) + recordTag(t, db, testTagRepo, "latest", "sha256:old", now.Add(-time.Hour)) + recordTag(t, db, testTagRepo, "2026.11.0", "sha256:new", now.Add(-time.Minute)) + recordTag(t, db, "pkg:oci/library/alpine", "latest", "sha256:alpine", now) + + // latest moves to the new image. + recordTag(t, db, testTagRepo, "latest", "sha256:new", now) + // A sighting older than the stored one changes nothing. + recordTag(t, db, testTagRepo, "2026.11.0", "sha256:other", now.Add(-time.Hour)) + + tags, err := db.ImageTags(testTagRepo, 10) + if err != nil { + t.Fatalf("ImageTags: %v", err) + } + want := []ImageTag{ + {Tag: "latest", Digest: "sha256:new", SeenAt: now}, + {Tag: "2026.11.0", Digest: "sha256:new", SeenAt: now.Add(-time.Minute)}, + {Tag: "2026.10.0", Digest: "sha256:old", SeenAt: now.Add(-48 * time.Hour)}, + } + if len(tags) != len(want) { + t.Fatalf("ImageTags = %+v, want %+v", tags, want) + } + for i := range want { + if tags[i].Tag != want[i].Tag || tags[i].Digest != want[i].Digest || !tags[i].SeenAt.Equal(want[i].SeenAt) { + t.Errorf("tag %d = %+v, want %+v", i, tags[i], want[i]) + } + } + + if tags, err := db.ImageTags(testTagRepo, 1); err != nil || len(tags) != 1 || tags[0].Tag != "latest" { + t.Errorf("ImageTags(limit 1) = %+v, %v; want latest only", tags, err) + } + }) +} + +func TestMigrateAddsImageTags(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if _, err := db.Exec("DROP TABLE image_tags"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DELETE FROM migrations WHERE name = '024_add_image_tags'"); err != nil { + t.Fatal(err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + recordTag(t, db, testTagRepo, "latest", "sha256:a", time.Now()) + if tags, err := db.ImageTags(testTagRepo, 10); err != nil || len(tags) != 1 { + t.Errorf("ImageTags after migration = %+v, %v; want one tag", tags, err) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index bca37bd4..39e4174a 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -151,6 +151,14 @@ CREATE TABLE IF NOT EXISTS metadata_dates ( PRIMARY KEY (ecosystem, proxy_url, name) ); +CREATE TABLE IF NOT EXISTS image_tags ( + package_purl TEXT NOT NULL, + tag TEXT NOT NULL, + digest TEXT NOT NULL, + seen_at INTEGER NOT NULL, + PRIMARY KEY (package_purl, tag) +); + CREATE TABLE IF NOT EXISTS upstream_holds ( host TEXT NOT NULL PRIMARY KEY, until_ms INTEGER NOT NULL @@ -330,6 +338,14 @@ CREATE TABLE IF NOT EXISTS metadata_dates ( PRIMARY KEY (ecosystem, proxy_url, name) ); +CREATE TABLE IF NOT EXISTS image_tags ( + package_purl TEXT NOT NULL, + tag TEXT NOT NULL, + digest TEXT NOT NULL, + seen_at BIGINT NOT NULL, + PRIMARY KEY (package_purl, tag) +); + CREATE TABLE IF NOT EXISTS upstream_holds ( host TEXT NOT NULL PRIMARY KEY, until_ms BIGINT NOT NULL @@ -546,6 +562,7 @@ var migrations = []migration{ {"021_add_projects", migrateAddProjects}, {"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt}, {"023_add_metadata_upstream_url", migrateAddMetadataUpstreamURL}, + {"024_add_image_tags", migrateAddImageTags}, } // isTableNotFound returns true if the error indicates a missing table. @@ -1030,6 +1047,26 @@ func migrateAddMetadataDates(db *DB) error { return nil } +// migrateAddImageTags creates the table of the tags of container image +// repositories that the web UI lists. +func migrateAddImageTags(db *DB) error { + millis := "INTEGER" + if db.dialect == DialectPostgres { + millis = "BIGINT" + } + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS image_tags ( + package_purl TEXT NOT NULL, + tag TEXT NOT NULL, + digest TEXT NOT NULL, + seen_at %s NOT NULL, + PRIMARY KEY (package_purl, tag) + )`, millis) + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating image_tags table: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") diff --git a/internal/handler/container_image_tags.go b/internal/handler/container_image_tags.go new file mode 100644 index 00000000..4cd4eb9e --- /dev/null +++ b/internal/handler/container_image_tags.go @@ -0,0 +1,29 @@ +package handler + +import ( + "regexp" + "time" +) + +// imageSubjectTagsPattern matches the tags that hold what refers to an +// image rather than an image: the referrers tag schema, and the +// sha256-.sig, .att and .sbom tags of cosign. +var imageSubjectTagsPattern = regexp.MustCompile(`^sha256-[a-f0-9]{64}(\.[a-z0-9]+)?$`) + +// recordImageTag notes which digest a tag of the repository pointed at when +// the upstream last confirmed it, for the web UI to list. A copy served stale +// during an outage confirms nothing. +func (h *ContainerHandler) recordImageTag(cacheName, reference string, manifest *cachedContainerManifest, stale bool) { + if h.proxy.DB == nil || stale || manifestDigestReferencePattern.MatchString(reference) || + imageSubjectTagsPattern.MatchString(reference) || !mirrorDigestPattern.MatchString(manifest.contentDigest) { + return + } + seen := manifest.fetchedAt + if seen.IsZero() { + seen = time.Now() + } + packagePURL := canonicalPackagePURL("oci", cacheName) + if err := h.proxy.DB.RecordImageTag(packagePURL, reference, manifest.contentDigest, seen); err != nil { + h.proxy.Logger.Warn("failed to record image tag", "package_purl", packagePURL, "tag", reference, "error", err) + } +} diff --git a/internal/handler/container_image_tags_test.go b/internal/handler/container_image_tags_test.go new file mode 100644 index 00000000..5c974fd3 --- /dev/null +++ b/internal/handler/container_image_tags_test.go @@ -0,0 +1,107 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +const imageTagsRepo = "team/app" + +func imageTagsHandler(t *testing.T) (*ContainerHandler, *fakeImageRegistry, fakeImage) { + t.Helper() + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + return NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL), registry, image +} + +func requestTagManifest(t *testing.T, h *ContainerHandler, method, reference string) { + t.Helper() + request := httptest.NewRequest(method, "/"+imageTagsRepo+"/manifests/"+reference, nil) + request.Header.Set("Accept", containerdManifestAccept) + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("%s %s: status %d: %s", method, reference, response.Code, response.Body.String()) + } +} + +func imageTags(t *testing.T, h *ContainerHandler) map[string]database.ImageTag { + t.Helper() + tags, err := h.proxy.DB.ImageTags(canonicalPackagePURL("oci", imageTagsRepo), 100) + if err != nil { + t.Fatalf("ImageTags: %v", err) + } + byTag := make(map[string]database.ImageTag, len(tags)) + for _, tag := range tags { + byTag[tag.Tag] = tag + } + return byTag +} + +// A pull by tag, with a GET or only a HEAD as Docker and containerd send, +// records the digest the tag points at. Pulls by digest name no tag. +func TestContainerPullRecordsImageTags(t *testing.T) { + h, _, image := imageTagsHandler(t) + requestTagManifest(t, h, http.MethodGet, "multi") + requestTagManifest(t, h, http.MethodHead, "single") + requestTagManifest(t, h, http.MethodGet, image.arm64Digest) + + tags := imageTags(t, h) + if len(tags) != 2 { + t.Fatalf("tags = %+v, want multi and single", tags) + } + for tag, digest := range map[string]string{"multi": image.indexDigest, "single": image.singleDigest} { + got := tags[tag] + if got.Digest != digest || time.Since(got.SeenAt) > time.Minute { + t.Errorf("tag %s = %+v, want %s seen just now", tag, got, digest) + } + } +} + +// Referrers and cosign tags hold what refers to an image, not an image, so +// they are not listed. +func TestContainerPullOfReferrersTagRecordsNoTag(t *testing.T) { + h, registry, image := imageTagsHandler(t) + subject := "sha256-" + image.indexDigest[len("sha256:"):] + for _, tag := range []string{subject, subject + ".sig", subject + ".att"} { + registry.mu.Lock() + registry.manifests["/v2/"+imageTagsRepo+"/manifests/"+tag] = registry.manifests["/v2/"+imageTagsRepo+"/manifests/"+image.singleDigest] + registry.mu.Unlock() + requestTagManifest(t, h, http.MethodGet, tag) + } + if tags := imageTags(t, h); len(tags) != 0 { + t.Errorf("tags = %+v, want none", tags) + } +} + +// A copy served stale during an outage says nothing new about the tag. +func TestContainerStaleManifestKeepsTagTime(t *testing.T) { + h, registry, _ := imageTagsHandler(t) + requestTagManifest(t, h, http.MethodGet, "multi") + db := h.proxy.DB + if _, err := db.Exec(db.Rebind("UPDATE image_tags SET seen_at = ?"), 1000); err != nil { + t.Fatal(err) + } + registry.offline.Store(true) + requestTagManifest(t, h, http.MethodGet, "multi") + if got := imageTags(t, h)["multi"].SeenAt; !got.Equal(time.UnixMilli(1000)) { + t.Errorf("seen at = %v after a stale serve, want unchanged", got) + } +} + +// A scheduled mirror run resolves tags itself and records them like a pull. +func TestMirrorImageRecordsImageTags(t *testing.T) { + h, _, image := imageTagsHandler(t) + ref := ImageRef{Name: imageTagsRepo, References: []string{"multi"}} + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if got := imageTags(t, h)["multi"]; got.Digest != image.indexDigest { + t.Errorf("tag multi after a mirror run = %+v, want %s", got, image.indexDigest) + } +} diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 6d129d0a..c2868b0e 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -63,14 +63,14 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, } cached, _ := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, req.accept, req.cacheAccept) if cached != nil && (req.immutable() || h.containerManifestFresh(cached)) { - h.writeManifest(w, r, cacheName, cached, false) + h.writeManifest(w, r, cacheName, reference, cached, false) return } outcome := h.coalescedManifest(r.Context(), req) switch { case outcome.manifest != nil: - h.writeManifest(w, r, cacheName, outcome.manifest, false) + h.writeManifest(w, r, cacheName, reference, outcome.manifest, false) case outcome.relay != nil: outcome.relay.write(h.proxy, w, r) case outcome.digestInvalid: @@ -83,7 +83,7 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cacheName, reference string, cached *cachedContainerManifest, err error) { if cached != nil { h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err) - h.writeManifest(w, r, cacheName, cached, true) + h.writeManifest(w, r, cacheName, reference, cached, true) return } if referrersTagPattern.MatchString(reference) { @@ -110,10 +110,11 @@ func (h *ContainerHandler) manifestDenied(w http.ResponseWriter, cacheName, refe } // writeManifest writes manifest unless the denylist names its digest. -func (h *ContainerHandler) writeManifest(w http.ResponseWriter, r *http.Request, cacheName string, manifest *cachedContainerManifest, stale bool) { +func (h *ContainerHandler) writeManifest(w http.ResponseWriter, r *http.Request, cacheName, reference string, manifest *cachedContainerManifest, stale bool) { if h.manifestDenied(w, cacheName, manifest.contentDigest) { return } + h.recordImageTag(cacheName, reference, manifest, stale) writeContainerManifest(w, r, manifest, stale) } diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index f19ac736..624b8eb6 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -255,6 +255,7 @@ func (m *imageMirror) mirrorTag(ctx context.Context, tag string) (string, error) return digest, nil } m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept, manifest) + m.h.recordImageTag(m.cacheName, tag, manifest, false) return digest, nil } diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 604a4bf0..bc72aef2 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -74,6 +74,8 @@ type PackageShowData struct { UsedBy []VersionUsage // Cooldown is nil with cooldown off. Cooldown *PackageCooldown + // Tags lists a container image's tags, the most recently seen first. + Tags []database.ImageTag } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/image_tags_ui_test.go b/internal/server/image_tags_ui_test.go new file mode 100644 index 00000000..7ed68628 --- /dev/null +++ b/internal/server/image_tags_ui_test.go @@ -0,0 +1,74 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// The page of a container image lists its tags, most recently seen first, +// with the digest each one points at. +func TestPackageShowPageListsImageTags(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + pkg := &database.Package{PURL: "pkg:oci/pihole/pihole", Ecosystem: "oci", Name: "pihole/pihole"} + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + const layer = "sha256:7373c5bccb12b43ec422ccd65b3cc9b55ee6e86ad3cee28189322c6419a72b02" + if err := ts.db.UpsertVersion(&database.Version{PURL: pkg.PURL + "@" + layer, PackagePURL: pkg.PURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + const newDigest = "sha256:66c798d79a95c2691f280329c3611dba1746117c61c9f331907ca35a06a52778" + const oldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + now := time.Now() + for tag, at := range map[string]struct { + digest string + seen time.Time + }{ + "2026.10.0": {oldDigest, now.Add(-48 * time.Hour)}, + "2026.11.0": {newDigest, now.Add(-time.Minute)}, + "latest": {newDigest, now}, + } { + if err := ts.db.RecordImageTag(pkg.PURL, tag, at.digest, at.seen); err != nil { + t.Fatalf("RecordImageTag: %v", err) + } + } + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/oci/pihole/pihole", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + body := w.Body.String() + if !strings.Contains(body, "Tags (3)") { + t.Error("page has no Tags (3) section") + } + latest, current, older := strings.Index(body, ">latest<"), strings.Index(body, ">2026.11.0<"), strings.Index(body, ">2026.10.0<") + if latest < 0 || current < 0 || older < 0 || latest > current || current > older { + t.Errorf("tags at %d, %d, %d; want latest, 2026.11.0, 2026.10.0 in that order", latest, current, older) + } + if !strings.Contains(body, newDigest) || !strings.Contains(body, oldDigest) { + t.Error("page does not show the digests the tags point at") + } +} + +// Packages of other ecosystems have no tags section. +func TestPackageShowPageHasNoTagsOutsideOCI(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + pkg := &database.Package{PURL: "pkg:npm/no-tags", Ecosystem: "npm", Name: "no-tags"} + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/npm/no-tags", nil)) + if strings.Contains(w.Body.String(), "Tags (") { + t.Error("npm package page has a Tags section") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 19ba5bee..19b75c17 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1217,6 +1217,9 @@ func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) { } } +// maxImageTagsShown is how many tags a container image's page lists. +const maxImageTagsShown = 100 + func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) { pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name) if err != nil { @@ -1259,6 +1262,11 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), } data.UsedBy = s.packageUsedBy(pkg, versions) + if pkg.Ecosystem == "oci" { + if data.Tags, err = s.db.ImageTags(pkg.PURL, maxImageTagsShown); err != nil { + s.logger.Error("failed to get image tags", "error", err) + } + } if err := s.templates.Render(w, "package_show", data); err != nil { s.logger.Error("failed to render package show", "error", err) diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index afbdfa33..95a09011 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -80,6 +80,26 @@

Used by

{{end}} +{{if .Tags}} +
+
+

Tags ({{len .Tags}})

+

The digest each tag pointed at when the upstream last confirmed it

+
+
+ {{range .Tags}} +
+
+ {{.Tag}} + {{.Digest}} +
+ {{.SeenAt.UTC.Format "2006-01-02 15:04"}} UTC +
+ {{end}} +
+
+{{end}} + {{if .Versions}}
From 3370c7a80812e7940567bd95b980e9a1eff70c77 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 11 Nov 2026 21:33:02 +0100 Subject: [PATCH 2/7] Name the tags that contain a blob on its page A container image's versions in the web UI are its blobs. The page of one now names, under Version Info, the tags whose cached image contains it, following each tag's manifest to its platform manifests and their configs and layers. The manifests are read from the cache without counting as a use, so viewing a page does not change what eviction keeps. --- README.md | 3 +- internal/handler/container_image_tags.go | 74 +++++++++++++++++++ internal/handler/container_image_tags_test.go | 49 ++++++++++++ internal/handler/container_manifest.go | 23 ++++-- internal/server/dashboard.go | 3 + internal/server/image_tags_ui_test.go | 44 +++++++++++ internal/server/server.go | 27 ++++++- .../server/templates/pages/version_show.html | 3 + 8 files changed, 217 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6c2ea70e..d13cd711 100644 --- a/README.md +++ b/README.md @@ -562,7 +562,8 @@ and other pushes need a separate registry. An image's page in the web UI (`/ui/package/oci/{name}`) lists its tags, the most recently seen first, with the digest each one pointed at when the upstream last confirmed it, through a pull or a mirror run. Its versions are -the cached blobs, by digest. +the cached blobs, by digest, and a blob's page names the tags whose cached +image contains it. SBOMs, signatures and attestations attached to an image are found through the OCI 1.1 referrers API (`GET /v2/{name}/referrers/{digest}`). `oras discover` diff --git a/internal/handler/container_image_tags.go b/internal/handler/container_image_tags.go index 4cd4eb9e..a3e07458 100644 --- a/internal/handler/container_image_tags.go +++ b/internal/handler/container_image_tags.go @@ -1,8 +1,13 @@ package handler import ( + "context" + "encoding/json" "regexp" + "slices" "time" + + "github.com/git-pkgs/proxy/internal/database" ) // imageSubjectTagsPattern matches the tags that hold what refers to an @@ -27,3 +32,72 @@ func (h *ContainerHandler) recordImageTag(cacheName, reference string, manifest h.proxy.Logger.Warn("failed to record image tag", "package_purl", packagePURL, "tag", reference, "error", err) } } + +// imageTagWalkDepth is how many levels below a tag's manifest +// ImageTagsContaining looks: an index, its platform manifests, and their +// configs and layers. +const imageTagWalkDepth = 2 + +// ImageTagsContaining returns those of tags whose image, as cached under +// cacheName, contains digest: the manifest the tag points at, a platform +// manifest it lists, or a config or layer of one. Only manifests clients or +// mirror runs pulled are known. Reading them does not count as a use of the +// cache. +func (h *ContainerHandler) ImageTagsContaining(ctx context.Context, cacheName, digest string, tags []database.ImageTag) []string { + registryURL, upstreamName, _, ok := h.registryForName(cacheName) + if !ok { + return nil + } + // The manifests a walk read, by digest, with the digests they reference. + refs := map[string][]string{} + references := func(d string) []string { + if known, done := refs[d]; done { + return known + } + // A manifest fetched by tag is cached under its digest too. + manifest, _, err := h.readContainerManifest(ctx, h.containerManifestSharedCacheKey(registryURL, upstreamName, d, "")) + if err == nil && manifest != nil && manifest.contentDigest == d { + refs[d] = imageManifestReferences(manifest.body) + } else { + refs[d] = nil + } + return refs[d] + } + + var found []string + for _, tag := range tags { + level := []string{tag.Digest} + for depth := 0; len(level) > 0 && !slices.Contains(level, digest); depth++ { + if depth == imageTagWalkDepth { + level = nil + break + } + var next []string + for _, d := range level { + next = append(next, references(d)...) + } + level = next + } + if slices.Contains(level, digest) { + found = append(found, tag.Tag) + } + } + return found +} + +// imageManifestReferences returns the digests an index or image manifest +// references. +func imageManifestReferences(body []byte) []string { + var manifest ociManifest + if json.Unmarshal(body, &manifest) != nil { + return nil + } + var refs []string + for _, descriptor := range slices.Concat(manifest.Manifests, manifest.Layers) { + refs = append(refs, descriptor.Digest) + } + if manifest.Config != nil { + refs = append(refs, manifest.Config.Digest) + } + return refs +} diff --git a/internal/handler/container_image_tags_test.go b/internal/handler/container_image_tags_test.go index 5c974fd3..6feac6b7 100644 --- a/internal/handler/container_image_tags_test.go +++ b/internal/handler/container_image_tags_test.go @@ -4,6 +4,8 @@ import ( "context" "net/http" "net/http/httptest" + "slices" + "strings" "testing" "time" @@ -105,3 +107,50 @@ func TestMirrorImageRecordsImageTags(t *testing.T) { t.Errorf("tag multi after a mirror run = %+v, want %s", got, image.indexDigest) } } + +// A blob's page names the tags whose cached image contains it: the manifest +// the tag points at, the platform manifests it lists, and their configs and +// layers, as far as clients pulled them. +func TestImageTagsContaining(t *testing.T) { + h, _, image := imageTagsHandler(t) + // containerd: a GET of the tag, then the platform manifest by digest. + requestTagManifest(t, h, http.MethodGet, "multi") + requestTagManifest(t, h, http.MethodGet, image.arm64Digest) + // Docker: a HEAD of the tag, then the manifest by digest. + requestTagManifest(t, h, http.MethodHead, "single") + requestTagManifest(t, h, http.MethodGet, image.singleDigest) + tags, err := h.proxy.DB.ImageTags(canonicalPackagePURL("oci", imageTagsRepo), 100) + if err != nil { + t.Fatal(err) + } + + db := h.proxy.DB + if _, err := db.Exec("UPDATE metadata_cache SET last_accessed_at = NULL"); err != nil { + t.Fatal(err) + } + for name, tt := range map[string]struct { + digest string + want []string + }{ + "index": {image.indexDigest, []string{"multi"}}, + "platform": {image.arm64Digest, []string{"multi"}}, + "layer": {image.arm64Layer, []string{"multi"}}, + "single layer": {image.singleLayer, []string{"single"}}, + "unpulled layer": {image.amd64Layer, nil}, + "unknown": {"sha256:" + strings.Repeat("0", 64), nil}, + } { + got := h.ImageTagsContaining(context.Background(), imageTagsRepo, tt.digest, tags) + if !slices.Equal(got, tt.want) { + t.Errorf("%s: tags = %v, want %v", name, got, tt.want) + } + } + + // Looking does not count as a use of the cached manifests. + var touched int + if err := db.Get(&touched, "SELECT COUNT(*) FROM metadata_cache WHERE last_accessed_at IS NOT NULL"); err != nil { + t.Fatal(err) + } + if touched != 0 { + t.Errorf("%d cached manifests were marked as used", touched) + } +} diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index c2868b0e..c4dade21 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" "time" + + "github.com/git-pkgs/proxy/internal/database" ) const ( @@ -221,23 +223,32 @@ func (h *ContainerHandler) storeContainerManifestLegacyEntries(ctx context.Conte } func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) { + manifest, entry, err := h.readContainerManifest(ctx, cacheKey) + if manifest != nil { + h.proxy.recordMetadataRead(entry) + } + return manifest, err +} + +// readContainerManifest is loadContainerManifest without counting the read +// as a use of the cached entry. +func (h *ContainerHandler) readContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, *database.MetadataCacheEntry, error) { if h.proxy.DB == nil || h.proxy.Storage == nil { - return nil, nil + return nil, nil, nil } entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) if err != nil || entry == nil { - return nil, err + return nil, nil, err } reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) if err != nil { - return nil, nil + return nil, nil, nil } defer func() { _ = reader.Close() }() body, err := h.proxy.ReadMetadata(reader) if err != nil { - return nil, err + return nil, nil, err } - h.proxy.recordMetadataRead(entry) manifest := &cachedContainerManifest{body: body, size: int64(len(body))} if entry.ContentType.Valid { @@ -260,7 +271,7 @@ func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey s if entry.FetchedAt.Valid { manifest.fetchedAt = entry.FetchedAt.Time } - return manifest, nil + return manifest, entry, nil } func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error { diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index bc72aef2..6c897e17 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -91,6 +91,9 @@ type VersionShowData struct { // HasDeletableArtifact is true when a stored artifact was fetched from // the upstream rather than pushed. HasDeletableArtifact bool + // ImageTags names the tags whose cached image contains a container + // image's blob. + ImageTags []string } // SearchPageData contains data for rendering the search results page. diff --git a/internal/server/image_tags_ui_test.go b/internal/server/image_tags_ui_test.go index 7ed68628..b366c449 100644 --- a/internal/server/image_tags_ui_test.go +++ b/internal/server/image_tags_ui_test.go @@ -1,6 +1,7 @@ package server import ( + "context" "net/http" "net/http/httptest" "strings" @@ -72,3 +73,46 @@ func TestPackageShowPageHasNoTagsOutsideOCI(t *testing.T) { t.Error("npm package page has a Tags section") } } + +type fakeImageTags struct { + cacheName, digest string + tags []database.ImageTag +} + +func (f *fakeImageTags) ImageTagsContaining(_ context.Context, cacheName, digest string, tags []database.ImageTag) []string { + f.cacheName, f.digest, f.tags = cacheName, digest, tags + return []string{"latest", "2026.11.0"} +} + +// The page of an image's blob names the tags whose image contains it. +func TestVersionShowPageListsImageTagsContainingTheBlob(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + finder := &fakeImageTags{} + ts.server.images = finder + + pkg := &database.Package{PURL: "pkg:oci/pihole/pihole", Ecosystem: "oci", Name: "pihole/pihole"} + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + const layer = "sha256:7373c5bccb12b43ec422ccd65b3cc9b55ee6e86ad3cee28189322c6419a72b02" + if err := ts.db.UpsertVersion(&database.Version{PURL: pkg.PURL + "@" + layer, PackagePURL: pkg.PURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + if err := ts.db.RecordImageTag(pkg.PURL, "latest", "sha256:66c798d79a95c2691f280329c3611dba1746117c61c9f331907ca35a06a52778", time.Now()); err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/oci/pihole/pihole/"+layer, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + if finder.cacheName != "pihole/pihole" || finder.digest != layer || len(finder.tags) != 1 { + t.Errorf("looked up %q %q with %d tags, want pihole/pihole, the layer and 1 tag", finder.cacheName, finder.digest, len(finder.tags)) + } + body := w.Body.String() + if !strings.Contains(body, "Tags:") || !strings.Contains(body, ">latest<") || !strings.Contains(body, ">2026.11.0<") { + t.Error("version page does not name the tags containing the blob") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 19b75c17..4a147d45 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -134,6 +134,9 @@ type Server struct { scheduler *mirror.Scheduler git *gitcache.Cache cooldown *cooldownView + // images finds the tags whose image contains a blob, for the version + // page of a container image. Nil until the protocol handlers are mounted. + images imageTagFinder // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -470,10 +473,17 @@ func (s *Server) serve(listener net.Listener) error { // mountProtocolHandlers constructs every ecosystem handler and mounts it on // r under its protocol prefix. func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { - s.mountProtocolHandlersAt(r, proxy, s.cfg.BaseURL) + s.images = s.mountProtocolHandlersAt(r, proxy, s.cfg.BaseURL) } -func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, baseURL string) { +// imageTagFinder is what the version page needs of the container handler. +type imageTagFinder interface { + ImageTagsContaining(ctx context.Context, cacheName, digest string, tags []database.ImageTag) []string +} + +// mountProtocolHandlersAt mounts every ecosystem handler, and returns the +// container handler. +func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, baseURL string) *handler.ContainerHandler { npmHandler := handler.NewNPMHandler(proxy, baseURL, s.cfg.Upstream.NPM). WithPackageRoutes(s.cfg.Upstream.NPMRoutes) cargoHandler := handler.NewCargoHandler( @@ -573,6 +583,7 @@ func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, bas gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(baseURL) r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) } + return containerHandler } // configureScanning builds the scanner group from cfg and wires it into @@ -1307,6 +1318,17 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, // browsed, so it must not be advertised as cached. hasCached := firstBrowsableArtifact(artifacts) != nil + var imageTags []string + if pkg.Ecosystem == "oci" && s.images != nil { + tags, err := s.db.ImageTags(pkg.PURL, maxImageTagsShown) + if err != nil { + s.logger.Error("failed to get image tags", "error", err) + } + if len(tags) > 0 { + imageTags = s.images.ImageTagsContaining(r.Context(), pkg.Name, version, tags) + } + } + data := VersionShowData{ Layout: s.layoutFor(r), Package: pkg, @@ -1317,6 +1339,7 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, LicenseCategory: categorizeLicense(ver.License), HasCachedArtifact: hasCached, HasDeletableArtifact: hasDeletableArtifact(artifacts), + ImageTags: imageTags, } if err := s.templates.Render(w, "version_show", data); err != nil { diff --git a/internal/server/templates/pages/version_show.html b/internal/server/templates/pages/version_show.html index 468f24cd..96499d84 100644 --- a/internal/server/templates/pages/version_show.html +++ b/internal/server/templates/pages/version_show.html @@ -43,6 +43,9 @@

Version Info

{{if .Version.PublishedAt.Valid}}
Published:
{{.Version.PublishedAt.Time.Format "2006-01-02 15:04:05"}}
{{end}} + {{if .ImageTags}} +
Tags:
{{range $i, $tag := .ImageTags}}{{if $i}}, {{end}}{{$tag}}{{end}}
+ {{end}} {{if .Version.Integrity.Valid}}
Integrity:
{{.Version.Integrity.String}}
{{end}} From f09d60125439eef236f3d71d542be65aa4612ecc Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 11 Nov 2026 21:43:19 +0100 Subject: [PATCH 3/7] Search past the listed tags for the ones containing a blob A blob's page looked only through the 100 tags an image's page lists, so a blob of an older tag named none. It now looks through up to 1000. --- internal/server/image_tags_ui_test.go | 45 +++++++++++++++++++++++++++ internal/server/server.go | 12 +++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/internal/server/image_tags_ui_test.go b/internal/server/image_tags_ui_test.go index b366c449..84905c0a 100644 --- a/internal/server/image_tags_ui_test.go +++ b/internal/server/image_tags_ui_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "fmt" "net/http" "net/http/httptest" "strings" @@ -116,3 +117,47 @@ func TestVersionShowPageListsImageTagsContainingTheBlob(t *testing.T) { t.Error("version page does not name the tags containing the blob") } } + +// matchOldest finds only the tag "oldest" in a blob's image. +type matchOldest struct{} + +func (matchOldest) ImageTagsContaining(_ context.Context, _, _ string, tags []database.ImageTag) []string { + for _, tag := range tags { + if tag.Tag == "oldest" { + return []string{"oldest"} + } + } + return nil +} + +// The tags searched for a blob are not cut to the ones the image's page +// lists, so a blob of an old tag still names it. +func TestVersionShowPageSearchesPastTheListedTags(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.images = matchOldest{} + + pkg := &database.Package{PURL: "pkg:oci/team/app", Ecosystem: "oci", Name: "team/app"} + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatal(err) + } + const layer = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + if err := ts.db.UpsertVersion(&database.Version{PURL: pkg.PURL + "@" + layer, PackagePURL: pkg.PURL}); err != nil { + t.Fatal(err) + } + now := time.Now() + if err := ts.db.RecordImageTag(pkg.PURL, "oldest", "sha256:old", now.Add(-time.Hour)); err != nil { + t.Fatal(err) + } + for i := range maxImageTagsShown + 1 { + if err := ts.db.RecordImageTag(pkg.PURL, fmt.Sprintf("v%d", i), "sha256:new", now); err != nil { + t.Fatal(err) + } + } + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/oci/team/app/"+layer, nil)) + if !strings.Contains(w.Body.String(), ">oldest<") { + t.Errorf("status %d: version page does not name the oldest tag", w.Code) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 4a147d45..b94bd037 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1228,8 +1228,14 @@ func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) { } } -// maxImageTagsShown is how many tags a container image's page lists. -const maxImageTagsShown = 100 +const ( + // maxImageTagsShown is how many tags a container image's page lists. + maxImageTagsShown = 100 + // maxImageTagsSearched is how many tags, the most recently seen first, + // a blob's page looks through for those whose image contains it. Tags + // pointing at the same image share one read of its manifests. + maxImageTagsSearched = 1000 +) func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) { pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name) @@ -1320,7 +1326,7 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, var imageTags []string if pkg.Ecosystem == "oci" && s.images != nil { - tags, err := s.db.ImageTags(pkg.PURL, maxImageTagsShown) + tags, err := s.db.ImageTags(pkg.PURL, maxImageTagsSearched) if err != nil { s.logger.Error("failed to get image tags", "error", err) } From 15d786184da4047eb6fbd8b5c28cd447150f23ff Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 13:55:08 +0100 Subject: [PATCH 4/7] Find a blob's tags through recorded manifest references Each served manifest records the digests it references, so a blob's page finds the tags whose image contains it with one query instead of reading up to 1000 manifests from storage, and the limit applies to the matches. --- internal/database/image_tags.go | 51 ++++++++ internal/database/image_tags_test.go | 110 ++++++++++++++++++ internal/database/schema.go | 36 ++++++ internal/handler/container_image_tags.go | 64 +++------- internal/handler/container_image_tags_test.go | 52 ++++++--- internal/handler/container_manifest.go | 24 ++-- internal/server/image_tags_ui_test.go | 42 +++---- internal/server/server.go | 30 +---- 8 files changed, 270 insertions(+), 139 deletions(-) diff --git a/internal/database/image_tags.go b/internal/database/image_tags.go index b5ada1ed..367ae093 100644 --- a/internal/database/image_tags.go +++ b/internal/database/image_tags.go @@ -1,6 +1,8 @@ package database import ( + "database/sql" + "errors" "time" ) @@ -47,3 +49,52 @@ func (db *DB) ImageTags(packagePURL string, limit int) ([]ImageTag, error) { } return tags, nil } + +// RecordImageManifestReferences stores the digests a manifest of a +// repository references: the platform manifests of an index, or the config +// and layers of an image. A digest fixes the manifest's content, so a +// manifest already recorded is left as it is. +func (db *DB) RecordImageManifestReferences(packagePURL, manifestDigest string, references []string) error { + var known int + err := db.Get(&known, db.Rebind(` + SELECT 1 FROM image_manifest_references WHERE package_purl = ? AND manifest_digest = ? LIMIT 1 + `), packagePURL, manifestDigest) + if err == nil || !errors.Is(err, sql.ErrNoRows) { + return err + } + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + insert := db.Rebind(` + INSERT INTO image_manifest_references (package_purl, manifest_digest, reference_digest) VALUES (?, ?, ?) + ON CONFLICT DO NOTHING + `) + for _, reference := range references { + if _, err := tx.Exec(insert, packagePURL, manifestDigest, reference); err != nil { + return err + } + } + return tx.Commit() +} + +// ImageTagsContaining returns up to limit tags of a repository whose image +// contains digest, the most recently seen first: tags pointing at digest, +// at a manifest that references it, or at an index listing such a manifest. +func (db *DB) ImageTagsContaining(packagePURL, digest string, limit int) ([]string, error) { + var tags []string + err := db.Select(&tags, db.Rebind(` + SELECT tag FROM image_tags WHERE package_purl = ? AND (digest = ? OR digest IN ( + SELECT manifest_digest FROM image_manifest_references + WHERE package_purl = ? AND reference_digest = ? + UNION + SELECT parent.manifest_digest FROM image_manifest_references child + JOIN image_manifest_references parent + ON parent.package_purl = child.package_purl AND parent.reference_digest = child.manifest_digest + WHERE child.package_purl = ? AND child.reference_digest = ? + )) + ORDER BY seen_at DESC, tag LIMIT ? + `), packagePURL, digest, packagePURL, digest, packagePURL, digest, limit) + return tags, err +} diff --git a/internal/database/image_tags_test.go b/internal/database/image_tags_test.go index ccc03f25..ec2aadf1 100644 --- a/internal/database/image_tags_test.go +++ b/internal/database/image_tags_test.go @@ -1,6 +1,8 @@ package database import ( + "fmt" + "slices" "testing" "time" ) @@ -70,3 +72,111 @@ func TestMigrateAddsImageTags(t *testing.T) { } }) } + +func recordReferences(t *testing.T, db *DB, repo, manifest string, references ...string) { + t.Helper() + if err := db.RecordImageManifestReferences(repo, manifest, references); err != nil { + t.Fatalf("RecordImageManifestReferences(%s): %v", manifest, err) + } +} + +// The tags whose image contains a digest: the manifest a tag points at, the +// platform manifests of an index, and their configs and layers. +func TestImageTagsContaining(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + recordReferences(t, db, testTagRepo, "sha256:index", "sha256:amd64", "sha256:arm64") + recordReferences(t, db, testTagRepo, "sha256:amd64", "sha256:amd64-config", "sha256:shared", "sha256:amd64-layer") + recordReferences(t, db, testTagRepo, "sha256:single", "sha256:single-config", "sha256:shared") + // The same digests in another repository say nothing about this one. + recordReferences(t, db, "pkg:oci/library/alpine", "sha256:alpine", "sha256:single-config") + recordReferences(t, db, "pkg:oci/library/alpine", "sha256:index", "sha256:alpine-layer") + recordReferences(t, db, "pkg:oci/library/alpine", "sha256:single", "sha256:amd64") + recordTag(t, db, "pkg:oci/library/alpine", "latest", "sha256:alpine", now) + + recordTag(t, db, testTagRepo, "multi", "sha256:index", now) + recordTag(t, db, testTagRepo, "single", "sha256:single", now.Add(-time.Minute)) + recordTag(t, db, testTagRepo, "old", "sha256:index", now.Add(-time.Hour)) + + for _, tt := range []struct { + digest string + want []string + }{ + {"sha256:index", []string{"multi", "old"}}, + {"sha256:amd64", []string{"multi", "old"}}, + {"sha256:arm64", []string{"multi", "old"}}, + {"sha256:amd64-layer", []string{"multi", "old"}}, + {"sha256:shared", []string{"multi", "single", "old"}}, + {"sha256:single-config", []string{"single"}}, + {"sha256:alpine-layer", nil}, + {"sha256:unknown", nil}, + } { + got, err := db.ImageTagsContaining(testTagRepo, tt.digest, 10) + if err != nil { + t.Fatalf("ImageTagsContaining(%s): %v", tt.digest, err) + } + if !slices.Equal(got, tt.want) { + t.Errorf("ImageTagsContaining(%s) = %v, want %v", tt.digest, got, tt.want) + } + } + + if got, _ := db.ImageTagsContaining(testTagRepo, "sha256:shared", 2); !slices.Equal(got, []string{"multi", "single"}) { + t.Errorf("ImageTagsContaining(limit 2) = %v, want multi and single", got) + } + }) +} + +// The limit applies to the tags that match, so a match among older tags is +// found however many newer tags there are. +func TestImageTagsContainingLimitsTheMatches(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + recordReferences(t, db, testTagRepo, "sha256:old", "sha256:layer") + recordTag(t, db, testTagRepo, "oldest", "sha256:old", now.Add(-time.Hour)) + for i := range 5 { + recordTag(t, db, testTagRepo, fmt.Sprintf("v%d", i), "sha256:new", now) + } + if got, _ := db.ImageTagsContaining(testTagRepo, "sha256:layer", 2); !slices.Equal(got, []string{"oldest"}) { + t.Errorf("ImageTagsContaining = %v, want oldest", got) + } + }) +} + +// A manifest's digest fixes its content, so the references recorded first +// stand, and recording them again adds nothing. +func TestRecordImageManifestReferencesKeepsTheFirstRecord(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + recordReferences(t, db, testTagRepo, "sha256:m", "sha256:a", "sha256:b", "sha256:a") + recordReferences(t, db, testTagRepo, "sha256:m", "sha256:c") + recordTag(t, db, testTagRepo, "latest", "sha256:m", time.Now()) + if got, _ := db.ImageTagsContaining(testTagRepo, "sha256:c", 10); len(got) != 0 { + t.Errorf("a second record added a reference: %v", got) + } + var rows int + if err := db.Get(&rows, "SELECT COUNT(*) FROM image_manifest_references"); err != nil { + t.Fatal(err) + } + if rows != 2 { + t.Errorf("%d rows, want 2", rows) + } + }) +} + +func TestMigrateAddsImageManifestReferences(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if _, err := db.Exec("DROP TABLE image_manifest_references"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DELETE FROM migrations WHERE name = '025_add_image_manifest_references'"); err != nil { + t.Fatal(err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + recordReferences(t, db, testTagRepo, "sha256:m", "sha256:a") + recordTag(t, db, testTagRepo, "latest", "sha256:m", time.Now()) + if got, err := db.ImageTagsContaining(testTagRepo, "sha256:a", 10); err != nil || len(got) != 1 { + t.Errorf("ImageTagsContaining after migration = %v, %v; want latest", got, err) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 39e4174a..7ddb502a 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -159,6 +159,14 @@ CREATE TABLE IF NOT EXISTS image_tags ( PRIMARY KEY (package_purl, tag) ); +CREATE TABLE IF NOT EXISTS image_manifest_references ( + package_purl TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + reference_digest TEXT NOT NULL, + PRIMARY KEY (package_purl, manifest_digest, reference_digest) +); +CREATE INDEX IF NOT EXISTS idx_image_manifest_references_reference ON image_manifest_references(package_purl, reference_digest); + CREATE TABLE IF NOT EXISTS upstream_holds ( host TEXT NOT NULL PRIMARY KEY, until_ms INTEGER NOT NULL @@ -346,6 +354,14 @@ CREATE TABLE IF NOT EXISTS image_tags ( PRIMARY KEY (package_purl, tag) ); +CREATE TABLE IF NOT EXISTS image_manifest_references ( + package_purl TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + reference_digest TEXT NOT NULL, + PRIMARY KEY (package_purl, manifest_digest, reference_digest) +); +CREATE INDEX IF NOT EXISTS idx_image_manifest_references_reference ON image_manifest_references(package_purl, reference_digest); + CREATE TABLE IF NOT EXISTS upstream_holds ( host TEXT NOT NULL PRIMARY KEY, until_ms BIGINT NOT NULL @@ -563,6 +579,7 @@ var migrations = []migration{ {"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt}, {"023_add_metadata_upstream_url", migrateAddMetadataUpstreamURL}, {"024_add_image_tags", migrateAddImageTags}, + {"025_add_image_manifest_references", migrateAddImageManifestReferences}, } // isTableNotFound returns true if the error indicates a missing table. @@ -1067,6 +1084,25 @@ func migrateAddImageTags(db *DB) error { return nil } +// migrateAddImageManifestReferences creates the table of the digests each +// cached image manifest references, which finds the tags whose image +// contains a blob. +func migrateAddImageManifestReferences(db *DB) error { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS image_manifest_references ( + package_purl TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + reference_digest TEXT NOT NULL, + PRIMARY KEY (package_purl, manifest_digest, reference_digest) + )`); err != nil { + return fmt.Errorf("creating image_manifest_references table: %w", err) + } + if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_image_manifest_references_reference + ON image_manifest_references(package_purl, reference_digest)`); err != nil { + return fmt.Errorf("creating image_manifest_references index: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") diff --git a/internal/handler/container_image_tags.go b/internal/handler/container_image_tags.go index a3e07458..b861d308 100644 --- a/internal/handler/container_image_tags.go +++ b/internal/handler/container_image_tags.go @@ -1,13 +1,10 @@ package handler import ( - "context" "encoding/json" "regexp" "slices" "time" - - "github.com/git-pkgs/proxy/internal/database" ) // imageSubjectTagsPattern matches the tags that hold what refers to an @@ -33,56 +30,23 @@ func (h *ContainerHandler) recordImageTag(cacheName, reference string, manifest } } -// imageTagWalkDepth is how many levels below a tag's manifest -// ImageTagsContaining looks: an index, its platform manifests, and their -// configs and layers. -const imageTagWalkDepth = 2 - -// ImageTagsContaining returns those of tags whose image, as cached under -// cacheName, contains digest: the manifest the tag points at, a platform -// manifest it lists, or a config or layer of one. Only manifests clients or -// mirror runs pulled are known. Reading them does not count as a use of the -// cache. -func (h *ContainerHandler) ImageTagsContaining(ctx context.Context, cacheName, digest string, tags []database.ImageTag) []string { - registryURL, upstreamName, _, ok := h.registryForName(cacheName) - if !ok { - return nil +// recordImageManifestReferences notes the digests a served manifest +// references, which finds the tags whose image contains a blob. A digest +// fixes the content, so a stale copy is as good as a fresh one. +func (h *ContainerHandler) recordImageManifestReferences(cacheName string, manifest *cachedContainerManifest) { + // The first record of a digest stands, so only a body that hashes to it + // is recorded. + if h.proxy.DB == nil || sha256Digest(manifest.body) != manifest.contentDigest { + return } - // The manifests a walk read, by digest, with the digests they reference. - refs := map[string][]string{} - references := func(d string) []string { - if known, done := refs[d]; done { - return known - } - // A manifest fetched by tag is cached under its digest too. - manifest, _, err := h.readContainerManifest(ctx, h.containerManifestSharedCacheKey(registryURL, upstreamName, d, "")) - if err == nil && manifest != nil && manifest.contentDigest == d { - refs[d] = imageManifestReferences(manifest.body) - } else { - refs[d] = nil - } - return refs[d] + references := imageManifestReferences(manifest.body) + if len(references) == 0 { + return } - - var found []string - for _, tag := range tags { - level := []string{tag.Digest} - for depth := 0; len(level) > 0 && !slices.Contains(level, digest); depth++ { - if depth == imageTagWalkDepth { - level = nil - break - } - var next []string - for _, d := range level { - next = append(next, references(d)...) - } - level = next - } - if slices.Contains(level, digest) { - found = append(found, tag.Tag) - } + packagePURL := canonicalPackagePURL("oci", cacheName) + if err := h.proxy.DB.RecordImageManifestReferences(packagePURL, manifest.contentDigest, references); err != nil { + h.proxy.Logger.Warn("failed to record image manifest references", "package_purl", packagePURL, "digest", manifest.contentDigest, "error", err) } - return found } // imageManifestReferences returns the digests an index or image manifest diff --git a/internal/handler/container_image_tags_test.go b/internal/handler/container_image_tags_test.go index 6feac6b7..eb0de105 100644 --- a/internal/handler/container_image_tags_test.go +++ b/internal/handler/container_image_tags_test.go @@ -108,10 +108,11 @@ func TestMirrorImageRecordsImageTags(t *testing.T) { } } -// A blob's page names the tags whose cached image contains it: the manifest -// the tag points at, the platform manifests it lists, and their configs and -// layers, as far as clients pulled them. -func TestImageTagsContaining(t *testing.T) { +// Pulled manifests record what they reference, so a blob's page finds the +// tags whose image contains it: the manifest the tag points at, the platform +// manifests it lists, and their configs and layers, as far as clients pulled +// them. +func TestContainerPullRecordsWhatImagesContain(t *testing.T) { h, _, image := imageTagsHandler(t) // containerd: a GET of the tag, then the platform manifest by digest. requestTagManifest(t, h, http.MethodGet, "multi") @@ -119,15 +120,7 @@ func TestImageTagsContaining(t *testing.T) { // Docker: a HEAD of the tag, then the manifest by digest. requestTagManifest(t, h, http.MethodHead, "single") requestTagManifest(t, h, http.MethodGet, image.singleDigest) - tags, err := h.proxy.DB.ImageTags(canonicalPackagePURL("oci", imageTagsRepo), 100) - if err != nil { - t.Fatal(err) - } - db := h.proxy.DB - if _, err := db.Exec("UPDATE metadata_cache SET last_accessed_at = NULL"); err != nil { - t.Fatal(err) - } for name, tt := range map[string]struct { digest string want []string @@ -139,18 +132,39 @@ func TestImageTagsContaining(t *testing.T) { "unpulled layer": {image.amd64Layer, nil}, "unknown": {"sha256:" + strings.Repeat("0", 64), nil}, } { - got := h.ImageTagsContaining(context.Background(), imageTagsRepo, tt.digest, tags) + got, err := h.proxy.DB.ImageTagsContaining(canonicalPackagePURL("oci", imageTagsRepo), tt.digest, 10) + if err != nil { + t.Fatal(err) + } if !slices.Equal(got, tt.want) { t.Errorf("%s: tags = %v, want %v", name, got, tt.want) } } +} - // Looking does not count as a use of the cached manifests. - var touched int - if err := db.Get(&touched, "SELECT COUNT(*) FROM metadata_cache WHERE last_accessed_at IS NOT NULL"); err != nil { - t.Fatal(err) +// A mirror run reads the manifests through the handler, so it records what +// they reference like a pull. +func TestMirrorImageRecordsWhatImagesContain(t *testing.T) { + h, _, image := imageTagsHandler(t) + ref := ImageRef{Name: imageTagsRepo, References: []string{"multi"}} + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("MirrorImage: %v", err) } - if touched != 0 { - t.Errorf("%d cached manifests were marked as used", touched) + got, err := h.proxy.DB.ImageTagsContaining(canonicalPackagePURL("oci", imageTagsRepo), image.amd64Layer, 10) + if err != nil || !slices.Equal(got, []string{"multi"}) { + t.Errorf("tags containing the amd64 layer after a mirror run = %v, %v; want multi", got, err) + } +} + +// The first record of a manifest stands, so a body that is not the one its +// digest names is not recorded under that digest. +func TestImageManifestReferencesNeedTheBodyTheDigestNames(t *testing.T) { + h, _, image := imageTagsHandler(t) + body := []byte(`{"schemaVersion":2,"layers":[{"digest":"sha256:` + strings.Repeat("1", 64) + `"}]}`) + h.recordImageManifestReferences(imageTagsRepo, &cachedContainerManifest{body: body, contentDigest: image.singleDigest}) + requestTagManifest(t, h, http.MethodGet, "single") + got, err := h.proxy.DB.ImageTagsContaining(canonicalPackagePURL("oci", imageTagsRepo), image.singleLayer, 10) + if err != nil || !slices.Equal(got, []string{"single"}) { + t.Errorf("tags containing the single layer = %v, %v; want single", got, err) } } diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index c4dade21..f17f5b61 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -13,8 +13,6 @@ import ( "strconv" "strings" "time" - - "github.com/git-pkgs/proxy/internal/database" ) const ( @@ -116,6 +114,7 @@ func (h *ContainerHandler) writeManifest(w http.ResponseWriter, r *http.Request, if h.manifestDenied(w, cacheName, manifest.contentDigest) { return } + h.recordImageManifestReferences(cacheName, manifest) h.recordImageTag(cacheName, reference, manifest, stale) writeContainerManifest(w, r, manifest, stale) } @@ -223,32 +222,23 @@ func (h *ContainerHandler) storeContainerManifestLegacyEntries(ctx context.Conte } func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) { - manifest, entry, err := h.readContainerManifest(ctx, cacheKey) - if manifest != nil { - h.proxy.recordMetadataRead(entry) - } - return manifest, err -} - -// readContainerManifest is loadContainerManifest without counting the read -// as a use of the cached entry. -func (h *ContainerHandler) readContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, *database.MetadataCacheEntry, error) { if h.proxy.DB == nil || h.proxy.Storage == nil { - return nil, nil, nil + return nil, nil } entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) if err != nil || entry == nil { - return nil, nil, err + return nil, err } reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) if err != nil { - return nil, nil, nil + return nil, nil } defer func() { _ = reader.Close() }() body, err := h.proxy.ReadMetadata(reader) if err != nil { - return nil, nil, err + return nil, err } + h.proxy.recordMetadataRead(entry) manifest := &cachedContainerManifest{body: body, size: int64(len(body))} if entry.ContentType.Valid { @@ -271,7 +261,7 @@ func (h *ContainerHandler) readContainerManifest(ctx context.Context, cacheKey s if entry.FetchedAt.Valid { manifest.fetchedAt = entry.FetchedAt.Time } - return manifest, entry, nil + return manifest, nil } func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error { diff --git a/internal/server/image_tags_ui_test.go b/internal/server/image_tags_ui_test.go index 84905c0a..e606b11d 100644 --- a/internal/server/image_tags_ui_test.go +++ b/internal/server/image_tags_ui_test.go @@ -1,7 +1,6 @@ package server import ( - "context" "fmt" "net/http" "net/http/httptest" @@ -75,22 +74,10 @@ func TestPackageShowPageHasNoTagsOutsideOCI(t *testing.T) { } } -type fakeImageTags struct { - cacheName, digest string - tags []database.ImageTag -} - -func (f *fakeImageTags) ImageTagsContaining(_ context.Context, cacheName, digest string, tags []database.ImageTag) []string { - f.cacheName, f.digest, f.tags = cacheName, digest, tags - return []string{"latest", "2026.11.0"} -} - // The page of an image's blob names the tags whose image contains it. func TestVersionShowPageListsImageTagsContainingTheBlob(t *testing.T) { ts := newTestServer(t) defer ts.close() - finder := &fakeImageTags{} - ts.server.images = finder pkg := &database.Package{PURL: "pkg:oci/pihole/pihole", Ecosystem: "oci", Name: "pihole/pihole"} if err := ts.db.UpsertPackage(pkg); err != nil { @@ -100,34 +87,29 @@ func TestVersionShowPageListsImageTagsContainingTheBlob(t *testing.T) { if err := ts.db.UpsertVersion(&database.Version{PURL: pkg.PURL + "@" + layer, PackagePURL: pkg.PURL}); err != nil { t.Fatalf("UpsertVersion: %v", err) } - if err := ts.db.RecordImageTag(pkg.PURL, "latest", "sha256:66c798d79a95c2691f280329c3611dba1746117c61c9f331907ca35a06a52778", time.Now()); err != nil { + const image = "sha256:66c798d79a95c2691f280329c3611dba1746117c61c9f331907ca35a06a52778" + if err := ts.db.RecordImageManifestReferences(pkg.PURL, image, []string{layer}); err != nil { t.Fatal(err) } + now := time.Now() + for tag, digest := range map[string]string{"latest": image, "2026.11.0": image, "2026.10.0": "sha256:other"} { + if err := ts.db.RecordImageTag(pkg.PURL, tag, digest, now); err != nil { + t.Fatal(err) + } + } w := httptest.NewRecorder() ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/oci/pihole/pihole/"+layer, nil)) if w.Code != http.StatusOK { t.Fatalf("status %d: %s", w.Code, w.Body.String()) } - if finder.cacheName != "pihole/pihole" || finder.digest != layer || len(finder.tags) != 1 { - t.Errorf("looked up %q %q with %d tags, want pihole/pihole, the layer and 1 tag", finder.cacheName, finder.digest, len(finder.tags)) - } body := w.Body.String() if !strings.Contains(body, "Tags:") || !strings.Contains(body, ">latest<") || !strings.Contains(body, ">2026.11.0<") { t.Error("version page does not name the tags containing the blob") } -} - -// matchOldest finds only the tag "oldest" in a blob's image. -type matchOldest struct{} - -func (matchOldest) ImageTagsContaining(_ context.Context, _, _ string, tags []database.ImageTag) []string { - for _, tag := range tags { - if tag.Tag == "oldest" { - return []string{"oldest"} - } + if strings.Contains(body, ">2026.10.0<") { + t.Error("version page names a tag whose image lacks the blob") } - return nil } // The tags searched for a blob are not cut to the ones the image's page @@ -135,7 +117,6 @@ func (matchOldest) ImageTagsContaining(_ context.Context, _, _ string, tags []da func TestVersionShowPageSearchesPastTheListedTags(t *testing.T) { ts := newTestServer(t) defer ts.close() - ts.server.images = matchOldest{} pkg := &database.Package{PURL: "pkg:oci/team/app", Ecosystem: "oci", Name: "team/app"} if err := ts.db.UpsertPackage(pkg); err != nil { @@ -145,6 +126,9 @@ func TestVersionShowPageSearchesPastTheListedTags(t *testing.T) { if err := ts.db.UpsertVersion(&database.Version{PURL: pkg.PURL + "@" + layer, PackagePURL: pkg.PURL}); err != nil { t.Fatal(err) } + if err := ts.db.RecordImageManifestReferences(pkg.PURL, "sha256:old", []string{layer}); err != nil { + t.Fatal(err) + } now := time.Now() if err := ts.db.RecordImageTag(pkg.PURL, "oldest", "sha256:old", now.Add(-time.Hour)); err != nil { t.Fatal(err) diff --git a/internal/server/server.go b/internal/server/server.go index 911d0e93..7537a10a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -134,9 +134,6 @@ type Server struct { scheduler *mirror.Scheduler git *gitcache.Cache cooldown *cooldownView - // images finds the tags whose image contains a blob, for the version - // page of a container image. Nil until the protocol handlers are mounted. - images imageTagFinder // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -479,17 +476,10 @@ func (s *Server) serve(listener net.Listener) error { // mountProtocolHandlers constructs every ecosystem handler and mounts it on // r under its protocol prefix. func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { - s.images = s.mountProtocolHandlersAt(r, proxy, s.cfg.BaseURL) + s.mountProtocolHandlersAt(r, proxy, s.cfg.BaseURL) } -// imageTagFinder is what the version page needs of the container handler. -type imageTagFinder interface { - ImageTagsContaining(ctx context.Context, cacheName, digest string, tags []database.ImageTag) []string -} - -// mountProtocolHandlersAt mounts every ecosystem handler, and returns the -// container handler. -func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, baseURL string) *handler.ContainerHandler { +func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, baseURL string) { npmHandler := handler.NewNPMHandler(proxy, baseURL, s.cfg.Upstream.NPM). WithPackageRoutes(s.cfg.Upstream.NPMRoutes) cargoHandler := handler.NewCargoHandler( @@ -589,7 +579,6 @@ func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, bas gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(baseURL) r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) } - return containerHandler } // configureScanning builds the scanner group from cfg and wires it into @@ -1235,12 +1224,9 @@ func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) { } const ( - // maxImageTagsShown is how many tags a container image's page lists. + // maxImageTagsShown is how many tags a container image's page lists, + // and how many a blob's page names. maxImageTagsShown = 100 - // maxImageTagsSearched is how many tags, the most recently seen first, - // a blob's page looks through for those whose image contains it. Tags - // pointing at the same image share one read of its manifests. - maxImageTagsSearched = 1000 ) func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) { @@ -1331,14 +1317,10 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, hasCached := firstBrowsableArtifact(artifacts) != nil var imageTags []string - if pkg.Ecosystem == "oci" && s.images != nil { - tags, err := s.db.ImageTags(pkg.PURL, maxImageTagsSearched) - if err != nil { + if pkg.Ecosystem == "oci" { + if imageTags, err = s.db.ImageTagsContaining(pkg.PURL, version, maxImageTagsShown); err != nil { s.logger.Error("failed to get image tags", "error", err) } - if len(tags) > 0 { - imageTags = s.images.ImageTagsContaining(r.Context(), pkg.Name, version, tags) - } } data := VersionShowData{ From 37b816a07993b8486047d71c50783364073b2a6d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 14:45:08 +0100 Subject: [PATCH 5/7] Reset the image tag tables between Postgres tests --- internal/database/database_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 500ad0d8..91ca3970 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "projects", "project_components", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "projects", "project_components", "image_tags", "image_manifest_references", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } From ccf080ebfd38e3e63d708a56e653dbc3f49d8f70 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 14:47:36 +0100 Subject: [PATCH 6/7] Check a manifest's references with its digest's own algorithm --- internal/handler/container_image_tags.go | 2 +- internal/handler/container_image_tags_test.go | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/handler/container_image_tags.go b/internal/handler/container_image_tags.go index b861d308..61f5ca8c 100644 --- a/internal/handler/container_image_tags.go +++ b/internal/handler/container_image_tags.go @@ -36,7 +36,7 @@ func (h *ContainerHandler) recordImageTag(cacheName, reference string, manifest func (h *ContainerHandler) recordImageManifestReferences(cacheName string, manifest *cachedContainerManifest) { // The first record of a digest stands, so only a body that hashes to it // is recorded. - if h.proxy.DB == nil || sha256Digest(manifest.body) != manifest.contentDigest { + if h.proxy.DB == nil || verifyDigest(manifest.contentDigest, manifest.body) != nil { return } references := imageManifestReferences(manifest.body) diff --git a/internal/handler/container_image_tags_test.go b/internal/handler/container_image_tags_test.go index eb0de105..c423fe5a 100644 --- a/internal/handler/container_image_tags_test.go +++ b/internal/handler/container_image_tags_test.go @@ -2,6 +2,8 @@ package handler import ( "context" + "crypto/sha512" + "encoding/hex" "net/http" "net/http/httptest" "slices" @@ -168,3 +170,21 @@ func TestImageManifestReferencesNeedTheBodyTheDigestNames(t *testing.T) { t.Errorf("tags containing the single layer = %v, %v; want single", got, err) } } + +// A manifest named by a sha512 digest is checked with sha512. +func TestImageManifestReferencesOfASHA512Manifest(t *testing.T) { + h, _, _ := imageTagsHandler(t) + layer := "sha256:" + strings.Repeat("1", 64) + body := []byte(`{"schemaVersion":2,"layers":[{"digest":"` + layer + `"}]}`) + sum := sha512.Sum512(body) + digest := "sha512:" + hex.EncodeToString(sum[:]) + h.recordImageManifestReferences(imageTagsRepo, &cachedContainerManifest{body: body, contentDigest: digest}) + repo := canonicalPackagePURL("oci", imageTagsRepo) + if err := h.proxy.DB.RecordImageTag(repo, "v1", digest, time.Now()); err != nil { + t.Fatal(err) + } + got, err := h.proxy.DB.ImageTagsContaining(repo, layer, 10) + if err != nil || !slices.Equal(got, []string{"v1"}) { + t.Errorf("tags containing the layer = %v, %v; want v1", got, err) + } +} From 7b36b5dd03544eddd93212a4fc224d067420e95f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 15:06:36 +0100 Subject: [PATCH 7/7] Prune container records without a retention rule and bound them Manifests are now recorded on every install, but were only pruned by the retention sweep. An hourly prune of its own now runs whatever the retention config. It drops tags the upstream has not confirmed for 90 days, which also bounds the manifests kept for tags, and deletes manifests by name in batches rather than with one NOT IN list. A tag is only written when the upstream confirmed it later than stored, so a fresh cache hit writes nothing. One platform's manifest of the index a tag points at keeps the index's digest and only moves the time. The tag list heading no longer shows a count capped at 100. --- docs/architecture.md | 4 +- docs/configuration.md | 2 +- internal/database/image_manifests.go | 53 ++++++++-- internal/database/image_tags.go | 33 +++++++ internal/database/image_tags_test.go | 99 +++++++++++++++++++ internal/handler/container_image_tags.go | 26 ++++- internal/handler/container_image_tags_test.go | 46 +++++++++ internal/server/image_records_prune.go | 45 +++++++++ internal/server/image_records_prune_test.go | 66 +++++++++++++ internal/server/image_tags_ui_test.go | 4 +- internal/server/retention.go | 11 ++- .../server/retention_keep_latest_oci_test.go | 10 +- internal/server/server.go | 1 + .../server/templates/pages/package_show.html | 4 +- 14 files changed, 379 insertions(+), 25 deletions(-) create mode 100644 internal/server/image_records_prune.go create mode 100644 internal/server/image_records_prune_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 0642613a..d8b8d2e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -226,9 +226,9 @@ image_tags ( `metadata_dates` holds the `Last-Modified` served with rewritten metadata, such as npm packuments, Composer documents, PyPI simple pages and Helm indexes. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have. -`image_manifests` holds which manifests and blobs each served container manifest references. The retention sweep reads it to keep the blobs of the newest images of a repository, and the web UI to name the tags whose image contains a blob. The sweep drops the rows of images that are no longer among the newest and that no tag points at. +`image_manifests` holds which manifests and blobs each served container manifest references. The retention sweep reads it to keep the blobs of the newest images of a repository, and the web UI to name the tags whose image contains a blob. An hourly prune, which runs with or without a retention rule, drops the rows of images that are no longer among the newest and that no tag points at. -`image_tags` holds the tags of a container image repository, each with the digest it pointed at when the upstream last confirmed it, for the web UI. +`image_tags` holds the tags of a container image repository, each with the digest it pointed at when the upstream last confirmed it, for the web UI. The same prune drops tags the upstream has not confirmed for 90 days. A tag served as one platform's manifest of the index it points at, to a client that accepts no index, keeps the index's digest. On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`. diff --git a/docs/configuration.md b/docs/configuration.md index ddd5ad8b..4f0d61fc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -133,7 +133,7 @@ For `oci` there are no versions to order, so `keep_latest` counts images instead To know which blobs an image needs, the proxy records what each manifest it serves references, in the `image_manifests` table. This has a few consequences: -- An image only counts once its tag was pulled or mirrored, and its blobs are known once its platform manifest was pulled too. Until then they expire by their rule as before. While `keep_latest` is off, the sweep drops the records of images that no tag points at any more, so those don't count when it is turned on. +- An image only counts once its tag was pulled or mirrored, and its blobs are known once its platform manifest was pulled too. Until then they expire by their rule as before. The records are pruned hourly, also without a retention rule: while `keep_latest` is off, those of images that no tag points at any more go, so they don't count when it is turned on, and so do tags not pulled for 90 days. - An image pulled only by digest is not counted and not kept. - A multi-platform tag counts as one image, also when a client that accepts no image index was served one platform's manifest for it. - Homebrew bottles and Helm charts pulled over OCI are images here: the newest versions of a bottle or chart are those whose tags were last asked for. diff --git a/internal/database/image_manifests.go b/internal/database/image_manifests.go index cdd75ef2..0db9e8e0 100644 --- a/internal/database/image_manifests.go +++ b/internal/database/image_manifests.go @@ -21,6 +21,10 @@ const ( imageWalkBatch = 500 ) +// imagePruneBatch is how many manifests one prune query deletes, well below +// the bind limits of SQLite and PostgreSQL. A variable so tests can lower it. +var imagePruneBatch = 500 + // ImageManifest is what one container image manifest of a repository // references. type ImageManifest struct { @@ -121,16 +125,53 @@ func (db *DB) PruneImageManifests(packagePURL string, n int, before time.Time) e if err != nil { return err } - query := `DELETE FROM image_manifests WHERE package_purl = ? AND recorded_at < ?` - args := []any{packagePURL, before.UnixMilli()} - if len(manifests) > 0 { - query, args, err = sqlx.In(query+` AND digest NOT IN (?)`, packagePURL, before.UnixMilli(), manifests) + keep := make(map[string]struct{}, len(manifests)) + for _, digest := range manifests { + keep[digest] = struct{}{} + } + + var old []string + err = db.Select(&old, db.Rebind(` + SELECT digest FROM image_manifests WHERE package_purl = ? AND recorded_at < ? + `), packagePURL, before.UnixMilli()) + if err != nil { + return err + } + drop := old[:0] + for _, digest := range old { + if _, ok := keep[digest]; !ok { + drop = append(drop, digest) + } + } + // Deleted by name in batches: a list of the kept ones can outgrow what + // one query may bind. + for start := 0; start < len(drop); start += imagePruneBatch { + batch := drop[start:min(start+imagePruneBatch, len(drop))] + query, args, err := sqlx.In(`DELETE FROM image_manifests WHERE package_purl = ? AND digest IN (?)`, packagePURL, batch) if err != nil { return fmt.Errorf("building prune query: %w", err) } + if _, err := db.Exec(db.Rebind(query), args...); err != nil { + return err + } } - _, err = db.Exec(db.Rebind(query), args...) - return err + return nil +} + +// ImageManifestRefs returns the digests a recorded manifest of a repository +// references, or none for one that is not recorded. +func (db *DB) ImageManifestRefs(packagePURL, digest string) ([]string, error) { + var row imageManifestRow + err := db.Get(&row, db.Rebind(` + SELECT digest, refs FROM image_manifests WHERE package_purl = ? AND digest = ? + `), packagePURL, digest) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return row.refs(), nil } // walkNewestImages returns every digest the n newest images of a repository diff --git a/internal/database/image_tags.go b/internal/database/image_tags.go index 47349c1b..e0d55d3e 100644 --- a/internal/database/image_tags.go +++ b/internal/database/image_tags.go @@ -1,6 +1,8 @@ package database import ( + "database/sql" + "errors" "time" ) @@ -30,6 +32,37 @@ func (db *DB) RecordImageTag(packagePURL, tag, digest string, seenAt time.Time) return err } +// ImageTag returns a tag of a repository as stored, and whether there is one. +func (db *DB) ImageTag(packagePURL, tag string) (ImageTag, bool, error) { + var row imageTagRow + err := db.Get(&row, db.Rebind(` + SELECT tag, digest, seen_at FROM image_tags WHERE package_purl = ? AND tag = ? + `), packagePURL, tag) + if errors.Is(err, sql.ErrNoRows) { + return ImageTag{}, false, nil + } + if err != nil { + return ImageTag{}, false, err + } + return ImageTag{Tag: row.Tag, Digest: row.Digest, SeenAt: time.UnixMilli(row.SeenAt)}, true, nil +} + +// TouchImageTag moves the time a tag was last seen forward and leaves its +// digest. +func (db *DB) TouchImageTag(packagePURL, tag string, seenAt time.Time) error { + _, err := db.Exec(db.Rebind(` + UPDATE image_tags SET seen_at = ? WHERE package_purl = ? AND tag = ? AND seen_at < ? + `), seenAt.UnixMilli(), packagePURL, tag, seenAt.UnixMilli()) + return err +} + +// PruneImageTags removes the tags the upstream has not confirmed since the +// given time, in every repository. +func (db *DB) PruneImageTags(before time.Time) error { + _, err := db.Exec(db.Rebind(`DELETE FROM image_tags WHERE seen_at < ?`), before.UnixMilli()) + return err +} + // ImageTags returns up to limit tags of a repository, the most recently seen // first. func (db *DB) ImageTags(packagePURL string, limit int) ([]ImageTag, error) { diff --git a/internal/database/image_tags_test.go b/internal/database/image_tags_test.go index 7b3d2a00..aa2948b6 100644 --- a/internal/database/image_tags_test.go +++ b/internal/database/image_tags_test.go @@ -188,3 +188,102 @@ func TestPruneImageManifestsKeepsWhatTagsPointAt(t *testing.T) { } }) } + +// Tags the upstream has not confirmed since the given time are dropped, in +// every repository. +func TestPruneImageTags(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + recordTag(t, db, testTagRepo, "old", "sha256:a", now.Add(-100*24*time.Hour)) + recordTag(t, db, testTagRepo, "latest", "sha256:b", now) + recordTag(t, db, "pkg:oci/library/alpine", "3.19", "sha256:c", now.Add(-100*24*time.Hour)) + + if err := db.PruneImageTags(now.Add(-90 * 24 * time.Hour)); err != nil { + t.Fatalf("PruneImageTags: %v", err) + } + if tags, _ := db.ImageTags(testTagRepo, 10); len(tags) != 1 || tags[0].Tag != "latest" { + t.Errorf("tags = %+v, want latest only", tags) + } + if tags, _ := db.ImageTags("pkg:oci/library/alpine", 10); len(tags) != 0 { + t.Errorf("alpine tags = %+v, want none", tags) + } + }) +} + +// A tag's stored digest and time are read back, and a missing tag says so. +func TestImageTag(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().Truncate(time.Millisecond) + recordTag(t, db, testTagRepo, "latest", "sha256:a", now) + tag, ok, err := db.ImageTag(testTagRepo, "latest") + if err != nil || !ok || tag.Digest != "sha256:a" || !tag.SeenAt.Equal(now) { + t.Errorf("ImageTag(latest) = %+v, %v, %v; want sha256:a at %v", tag, ok, err, now) + } + if _, ok, err := db.ImageTag(testTagRepo, "missing"); err != nil || ok { + t.Errorf("ImageTag(missing) ok = %v, %v; want false", ok, err) + } + if _, ok, err := db.ImageTag("pkg:oci/library/alpine", "latest"); err != nil || ok { + t.Errorf("ImageTag in another repository ok = %v, %v; want false", ok, err) + } + }) +} + +// TouchImageTag moves a tag's time forward and leaves its digest. +func TestTouchImageTag(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().Truncate(time.Millisecond) + recordTag(t, db, testTagRepo, "latest", "sha256:a", now.Add(-time.Hour)) + if err := db.TouchImageTag(testTagRepo, "latest", now); err != nil { + t.Fatal(err) + } + if err := db.TouchImageTag(testTagRepo, "latest", now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + tag, _, err := db.ImageTag(testTagRepo, "latest") + if err != nil || tag.Digest != "sha256:a" || !tag.SeenAt.Equal(now) { + t.Errorf("tag = %+v, %v; want sha256:a at %v", tag, err, now) + } + }) +} + +// The references a manifest records are read back; an unknown one has none. +func TestImageManifestRefs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + recordReferences(t, db, testTagRepo, "sha256:index", "sha256:amd64", "sha256:arm64") + if refs, err := db.ImageManifestRefs(testTagRepo, "sha256:index"); err != nil || !slices.Equal(refs, []string{"sha256:amd64", "sha256:arm64"}) { + t.Errorf("refs = %v, %v", refs, err) + } + if refs, err := db.ImageManifestRefs(testTagRepo, "sha256:unknown"); err != nil || refs != nil { + t.Errorf("unknown refs = %v, %v; want none", refs, err) + } + if refs, err := db.ImageManifestRefs("pkg:oci/library/alpine", "sha256:index"); err != nil || refs != nil { + t.Errorf("refs in another repository = %v, %v; want none", refs, err) + } + }) +} + +// The delete runs in batches, so a repository with more manifests to keep +// than a query can bind is pruned all the same. +func TestPruneImageManifestsInBatches(t *testing.T) { + defer func(n int) { imagePruneBatch = n }(imagePruneBatch) + imagePruneBatch = 2 + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + for i := range 5 { + kept := fmt.Sprintf("sha256:kept-%d", i) + recordReferences(t, db, testTagRepo, kept, "sha256:layer") + recordTag(t, db, testTagRepo, fmt.Sprintf("v%d", i), kept, now) + recordReferences(t, db, testTagRepo, fmt.Sprintf("sha256:gone-%d", i), "sha256:layer") + } + if err := db.PruneImageManifests(testTagRepo, 0, now.Add(time.Minute)); err != nil { + t.Fatalf("PruneImageManifests: %v", err) + } + for i := range 5 { + for digest, want := range map[string]bool{fmt.Sprintf("sha256:kept-%d", i): true, fmt.Sprintf("sha256:gone-%d", i): false} { + if _, ok, _, err := db.ImageManifestState(testTagRepo, digest); err != nil || ok != want { + t.Errorf("%s recorded = %v, %v; want %v", digest, ok, err, want) + } + } + } + }) +} diff --git a/internal/handler/container_image_tags.go b/internal/handler/container_image_tags.go index b30252ab..88277825 100644 --- a/internal/handler/container_image_tags.go +++ b/internal/handler/container_image_tags.go @@ -1,6 +1,7 @@ package handler import ( + "slices" "time" ) @@ -17,7 +18,30 @@ func (h *ContainerHandler) recordImageTag(cacheName, reference string, manifest seen = time.Now() } packagePURL := canonicalPackagePURL("oci", cacheName) - if err := h.proxy.DB.RecordImageTag(packagePURL, reference, manifest.contentDigest, seen); err != nil { + stored, known, err := h.proxy.DB.ImageTag(packagePURL, reference) + if err != nil { + h.proxy.Logger.Warn("failed to read image tag", "package_purl", packagePURL, "tag", reference, "error", err) + return + } + // Times are stored in milliseconds. A fresh cache hit repeats what is + // stored, and the one writer SQLite has should not hold the pull up. + if known && seen.UnixMilli() <= stored.SeenAt.UnixMilli() { + return + } + if known && stored.Digest != manifest.contentDigest && h.referencedBy(packagePURL, stored.Digest, manifest.contentDigest) { + // One platform's manifest of the index the tag points at, served to + // a client that accepts no index: the same image seen another way. + err = h.proxy.DB.TouchImageTag(packagePURL, reference, seen) + } else { + err = h.proxy.DB.RecordImageTag(packagePURL, reference, manifest.contentDigest, seen) + } + if err != nil { h.proxy.Logger.Warn("failed to record image tag", "package_purl", packagePURL, "tag", reference, "error", err) } } + +// referencedBy reports whether the recorded manifest parent references digest. +func (h *ContainerHandler) referencedBy(packagePURL, parent, digest string) bool { + refs, err := h.proxy.DB.ImageManifestRefs(packagePURL, parent) + return err == nil && slices.Contains(refs, digest) +} diff --git a/internal/handler/container_image_tags_test.go b/internal/handler/container_image_tags_test.go index 843eb1ae..1f381cea 100644 --- a/internal/handler/container_image_tags_test.go +++ b/internal/handler/container_image_tags_test.go @@ -155,3 +155,49 @@ func TestMirrorImageRecordsWhatImagesContain(t *testing.T) { t.Errorf("tags containing the amd64 layer after a mirror run = %v, %v; want multi", got, err) } } + +// A fresh cache hit repeats what is stored, so it writes nothing: SQLite has +// one writer, and every cached pull would queue behind it. +func TestContainerCachedPullWritesNoTag(t *testing.T) { + h, _, _ := imageTagsHandler(t) + h.proxy.MetadataTTL = time.Hour + db := h.proxy.DB + if _, err := db.Exec(`CREATE TABLE tag_writes (n INTEGER)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TRIGGER count_tag_writes BEFORE INSERT ON image_tags BEGIN INSERT INTO tag_writes VALUES (1); END`); err != nil { + t.Fatal(err) + } + for range 3 { + requestTagManifest(t, h, http.MethodGet, "multi") + requestTagManifest(t, h, http.MethodHead, "multi") + } + var writes int + if err := db.Get(&writes, `SELECT COUNT(*) FROM tag_writes`); err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Errorf("%d tag writes, want 1", writes) + } +} + +// A client that accepts no index is served one platform's manifest for a +// tag. That is the image the tag points at seen another way, so the tag keeps +// the index's digest and only its time moves. A digest the stored one does +// not reference is a new image. +func TestContainerPlatformManifestKeepsTheTagsIndex(t *testing.T) { + h, _, image := imageTagsHandler(t) + requestTagManifest(t, h, http.MethodGet, "multi") + later := imageTags(t, h)["multi"].SeenAt.Add(time.Minute) + + h.recordImageTag(imageTagsRepo, "multi", &cachedContainerManifest{contentDigest: image.amd64Digest, fetchedAt: later}, false) + if got := imageTags(t, h)["multi"]; got.Digest != image.indexDigest || !got.SeenAt.Equal(later) { + t.Errorf("tag after a platform manifest = %+v, want %s at %v", got, image.indexDigest, later) + } + + moved := later.Add(time.Minute) + h.recordImageTag(imageTagsRepo, "multi", &cachedContainerManifest{contentDigest: image.singleDigest, fetchedAt: moved}, false) + if got := imageTags(t, h)["multi"]; got.Digest != image.singleDigest || !got.SeenAt.Equal(moved) { + t.Errorf("tag after another image = %+v, want %s at %v", got, image.singleDigest, moved) + } +} diff --git a/internal/server/image_records_prune.go b/internal/server/image_records_prune.go new file mode 100644 index 00000000..16be3bc0 --- /dev/null +++ b/internal/server/image_records_prune.go @@ -0,0 +1,45 @@ +package server + +import ( + "context" + "time" + + "github.com/git-pkgs/proxy/internal/retention" +) + +const ( + // imageRecordPruneInterval is how often container tag and manifest + // records are pruned. + imageRecordPruneInterval = time.Hour + // imageTagMaxAge is how long a tag the upstream has not confirmed is + // kept, and with it what its image references. + imageTagMaxAge = 90 * 24 * time.Hour +) + +// startImageRecordPruning prunes the container tag and manifest records once +// at startup and then every imageRecordPruneInterval. They are written on +// every install, so this runs whether or not a retention rule is set. +func (s *Server) startImageRecordPruning(ctx context.Context) { + keep := 0 + if rules, err := s.cfg.RetentionRules(retention.Default); err == nil { + keep = rules.KeepLatest + } + prune := func() { + s.runExclusive(ctx, "image-record-pruning", func() { + pruneImageRecords(s.db, s.logger, keep, time.Now()) + }) + } + s.background.Go(func() { + prune() + ticker := time.NewTicker(imageRecordPruneInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + prune() + } + } + }) +} diff --git a/internal/server/image_records_prune_test.go b/internal/server/image_records_prune_test.go new file mode 100644 index 00000000..24703c7a --- /dev/null +++ b/internal/server/image_records_prune_test.go @@ -0,0 +1,66 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/retention" +) + +// Container records are pruned without any retention rule: they are written +// on every install, for the web UI. +func TestImageRecordPruningRunsWithoutRetention(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + if rules, err := ts.server.cfg.RetentionRules(retention.Default); err == nil && rules.MinPositive() != 0 { + t.Fatalf("test config has retention rules: %+v", rules) + } + const repo = "pkg:oci/team/app" + db := ts.db + now := time.Now() + if err := db.RecordImageManifest(database.ImageManifest{PackagePURL: repo, Digest: "sha256:tagged", Refs: []string{"sha256:layer"}}); err != nil { + t.Fatal(err) + } + if err := db.RecordImageManifest(database.ImageManifest{PackagePURL: repo, Digest: "sha256:untagged", Refs: []string{"sha256:layer"}}); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(db.Rebind(`UPDATE image_manifests SET recorded_at = ?`), now.Add(-2*imageRecordGrace).UnixMilli()); err != nil { + t.Fatal(err) + } + if err := db.RecordImageTag(repo, "latest", "sha256:tagged", now); err != nil { + t.Fatal(err) + } + if err := db.RecordImageTag(repo, "stale", "sha256:untagged", now.Add(-imageTagMaxAge-time.Hour)); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + ts.server.startImageRecordPruning(ctx) + defer func() { + cancel() + ts.server.background.Wait() + }() + + deadline := time.Now().Add(5 * time.Second) + for { + _, recorded, _, err := db.ImageManifestState(repo, "sha256:untagged") + if err != nil { + t.Fatal(err) + } + if !recorded { + break + } + if time.Now().After(deadline) { + t.Fatal("the manifest of a tag not confirmed for too long was not pruned") + } + time.Sleep(10 * time.Millisecond) + } + if _, recorded, _, err := db.ImageManifestState(repo, "sha256:tagged"); err != nil || !recorded { + t.Errorf("the manifest latest points at was pruned: %v, %v", recorded, err) + } + if tags, err := db.ImageTags(repo, 10); err != nil || len(tags) != 1 || tags[0].Tag != "latest" { + t.Errorf("tags = %+v, %v; want latest only", tags, err) + } +} diff --git a/internal/server/image_tags_ui_test.go b/internal/server/image_tags_ui_test.go index 5301b352..9fdde3c7 100644 --- a/internal/server/image_tags_ui_test.go +++ b/internal/server/image_tags_ui_test.go @@ -47,8 +47,8 @@ func TestPackageShowPageListsImageTags(t *testing.T) { t.Fatalf("status %d", w.Code) } body := w.Body.String() - if !strings.Contains(body, "Tags (3)") { - t.Error("page has no Tags (3) section") + if !strings.Contains(body, ">Tags<") { + t.Error("page has no Tags section") } latest, current, older := strings.Index(body, ">latest<"), strings.Index(body, ">2026.11.0<"), strings.Index(body, ">2026.10.0<") if latest < 0 || current < 0 || older < 0 || latest > current || current > older { diff --git a/internal/server/retention.go b/internal/server/retention.go index 25f0ad4c..179a05a5 100644 --- a/internal/server/retention.go +++ b/internal/server/retention.go @@ -101,8 +101,6 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, return } - pruneImageRecords(db, logger, rules.KeepLatest, now) - // Without the buffered hits written, an artifact being served right now // can look expired, so skip this sweep rather than clear it. if err := db.FlushHits(); err != nil { @@ -159,10 +157,13 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, } } -// pruneImageRecords drops the container manifest records that neither the -// keep newest images of their repository nor the images its tags point at -// reference. +// pruneImageRecords drops the container tags the upstream has not confirmed +// for imageTagMaxAge, then the manifest records that neither the keep newest +// images of their repository nor the images its tags point at reference. func pruneImageRecords(db *database.DB, logger *slog.Logger, keep int, now time.Time) { + if err := db.PruneImageTags(now.Add(-imageTagMaxAge)); err != nil { + logger.Warn("retention: failed to prune image tags", "error", err) + } repositories, err := db.ImageManifestRepositories() if err != nil { logger.Warn("retention: failed to list image manifest records", "error", err) diff --git a/internal/server/retention_keep_latest_oci_test.go b/internal/server/retention_keep_latest_oci_test.go index 5f5c29dd..10e2ea5b 100644 --- a/internal/server/retention_keep_latest_oci_test.go +++ b/internal/server/retention_keep_latest_oci_test.go @@ -222,9 +222,9 @@ func TestSweepRetentionKeepsBlobsOfNewestImages(t *testing.T) { } } -// A sweep drops the records of images that are no longer among the newest +// The prune drops the records of images that are no longer among the newest // and that no tag points at, so the table holds a few images per repository. -func TestSweepRetentionPrunesImageRecords(t *testing.T) { +func TestPruneImageRecords(t *testing.T) { registry := newKeepImagesRegistry(t) srv, db := newKeepImagesProxy(t, registry) images := []keepImage{registry.addImage(t, "first"), registry.addImage(t, "second")} @@ -243,8 +243,7 @@ func TestSweepRetentionPrunesImageRecords(t *testing.T) { return ok } - rules := retention.Rules{Ecosystems: map[string]time.Duration{"oci": retentionDay}, KeepLatest: 1} - sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + pruneImageRecords(db, discardLogger(), 1, now) for _, digest := range []string{images[0].index, images[0].manifest} { if recorded(digest) { t.Errorf("manifest %s of the older image is still recorded", digest) @@ -258,8 +257,7 @@ func TestSweepRetentionPrunesImageRecords(t *testing.T) { // Without keep_latest the records of the image the tag points at stay, // for the web UI. - rules.KeepLatest = 0 - sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + pruneImageRecords(db, discardLogger(), 0, now) if !recorded(images[1].index) || !recorded(images[1].manifest) { t.Error("the records of the image the tag points at are gone after keep_latest was turned off") } diff --git a/internal/server/server.go b/internal/server/server.go index 7537a10a..ffcb65ad 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -464,6 +464,7 @@ func (s *Server) serve(listener net.Listener) error { s.background.Go(func() { s.startEvictionLoop(bgCtx) }) retentionResolvers := s.retentionResolvers(proxy) s.background.Go(func() { s.startRetentionLoop(bgCtx, retentionResolvers) }) + s.startImageRecordPruning(bgCtx) s.background.Go(func() { s.startAPKSnapshotPruning(bgCtx) }) s.background.Go(func() { s.startReclaimLoop(bgCtx) }) diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index 95a09011..815016df 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -83,8 +83,8 @@

Used by

{{if .Tags}}
-

Tags ({{len .Tags}})

-

The digest each tag pointed at when the upstream last confirmed it

+

Tags

+

The most recently seen tags, up to 100, with the digest each pointed at when the upstream last confirmed it

{{range .Tags}}