List a container image's tags on its page

README.md

@@ -559,6 +559,12 @@ docker pull localhost:8080/library/nginx:latest
 The registry is pull-only. BuildKit cache exports (`--cache-to type=registry`)
 and other pushes need a separate registry.
 
+An image's page in the web UI (`/ui/package/oci/{name}`) lists its tags, the
+most recently seen first, with the digest each one pointed at when the
+upstream last confirmed it, through a pull or a mirror run. Its versions are
+the cached blobs, by digest, and a blob's page names the tags whose cached
+image contains it.
+
 SBOMs, signatures and attestations attached to an image are found through the
 OCI 1.1 referrers API (`GET /v2/{name}/referrers/{digest}`). `oras discover`
 and `notation verify` use it, and so does cosign for its newer signature

docs/architecture.md

@@ -214,11 +214,21 @@ image_manifests (
     recorded_at   INTEGER NOT NULL,   -- Unix ms; BIGINT on Postgres
     PRIMARY KEY (package_purl, digest)
 )
+
+image_tags (
+    package_purl  TEXT NOT NULL,      -- the repository, as its blobs are cached
+    tag           TEXT NOT NULL,
+    digest        TEXT NOT NULL,      -- the manifest the tag pointed at
+    seen_at       INTEGER NOT NULL,   -- Unix ms the upstream last confirmed it; BIGINT on Postgres
+    PRIMARY KEY (package_purl, tag)
+)
 ```
 
 `metadata_dates` holds the `Last-Modified` served with rewritten metadata, such as npm packuments, Composer documents, PyPI simple pages and Helm indexes. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have.
 
-`image_manifests` is filled only with `storage.retention.keep_latest` set and a retention rule for `oci`. It holds which manifests and blobs each served container manifest references, so the retention sweep can keep the blobs of the newest images of a repository. The sweep also drops the rows of images that are no longer among the newest.
+`image_manifests` holds which manifests and blobs each served container manifest references. The retention sweep reads it to keep the blobs of the newest images of a repository, and the web UI to name the tags whose image contains a blob. An hourly prune, which runs with or without a retention rule, drops the rows of images that are no longer among the newest and that no tag points at.
+
+`image_tags` holds the tags of a container image repository, each with the digest it pointed at when the upstream last confirmed it, for the web UI. The same prune drops tags the upstream has not confirmed for 90 days. A tag served as one platform's manifest of the index it points at, to a client that accepts no index, keeps the index's digest.
 
 On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`.
 

docs/configuration.md

@@ -131,9 +131,9 @@ With this, the files of the two highest cached versions of a package are never e
 
 For `oci` there are no versions to order, so `keep_latest` counts images instead: it keeps the blobs of the images of a repository that a tag most recently pointed at. With `keep_latest: 2` and one tag such as `beta` in use, that is the image `beta` points at now and the one it pointed at before. The count is per repository, not per tag, so with several tags in use set it at least that high. A blob that a kept image shares with an older one stays.
 
-To know which blobs an image needs, the proxy records what each manifest it serves references, in the `image_manifests` table, while `keep_latest` is set and a retention rule applies to `oci`. This has a few consequences:
+To know which blobs an image needs, the proxy records what each manifest it serves references, in the `image_manifests` table. This has a few consequences:
 
-- An image only counts once its tag was pulled or mirrored with `keep_latest` set, and its blobs are known once its platform manifest was pulled too. Until then they expire by their rule as before.
+- An image only counts once its tag was pulled or mirrored, and its blobs are known once its platform manifest was pulled too. Until then they expire by their rule as before. The records are pruned hourly, also without a retention rule: while `keep_latest` is off, those of images that no tag points at any more go, so they don't count when it is turned on, and so do tags not pulled for 90 days.
 - An image pulled only by digest is not counted and not kept.
 - A multi-platform tag counts as one image, also when a client that accepts no image index was served one platform's manifest for it.
 - Homebrew bottles and Helm charts pulled over OCI are images here: the newest versions of a bottle or chart are those whose tags were last asked for.

internal/database/database_test.go

@@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB {
 	// Drop and recreate every table CreateSchema creates for clean test state;
 	// leftover migration records make the next CreateSchema fail on the
 	// migrations primary key.
-	tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "image_manifests", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "projects", "project_components", "migrations", "schema_info"}
+	tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "image_manifests", "image_tags", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "projects", "project_components", "migrations", "schema_info"}
 	for _, table := range tables {
 		_, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE")
 	}

internal/database/image_manifests.go

@@ -21,6 +21,10 @@ const (
 	imageWalkBatch = 500
 )
 
+// imagePruneBatch is how many manifests one prune query deletes, well below
+// the bind limits of SQLite and PostgreSQL. A variable so tests can lower it.
+var imagePruneBatch = 500
+
 // ImageManifest is what one container image manifest of a repository
 // references.
 type ImageManifest struct {
@@ -99,45 +103,100 @@ func (db *DB) ImageManifestRepositories() ([]string, error) {
 	return repositories, err
 }
 
-// PruneImageManifests removes the manifests of a repository that its n
-// newest images do not reference and that were recorded before the given
-// time. The time leaves a manifest recorded shortly before the one that
-// references it in place.
+// PruneImageManifests removes the manifests of a repository that neither
+// its n newest images nor the images its tags point at reference, and that
+// were recorded before the given time. The time leaves a manifest recorded
+// shortly before the one that references it in place. What the tags point
+// at stays for ImageTagsContaining.
 func (db *DB) PruneImageManifests(packagePURL string, n int, before time.Time) error {
-	_, manifests, err := db.walkNewestImages(packagePURL, n)
+	roots, err := db.newestImages(packagePURL, n)
+	if err != nil {
+		return err
+	}
+	var tagged []imageManifestRow
+	err = db.Select(&tagged, db.Rebind(`
+		SELECT digest, refs FROM image_manifests WHERE package_purl = ?
+			AND digest IN (SELECT digest FROM image_tags WHERE package_purl = ?)
+	`), packagePURL, packagePURL)
+	if err != nil {
+		return err
+	}
+	_, manifests, err := db.walkImages(packagePURL, append(roots, tagged...))
 	if err != nil {
 		return err
 	}
-	query := `DELETE FROM image_manifests WHERE package_purl = ? AND recorded_at < ?`
-	args := []any{packagePURL, before.UnixMilli()}
-	if len(manifests) > 0 {
-		query, args, err = sqlx.In(query+` AND digest NOT IN (?)`, packagePURL, before.UnixMilli(), manifests)
+	keep := make(map[string]struct{}, len(manifests))
+	for _, digest := range manifests {
+		keep[digest] = struct{}{}
+	}
+
+	var old []string
+	err = db.Select(&old, db.Rebind(`
+		SELECT digest FROM image_manifests WHERE package_purl = ? AND recorded_at < ?
+	`), packagePURL, before.UnixMilli())
+	if err != nil {
+		return err
+	}
+	drop := old[:0]
+	for _, digest := range old {
+		if _, ok := keep[digest]; !ok {
+			drop = append(drop, digest)
+		}
+	}
+	// Deleted by name in batches: a list of the kept ones can outgrow what
+	// one query may bind.
+	for start := 0; start < len(drop); start += imagePruneBatch {
+		batch := drop[start:min(start+imagePruneBatch, len(drop))]
+		query, args, err := sqlx.In(`DELETE FROM image_manifests WHERE package_purl = ? AND digest IN (?)`, packagePURL, batch)
 		if err != nil {
 			return fmt.Errorf("building prune query: %w", err)
 		}
+		if _, err := db.Exec(db.Rebind(query), args...); err != nil {
+			return err
+		}
 	}
-	_, err = db.Exec(db.Rebind(query), args...)
-	return err
+	return nil
+}
+
+// ImageManifestRefs returns the digests a recorded manifest of a repository
+// references, or none for one that is not recorded.
+func (db *DB) ImageManifestRefs(packagePURL, digest string) ([]string, error) {
+	var row imageManifestRow
+	err := db.Get(&row, db.Rebind(`
+		SELECT digest, refs FROM image_manifests WHERE package_purl = ? AND digest = ?
+	`), packagePURL, digest)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return row.refs(), nil
 }
 
 // walkNewestImages returns every digest the n newest images of a repository
 // reference, themselves included, and those among them that are recorded
 // manifests.
 func (db *DB) walkNewestImages(packagePURL string, n int) (map[string]struct{}, []string, error) {
-	digests := map[string]struct{}{}
-	if n <= 0 {
-		return digests, nil, nil
-	}
 	rows, err := db.newestImages(packagePURL, n)
 	if err != nil {
 		return nil, nil, err
 	}
+	return db.walkImages(packagePURL, rows)
+}
 
-	var manifests []string
-	for _, row := range rows {
+// walkImages returns every digest the given manifests of a repository
+// reference, themselves included, and those among them that are recorded
+// manifests.
+func (db *DB) walkImages(packagePURL string, roots []imageManifestRow) (map[string]struct{}, []string, error) {
+	digests := map[string]struct{}{}
+	for _, row := range roots {
 		digests[row.Digest] = struct{}{}
 	}
-	for depth := 0; len(rows) > 0; depth++ {
+
+	var manifests []string
+	var err error
+	for depth, rows := 0, roots; len(rows) > 0; depth++ {
 		var next []string
 		for _, row := range rows {
 			manifests = append(manifests, row.Digest)
@@ -165,6 +224,9 @@ func (db *DB) walkNewestImages(packagePURL string, n int) (map[string]struct{},
 // manifest that another tagged one references is that image seen another
 // way: it is no image of its own and lends the other its tag time.
 func (db *DB) newestImages(packagePURL string, n int) ([]imageManifestRow, error) {
+	if n <= 0 {
+		return nil, nil
+	}
 	var rows []imageManifestRow
 	err := db.Select(&rows, db.Rebind(`
 		SELECT digest, refs, tagged_at FROM image_manifests

internal/database/image_tags.go

@@ -0,0 +1,133 @@
+package database
+
+import (
+	"database/sql"
+	"errors"
+	"time"
+)
+
+// ImageTag is a tag of a container image repository as the proxy last saw
+// it upstream.
+type ImageTag struct {
+	Tag    string `db:"tag"`
+	Digest string `db:"digest"`
+	// SeenAt is when the upstream last said the tag points at Digest.
+	SeenAt time.Time `db:"-"`
+}
+
+type imageTagRow struct {
+	Tag    string `db:"tag"`
+	Digest string `db:"digest"`
+	SeenAt int64  `db:"seen_at"`
+}
+
+// RecordImageTag stores that a tag of a repository pointed at digest at the
+// given time. An older sighting than the stored one changes nothing.
+func (db *DB) RecordImageTag(packagePURL, tag, digest string, seenAt time.Time) error {
+	_, err := db.Exec(db.Rebind(`
+		INSERT INTO image_tags (package_purl, tag, digest, seen_at) VALUES (?, ?, ?, ?)
+		ON CONFLICT (package_purl, tag) DO UPDATE SET digest = excluded.digest, seen_at = excluded.seen_at
+		WHERE image_tags.seen_at < excluded.seen_at
+	`), packagePURL, tag, digest, seenAt.UnixMilli())
+	return err
+}
+
+// ImageTag returns a tag of a repository as stored, and whether there is one.
+func (db *DB) ImageTag(packagePURL, tag string) (ImageTag, bool, error) {
+	var row imageTagRow
+	err := db.Get(&row, db.Rebind(`
+		SELECT tag, digest, seen_at FROM image_tags WHERE package_purl = ? AND tag = ?
+	`), packagePURL, tag)
+	if errors.Is(err, sql.ErrNoRows) {
+		return ImageTag{}, false, nil
+	}
+	if err != nil {
+		return ImageTag{}, false, err
+	}
+	return ImageTag{Tag: row.Tag, Digest: row.Digest, SeenAt: time.UnixMilli(row.SeenAt)}, true, nil
+}
+
+// TouchImageTag moves the time a tag was last seen forward and leaves its
+// digest.
+func (db *DB) TouchImageTag(packagePURL, tag string, seenAt time.Time) error {
+	_, err := db.Exec(db.Rebind(`
+		UPDATE image_tags SET seen_at = ? WHERE package_purl = ? AND tag = ? AND seen_at < ?
+	`), seenAt.UnixMilli(), packagePURL, tag, seenAt.UnixMilli())
+	return err
+}
+
+// PruneImageTags removes the tags the upstream has not confirmed since the
+// given time, in every repository.
+func (db *DB) PruneImageTags(before time.Time) error {
+	_, err := db.Exec(db.Rebind(`DELETE FROM image_tags WHERE seen_at < ?`), before.UnixMilli())
+	return err
+}
+
+// ImageTags returns up to limit tags of a repository, the most recently seen
+// first.
+func (db *DB) ImageTags(packagePURL string, limit int) ([]ImageTag, error) {
+	var rows []imageTagRow
+	err := db.Select(&rows, db.Rebind(`
+		SELECT tag, digest, seen_at FROM image_tags WHERE package_purl = ?
+		ORDER BY seen_at DESC, tag LIMIT ?
+	`), packagePURL, limit)
+	if err != nil {
+		return nil, err
+	}
+	tags := make([]ImageTag, len(rows))
+	for i, row := range rows {
+		tags[i] = ImageTag{Tag: row.Tag, Digest: row.Digest, SeenAt: time.UnixMilli(row.SeenAt)}
+	}
+	return tags, nil
+}
+
+// ImageTagsContaining returns up to limit tags of a repository whose image
+// contains digest, the most recently seen first: tags pointing at digest,
+// at a manifest that references it, or at an index listing such a manifest.
+// It reads the references recorded in image_manifests.
+func (db *DB) ImageTagsContaining(packagePURL, digest string, limit int) ([]string, error) {
+	var manifests []imageManifestRow
+	err := db.Select(&manifests, db.Rebind(`
+		SELECT digest, refs FROM image_manifests WHERE package_purl = ?
+	`), packagePURL)
+	if err != nil {
+		return nil, err
+	}
+	referencedBy := make(map[string][]string)
+	for _, manifest := range manifests {
+		for _, ref := range manifest.refs() {
+			referencedBy[ref] = append(referencedBy[ref], manifest.Digest)
+		}
+	}
+	containing := map[string]struct{}{digest: {}}
+	for next := []string{digest}; len(next) > 0; {
+		var parents []string
+		for _, child := range next {
+			for _, parent := range referencedBy[child] {
+				if _, seen := containing[parent]; !seen {
+					containing[parent] = struct{}{}
+					parents = append(parents, parent)
+				}
+			}
+		}
+		next = parents
+	}
+
+	var rows []imageTagRow
+	err = db.Select(&rows, db.Rebind(`
+		SELECT tag, digest, seen_at FROM image_tags WHERE package_purl = ? ORDER BY seen_at DESC, tag
+	`), packagePURL)
+	if err != nil {
+		return nil, err
+	}
+	var tags []string
+	for _, row := range rows {
+		if len(tags) == limit {
+			break
+		}
+		if _, ok := containing[row.Digest]; ok {
+			tags = append(tags, row.Tag)
+		}
+	}
+	return tags, nil
+}