Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cspell-words.txt
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ distroless
docstrings
dunder
dupl
embeddedstructfieldcheck
endfor
errchkjson
esac
Expand Down
2 changes: 2 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ nogo(
GO_TARGETS = [
"//cmd/github-metrics:github-metrics_lib",
"//cmd/github-metrics:github-metrics_test",
"//internal/backfill",
"//internal/backfill:backfill_test",
"//internal/budget",
"//internal/budget:budget_test",
"//internal/ghclient",
Expand Down
29 changes: 29 additions & 0 deletions internal/backfill/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright Tony Burns

load("@rules_go//go:def.bzl", "go_library", "go_test")
load("@tbhb_rules//repo:defs.bzl", "repo_files")

repo_files()

go_library(
name = "backfill",
srcs = ["backfill.go"],
importpath = "github.com/tbhb-dev/github-metrics/internal/backfill",
visibility = ["//:__subpackages__"],
deps = [
"//internal/budget",
"//internal/ghclient",
],
)

go_test(
name = "backfill_test",
srcs = ["backfill_test.go"],
visibility = ["//:__pkg__"],
deps = [
":backfill",
"//internal/budget",
"//internal/ghclient",
],
)
105 changes: 105 additions & 0 deletions internal/backfill/backfill.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright Tony Burns

// Package backfill lists a repository's pull requests in creation order.
package backfill

import (
"context"
"encoding/json"
"errors"
"fmt"
"time"

"github.com/tbhb-dev/github-metrics/internal/budget"
"github.com/tbhb-dev/github-metrics/internal/ghclient"
)

// ErrMaxRequests means the request cap isn't positive.
var ErrMaxRequests = errors.New("max requests must be positive")

// Options sets the page request cap for one run.
type Options struct {
MaxRequests int
}

// Record contains only the pull request fields needed for backfill.
type Record struct {
CreatedAt time.Time `json:"created_at"` //nolint:tagliatelle // GitHub field name.
MergedAt time.Time `json:"merged_at"` //nolint:tagliatelle // GitHub field name.
ClosedAt time.Time `json:"closed_at"` //nolint:tagliatelle // GitHub field name.
State string `json:"state"`
HeadSHA string `json:"-"`
BaseRef string `json:"-"`
Number int `json:"number"`
Draft bool `json:"draft"`
}

// Result holds records in page order and the next URL when capped.
type Result struct {
ResumeURL string
Records []Record
Truncated bool
}

type pageRecord struct {
Head struct {
SHA string `json:"sha"`
} `json:"head"`
Base struct {
Ref string `json:"ref"`
} `json:"base"`

Record //nolint:embeddedstructfieldcheck // Keep pointer fields together for field alignment.
}

// List reads all pages, or stops at the cap. The client manages ETags and
// retries. A 304 ends the run without adding records from that page. Reuse
// the client's tag store only when previously read records have been kept.
func List(ctx context.Context, client *ghclient.Client, repo string, opts Options) (Result, error) {
var result Result

if opts.MaxRequests <= 0 {
return result, ErrMaxRequests
}

next := "/repos/" + repo + "/pulls?state=all&sort=created&direction=asc&per_page=100"

for range opts.MaxRequests {
response, err := client.Get(ctx, budget.Backfill, next)
if err != nil {
return result, fmt.Errorf("list pull requests: %w", err)
}

if response.NotModified() {
return result, nil
}

var page []pageRecord

err = json.Unmarshal(response.Body, &page)
if err != nil {
return result, fmt.Errorf("decode pull requests: %w", err)
}

if len(page) == 0 {
return result, nil
}

for _, item := range page {
item.Record.HeadSHA = item.Head.SHA
item.Record.BaseRef = item.Base.Ref
result.Records = append(result.Records, item.Record)
}

next = response.Next
if next == "" {
return result, nil
}
}

result.Truncated = true
result.ResumeURL = next

return result, nil
}
208 changes: 208 additions & 0 deletions internal/backfill/backfill_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright Tony Burns

package backfill_test

import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strconv"
"sync/atomic"
"testing"
"time"

"github.com/tbhb-dev/github-metrics/internal/backfill"
"github.com/tbhb-dev/github-metrics/internal/budget"
"github.com/tbhb-dev/github-metrics/internal/ghclient"
)

type fakeTokens struct{}

func (fakeTokens) Token(context.Context) (string, error) { return "test-token", nil }
func (fakeTokens) Renew(context.Context) (string, error) { return "test-token", nil }

const firstPage = "/repos/o/r/pulls?state=all&sort=created&direction=asc&per_page=100"

func newClient(t *testing.T, handler http.HandlerFunc) (*ghclient.Client, *httptest.Server) {
t.Helper()

server := httptest.NewTLSServer(handler)
t.Cleanup(server.Close)

client, err := ghclient.New(server.URL, server.Client(), budget.New(), fakeTokens{}, ghclient.NewETags())
if err != nil {
t.Fatal(err)
}

return client, server
}

func write(t *testing.T, resp http.ResponseWriter, body string) {
t.Helper()

_, err := resp.Write([]byte(body))
if err != nil {
t.Error(err)
}
}

func TestListThreePages(t *testing.T) {
t.Parallel()

var calls atomic.Int32

client, _ := newClient(t, func(resp http.ResponseWriter, req *http.Request) {
page := calls.Add(1)
want := firstPage

if page > 1 {
want += "&page=" + strconv.Itoa(int(page))
}

if req.URL.RequestURI() != want {
t.Errorf("request = %q, want %q", req.URL.RequestURI(), want)
}

if page < 3 {
resp.Header().Set("Link", "<https://"+req.Host+firstPage+"&page="+
strconv.Itoa(int(page+1))+`>; rel="next"`)
}

write(t, resp, fmt.Sprintf(`[{"number":%d},{"number":%d}]`, page*2, page*2+1))
})

result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 3})
if err != nil || result.Truncated || result.ResumeURL != "" || calls.Load() != 3 {
t.Fatalf("result = %+v, err = %v, calls = %d", result, err, calls.Load())
}

numbers := make([]int, 0, len(result.Records))
for _, record := range result.Records {
numbers = append(numbers, record.Number)
}

if !reflect.DeepEqual(numbers, []int{2, 3, 4, 5, 6, 7}) {
t.Errorf("numbers = %v", numbers)
}
}

func TestListCapAndEmpty(t *testing.T) {
t.Parallel()

for _, test := range []struct {
name string
body string
count int
truncated bool
}{
{name: "cap", body: `[{"number":1}]`, count: 1, truncated: true},
{name: "empty", body: "[]", count: 0, truncated: false},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()

var calls atomic.Int32

client, server := newClient(t, func(resp http.ResponseWriter, req *http.Request) {
calls.Add(1)
resp.Header().Set("Link", "<https://"+req.Host+firstPage+`&page=2>; rel="next"`)
write(t, resp, test.body)
})

result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1})
wantResume := ""

if test.truncated {
wantResume = server.URL + firstPage + "&page=2"
}

if err != nil || result.Truncated != test.truncated || result.ResumeURL != wantResume ||
len(result.Records) != test.count || calls.Load() != 1 {
t.Fatalf("result = %+v, err = %v, calls = %d", result, err, calls.Load())
}
})
}
}

func TestListNotModified(t *testing.T) {
t.Parallel()

const etag = `"page-one"`

client, _ := newClient(t, func(resp http.ResponseWriter, req *http.Request) {
if req.Header.Get("If-None-Match") == etag {
resp.WriteHeader(http.StatusNotModified)

return
}

resp.Header().Set("ETag", etag)
write(t, resp, `[{"number":1}]`)
})

first, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1})
if err != nil || len(first.Records) != 1 {
t.Fatalf("first = %+v, err = %v", first, err)
}

result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1})
if err != nil || len(result.Records) != 0 || result.Truncated || result.ResumeURL != "" {
t.Fatalf("result = %+v, err = %v", result, err)
}
}

func TestListZeroCap(t *testing.T) {
t.Parallel()

_, err := backfill.List(t.Context(), nil, "o/r", backfill.Options{MaxRequests: 0})
if !errors.Is(err, backfill.ErrMaxRequests) {
t.Fatalf("err = %v", err)
}
}

func TestListFields(t *testing.T) {
t.Parallel()

client, _ := newClient(t, func(resp http.ResponseWriter, _ *http.Request) {
write(t, resp, `[{"number":7,"state":"closed","draft":true,
"created_at":"2026-01-01T00:00:00Z","merged_at":"2026-01-02T00:00:00Z",
"closed_at":"2026-01-03T00:00:00Z","head":{"sha":"abc","ref":{}},
"base":{"ref":"main","label":[]},"title":{},"body":[],"user":{"login":{}}},
{"number":8,"merged_at":null,"closed_at":null}]`)
})

result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1})
if err != nil || len(result.Records) != 2 {
t.Fatalf("result = %+v, err = %v", result, err)
}

want := backfill.Record{
CreatedAt: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
MergedAt: time.Date(2026, time.January, 2, 0, 0, 0, 0, time.UTC),
ClosedAt: time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC),
State: "closed", HeadSHA: "abc", BaseRef: "main", Number: 7, Draft: true,
}
if !reflect.DeepEqual(result.Records[0], want) || !result.Records[1].MergedAt.IsZero() ||
!result.Records[1].ClosedAt.IsZero() {
t.Errorf("records = %+v", result.Records)
}
}

func TestListWrapsClientError(t *testing.T) {
t.Parallel()

client, _ := newClient(t, func(resp http.ResponseWriter, _ *http.Request) {
resp.WriteHeader(http.StatusNotFound)
})

_, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1})

var status *ghclient.StatusError
if !errors.As(err, &status) || status.Status != http.StatusNotFound {
t.Fatalf("err = %v", err)
}
}