From 8a1c133402afe789465aa7926ca0f63f9aa83621 Mon Sep 17 00:00:00 2001 From: "tbhb.dev Agent" Date: Sun, 15 Nov 2026 21:23:44 -0500 Subject: [PATCH] feat: list pull requests for repository backfill Generated-by: LLM effort=high;harness=codex:0.164.0;model=openai:gpt-6.1-sol;session=sha-256-64:e3ba5e0494e132ff;surface=exec --- .cspell-words.txt | 1 + BUILD.bazel | 2 + internal/backfill/BUILD.bazel | 29 ++++ internal/backfill/backfill.go | 105 +++++++++++++++ internal/backfill/backfill_test.go | 208 +++++++++++++++++++++++++++++ 5 files changed, 345 insertions(+) create mode 100644 internal/backfill/BUILD.bazel create mode 100644 internal/backfill/backfill.go create mode 100644 internal/backfill/backfill_test.go diff --git a/.cspell-words.txt b/.cspell-words.txt index 8c28479..b82f691 100644 --- a/.cspell-words.txt +++ b/.cspell-words.txt @@ -39,6 +39,7 @@ distroless docstrings dunder dupl +embeddedstructfieldcheck endfor errchkjson esac diff --git a/BUILD.bazel b/BUILD.bazel index 807d9b0..189d047 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -102,6 +102,8 @@ nogo( GO_TARGETS = [ "//cmd/github-metrics:github-metrics_lib", "//cmd/github-metrics:github-metrics_test", + "//internal/backfill", + "//internal/backfill:backfill_test", "//internal/budget", "//internal/budget:budget_test", "//internal/ghclient", diff --git a/internal/backfill/BUILD.bazel b/internal/backfill/BUILD.bazel new file mode 100644 index 0000000..ff3c5f3 --- /dev/null +++ b/internal/backfill/BUILD.bazel @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Tony Burns + +load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@tbhb_rules//repo:defs.bzl", "repo_files") + +repo_files() + +go_library( + name = "backfill", + srcs = ["backfill.go"], + importpath = "github.com/tbhb-dev/github-metrics/internal/backfill", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/budget", + "//internal/ghclient", + ], +) + +go_test( + name = "backfill_test", + srcs = ["backfill_test.go"], + visibility = ["//:__pkg__"], + deps = [ + ":backfill", + "//internal/budget", + "//internal/ghclient", + ], +) diff --git a/internal/backfill/backfill.go b/internal/backfill/backfill.go new file mode 100644 index 0000000..6571ab0 --- /dev/null +++ b/internal/backfill/backfill.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Tony Burns + +// Package backfill lists a repository's pull requests in creation order. +package backfill + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/tbhb-dev/github-metrics/internal/budget" + "github.com/tbhb-dev/github-metrics/internal/ghclient" +) + +// ErrMaxRequests means the request cap isn't positive. +var ErrMaxRequests = errors.New("max requests must be positive") + +// Options sets the page request cap for one run. +type Options struct { + MaxRequests int +} + +// Record contains only the pull request fields needed for backfill. +type Record struct { + CreatedAt time.Time `json:"created_at"` //nolint:tagliatelle // GitHub field name. + MergedAt time.Time `json:"merged_at"` //nolint:tagliatelle // GitHub field name. + ClosedAt time.Time `json:"closed_at"` //nolint:tagliatelle // GitHub field name. + State string `json:"state"` + HeadSHA string `json:"-"` + BaseRef string `json:"-"` + Number int `json:"number"` + Draft bool `json:"draft"` +} + +// Result holds records in page order and the next URL when capped. +type Result struct { + ResumeURL string + Records []Record + Truncated bool +} + +type pageRecord struct { + Head struct { + SHA string `json:"sha"` + } `json:"head"` + Base struct { + Ref string `json:"ref"` + } `json:"base"` + + Record //nolint:embeddedstructfieldcheck // Keep pointer fields together for field alignment. +} + +// List reads all pages, or stops at the cap. The client manages ETags and +// retries. A 304 ends the run without adding records from that page. Reuse +// the client's tag store only when previously read records have been kept. +func List(ctx context.Context, client *ghclient.Client, repo string, opts Options) (Result, error) { + var result Result + + if opts.MaxRequests <= 0 { + return result, ErrMaxRequests + } + + next := "/repos/" + repo + "/pulls?state=all&sort=created&direction=asc&per_page=100" + + for range opts.MaxRequests { + response, err := client.Get(ctx, budget.Backfill, next) + if err != nil { + return result, fmt.Errorf("list pull requests: %w", err) + } + + if response.NotModified() { + return result, nil + } + + var page []pageRecord + + err = json.Unmarshal(response.Body, &page) + if err != nil { + return result, fmt.Errorf("decode pull requests: %w", err) + } + + if len(page) == 0 { + return result, nil + } + + for _, item := range page { + item.Record.HeadSHA = item.Head.SHA + item.Record.BaseRef = item.Base.Ref + result.Records = append(result.Records, item.Record) + } + + next = response.Next + if next == "" { + return result, nil + } + } + + result.Truncated = true + result.ResumeURL = next + + return result, nil +} diff --git a/internal/backfill/backfill_test.go b/internal/backfill/backfill_test.go new file mode 100644 index 0000000..bad7706 --- /dev/null +++ b/internal/backfill/backfill_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Tony Burns + +package backfill_test + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/tbhb-dev/github-metrics/internal/backfill" + "github.com/tbhb-dev/github-metrics/internal/budget" + "github.com/tbhb-dev/github-metrics/internal/ghclient" +) + +type fakeTokens struct{} + +func (fakeTokens) Token(context.Context) (string, error) { return "test-token", nil } +func (fakeTokens) Renew(context.Context) (string, error) { return "test-token", nil } + +const firstPage = "/repos/o/r/pulls?state=all&sort=created&direction=asc&per_page=100" + +func newClient(t *testing.T, handler http.HandlerFunc) (*ghclient.Client, *httptest.Server) { + t.Helper() + + server := httptest.NewTLSServer(handler) + t.Cleanup(server.Close) + + client, err := ghclient.New(server.URL, server.Client(), budget.New(), fakeTokens{}, ghclient.NewETags()) + if err != nil { + t.Fatal(err) + } + + return client, server +} + +func write(t *testing.T, resp http.ResponseWriter, body string) { + t.Helper() + + _, err := resp.Write([]byte(body)) + if err != nil { + t.Error(err) + } +} + +func TestListThreePages(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + client, _ := newClient(t, func(resp http.ResponseWriter, req *http.Request) { + page := calls.Add(1) + want := firstPage + + if page > 1 { + want += "&page=" + strconv.Itoa(int(page)) + } + + if req.URL.RequestURI() != want { + t.Errorf("request = %q, want %q", req.URL.RequestURI(), want) + } + + if page < 3 { + resp.Header().Set("Link", "; rel="next"`) + write(t, resp, test.body) + }) + + result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1}) + wantResume := "" + + if test.truncated { + wantResume = server.URL + firstPage + "&page=2" + } + + if err != nil || result.Truncated != test.truncated || result.ResumeURL != wantResume || + len(result.Records) != test.count || calls.Load() != 1 { + t.Fatalf("result = %+v, err = %v, calls = %d", result, err, calls.Load()) + } + }) + } +} + +func TestListNotModified(t *testing.T) { + t.Parallel() + + const etag = `"page-one"` + + client, _ := newClient(t, func(resp http.ResponseWriter, req *http.Request) { + if req.Header.Get("If-None-Match") == etag { + resp.WriteHeader(http.StatusNotModified) + + return + } + + resp.Header().Set("ETag", etag) + write(t, resp, `[{"number":1}]`) + }) + + first, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1}) + if err != nil || len(first.Records) != 1 { + t.Fatalf("first = %+v, err = %v", first, err) + } + + result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1}) + if err != nil || len(result.Records) != 0 || result.Truncated || result.ResumeURL != "" { + t.Fatalf("result = %+v, err = %v", result, err) + } +} + +func TestListZeroCap(t *testing.T) { + t.Parallel() + + _, err := backfill.List(t.Context(), nil, "o/r", backfill.Options{MaxRequests: 0}) + if !errors.Is(err, backfill.ErrMaxRequests) { + t.Fatalf("err = %v", err) + } +} + +func TestListFields(t *testing.T) { + t.Parallel() + + client, _ := newClient(t, func(resp http.ResponseWriter, _ *http.Request) { + write(t, resp, `[{"number":7,"state":"closed","draft":true, + "created_at":"2026-01-01T00:00:00Z","merged_at":"2026-01-02T00:00:00Z", + "closed_at":"2026-01-03T00:00:00Z","head":{"sha":"abc","ref":{}}, + "base":{"ref":"main","label":[]},"title":{},"body":[],"user":{"login":{}}}, + {"number":8,"merged_at":null,"closed_at":null}]`) + }) + + result, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1}) + if err != nil || len(result.Records) != 2 { + t.Fatalf("result = %+v, err = %v", result, err) + } + + want := backfill.Record{ + CreatedAt: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC), + MergedAt: time.Date(2026, time.January, 2, 0, 0, 0, 0, time.UTC), + ClosedAt: time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC), + State: "closed", HeadSHA: "abc", BaseRef: "main", Number: 7, Draft: true, + } + if !reflect.DeepEqual(result.Records[0], want) || !result.Records[1].MergedAt.IsZero() || + !result.Records[1].ClosedAt.IsZero() { + t.Errorf("records = %+v", result.Records) + } +} + +func TestListWrapsClientError(t *testing.T) { + t.Parallel() + + client, _ := newClient(t, func(resp http.ResponseWriter, _ *http.Request) { + resp.WriteHeader(http.StatusNotFound) + }) + + _, err := backfill.List(t.Context(), client, "o/r", backfill.Options{MaxRequests: 1}) + + var status *ghclient.StatusError + if !errors.As(err, &status) || status.Status != http.StatusNotFound { + t.Fatalf("err = %v", err) + } +}