Implements spec #135 (spec 2 of 4, derived from wayfinder map #114; decisions settled in #120/#121/#125/#131). Blocked-by #134 is merged, so this lands on `main`. Four owner actions the dashboard can now perform, one ticket each: - **#149** — Latest Chapter correction: one numeric input, overwritten by the next machine write. - **#151** — Series URL repair: owner-typed, gated by the poller's own fetch gate. - **#150 / #153 / #154** — Cover replacement: addresses derived from bytes (`#150`), a Forced Poll replaces the Cover while an ordinary pass still only fills a blank one (`#153`), and byte reclamation is one guarded helper, file first / covers row last (`#154`). - **#155** — Orphan removal: one Series at a time, with the foreign key as the guard. Plus **#152** — Latest Chapter provenance: one derived line naming the actor class, so an owner can tell a hand-edited number from a machine read. - Migration `0015_latest_correction.sql` adds the correction/provenance columns; `0009` now derives cover addresses from bytes. - ADR `0014-cover-addresses-from-bytes.md` records the address scheme. Backend tests cover the store, poller, admin handlers, and web routes (`go test ./...`, needs Docker). Reviewed-on: #156 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #156.
This commit is contained in:
@@ -6,8 +6,10 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -214,9 +216,9 @@ func TestRunOncePrefetchesPublicCover(t *testing.T) {
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||
}
|
||||
body, contentType, found, err := s.GetCover(coverURL)
|
||||
body, contentType, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetCover: %v found=%v", err, found)
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, found)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
||||
@@ -588,8 +590,8 @@ func TestFetchableSeriesURL(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
||||
t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
|
||||
if got := FetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
||||
t.Errorf("FetchableSeriesURL(%q, %q) = %v, want %v",
|
||||
tt.site, tt.seriesURL, got, tt.want)
|
||||
}
|
||||
})
|
||||
@@ -663,7 +665,7 @@ func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: %v found=%v", err, found)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
||||
@@ -811,7 +813,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
||||
}
|
||||
@@ -821,7 +823,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||
}
|
||||
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) {
|
||||
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddressForBytes([]byte("cover-bytes")) {
|
||||
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
||||
}
|
||||
}
|
||||
@@ -914,7 +916,7 @@ func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("not an image"))); err != nil || found {
|
||||
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
||||
}
|
||||
}
|
||||
@@ -940,7 +942,7 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || found {
|
||||
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
||||
}
|
||||
}
|
||||
@@ -1019,8 +1021,8 @@ func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
||||
t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
||||
if got := FetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
||||
t.Fatalf("FetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1038,7 +1040,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
seriesURL string
|
||||
kind string
|
||||
body string
|
||||
wantCover string
|
||||
browser bool
|
||||
}{
|
||||
{
|
||||
@@ -1047,14 +1048,12 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
||||
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
||||
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
||||
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld novel",
|
||||
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
|
||||
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
||||
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
|
||||
},
|
||||
{
|
||||
name: "kagane manga",
|
||||
@@ -1062,7 +1061,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
||||
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
||||
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
||||
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
@@ -1086,7 +1084,7 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
p.waitCovers()
|
||||
|
||||
got := readBookmark(t, s, tc.key)
|
||||
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
|
||||
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes"))
|
||||
if got.Cover != wantWire {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
||||
}
|
||||
@@ -1143,7 +1141,7 @@ func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
|
||||
t.Fatalf("cover fetch calls = %d, want 0", got)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -1190,7 +1188,7 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
|
||||
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -2337,3 +2335,363 @@ func TestForcedSeriesWakesSleepingBrowser(t *testing.T) {
|
||||
t.Fatalf("browser fetches with a forced series = %d, want 2 (the lane wakes)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A forced pass accepts the page as it now stands, so it writes the Cover
|
||||
// through the replace path; an ordinary pass still only fills a blank one
|
||||
// (issue #135).
|
||||
func TestRunOnceForcedPassReplacesExistingCover(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
first = "https://cdn.example/covers/first.jpg"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
||||
t.Fatalf("seed cover: %v", err)
|
||||
}
|
||||
at := time.UnixMilli(5_000_000)
|
||||
public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}
|
||||
|
||||
t.Run("unforced pass leaves the cover alone", func(t *testing.T) {
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
||||
CoverBytesFetch: public,
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
if got := public.callCount(); got != 0 {
|
||||
t.Fatalf("cover fetch calls = %d, want 0", got)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("forced pass replaces the cover", func(t *testing.T) {
|
||||
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
||||
CoverBytesFetch: public,
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
if got := public.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("second")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the second one %q", got.Cover, want)
|
||||
}
|
||||
body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("second")))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
||||
}
|
||||
if string(body) != "second" {
|
||||
t.Fatalf("stored cover = %q, want second", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Identical artwork re-served is an honest no-op the caller can tell apart
|
||||
// from a replacement: the address comes from the bytes, so the row cannot
|
||||
// change in substance, and the replace call site reports the three outcomes
|
||||
// distinctly (issue #135).
|
||||
func TestRunOnceForcedPassIdenticalBytesLogsNoOp(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
first = "https://cdn.example/covers/first.jpg"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.SetSeriesCover("asura", seriesID, first, []byte("cover-bytes"), "image/jpeg"); err != nil {
|
||||
t.Fatalf("seed cover: %v", err)
|
||||
}
|
||||
at := time.UnixMilli(5_000_000)
|
||||
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
||||
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want unchanged %q", got.Cover, want)
|
||||
}
|
||||
if body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || !ok || string(body) != "cover-bytes" {
|
||||
t.Fatalf("stored cover after no-op: found=%v err=%v", ok, err)
|
||||
}
|
||||
logged := logs.String()
|
||||
if !strings.Contains(logged, "cover unchanged") {
|
||||
t.Fatalf("no-op not reported as unchanged; log:\n%s", logged)
|
||||
}
|
||||
if strings.Contains(logged, "cover replaced") {
|
||||
t.Fatalf("no-op reported as a replacement; log:\n%s", logged)
|
||||
}
|
||||
}
|
||||
|
||||
// A Series whose sharded Cover file was unlinked out from under it is
|
||||
// repaired by one forced pass: same bytes mean the same address and the file
|
||||
// re-linked (issue #135, story 32).
|
||||
func TestRunOnceForcedPassRelinksUnlinkedCoverFile(t *testing.T) {
|
||||
coverDir := t.TempDir()
|
||||
url := pgtest.URL(t)
|
||||
s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
const (
|
||||
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
first = "https://cdn.example/covers/first.jpg"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.SetSeriesCover("asura", seriesID, first, []byte("cover-bytes"), "image/jpeg"); err != nil {
|
||||
t.Fatalf("seed cover: %v", err)
|
||||
}
|
||||
address := store.CoverAddressForBytes([]byte("cover-bytes"))
|
||||
coverPath := filepath.Join(coverDir, filepath.FromSlash(address[:2]+"/"+address[2:4]+"/"+address))
|
||||
if err := os.Remove(coverPath); err != nil {
|
||||
t.Fatalf("unlink cover file: %v", err)
|
||||
}
|
||||
if _, _, ok, err := s.CoverByAddress(address); err != nil || ok {
|
||||
t.Fatalf("CoverByAddress after unlink = found %v err %v; want missing (file gone)", ok, err)
|
||||
}
|
||||
|
||||
at := time.UnixMilli(5_000_000)
|
||||
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
||||
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
body, contentType, ok, err := s.CoverByAddress(address)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress after forced pass: found=%v err=%v; want the file re-linked", ok, err)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("re-linked cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
||||
}
|
||||
}
|
||||
|
||||
// A forced pass degrades exactly like an ordinary one when the cover sidecar
|
||||
// is unreachable: the fetch is skipped and logged, the chapter poll is
|
||||
// untouched, and the stored Cover is not moved (issue #135, story 6).
|
||||
func TestRunOnceForcedPassWithoutCoverFetcherStillPolls(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "kagane", SeriesID: seriesID,
|
||||
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.SetSeriesCover("kagane", seriesID, "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed", []byte("existing"), "image/webp"); err != nil {
|
||||
t.Fatalf("seed cover: %v", err)
|
||||
}
|
||||
at := time.UnixMilli(5_000_000)
|
||||
if err := s.ForceSeriesPoll("kagane", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
|
||||
p := &Poller{
|
||||
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixtureWithCover, status: 200},
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
got := readBookmark(t, s, key)
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("existing")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the existing one %q untouched", got.Cover, want)
|
||||
}
|
||||
if checked := readLatestCheckedAt(t, s, key); checked != at.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d", checked, at.UnixMilli())
|
||||
}
|
||||
if !strings.Contains(logs.String(), "fetch cover") {
|
||||
t.Fatalf("missing skipped-cover log; log:\n%s", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A forced replacement strands the previous address, and the poller reclaims
|
||||
// it from the stranded branch: the old sharded file and covers row are both
|
||||
// gone once the pass lands while the new bytes read back (issue #154). The
|
||||
// identical-bytes no-op that follows reclaims nothing: previous == current
|
||||
// there, and a reclamation would delete the Cover the pass just wrote.
|
||||
func TestRunOnceForcedPassReclaimsSupersededCover(t *testing.T) {
|
||||
coverDir := t.TempDir()
|
||||
url := pgtest.URL(t)
|
||||
s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
const (
|
||||
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
||||
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
first = "https://cdn.example/covers/first.jpg"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
||||
t.Fatalf("seed cover: %v", err)
|
||||
}
|
||||
stale := store.CoverAddressForBytes([]byte("first"))
|
||||
stalePath := filepath.Join(coverDir, filepath.FromSlash(stale[:2]+"/"+stale[2:4]+"/"+stale))
|
||||
|
||||
at := time.UnixMilli(5_000_000)
|
||||
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
||||
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"},
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
// The stranded address is gone on disk and in SQL; the new one reads back.
|
||||
if _, err := os.Stat(stalePath); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("stale sharded file after replacement = %v, want fs.ErrNotExist", err)
|
||||
}
|
||||
if _, _, ok, err := s.CoverByAddress(stale); err != nil || ok {
|
||||
t.Fatalf("stale bytes after replacement = found %v err %v, want reclaimed", ok, err)
|
||||
}
|
||||
if body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("second"))); err != nil || !ok || string(body) != "second" {
|
||||
t.Fatalf("new bytes after replacement = found %v err %v, want served", ok, err)
|
||||
}
|
||||
|
||||
// The identical-bytes pass is an honest no-op and reclaims nothing.
|
||||
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(2*time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
current := store.CoverAddressForBytes([]byte("second"))
|
||||
currentPath := filepath.Join(coverDir, filepath.FromSlash(current[:2]+"/"+current[2:4]+"/"+current))
|
||||
if _, err := os.Stat(currentPath); err != nil {
|
||||
t.Fatalf("live sharded file after no-op pass = %v, want present", err)
|
||||
}
|
||||
if body, _, ok, err := s.CoverByAddress(current); err != nil || !ok || string(body) != "second" {
|
||||
t.Fatalf("bytes after no-op pass = found %v err %v, want still served", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A reclamation that fails must not fail the Poll: the failure is logged
|
||||
// against the Series and the replacement still lands, so the stranded bytes
|
||||
// stay reachable for a retry and the owner's act succeeded (issue #154).
|
||||
func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) {
|
||||
coverDir := t.TempDir()
|
||||
url := pgtest.URL(t)
|
||||
s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
const (
|
||||
key = "asura:chronicles-of-the-fallen-f886a8af"
|
||||
seriesID = "chronicles-of-the-fallen-f886a8af"
|
||||
seriesURL = "https://asurascans.com/comics/chronicles-of-the-fallen-f886a8af"
|
||||
first = "https://cdn.example/covers/first.jpg"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
||||
t.Fatalf("seed cover: %v", err)
|
||||
}
|
||||
// Make the stranded file unremovable: a non-empty directory in its place.
|
||||
stale := store.CoverAddressForBytes([]byte("first"))
|
||||
stalePath := filepath.Join(coverDir, filepath.FromSlash(stale[:2]+"/"+stale[2:4]+"/"+stale))
|
||||
if err := os.Remove(stalePath); err != nil {
|
||||
t.Fatalf("clear file: %v", err)
|
||||
}
|
||||
if err := os.Mkdir(stalePath, 0o755); err != nil {
|
||||
t.Fatalf("replace file with dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(stalePath, "blob"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("fill dir: %v", err)
|
||||
}
|
||||
|
||||
at := time.UnixMilli(5_000_000)
|
||||
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
||||
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"},
|
||||
Now: func() time.Time { return at },
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
logged := logs.String()
|
||||
if !strings.Contains(logged, "reclaim cover") {
|
||||
t.Fatalf("failed reclamation not logged; log:\n%s", logged)
|
||||
}
|
||||
if !strings.Contains(logged, "cover replaced") {
|
||||
t.Fatalf("replacement not reported after a failed reclaim; log:\n%s", logged)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("second")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user