diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 0a344d9..d960979 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -2,6 +2,7 @@ package latest import ( "context" + "errors" "log" "net/url" "slices" @@ -57,60 +58,85 @@ type Poller struct { var browserBackedSites = []string{"kagane", "novelfull"} +// fillBlankCover gives a Series its Cover when it has none. The blank state is +// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows +// created before acquisition existed, and rows whose creation-time fetch +// failed, both heal here. A non-blank CoverAddress is left alone — refetching +// would add a request per Series per cycle and change artwork under the Reader +// for no visible reason. A row that already carries a source URL is owned by +// prefetchCover instead; this path only extracts from the series page. +// +// Failures are logged against the Series and never returned: the chapter poll +// must not notice. A failed fill is retried the next time this Series is due; +// there is no separate retry queue. +func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, body string) { + if sr.CoverAddress != "" || sr.Cover != "" { + return + } + cover, ok := coverFrom(sr.Site, sr.SeriesURL, body) + if !ok { + return + } + p.storeCover(ctx, sr, cover) +} + +// prefetchCover heals Series that already carry a third-party source URL but +// no stored address — the state left by client-supplied covers before +// acquisition moved server-side. Every Site takes the same path; only the +// byte fetcher differs (kagane needs the browser). New blanks have no source +// URL and go through fillBlankCover from the series page instead. func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) { - if sr.Cover == "" { + if sr.Cover == "" || sr.CoverAddress != "" { return } - if sr.Site == "kagane" { - if p.CoverFetch == nil { - return - } - imageID, ok := store.KaganeImageID(sr.Cover) - if !ok { - return - } - _, _, found, err := p.Store.GetKaganeCover(imageID) - if err != nil { - log.Printf("latest poll %q: read cover: %v", sr.Key(), err) - return - } - if found { - return - } - body, contentType, err := p.CoverFetch.Image(ctx, imageID) - if err != nil { - log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err) - return - } - if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil { - log.Printf("latest poll %q: persist cover: %v", sr.Key(), err) - } - return - } - if p.CoverBytesFetch == nil || sr.CoverAddress != "" { - return - } - // Bytes may already be stored from an earlier poll that ran before the - // Series carried an address; storing them again is free (they are - // content-addressed and immutable), and the point of the second call is - // the address, which is what makes the Cover visible on the wire. body, contentType, found, err := p.Store.GetCover(sr.Cover) if err != nil { log.Printf("latest poll %q: read cover: %v", sr.Key(), err) return } - if !found { - body, contentType, err = p.CoverBytesFetch.Fetch(ctx, sr.Cover) - if err != nil { - log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err) - return + if found { + if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil { + log.Printf("latest poll %q: persist cover: %v", sr.Key(), err) } + return } - if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil { + p.storeCover(ctx, sr, sr.Cover) +} + +// storeCover fetches bytes for sourceURL and points the Series at them. Every +// failure is logged against the Series and swallowed so the chapter poll +// cannot see it. +func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL string) { + bytes, contentType, err := p.fetchCoverBytes(ctx, sr, sourceURL) + if err != nil { + log.Printf("latest poll %q: fetch cover %s: %v", sr.Key(), sourceURL, err) + return + } + if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sourceURL, bytes, contentType); err != nil { log.Printf("latest poll %q: persist cover: %v", sr.Key(), err) } } +// fetchCoverBytes routes by Site: only kagane needs the browser for image +// bytes; every other Site's CDN answers plain TLS. Missing fetchers degrade to +// a blank Cover rather than falling back onto a path that cannot succeed. +func (p *Poller) fetchCoverBytes(ctx context.Context, sr store.Series, cover string) ([]byte, string, error) { + if sr.Site == "kagane" { + if p.CoverFetch == nil { + return nil, "", errors.New("no cover fetcher") + } + imageID, ok := store.KaganeImageID(cover) + if !ok { + return nil, "", errors.New("invalid kagane cover URL") + } + return p.CoverFetch.Image(ctx, imageID) + } + if p.CoverBytesFetch == nil { + return nil, "", errors.New("no cover fetcher") + } + return p.CoverBytesFetch.Fetch(ctx, cover) +} + // fetcherFor returns the fetcher a site needs, or nil when the site cannot be // fetched at all right now. kagane and novelfull both sit behind a Cloudflare // JavaScript challenge that no TLS fingerprint clears — kagane verified @@ -234,6 +260,9 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) { } latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body) + // Cover fill is independent of the chapter signal: a page that lost its + // chapter list may keep its og:image, and a blank Series heals either way. + p.fillBlankCover(ctx, sr, body) if !ok { // Most likely a challenge page or a layout change. Either way the row is // already stamped, so this waits out a cooldown instead of hot-looping. diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 2bcc8b7..a8d5353 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -743,6 +743,9 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) { if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1", got) } + if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) { + t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover) + } } func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) { @@ -932,3 +935,219 @@ func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) { }) } } + +// A Series that has been blank since creation has no source URL to refetch. +// The poll extracts the Cover from the same series page it already fetched +// for the chapter signal and stores the bytes — every Site, both Libraries. +func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) { + cases := []struct { + name string + key string + site string + seriesID string + seriesURL string + kind string + body string + wantCover string + browser bool + }{ + { + name: "asura manga", + key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura", + seriesID: "chronicles-of-the-demon-faction-f886a8af", + seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", + kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture, + wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp", + }, + { + name: "lightnovelworld novel", + key: "lightnovelworld:a-will-eternal", site: "lightnovelworld", + seriesID: "a-will-eternal", seriesURL: "https://lightnovelworld.net/novel/a-will-eternal/", + kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture, + wantCover: "https://lightnovelworld.net/wp-content/uploads/2026/03/a-will-eternal-1.webp", + }, + { + name: "kagane manga", + key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane", + seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787", + seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787", + kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true, + wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s, _ := newTestStore(t) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: tc.key, Site: tc.site, SeriesID: tc.seriesID, + SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + page := &fakeFetcher{body: tc.body, status: 200} + public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} + browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} + p := &Poller{ + Store: s, Fetch: page, BrowserFetch: page, + CoverBytesFetch: public, CoverFetch: browser, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, + Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + got := readBookmark(t, s, tc.key) + wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover) + if got.Cover != wantWire { + t.Fatalf("Cover = %q, want %q", got.Cover, wantWire) + } + if tc.browser { + if got := browser.callCount(); got != 1 { + t.Fatalf("browser cover fetches = %d, want 1", got) + } + if got := public.callCount(); got != 0 { + t.Fatalf("public cover fetches = %d, want 0", got) + } + } else { + if got := public.callCount(); got != 1 { + t.Fatalf("public cover fetches = %d, want 1", got) + } + if got := browser.callCount(); got != 0 { + t.Fatalf("browser cover fetches = %d, want 0", got) + } + } + }) + } +} + +// Once a Cover exists the poll must leave it alone: refetching every cycle is +// noise for the Reader and a request per Series against Sites that already +// bot-score the deployment's single IP. +func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + first = "https://cdn.example/covers/first.jpg" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil { + t.Fatalf("seed cover: %v", err) + } + public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"} + at := time.UnixMilli(5_000_000) + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, + CoverBytesFetch: public, + Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + at = at.Add(2 * time.Hour) + p.runOnce(context.Background()) + + if got := public.callCount(); got != 0 { + t.Fatalf("cover fetch calls = %d, want 0", got) + } + got := readBookmark(t, s, key) + if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want { + t.Fatalf("Cover = %q, want the first one %q", got.Cover, want) + } +} + +// A blank Cover whose byte fetch fails is retried the next time the Series is +// polled. There is no separate retry queue — the due cycle is the queue. +func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + public := &fakeBytesCoverFetcher{err: errors.New("cdn down")} + at := time.UnixMilli(5_000_000) + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, + CoverBytesFetch: public, + Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + if got := readBookmark(t, s, key); got.Cover != "" { + t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover) + } + if got := public.callCount(); got != 1 { + t.Fatalf("cover fetch calls after fail = %d, want 1", got) + } + + public.err = nil + public.body = []byte("cover-bytes") + public.contentType = "image/jpeg" + at = at.Add(2 * time.Hour) + p.runOnce(context.Background()) + + if got := public.callCount(); got != 2 { + t.Fatalf("cover fetch calls after retry = %d, want 2", got) + } + got := readBookmark(t, s, key) + if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want { + t.Fatalf("Cover after retry = %q, want %q", got.Cover, want) + } +} + +// Cover work is cosmetic: a failed blank fill must leave the chapter poll's +// result intact for every Site, not only kagane. +func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + now := time.UnixMilli(5_000_000) + var logs strings.Builder + prev := log.Writer() + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(prev) }) + + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, + CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")}, + Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + got := readBookmark(t, s, key) + if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 { + t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum) + } + if got.Cover != "" { + t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover) + } + if !strings.Contains(logs.String(), key) { + t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String()) + } +} + +// kaganeAPIFixture carries chapter data only. The blank-fill path needs a +// cover image id in the same body the chapter poll already retrieved. +const kaganeAPIFixtureWithCover = ` +{"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption", +"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}], +"series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1}, +{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41}, +{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]} +`