Files
mangaBookmark/backend/internal/latest/poller_test.go
T
sulthan 21615be2bd feat: one registry entry per Site, one shared Series-page read (#95)
Closes #94.

## What

Two phases per the spec, in three feature commits plus two review-fix commits:

**Phase one — one registry entry per Site** (`2d134fb`)
The six per-site comparison points that used to live across three files collapse
into one `sites` map in `backend/internal/latest/sites.go`: Latest Chapter parse,
Cover parse, browser-backed list, fetcher route, host pins, and the browser
payload read all become lookups into it. `browserBackedSites()` is derived from
the registry (sorted, deterministic); `fetcherFor` and `fetchableSeriesURL` keep
their signatures and become lookups; `BrowserFetcher.Get` dispatches through the
entries' `Read`/`Done` while the tab lifecycle stays in `BrowserFetcher.run`.

**Phase two — one shared Series-page read** (`f215130`)
`readSeriesPage` (new `read.go`) performs the read the Poll and the Acquisition
have in common: gate, route, fetch, parse Latest Chapter, parse Cover address.
It returns facts only — polling and persistence policies (stamp order, cooldowns,
cover policy) stay with the callers; `acquire.go` gained the comment naming the
deliberate post-fetch stamp order. The poll's legacy cover heal and the
no-chapter byte-count diagnostic were restored after review (`d998f87`) so the
claims "the Poll keeps its own Cover policy" and "pinning is the only
behavioural change" both hold.

## Behaviour

- All six Sites now pin their host exactly; asura/demonic/comix previously
  accepted any https host. For asura this is a strict improvement: its dead old
  domain redirects deep links to the site root and would parse the wrong
  document.
- Everything else is unchanged: existing parse tables, the challenge-body table
  and the gate table pass unmodified except the one deliberate exception — the
  gate table gains the three new pin cases.

## Security invariants preserved

- The address gate is recognisably the same rule, now a single registry lookup:
  `https` + exact hostname match, all callers route through it. No fetch path
  was widened; asura/demonic/comix were narrowed.
- The second host pin inside each browser entry's Read is retained deliberately
  (browser = strong SSRF primitive, `series_url` is client-supplied) and is not
  deduplicated against the shared gate.
- Review hardening: `fetcherFor` now fails closed for unknown site strings
  (previously fell through to the TLS fetcher on an unreachable path), and the
  browser dispatch iterates a sorted list so outcomes cannot depend on map order.
- The security review's log-injection finding was checked against Go's
  `url.Parse` and does not hold: control characters are rejected anywhere in a
  URL, so a client-supplied value in a log line cannot carry a newline.

## Review

Reviewed on three axes (spec, standards, security) by read-only subagents over
`672c16f..f1b26f4`. No blocking findings; all minor/nit findings addressed in
`d998f87` and `700de20`. Verified end to end with `go test ./...` (Docker
Postgres per test package) on every commit.

## Out of scope (tracked separately)

- Dropping asuracomic.net (CORS allowlist, userscript match, API fixtures,
  live env) — separate issue, per spec.

Reviewed-on: #95
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
2026-08-12 05:43:37 +07:00

1212 lines
43 KiB
Go

package latest
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"log"
"os"
"strings"
"sync"
"testing"
"time"
"bookmarkmanager/backend/internal/pgtest"
"bookmarkmanager/backend/internal/store"
)
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
// testOwner is the owner every test store seeds. A second reader, where a
// test needs one, is created by opening the same database as a second owner.
var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}
// testCoverBaseURL is the public origin every stored cover URL is built from.
const testCoverBaseURL = "https://bookmarks.test"
// newTestStore opens a store on a Postgres database of this test's own and
// returns the URL, for helpers that need a second connection to the same
// database (see TestRunOnceFetchesSharedSeriesOnce).
func newTestStore(t *testing.T) (*store.Store, string) {
t.Helper()
url := pgtest.URL(t)
s, err := store.Open(url, testOwner, t.TempDir(), testCoverBaseURL)
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { s.Close() })
return s, url
}
// seedForCheck inserts a bookmark (and with it its series) and forces the
// series' latest_checked_at.
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
t.Helper()
site, seriesID, ok := strings.Cut(key, ":")
if !ok {
t.Fatalf("key %q: no ':' separator", key)
}
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key,
Site: site,
SeriesID: seriesID,
SeriesURL: seriesURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed %q: %v", key, err)
}
if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
t.Fatalf("seed mark %q: %v", key, err)
}
}
func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
t.Helper()
site, seriesID, ok := strings.Cut(key, ":")
if !ok {
t.Fatalf("key %q: no ':' separator", key)
}
ts, err := s.LatestCheckedAt(site, seriesID)
if err != nil {
t.Fatalf("LatestCheckedAt %q: %v", key, err)
}
return ts
}
// fakeFetcher stands in for the network. Every poller test uses it, so nothing
// in this file can reach tls-client or a real site.
type fakeFetcher struct {
mu sync.Mutex
calls []string
body string
status int
err error
// perURL overrides body/status/err for specific URLs.
perURL map[string]fakeResponse
}
type fakeResponse struct {
body string
status int
err error
}
func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
f.mu.Lock()
f.calls = append(f.calls, url)
f.mu.Unlock()
if r, ok := f.perURL[url]; ok {
return r.body, r.status, r.err
}
return f.body, f.status, f.err
}
func (f *fakeFetcher) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
type fakeCoverFetcher struct {
mu sync.Mutex
calls []string
body []byte
contentType string
err error
}
func (f *fakeCoverFetcher) Image(_ context.Context, imageURL string) ([]byte, string, error) {
f.mu.Lock()
f.calls = append(f.calls, imageURL)
f.mu.Unlock()
if f.err != nil {
return nil, "", f.err
}
return f.body, f.contentType, nil
}
func (f *fakeCoverFetcher) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
type fakeBytesCoverFetcher struct {
mu sync.Mutex
calls []string
body []byte
contentType string
err error
}
func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) {
f.mu.Lock()
f.calls = append(f.calls, sourceURL)
f.mu.Unlock()
if f.err != nil {
return nil, "", f.err
}
return f.body, f.contentType, nil
}
func (f *fakeBytesCoverFetcher) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
// instantly and deterministically.
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
t.Helper()
return &Poller{
Store: s,
Fetch: f,
Now: func() time.Time { return at },
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute,
Stagger: 0,
Batch: 14,
}
}
// seedCoverSource writes a Series' cover source address without any stored
// bytes. Nothing in production produces that state any more — a client cover
// is discarded and an acquired one arrives with its bytes — but rows created
// before covers moved server-side still carry one, and the prefetch is what
// heals them.
func seedCoverSource(t *testing.T, dbURL, site, seriesID, coverURL string) {
t.Helper()
db, err := sql.Open("pgx", dbURL)
if err != nil {
t.Fatalf("open %s: %v", dbURL, err)
}
defer db.Close()
if _, err := db.Exec(
`UPDATE series SET cover = $3 WHERE site = $1 AND series_id = $2`,
site, seriesID, coverURL); err != nil {
t.Fatalf("seed cover source %s:%s: %v", site, seriesID, err)
}
}
func TestRunOncePrefetchesPublicCover(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "asura:chronicles-of-the-demon-faction-f886a8af"
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
coverURL = "https://cdn.example/covers/chronicles.jpg"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af",
SeriesURL: seriesURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af", coverURL)
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got)
}
body, contentType, found, err := s.GetCover(coverURL)
if err != nil || !found {
t.Fatalf("GetCover: %v found=%v", err, found)
}
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
}
}
func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "asura:non-image-cover"
seriesURL = "https://asurascans.com/comics/non-image-cover"
coverURL = "https://cdn.example/covers/challenge"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "asura", "non-image-cover", coverURL)
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
}
}
func TestRunOnceRecordsLatestChapter(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
now := time.UnixMilli(5_000_000)
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).runOnce(context.Background())
b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af")
if err != nil || !ok {
t.Fatalf("Get: %v ok=%v", err, ok)
}
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
}
if b.LatestChapter != "Chapter 181" {
t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
}
if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
}
}
// The whole point of the updated_at CASE in Upsert: a newly published chapter is
// not reading progress and must not move the series up the list.
func TestRunOnceDoesNotReorderList(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
const key = "asura:chronicles-of-the-demon-faction-f886a8af"
// "other" is the most recently read, so it must stay at the top of List().
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: "asura:other", Site: "asura", SeriesID: "other",
SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
}); err != nil {
t.Fatalf("seed other: %v", err)
}
seedForCheck(t, s, key, url, 0)
before, _, err := s.Get(s.OwnerID(), key)
if err != nil {
t.Fatalf("Get before: %v", err)
}
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
after, _, err := s.Get(s.OwnerID(), key)
if err != nil {
t.Fatalf("Get after: %v", err)
}
if after.UpdatedAt != before.UpdatedAt {
t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
before.UpdatedAt, after.UpdatedAt)
}
list, err := s.List(s.OwnerID())
if err != nil {
t.Fatalf("List: %v", err)
}
if list[0].Key != "asura:other" {
t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
}
}
// A failed fetch must still consume the cooldown, or a renamed series gets
// retried on every tick forever.
func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
tests := []struct {
name string
resp fakeResponse
}{
{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
{"non-200", fakeResponse{body: "nope", status: 503}},
{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
{"empty body", fakeResponse{body: "", status: 200}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/x"
seedForCheck(t, s, "asura:x", url, 0)
now := time.UnixMilli(7_000_000)
f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
newTestPoller(t, s, f, now).runOnce(context.Background())
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
}
b, _, err := s.Get(s.OwnerID(), "asura:x")
if err != nil {
t.Fatalf("Get: %v", err)
}
if b.LatestChapterNum != nil {
t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
}
})
}
}
func TestRunOnceRespectsBatchLimit(t *testing.T) {
s, _ := newTestStore(t)
for i := 0; i < 20; i++ {
key := "asura:s" + string(rune('a'+i))
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
}
f := &fakeFetcher{body: "", status: 200}
p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
p.Batch = 5
p.runOnce(context.Background())
if got := f.callCount(); got != 5 {
t.Fatalf("fetched %d series, want 5 (batch limit)", got)
}
}
// The point of the split (ADR-0003): a series referenced by several bookmarks
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
// series when two readers track it (issue #22).
func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
s, url := newTestStore(t)
// The slug must match the fixture's own anchors: asura's parser scopes
// chapter links to the stored slug.
const slug = "chronicles-of-the-demon-faction-f886a8af"
const seriesURL = "https://asurascans.com/comics/" + slug
seedForCheck(t, s, "asura:"+slug, seriesURL, 0)
// A second reader tracks the same series. The seed is the only
// reader-creation path, so a second Open as a different owner is how a
// test gets a second reader on the same database.
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
if err != nil {
t.Fatalf("Open second reader: %v", err)
}
t.Cleanup(func() { other.Close() })
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000,
}); err != nil {
t.Fatalf("seed second reader: %v", err)
}
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched shared series %d times, want 1", got)
}
// Both bookmarks join to the same updated series row.
for _, st := range []*store.Store{s, other} {
b, ok, err := st.Get(st.OwnerID(), "asura:"+slug)
if err != nil || !ok {
t.Fatalf("Get: %v ok=%v", err, ok)
}
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
}
}
}
// One unreachable series must not abandon the rest of the batch.
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
s, _ := newTestStore(t)
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
for _, k := range keys {
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
}
now := time.UnixMilli(6_000_000)
f := &fakeFetcher{
body: "", status: 200,
perURL: map[string]fakeResponse{
"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
},
}
newTestPoller(t, s, f, now).runOnce(context.Background())
if got := f.callCount(); got != 5 {
t.Fatalf("fetched %d series, want all 5 attempted", got)
}
for _, k := range keys {
if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
}
}
}
func TestRunLogsCooldowns(t *testing.T) {
var logs strings.Builder
previous := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(previous) })
ctx, cancel := context.WithCancel(context.Background())
cancel()
(&Poller{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: time.Hour,
}).Run(ctx)
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
!strings.Contains(got, "browser-cooldown=6h") {
t.Fatalf("startup log = %q, want both cooldowns", got)
}
}
// The cooldown is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the tick interval independent of it.
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/x"
seedForCheck(t, s, "asura:x", url, 0)
now := time.UnixMilli(8_000_000)
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
p := newTestPoller(t, s, f, now)
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("first pass fetched %d, want 1", got)
}
// Same instant, and again 59 minutes later: both inside the 1h cooldown.
p.runOnce(context.Background())
p.Now = func() time.Time { return now.Add(59 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times inside the cooldown, want 1", got)
}
// Past the cooldown, it is due again.
p.Now = func() time.Time { return now.Add(61 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 2 {
t.Fatalf("fetched %d times after the cooldown, want 2", got)
}
}
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
s, _ := newTestStore(t)
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
hour := time.Hour
now := time.Unix(2*int64(hour/time.Second), 0)
tls := &fakeFetcher{status: 200}
browser := &fakeFetcher{status: 200}
p := &Poller{
Store: s,
Fetch: tls,
BrowserFetch: browser,
Now: func() time.Time { return now },
Cooldown: hour,
BrowserCooldown: 6 * hour,
Batch: 10,
}
p.runOnce(context.Background())
if got := tls.callCount(); got != 1 {
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
}
if got := browser.callCount(); got != 0 {
t.Fatalf("browser fetches after 2h = %d, want 0", got)
}
now = time.Unix(7*int64(hour/time.Second), 0)
p.runOnce(context.Background())
if got := tls.callCount(); got != 2 {
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
}
if got := browser.callCount(); got != 1 {
t.Fatalf("browser fetches after 7h = %d, want 1", got)
}
}
// A site that retracts a chapter should correct the stored number downward,
// mirroring the userscript's equality check (L427) rather than a >.
func TestRunOnceCorrectsDownward(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
const key = "demonic:Catastrophic-Necromancer"
high := 400.0
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
b, _, err := s.Get(s.OwnerID(), key)
if err != nil {
t.Fatalf("Get: %v", err)
}
if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
}
}
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
// reject anything that is not a known site with an https URL before spending a
// request on it — the cooldown still gets consumed either way.
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
tests := []struct {
name string
site string
seriesURL string
wantCalls int
}{
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
{"unparseable url", "asura", "http://[::1", 0},
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, _ := newTestStore(t)
key := tt.site + ":x"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
now := time.UnixMilli(4_000_000)
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{
Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
})
if got := f.callCount(); got != tt.wantCalls {
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
}
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
}
})
}
}
// A cancelled context must abandon the batch rather than run it to completion.
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
s, _ := newTestStore(t)
for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
f := &fakeFetcher{body: "", status: 200}
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)
if got := f.callCount(); got != 0 {
t.Fatalf("fetched %d series with a cancelled context, want 0", got)
}
}
func TestFetchableSeriesURL(t *testing.T) {
tests := []struct {
name string
site string
seriesURL string
want bool
}{
{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
// The three plain-TLS Sites are pinned too: a client-supplied
// series_url must not aim a fetcher at a lookalike host, even when
// the fetcher is only an HTTP GET.
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
{"comix on a foreign host", "comix", "https://evil.example/title/x", false},
// The browser fetcher runs JavaScript and carries cookies, so a
// client-supplied series_url must not be able to aim it anywhere else.
{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
{"non-https", "comix", "http://comix.to/title/x", false},
{"no host", "comix", "https:///title/x", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
tt.site, tt.seriesURL, got, tt.want)
}
})
}
}
// A kagane row must not be handed to the plain TLS fetcher: it would only ever
// receive a challenge page, and the browser fetcher is the whole reason kagane
// is pollable at all.
func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
s, _ := newTestStore(t)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
Site: "kagane",
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s,
Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(f.calls) != 0 {
t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
}
}
// novelfull without a browser is not skipped outright: its challenge is a
// live time-varying fact, so the plain-TLS page fetch is attempted and — when
// the body answers — fills both the chapter and the Cover, exactly the
// client-scraped rows #62 wants healed.
func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
s, _ := newTestStore(t)
key := "novelfull:reverend-insanity"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key,
Site: "novelfull",
SeriesID: "reverend-insanity",
SeriesURL: "https://novelfull.com/reverend-insanity.html",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(tlsF.calls) != 1 {
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetches = %d, want 1", got)
}
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want)
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum)
}
}
// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
func TestKaganeUsesBrowserFetcher(t *testing.T) {
s, _ := newTestStore(t)
key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key,
Site: "kagane",
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
tlsF := &fakeFetcher{body: "", status: 200}
browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(tlsF.calls) != 0 {
t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
}
if len(browserF.calls) != 1 {
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
}
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
}
}
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "kagane", "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", coverURL)
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
if err != nil || !ok {
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
}
if string(body) != "cover-bytes" || contentType != "image/webp" {
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got)
}
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) {
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
}
}
func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
at := time.UnixMilli(5_000_000)
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
}
}
func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
now := time.UnixMilli(5_000_000)
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
}
if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() {
t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli())
}
}
func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
}
}
func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
}
}
func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
s, dbURL := newTestStore(t)
const key = "asura:solo"
const coverURL = "https://asurascans.com/covers/solo.jpg"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "asura", "solo", coverURL)
browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
CoverBytesFetch: publicCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if got := publicCovers.callCount(); got != 1 {
t.Fatalf("public cover fetch calls = %d, want 1", got)
}
if got := browserCovers.callCount(); got != 0 {
t.Fatalf("browser cover fetch calls for asura = %d, want 0", got)
}
}
func TestFetcherForRoutesNovelSites(t *testing.T) {
tls := &fakeFetcher{}
browser := &fakeFetcher{}
cases := []struct {
site string
browser Fetcher
tls Fetcher
want Fetcher
}{
{"asura", browser, tls, tls},
{"lightnovelworld", browser, tls, tls},
{"kagane", browser, tls, browser},
{"novelfull", browser, tls, browser},
// browser-less deployment: kagane is nothing, novelfull degrades to TLS
{"kagane", nil, tls, nil},
{"novelfull", nil, tls, tls},
// unknown site: fail closed — nothing to fetch or parse
{"mangadex", browser, tls, nil},
}
for _, tc := range cases {
t.Run(tc.site, func(t *testing.T) {
if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want {
t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
}
})
}
}
func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
cases := []struct {
name string
site string
url string
want bool
}{
{"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true},
{"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false},
{"novelfull over http", "novelfull", "http://novelfull.com/x.html", false},
{"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true},
{"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false},
{"unknown site", "webnovel", "https://webnovel.com/x", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
}
})
}
}
// A Series that has been blank since creation has no source URL to refetch.
// The poll extracts the Cover from the same series page it already fetched
// for the chapter signal and stores the bytes — every Site, both Libraries.
func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
cases := []struct {
name string
key string
site string
seriesID string
seriesURL string
kind string
body string
wantCover string
browser bool
}{
{
name: "asura manga",
key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura",
seriesID: "chronicles-of-the-demon-faction-f886a8af",
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
},
{
name: "lightnovelworld novel",
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
},
{
name: "kagane manga",
key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane",
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s, _ := newTestStore(t)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: tc.key, Site: tc.site, SeriesID: tc.seriesID,
SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
page := &fakeFetcher{body: tc.body, status: 200}
public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: page, BrowserFetch: page,
CoverBytesFetch: public, CoverFetch: browser,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
got := readBookmark(t, s, tc.key)
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
if got.Cover != wantWire {
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
}
if tc.browser {
if got := browser.callCount(); got != 1 {
t.Fatalf("browser cover fetches = %d, want 1", got)
}
if got := public.callCount(); got != 0 {
t.Fatalf("public cover fetches = %d, want 0", got)
}
} else {
if got := public.callCount(); got != 1 {
t.Fatalf("public cover fetches = %d, want 1", got)
}
if got := browser.callCount(); got != 0 {
t.Fatalf("browser cover fetches = %d, want 0", got)
}
}
})
}
}
// Once a Cover exists the poll must leave it alone: refetching every cycle is
// noise for the Reader and a request per Series against Sites that already
// bot-score the deployment's single IP.
func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "asura:chronicles-of-the-demon-faction-f886a8af"
seriesID = "chronicles-of-the-demon-faction-f886a8af"
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
first = "https://cdn.example/covers/first.jpg"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
t.Fatalf("seed cover: %v", err)
}
public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}
at := time.UnixMilli(5_000_000)
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: public,
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
if got := public.callCount(); got != 0 {
t.Fatalf("cover fetch calls = %d, want 0", got)
}
got := readBookmark(t, s, key)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want {
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
}
}
// A blank Cover whose byte fetch fails is retried the next time the Series is
// polled. There is no separate retry queue — the due cycle is the queue.
func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "asura:chronicles-of-the-demon-faction-f886a8af"
seriesID = "chronicles-of-the-demon-faction-f886a8af"
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
public := &fakeBytesCoverFetcher{err: errors.New("cdn down")}
at := time.UnixMilli(5_000_000)
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: public,
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if got := readBookmark(t, s, key); got.Cover != "" {
t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover)
}
if got := public.callCount(); got != 1 {
t.Fatalf("cover fetch calls after fail = %d, want 1", got)
}
public.err = nil
public.body = []byte("cover-bytes")
public.contentType = "image/jpeg"
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
if got := public.callCount(); got != 2 {
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
}
got := readBookmark(t, s, key)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
}
}
// Cover work is cosmetic: a failed blank fill must leave the chapter poll's
// result intact for every Site, not only kagane.
func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "asura:chronicles-of-the-demon-faction-f886a8af"
seriesID = "chronicles-of-the-demon-faction-f886a8af"
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
now := time.UnixMilli(5_000_000)
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")},
Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
got := readBookmark(t, s, key)
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
}
if got.Cover != "" {
t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover)
}
if !strings.Contains(logs.String(), key) {
t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String())
}
}
// kaganeAPIFixture carries chapter data only. The blank-fill path needs a
// cover image id in the same body the chapter poll already retrieved.
const kaganeAPIFixtureWithCover = `
{"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption",
"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}],
"series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1},
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
`