package latest import ( "context" "crypto/sha256" "database/sql" "errors" "log" "os" "strings" "sync" "testing" "time" "bookmarkmanager/backend/internal/pgtest" "bookmarkmanager/backend/internal/store" ) func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } // testOwner is the owner every test store seeds. A second reader, where a // test needs one, is created by opening the same database as a second owner. var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))} // testCoverBaseURL is the public origin every stored cover URL is built from. const testCoverBaseURL = "https://bookmarks.test" // newTestStore opens a store on a Postgres database of this test's own and // returns the URL, for helpers that need a second connection to the same // database (see TestRunOnceFetchesSharedSeriesOnce). func newTestStore(t *testing.T) (*store.Store, string) { t.Helper() url := pgtest.URL(t) s, err := store.Open(url, testOwner, t.TempDir(), testCoverBaseURL) if err != nil { t.Fatalf("Open: %v", err) } t.Cleanup(func() { s.Close() }) return s, url } // seedForCheck inserts a bookmark (and with it its series) and forces the // series' latest_checked_at. func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) { t.Helper() site, seriesID, ok := strings.Cut(key, ":") if !ok { t.Fatalf("key %q: no ':' separator", key) } if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: site, SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed %q: %v", key, err) } if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil { t.Fatalf("seed mark %q: %v", key, err) } } func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 { t.Helper() site, seriesID, ok := strings.Cut(key, ":") if !ok { t.Fatalf("key %q: no ':' separator", key) } ts, err := s.LatestCheckedAt(site, seriesID) if err != nil { t.Fatalf("LatestCheckedAt %q: %v", key, err) } return ts } // fakeFetcher stands in for the network. Every poller test uses it, so nothing // in this file can reach tls-client or a real site. type fakeFetcher struct { mu sync.Mutex calls []string body string status int err error // perURL overrides body/status/err for specific URLs. perURL map[string]fakeResponse } type fakeResponse struct { body string status int err error } func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) { f.mu.Lock() f.calls = append(f.calls, url) f.mu.Unlock() if r, ok := f.perURL[url]; ok { return r.body, r.status, r.err } return f.body, f.status, f.err } func (f *fakeFetcher) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } type fakeCoverFetcher struct { mu sync.Mutex calls []string body []byte contentType string err error } func (f *fakeCoverFetcher) Image(_ context.Context, imageURL string) ([]byte, string, error) { f.mu.Lock() f.calls = append(f.calls, imageURL) f.mu.Unlock() if f.err != nil { return nil, "", f.err } return f.body, f.contentType, nil } func (f *fakeCoverFetcher) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } type fakeBytesCoverFetcher struct { mu sync.Mutex calls []string body []byte contentType string err error } func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) { f.mu.Lock() f.calls = append(f.calls, sourceURL) f.mu.Unlock() if f.err != nil { return nil, "", f.err } return f.body, f.contentType, nil } func (f *fakeBytesCoverFetcher) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } // newTestPoller wires a poller with a frozen clock and no stagger, so tests run // instantly and deterministically. func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller { t.Helper() return &Poller{ Store: s, Fetch: f, Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: 6 * time.Hour, Interval: 10 * time.Minute, Stagger: 0, Batch: 14, } } // seedCoverSource writes a Series' cover source address without any stored // bytes. Nothing in production produces that state any more — a client cover // is discarded and an acquired one arrives with its bytes — but rows created // before covers moved server-side still carry one, and the prefetch is what // heals them. func seedCoverSource(t *testing.T, dbURL, site, seriesID, coverURL string) { t.Helper() db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() if _, err := db.Exec( `UPDATE series SET cover = $3 WHERE site = $1 AND series_id = $2`, site, seriesID, coverURL); err != nil { t.Fatalf("seed cover source %s:%s: %v", site, seriesID, err) } } func TestRunOncePrefetchesPublicCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" coverURL = "https://cdn.example/covers/chronicles.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af", SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af", coverURL) covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1", got) } body, contentType, found, err := s.GetCover(coverURL) if err != nil || !found { t.Fatalf("GetCover: %v found=%v", err, found) } if string(body) != "cover-bytes" || contentType != "image/jpeg" { t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType) } } func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:non-image-cover" seriesURL = "https://asurascans.com/comics/non-image-cover" coverURL = "https://cdn.example/covers/challenge" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", "non-image-cover", coverURL) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"}, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) if _, _, found, err := s.GetCover(coverURL); err != nil || found { t.Fatalf("non-image cover = found %v, err %v; want missing", found, err) } } func TestRunOnceRecordsLatestChapter(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0) now := time.UnixMilli(5_000_000) f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, now).runOnce(context.Background()) b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af") if err != nil || !ok { t.Fatalf("Get: %v ok=%v", err, ok) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum) } if b.LatestChapter != "Chapter 181" { t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181") } if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) } } // The whole point of the updated_at CASE in Upsert: a newly published chapter is // not reading progress and must not move the series up the list. func TestRunOnceDoesNotReorderList(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" const key = "asura:chronicles-of-the-demon-faction-f886a8af" // "other" is the most recently read, so it must stay at the top of List(). if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: "asura:other", Site: "asura", SeriesID: "other", SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000, }); err != nil { t.Fatalf("seed other: %v", err) } seedForCheck(t, s, key, url, 0) before, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get before: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background()) after, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get after: %v", err) } if after.UpdatedAt != before.UpdatedAt { t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump", before.UpdatedAt, after.UpdatedAt) } list, err := s.List(s.OwnerID()) if err != nil { t.Fatalf("List: %v", err) } if list[0].Key != "asura:other" { t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key) } } // A failed fetch must still consume the cooldown, or a renamed series gets // retried on every tick forever. func TestRunOnceMarksCheckedOnFailure(t *testing.T) { tests := []struct { name string resp fakeResponse }{ {"network error", fakeResponse{err: errors.New("dial tcp: refused")}}, {"non-200", fakeResponse{body: "nope", status: 503}}, {"challenge page", fakeResponse{body: challengeFixture, status: 200}}, {"empty body", fakeResponse{body: "", status: 200}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/x" seedForCheck(t, s, "asura:x", url, 0) now := time.UnixMilli(7_000_000) f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}} newTestPoller(t, s, f, now).runOnce(context.Background()) if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) } b, _, err := s.Get(s.OwnerID(), "asura:x") if err != nil { t.Fatalf("Get: %v", err) } if b.LatestChapterNum != nil { t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum) } }) } } func TestRunOnceRespectsBatchLimit(t *testing.T) { s, _ := newTestStore(t) for i := 0; i < 20; i++ { key := "asura:s" + string(rune('a'+i)) seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0) } f := &fakeFetcher{body: "", status: 200} p := newTestPoller(t, s, f, time.UnixMilli(5_000_000)) p.Batch = 5 p.runOnce(context.Background()) if got := f.callCount(); got != 5 { t.Fatalf("fetched %d series, want 5 (batch limit)", got) } } // The point of the split (ADR-0003): a series referenced by several bookmarks // is fetched once per due cycle, not once per bookmark. Two bookmarks share a // series when two readers track it (issue #22). func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) { s, url := newTestStore(t) // The slug must match the fixture's own anchors: asura's parser scopes // chapter links to the stored slug. const slug = "chronicles-of-the-demon-faction-f886a8af" const seriesURL = "https://asurascans.com/comics/" + slug seedForCheck(t, s, "asura:"+slug, seriesURL, 0) // A second reader tracks the same series. The seed is the only // reader-creation path, so a second Open as a different owner is how a // test gets a second reader on the same database. other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL) if err != nil { t.Fatalf("Open second reader: %v", err) } t.Cleanup(func() { other.Close() }) if _, err := s.Upsert(other.OwnerID(), store.Bookmark{ Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000, }); err != nil { t.Fatalf("seed second reader: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background()) if got := f.callCount(); got != 1 { t.Fatalf("fetched shared series %d times, want 1", got) } // Both bookmarks join to the same updated series row. for _, st := range []*store.Store{s, other} { b, ok, err := st.Get(st.OwnerID(), "asura:"+slug) if err != nil || !ok { t.Fatalf("Get: %v ok=%v", err, ok) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum) } } } // One unreachable series must not abandon the rest of the batch. func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) { s, _ := newTestStore(t) keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"} for _, k := range keys { seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0) } now := time.UnixMilli(6_000_000) f := &fakeFetcher{ body: "", status: 200, perURL: map[string]fakeResponse{ "https://asurascans.com/comics/asura:b": {err: errors.New("boom")}, }, } newTestPoller(t, s, f, now).runOnce(context.Background()) if got := f.callCount(); got != 5 { t.Fatalf("fetched %d series, want all 5 attempted", got) } for _, k := range keys { if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() { t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli()) } } } func TestRunLogsCooldowns(t *testing.T) { var logs strings.Builder previous := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(previous) }) ctx, cancel := context.WithCancel(context.Background()) cancel() (&Poller{ Cooldown: time.Hour, BrowserCooldown: 6 * time.Hour, Interval: time.Hour, }).Run(ctx) if got := logs.String(); !strings.Contains(got, "cooldown=1h") || !strings.Contains(got, "browser-cooldown=6h") { t.Fatalf("startup log = %q, want both cooldowns", got) } } // The cooldown is enforced by the due query, so a second immediate pass must do // nothing at all — this is what makes the tick interval independent of it. func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/x" seedForCheck(t, s, "asura:x", url, 0) now := time.UnixMilli(8_000_000) f := &fakeFetcher{body: asuraSeriesFixture, status: 200} p := newTestPoller(t, s, f, now) p.runOnce(context.Background()) if got := f.callCount(); got != 1 { t.Fatalf("first pass fetched %d, want 1", got) } // Same instant, and again 59 minutes later: both inside the 1h cooldown. p.runOnce(context.Background()) p.Now = func() time.Time { return now.Add(59 * time.Minute) } p.runOnce(context.Background()) if got := f.callCount(); got != 1 { t.Fatalf("fetched %d times inside the cooldown, want 1", got) } // Past the cooldown, it is due again. p.Now = func() time.Time { return now.Add(61 * time.Minute) } p.runOnce(context.Background()) if got := f.callCount(); got != 2 { t.Fatalf("fetched %d times after the cooldown, want 2", got) } } func TestRunOnceUsesBrowserCooldown(t *testing.T) { s, _ := newTestStore(t) const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0) seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0) hour := time.Hour now := time.Unix(2*int64(hour/time.Second), 0) tls := &fakeFetcher{status: 200} browser := &fakeFetcher{status: 200} p := &Poller{ Store: s, Fetch: tls, BrowserFetch: browser, Now: func() time.Time { return now }, Cooldown: hour, BrowserCooldown: 6 * hour, Batch: 10, } p.runOnce(context.Background()) if got := tls.callCount(); got != 1 { t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got) } if got := browser.callCount(); got != 0 { t.Fatalf("browser fetches after 2h = %d, want 0", got) } now = time.Unix(7*int64(hour/time.Second), 0) p.runOnce(context.Background()) if got := tls.callCount(); got != 2 { t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got) } if got := browser.callCount(); got != 1 { t.Fatalf("browser fetches after 7h = %d, want 1", got) } } // A site that retracts a chapter should correct the stored number downward, // mirroring the userscript's equality check (L427) rather than a >. func TestRunOnceCorrectsDownward(t *testing.T) { s, _ := newTestStore(t) const url = "https://demonicscans.org/manga/Catastrophic-Necromancer" const key = "demonic:Catastrophic-Necromancer" high := 400.0 if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer", SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } f := &fakeFetcher{body: demonicSeriesFixture, status: 200} newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background()) b, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get: %v", err) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 { t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum) } } // series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must // reject anything that is not a known site with an https URL before spending a // request on it — the cooldown still gets consumed either way. func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) { tests := []struct { name string site string seriesURL string wantCalls int }{ {"unknown site", "mangadex", "https://mangadex.org/title/x", 0}, {"http scheme", "asura", "http://asurascans.com/comics/x", 0}, {"unparseable url", "asura", "http://[::1", 0}, {"valid https asura", "asura", "https://asurascans.com/comics/x", 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s, _ := newTestStore(t) key := tt.site + ":x" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } now := time.UnixMilli(4_000_000) f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{ Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL, }) if got := f.callCount(); got != tt.wantCalls { t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls) } if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli()) } }) } } // A cancelled context must abandon the batch rather than run it to completion. func TestRunOnceStopsOnCancelledContext(t *testing.T) { s, _ := newTestStore(t) for _, k := range []string{"asura:a", "asura:b", "asura:c"} { seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0) } ctx, cancel := context.WithCancel(context.Background()) cancel() f := &fakeFetcher{body: "", status: 200} newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx) if got := f.callCount(); got != 0 { t.Fatalf("fetched %d series with a cancelled context, want 0", got) } } func TestFetchableSeriesURL(t *testing.T) { tests := []struct { name string site string seriesURL string want bool }{ {"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true}, {"demonic https", "demonic", "https://demonicscans.org/manga/X", true}, {"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true}, {"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true}, // The browser fetcher runs JavaScript and carries cookies, so a // client-supplied series_url must not be able to aim it anywhere else. {"kagane on a foreign host", "kagane", "https://evil.example/series/x", false}, {"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false}, {"unknown site", "mangadex", "https://mangadex.org/title/x", false}, {"non-https", "comix", "http://comix.to/title/x", false}, {"no host", "comix", "https:///title/x", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want { t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v", tt.site, tt.seriesURL, got, tt.want) } }) } } // A kagane row must not be handed to the plain TLS fetcher: it would only ever // receive a challenge page, and the browser fetcher is the whole reason kagane // is pollable at all. func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } f := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: f, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Interval: time.Hour, Batch: 10, } p.runOnce(context.Background()) if len(f.calls) != 0 { t.Errorf("TLS fetcher was called for kagane: %v", f.calls) } } // novelfull without a browser is not skipped outright: its challenge is a // live time-varying fact, so the plain-TLS page fetch is attempted and — when // the body answers — fills both the chapter and the Cover, exactly the // client-scraped rows #62 wants healed. func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) key := "novelfull:reverend-insanity" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "novelfull", SeriesID: "reverend-insanity", SeriesURL: "https://novelfull.com/reverend-insanity.html", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200} covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: tlsF, CoverBytesFetch: covers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Interval: time.Hour, Batch: 10, } p.runOnce(context.Background()) if len(tlsF.calls) != 1 { t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls)) } if got := covers.callCount(); got != 1 { t.Fatalf("cover fetches = %d, want 1", got) } got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want { t.Fatalf("Cover = %q, want %q", got.Cover, want) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 { t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum) } } // With a browser fetcher wired up, kagane goes to it and not to the TLS one. func TestKaganeUsesBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } tlsF := &fakeFetcher{body: "", status: 200} browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: tlsF, BrowserFetch: browserF, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Interval: time.Hour, Batch: 10, } p.runOnce(context.Background()) if len(tlsF.calls) != 0 { t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls) } if len(browserF.calls) != 1 { t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls) } got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 { t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum) } } func TestRunOncePrefetchesKaganeCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", coverURL) covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL)) if err != nil || !ok { t.Fatalf("CoverByAddress: %v found=%v", err, ok) } if string(body) != "cover-bytes" || contentType != "image/webp" { t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType) } if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1", got) } if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) { t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover) } } func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) at := time.UnixMilli(5_000_000) covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers, Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) at = at.Add(2 * time.Hour) p.runOnce(context.Background()) if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got) } } func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) now := time.UnixMilli(5_000_000) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")}, Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 { t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum) } if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli()) } } func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"}, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found { t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err) } } func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found { t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err) } } func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) { s, dbURL := newTestStore(t) const key = "asura:solo" const coverURL = "https://asurascans.com/covers/solo.jpg" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", "solo", coverURL) browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"} publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers, CoverBytesFetch: publicCovers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) if got := publicCovers.callCount(); got != 1 { t.Fatalf("public cover fetch calls = %d, want 1", got) } if got := browserCovers.callCount(); got != 0 { t.Fatalf("browser cover fetch calls for asura = %d, want 0", got) } } func TestFetcherForRoutesNovelSites(t *testing.T) { tls := &fakeFetcher{} browser := &fakeFetcher{} cases := []struct { site string browser Fetcher tls Fetcher want Fetcher }{ {"asura", browser, tls, tls}, {"lightnovelworld", browser, tls, tls}, {"kagane", browser, tls, browser}, {"novelfull", browser, tls, browser}, // browser-less deployment: kagane is nothing, novelfull degrades to TLS {"kagane", nil, tls, nil}, {"novelfull", nil, tls, tls}, } for _, tc := range cases { t.Run(tc.site, func(t *testing.T) { if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want { t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want) } }) } } func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) { cases := []struct { name string site string url string want bool }{ {"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true}, {"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false}, {"novelfull over http", "novelfull", "http://novelfull.com/x.html", false}, {"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true}, {"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false}, {"unknown site", "webnovel", "https://webnovel.com/x", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want { t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want) } }) } } // A Series that has been blank since creation has no source URL to refetch. // The poll extracts the Cover from the same series page it already fetched // for the chapter signal and stores the bytes — every Site, both Libraries. func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) { cases := []struct { name string key string site string seriesID string seriesURL string kind string body string wantCover string browser bool }{ { name: "asura manga", key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura", seriesID: "chronicles-of-the-demon-faction-f886a8af", seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture, wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp", }, { name: "lightnovelworld novel", key: "lightnovelworld:a-will-eternal", site: "lightnovelworld", seriesID: "a-will-eternal", seriesURL: "https://lightnovelworld.net/novel/a-will-eternal/", kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture, wantCover: "https://lightnovelworld.net/wp-content/uploads/2026/03/a-will-eternal-1.webp", }, { name: "kagane manga", key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane", seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787", seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787", kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true, wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s, _ := newTestStore(t) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: tc.key, Site: tc.site, SeriesID: tc.seriesID, SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } page := &fakeFetcher{body: tc.body, status: 200} public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: page, BrowserFetch: page, CoverBytesFetch: public, CoverFetch: browser, Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) got := readBookmark(t, s, tc.key) wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover) if got.Cover != wantWire { t.Fatalf("Cover = %q, want %q", got.Cover, wantWire) } if tc.browser { if got := browser.callCount(); got != 1 { t.Fatalf("browser cover fetches = %d, want 1", got) } if got := public.callCount(); got != 0 { t.Fatalf("public cover fetches = %d, want 0", got) } } else { if got := public.callCount(); got != 1 { t.Fatalf("public cover fetches = %d, want 1", got) } if got := browser.callCount(); got != 0 { t.Fatalf("browser cover fetches = %d, want 0", got) } } }) } } // Once a Cover exists the poll must leave it alone: refetching every cycle is // noise for the Reader and a request per Series against Sites that already // bot-score the deployment's single IP. func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"} at := time.UnixMilli(5_000_000) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: public, Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) at = at.Add(2 * time.Hour) p.runOnce(context.Background()) if got := public.callCount(); got != 0 { t.Fatalf("cover fetch calls = %d, want 0", got) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want { t.Fatalf("Cover = %q, want the first one %q", got.Cover, want) } } // A blank Cover whose byte fetch fails is retried the next time the Series is // polled. There is no separate retry queue — the due cycle is the queue. func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } public := &fakeBytesCoverFetcher{err: errors.New("cdn down")} at := time.UnixMilli(5_000_000) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: public, Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) if got := readBookmark(t, s, key); got.Cover != "" { t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover) } if got := public.callCount(); got != 1 { t.Fatalf("cover fetch calls after fail = %d, want 1", got) } public.err = nil public.body = []byte("cover-bytes") public.contentType = "image/jpeg" at = at.Add(2 * time.Hour) p.runOnce(context.Background()) if got := public.callCount(); got != 2 { t.Fatalf("cover fetch calls after retry = %d, want 2", got) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want { t.Fatalf("Cover after retry = %q, want %q", got.Cover, want) } } // Cover work is cosmetic: a failed blank fill must leave the chapter poll's // result intact for every Site, not only kagane. func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } now := time.UnixMilli(5_000_000) var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")}, Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) got := readBookmark(t, s, key) if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 { t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum) } if got.Cover != "" { t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover) } if !strings.Contains(logs.String(), key) { t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String()) } } // kaganeAPIFixture carries chapter data only. The blank-fill path needs a // cover image id in the same body the chapter poll already retrieved. const kaganeAPIFixtureWithCover = ` {"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption", "series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}], "series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1}, {"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41}, {"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]} `