package latest import ( "context" "crypto/sha256" "database/sql" "errors" "fmt" "io/fs" "log" "os" "path/filepath" "strings" "sync" "testing" "time" "bookmarkmanager/backend/internal/pgtest" "bookmarkmanager/backend/internal/store" ) func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } // testOwner is the owner every test store seeds. A second reader, where a // test needs one, is created by opening the same database as a second owner. var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))} // testCoverBaseURL is the public origin every stored cover URL is built from. const testCoverBaseURL = "https://bookmarks.test" // newTestStore opens a store on a Postgres database of this test's own and // returns the URL, for helpers that need a second connection to the same // database (see TestRunOnceFetchesSharedSeriesOnce). func newTestStore(t *testing.T) (*store.Store, string) { t.Helper() url := pgtest.URL(t) s, err := store.Open(url, testOwner, t.TempDir(), testCoverBaseURL) if err != nil { t.Fatalf("Open: %v", err) } t.Cleanup(func() { s.Close() }) return s, url } // seedForCheck inserts a bookmark (and with it its series) and forces the // series' latest_checked_at. func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) { t.Helper() site, seriesID, ok := strings.Cut(key, ":") if !ok { t.Fatalf("key %q: no ':' separator", key) } if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: site, SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed %q: %v", key, err) } if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil { t.Fatalf("seed mark %q: %v", key, err) } } func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 { t.Helper() site, seriesID, ok := strings.Cut(key, ":") if !ok { t.Fatalf("key %q: no ':' separator", key) } ts, err := s.LatestCheckedAt(site, seriesID) if err != nil { t.Fatalf("LatestCheckedAt %q: %v", key, err) } return ts } // fakeFetcher stands in for the network. Every poller test uses it, so nothing // in this file can reach tls-client or a real site. type fakeFetcher struct { mu sync.Mutex calls []string body string status int err error // perURL overrides body/status/err for specific URLs. perURL map[string]fakeResponse } type fakeResponse struct { body string status int err error } func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) { f.mu.Lock() f.calls = append(f.calls, url) f.mu.Unlock() if r, ok := f.perURL[url]; ok { return r.body, r.status, r.err } return f.body, f.status, f.err } func (f *fakeFetcher) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } type fakeCoverFetcher struct { mu sync.Mutex calls []string body []byte contentType string err error } func (f *fakeCoverFetcher) Image(_ context.Context, imageURL string) ([]byte, string, error) { f.mu.Lock() f.calls = append(f.calls, imageURL) f.mu.Unlock() if f.err != nil { return nil, "", f.err } return f.body, f.contentType, nil } func (f *fakeCoverFetcher) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } type fakeBytesCoverFetcher struct { mu sync.Mutex calls []string body []byte contentType string err error } func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) { f.mu.Lock() f.calls = append(f.calls, sourceURL) f.mu.Unlock() if f.err != nil { return nil, "", f.err } return f.body, f.contentType, nil } func (f *fakeBytesCoverFetcher) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } // fakeNotifier records the owner notices the poller sends, standing in for // the Discord webhook the way fakeFetcher stands in for the network. An err // set after construction makes every subsequent send fail, for the retry // test. type fakeNotifier struct { mu sync.Mutex calls []Fault err error } func (f *fakeNotifier) Notify(_ context.Context, fault Fault, _, _ string) error { f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, fault) return f.err } func (f *fakeNotifier) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.calls) } func (f *fakeNotifier) fault(i int) Fault { f.mu.Lock() defer f.mu.Unlock() return f.calls[i] } // newTestPoller wires a poller with a frozen clock. Rest and gap come from the // Site registry, so tests seed checked_at relative to the one-hour rest; the // round entry point (runOnce) runs every Lane back to back with no real // pacing, so tests stay instant and deterministic. func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller { t.Helper() return &Poller{ Store: s, Fetch: f, Now: func() time.Time { return at }, } } // seedCoverSource writes a Series' cover source address without any stored // bytes. Nothing in production produces that state any more — a client cover // is discarded and an acquired one arrives with its bytes — but rows created // before covers moved server-side still carry one, and the prefetch is what // heals them. func seedCoverSource(t *testing.T, dbURL, site, seriesID, coverURL string) { t.Helper() db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() if _, err := db.Exec( `UPDATE series SET cover = $3 WHERE site = $1 AND series_id = $2`, site, seriesID, coverURL); err != nil { t.Fatalf("seed cover source %s:%s: %v", site, seriesID, err) } } func TestRunOncePrefetchesPublicCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" coverURL = "https://cdn.example/covers/chronicles.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af", SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af", coverURL) covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1", got) } body, contentType, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))) if err != nil || !found { t.Fatalf("CoverByAddress: %v found=%v", err, found) } if string(body) != "cover-bytes" || contentType != "image/jpeg" { t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType) } } func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:non-image-cover" seriesURL = "https://asurascans.com/comics/non-image-cover" coverURL = "https://cdn.example/covers/challenge" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", "non-image-cover", coverURL) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"}, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if _, _, found, err := s.GetCover(coverURL); err != nil || found { t.Fatalf("non-image cover = found %v, err %v; want missing", found, err) } } func TestRunOnceRecordsLatestChapter(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0) now := time.UnixMilli(5_000_000) f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, now).runOnce(context.Background()) b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af") if err != nil || !ok { t.Fatalf("Get: %v ok=%v", err, ok) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum) } if b.LatestChapter != "Chapter 181" { t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181") } if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) } } // The whole point of the updated_at CASE in Upsert: a newly published chapter is // not reading progress and must not move the series up the list. func TestRunOnceDoesNotReorderList(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" const key = "asura:chronicles-of-the-demon-faction-f886a8af" // "other" is the most recently read, so it must stay at the top of List(). if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: "asura:other", Site: "asura", SeriesID: "other", SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000, }); err != nil { t.Fatalf("seed other: %v", err) } seedForCheck(t, s, key, url, 0) before, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get before: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background()) after, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get after: %v", err) } if after.UpdatedAt != before.UpdatedAt { t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump", before.UpdatedAt, after.UpdatedAt) } list, err := s.List(s.OwnerID()) if err != nil { t.Fatalf("List: %v", err) } if list[0].Key != "asura:other" { t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key) } } // A failed fetch must still consume the cooldown, or a renamed series gets // retried on every tick forever. func TestRunOnceMarksCheckedOnFailure(t *testing.T) { tests := []struct { name string resp fakeResponse }{ {"network error", fakeResponse{err: errors.New("dial tcp: refused")}}, {"non-200", fakeResponse{body: "nope", status: 503}}, {"challenge page", fakeResponse{body: challengeFixture, status: 200}}, {"empty body", fakeResponse{body: "", status: 200}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/x" seedForCheck(t, s, "asura:x", url, 0) now := time.UnixMilli(7_000_000) f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}} newTestPoller(t, s, f, now).runOnce(context.Background()) if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) } b, _, err := s.Get(s.OwnerID(), "asura:x") if err != nil { t.Fatalf("Get: %v", err) } if b.LatestChapterNum != nil { t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum) } }) } } // The point of the split (ADR-0003): a series referenced by several bookmarks // is fetched once per due cycle, not once per bookmark. Two bookmarks share a // series when two readers track it (issue #22). func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) { s, url := newTestStore(t) // The slug must match the fixture's own anchors: asura's parser scopes // chapter links to the stored slug. const slug = "chronicles-of-the-demon-faction-f886a8af" const seriesURL = "https://asurascans.com/comics/" + slug seedForCheck(t, s, "asura:"+slug, seriesURL, 0) // A second reader tracks the same series. The seed is the only // reader-creation path, so a second Open as a different owner is how a // test gets a second reader on the same database. other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL) if err != nil { t.Fatalf("Open second reader: %v", err) } t.Cleanup(func() { other.Close() }) if _, err := s.Upsert(other.OwnerID(), store.Bookmark{ Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000, }); err != nil { t.Fatalf("seed second reader: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background()) if got := f.callCount(); got != 1 { t.Fatalf("fetched shared series %d times, want 1", got) } // Both bookmarks join to the same updated series row. for _, st := range []*store.Store{s, other} { b, ok, err := st.Get(st.OwnerID(), "asura:"+slug) if err != nil || !ok { t.Fatalf("Get: %v ok=%v", err, ok) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum) } } } // One unreachable series must not abandon the rest of the Lane. func TestRunOnceOneBadSeriesDoesNotStallLane(t *testing.T) { s, _ := newTestStore(t) keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"} for _, k := range keys { seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0) } now := time.UnixMilli(6_000_000) f := &fakeFetcher{ body: "", status: 200, perURL: map[string]fakeResponse{ "https://asurascans.com/comics/asura:b": {err: errors.New("boom")}, }, } newTestPoller(t, s, f, now).runOnce(context.Background()) if got := f.callCount(); got != 5 { t.Fatalf("fetched %d series, want all 5 attempted", got) } for _, k := range keys { if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() { t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli()) } } } // Pace now lives in the registry, not config: Run logs the lane defaults so a // deployment can see what the poller is doing without reading the source. func TestRunLogsLaneDefaults(t *testing.T) { var logs strings.Builder previous := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(previous) }) // The pass log writes through the store, so Run needs a real one — a // poller without a Store is not a poller. s, _ := newTestStore(t) ctx, cancel := context.WithCancel(context.Background()) cancel() (&Poller{Store: s, Now: func() time.Time { return time.Now() }}).Run(ctx) got := logs.String() for _, want := range []string{"6 lanes", "rest=1h0m0s", "gap=10s"} { if !strings.Contains(got, want) { t.Fatalf("startup log = %q, want %q", got, want) } } } // The rest is enforced by the due query, so a second immediate pass must do // nothing at all — this is what makes the Lane's sleep independent of it. func TestRunOnceHonoursRestAcrossPasses(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/x" seedForCheck(t, s, "asura:x", url, 0) now := time.UnixMilli(8_000_000) f := &fakeFetcher{body: asuraSeriesFixture, status: 200} p := newTestPoller(t, s, f, now) p.runOnce(context.Background()) if got := f.callCount(); got != 1 { t.Fatalf("first pass fetched %d, want 1", got) } // Same instant, and again 59 minutes later: both inside the 1h rest. p.runOnce(context.Background()) p.Now = func() time.Time { return now.Add(59 * time.Minute) } p.runOnce(context.Background()) if got := f.callCount(); got != 1 { t.Fatalf("fetched %d times inside the rest, want 1", got) } // Past the rest, it is due again. p.Now = func() time.Time { return now.Add(61 * time.Minute) } p.runOnce(context.Background()) if got := f.callCount(); got != 2 { t.Fatalf("fetched %d times after the rest, want 2", got) } } // A site that retracts a chapter should correct the stored number downward, // mirroring the userscript's equality check (L427) rather than a >. func TestRunOnceCorrectsDownward(t *testing.T) { s, _ := newTestStore(t) const url = "https://demonicscans.org/manga/Catastrophic-Necromancer" const key = "demonic:Catastrophic-Necromancer" high := 400.0 if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer", SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } f := &fakeFetcher{body: demonicSeriesFixture, status: 200} newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background()) b, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get: %v", err) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 { t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum) } } // series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must // reject anything that is not a known site with an https URL before spending a // request on it — the cooldown still gets consumed either way. func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) { tests := []struct { name string site string seriesURL string wantCalls int }{ {"unknown site", "mangadex", "https://mangadex.org/title/x", 0}, {"http scheme", "asura", "http://asurascans.com/comics/x", 0}, {"unparseable url", "asura", "http://[::1", 0}, {"valid https asura", "asura", "https://asurascans.com/comics/x", 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s, _ := newTestStore(t) key := tt.site + ":x" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } now := time.UnixMilli(4_000_000) f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{ Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL, }) if got := f.callCount(); got != tt.wantCalls { t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls) } if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli()) } }) } } // A cancelled context must abandon the batch rather than run it to completion. func TestRunOnceStopsOnCancelledContext(t *testing.T) { s, _ := newTestStore(t) for _, k := range []string{"asura:a", "asura:b", "asura:c"} { seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0) } ctx, cancel := context.WithCancel(context.Background()) cancel() f := &fakeFetcher{body: "", status: 200} newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx) if got := f.callCount(); got != 0 { t.Fatalf("fetched %d series with a cancelled context, want 0", got) } } func TestFetchableSeriesURL(t *testing.T) { tests := []struct { name string site string seriesURL string want bool }{ {"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true}, {"demonic https", "demonic", "https://demonicscans.org/manga/X", true}, {"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true}, {"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true}, // The three plain-TLS Sites are pinned too: a client-supplied // series_url must not aim a fetcher at a lookalike host, even when // the fetcher is only an HTTP GET. {"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false}, {"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false}, {"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false}, {"comix on a foreign host", "comix", "https://evil.example/title/x", false}, // The browser fetcher runs JavaScript and carries cookies, so a // client-supplied series_url must not be able to aim it anywhere else. {"kagane on a foreign host", "kagane", "https://evil.example/series/x", false}, {"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false}, {"unknown site", "mangadex", "https://mangadex.org/title/x", false}, {"non-https", "comix", "http://comix.to/title/x", false}, {"no host", "comix", "https:///title/x", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := FetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want { t.Errorf("FetchableSeriesURL(%q, %q) = %v, want %v", tt.site, tt.seriesURL, got, tt.want) } }) } } // A kagane row must not be handed to the plain TLS fetcher: it would only ever // receive a challenge page, and the browser fetcher is the whole reason kagane // is pollable at all. func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } f := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: f, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) if len(f.calls) != 0 { t.Errorf("TLS fetcher was called for kagane: %v", f.calls) } if got := readLatestCheckedAt(t, s, "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); got != 0 { t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got) } } // novelfull without a browser is not skipped outright: its challenge is a // live time-varying fact, so the plain-TLS page fetch is attempted and — when // the body answers — fills both the chapter and the Cover, exactly the // client-scraped rows #62 wants healed. func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) key := "novelfull:reverend-insanity" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "novelfull", SeriesID: "reverend-insanity", SeriesURL: "https://novelfull.com/reverend-insanity.html", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200} covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: tlsF, CoverBytesFetch: covers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if len(tlsF.calls) != 1 { t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls)) } if got := covers.callCount(); got != 1 { t.Fatalf("cover fetches = %d, want 1", got) } got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want { t.Fatalf("Cover = %q, want %q", got.Cover, want) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 { t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum) } } // With a browser fetcher wired up, kagane goes to it and not to the TLS one. func TestKaganeUsesBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } tlsF := &fakeFetcher{body: "", status: 200} browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: tlsF, BrowserFetch: browserF, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) if len(tlsF.calls) != 0 { t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls) } if len(browserF.calls) != 1 { t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls) } got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 { t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum) } } // comix joined kagane behind the challenge on 2026-08-12 (#98): its page goes // to the browser, its Cover bytes go through the browser's image route because // static.comix.to is gated the same way, and the TLS fetcher is never asked // for either. func TestComixUsesBrowserFetcher(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "comix:n8we-dungeons-and-crayons" seriesID = "n8we-dungeons-and-crayons" seriesURL = "https://comix.to/title/n8we-dungeons-and-crayons" coverURL = "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "comix", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "comix", seriesID, coverURL) tlsF := &fakeFetcher{body: "", status: 200} browserF := &fakeFetcher{body: comixSeriesFixture, status: 200} covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} tlsCovers := &fakeBytesCoverFetcher{body: []byte("tls-bytes"), contentType: "image/jpeg"} p := &Poller{ Store: s, Fetch: tlsF, BrowserFetch: browserF, CoverFetch: covers, CoverBytesFetch: tlsCovers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if len(tlsF.calls) != 0 { t.Errorf("TLS fetcher was called for comix: %v", tlsF.calls) } if len(browserF.calls) != 1 { t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls) } if got := tlsCovers.callCount(); got != 0 { t.Errorf("TLS cover fetches = %d, want 0: static.comix.to answers a challenge", got) } if got := covers.callCount(); got != 1 { t.Fatalf("browser cover fetches = %d, want 1", got) } got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 80 { t.Errorf("LatestChapterNum = %v, want 80", got.LatestChapterNum) } } // Without a browser, comix is skipped outright rather than handed to plain // TLS: a plain fetch retrieves only a challenge page (measured 2026-08-12). func TestComixSkippedWhenNoBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: "comix:n8we-dungeons-and-crayons", Site: "comix", SeriesID: "n8we-dungeons-and-crayons", SeriesURL: "https://comix.to/title/n8we-dungeons-and-crayons", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } f := &fakeFetcher{body: comixSeriesFixture, status: 200} p := &Poller{ Store: s, Fetch: f, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) if len(f.calls) != 0 { t.Errorf("TLS fetcher was called for comix: %v", f.calls) } if got := readLatestCheckedAt(t, s, "comix:n8we-dungeons-and-crayons"); got != 0 { t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got) } } func TestRunOncePrefetchesKaganeCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", coverURL) covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))) if err != nil || !ok { t.Fatalf("CoverByAddress: %v found=%v", err, ok) } if string(body) != "cover-bytes" || contentType != "image/webp" { t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType) } if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1", got) } if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddressForBytes([]byte("cover-bytes")) { t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover) } } func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) at := time.UnixMilli(5_000_000) covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() at = at.Add(2 * time.Hour) p.runOnce(context.Background()) p.waitCovers() if got := covers.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got) } } func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) now := time.UnixMilli(5_000_000) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")}, Now: func() time.Time { return now }, } p.runOnce(context.Background()) p.waitCovers() got, found, err := s.Get(s.OwnerID(), key) if err != nil || !found { t.Fatalf("Get: %v found=%v", err, found) } if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 { t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum) } if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli()) } } func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"}, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("not an image"))); err != nil || found { t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err) } } func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "kagane", seriesID, coverURL) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || found { t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err) } } func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) { s, dbURL := newTestStore(t) const key = "asura:solo" const coverURL = "https://asurascans.com/covers/solo.jpg" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo", UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", "solo", coverURL) browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"} publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers, CoverBytesFetch: publicCovers, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() if got := publicCovers.callCount(); got != 1 { t.Fatalf("public cover fetch calls = %d, want 1", got) } if got := browserCovers.callCount(); got != 0 { t.Fatalf("browser cover fetch calls for asura = %d, want 0", got) } } func TestFetcherForRoutesNovelSites(t *testing.T) { tls := &fakeFetcher{} browser := &fakeFetcher{} cases := []struct { site string browser Fetcher tls Fetcher want Fetcher }{ {"asura", browser, tls, tls}, {"lightnovelworld", browser, tls, tls}, {"kagane", browser, tls, browser}, {"novelfull", browser, tls, browser}, // browser-less deployment: kagane is nothing, novelfull degrades to TLS {"kagane", nil, tls, nil}, {"novelfull", nil, tls, tls}, // unknown site: fail closed — nothing to fetch or parse {"mangadex", browser, tls, nil}, } for _, tc := range cases { t.Run(tc.site, func(t *testing.T) { if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want { t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want) } }) } } func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) { cases := []struct { name string site string url string want bool }{ {"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true}, {"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false}, {"novelfull over http", "novelfull", "http://novelfull.com/x.html", false}, {"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true}, {"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false}, {"unknown site", "webnovel", "https://webnovel.com/x", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := FetchableSeriesURL(tc.site, tc.url); got != tc.want { t.Fatalf("FetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want) } }) } } // A Series that has been blank since creation has no source URL to refetch. // The poll extracts the Cover from the same series page it already fetched // for the chapter signal and stores the bytes — every Site, both Libraries. func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) { cases := []struct { name string key string site string seriesID string seriesURL string kind string body string browser bool }{ { name: "asura manga", key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura", seriesID: "chronicles-of-the-demon-faction-f886a8af", seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture, }, { name: "lightnovelworld novel", key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld", seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture, }, { name: "kagane manga", key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane", seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787", seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787", kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s, _ := newTestStore(t) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: tc.key, Site: tc.site, SeriesID: tc.seriesID, SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } page := &fakeFetcher{body: tc.body, status: 200} public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} p := &Poller{ Store: s, Fetch: page, BrowserFetch: page, CoverBytesFetch: public, CoverFetch: browser, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } p.runOnce(context.Background()) p.waitCovers() got := readBookmark(t, s, tc.key) wantWire := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")) if got.Cover != wantWire { t.Fatalf("Cover = %q, want %q", got.Cover, wantWire) } if tc.browser { if got := browser.callCount(); got != 1 { t.Fatalf("browser cover fetches = %d, want 1", got) } if got := public.callCount(); got != 0 { t.Fatalf("public cover fetches = %d, want 0", got) } } else { if got := public.callCount(); got != 1 { t.Fatalf("public cover fetches = %d, want 1", got) } if got := browser.callCount(); got != 0 { t.Fatalf("browser cover fetches = %d, want 0", got) } } }) } } // Once a Cover exists the poll must leave it alone: refetching every cycle is // noise for the Reader and a request per Series against Sites that already // bot-score the deployment's single IP. func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"} at := time.UnixMilli(5_000_000) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: public, Now: func() time.Time { return at }, } p.runOnce(context.Background()) at = at.Add(2 * time.Hour) p.runOnce(context.Background()) if got := public.callCount(); got != 0 { t.Fatalf("cover fetch calls = %d, want 0", got) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want { t.Fatalf("Cover = %q, want the first one %q", got.Cover, want) } } // A blank Cover whose byte fetch fails is retried the next time the Series is // polled. There is no separate retry queue — the due cycle is the queue. func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } public := &fakeBytesCoverFetcher{err: errors.New("cdn down")} at := time.UnixMilli(5_000_000) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: public, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() if got := readBookmark(t, s, key); got.Cover != "" { t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover) } if got := public.callCount(); got != 1 { t.Fatalf("cover fetch calls after fail = %d, want 1", got) } public.err = nil public.body = []byte("cover-bytes") public.contentType = "image/jpeg" at = at.Add(2 * time.Hour) p.runOnce(context.Background()) p.waitCovers() if got := public.callCount(); got != 2 { t.Fatalf("cover fetch calls after retry = %d, want 2", got) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want { t.Fatalf("Cover after retry = %q, want %q", got.Cover, want) } } // Cover work is cosmetic: a failed blank fill must leave the chapter poll's // result intact for every Site, not only kagane. func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } now := time.UnixMilli(5_000_000) var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")}, Now: func() time.Time { return now }, } p.runOnce(context.Background()) p.waitCovers() got := readBookmark(t, s, key) if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 { t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum) } if got.Cover != "" { t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover) } if !strings.Contains(logs.String(), key) { t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String()) } } // kaganeAPIFixture carries chapter data only. The blank-fill path needs a // cover image id in the same body the chapter poll already retrieved. const kaganeAPIFixtureWithCover = ` {"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption", "series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}], "series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1}, {"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41}, {"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]} ` func TestEffectiveGap(t *testing.T) { asura := sites["asura"] tests := []struct { eligible int want time.Duration clamped bool }{ {0, 10 * time.Second, false}, {5, 10 * time.Second, false}, {360, 10 * time.Second, false}, {720, 5 * time.Second, false}, {3600, time.Second, false}, {4000, time.Second, true}, } for _, tt := range tests { got, clamped := effectiveGap(asura, tt.eligible) if got != tt.want || clamped != tt.clamped { t.Errorf("effectiveGap(asura, %d) = (%s, %v), want (%s, %v)", tt.eligible, got, clamped, tt.want, tt.clamped) } } } // The due query orders by sharedness first, then age: a series two readers // track is polled before a single-reader series that has waited far longer // (ADR-0003, issue #100). func TestRunOnceOrdersBySharednessThenAge(t *testing.T) { s, url := newTestStore(t) other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL) if err != nil { t.Fatalf("Open second reader: %v", err) } t.Cleanup(func() { other.Close() }) now := time.UnixMilli(5_000_000) // popular: two readers, due for 2 minutes. loner: one reader, due for 23 // minutes. Popularity must win — the loner waited far longer. const ( popularKey = "asura:popular" lonerKey = "asura:loner" popularURL = "https://asurascans.com/comics/popular" lonerURL = "https://asurascans.com/comics/loner" ) seedForCheck(t, s, popularKey, popularURL, now.Add(-62*time.Minute).UnixMilli()) seedForCheck(t, s, lonerKey, lonerURL, 0) if _, err := s.Upsert(other.OwnerID(), store.Bookmark{ Key: popularKey, Site: "asura", SeriesID: "popular", UpdatedAt: 2000, }); err != nil { t.Fatalf("seed second reader: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} newTestPoller(t, s, f, now).runOnce(context.Background()) if len(f.calls) != 2 { t.Fatalf("fetches = %d, want 2", len(f.calls)) } if f.calls[0] != popularURL { t.Fatalf("first fetch = %q, want the shared series %q", f.calls[0], popularURL) } } // Two refusals in one pass stop the Lane: the remaining Series stay unstamped // and due, and the Lane backs off for RefuseBackoff before trying the Site // again. One hostile Site burns only its own Lane's budget (issue #100). func TestRunOnceSiteRefusalSkipsRestOfLaneAndBacksOff(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) const n = 4 for i := 0; i < n; i++ { key := fmt.Sprintf("kagane:s%d", i) seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0) } browser := &fakeFetcher{status: 403} tls := &fakeFetcher{status: 200} // An asura Series sits on its own Lane: it must still be polled while // kagane's Lane burns its budget on refusals (story 4). seedForCheck(t, s, "asura:still-polls", "https://asurascans.com/series/still-polls", 0) p := &Poller{ Store: s, Fetch: tls, BrowserFetch: browser, Now: func() time.Time { return now }, } p.runOnce(context.Background()) if got := browser.callCount(); got != 2 { t.Fatalf("browser fetches after first pass = %d, want 2 (refused twice)", got) } if got := tls.callCount(); got != 1 { t.Fatalf("asura fetches after first pass = %d, want 1 (its Lane is independent)", got) } stamped := 0 for i := 0; i < n; i++ { if readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)) == now.UnixMilli() { stamped++ } } if stamped != 2 { t.Fatalf("stamped series = %d, want 2; the remaining two stay due", stamped) } // Inside the backoff window nothing is attempted. p.Now = func() time.Time { return now.Add(14 * time.Minute) } p.runOnce(context.Background()) if got := browser.callCount(); got != 2 { t.Fatalf("browser fetches inside backoff = %d, want still 2", got) } // Past the backoff the Lane resumes and the two untried Series are polled. p.Now = func() time.Time { return now.Add(16 * time.Minute) } p.runOnce(context.Background()) if got := browser.callCount(); got != 4 { t.Fatalf("browser fetches after backoff = %d, want 4", got) } for i := 0; i < n; i++ { if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 { t.Fatalf("kagane:s%d still untried after backoff", i) } } } // Below five due Series with none waiting long, a browser Lane leaves Chrome // asleep; five due, or one waiting browserWakeAge, wakes it (ADR-0005). func TestBrowserLaneWakeThresholds(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) // Freshly due: checked two minutes before the rest elapses, so the wait // is far below browserWakeAge. seed := func(i int) { key := fmt.Sprintf("kagane:w%d", i) seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli()) } for i := 0; i < 3; i++ { seed(i) } browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser, Now: func() time.Time { return now }, } p.runOnce(context.Background()) if got := browser.callCount(); got != 0 { t.Fatalf("browser fetches with 3 freshly-due series = %d, want 0 (Chrome stays asleep)", got) } // The skipped pass still records its row, carrying the due count it never // read; the Lanes page (issue #145) reads that row — see // TestRunLanePassRecordsEveryExit/"asleep". // 5 due crosses the count threshold. for i := 3; i < 5; i++ { seed(i) } p.runOnce(context.Background()) if got := browser.callCount(); got != 5 { t.Fatalf("browser fetches with 5 due series = %d, want 5", got) } // A single long-neglected series wakes the browser by age alone. seedForCheck(t, s, "kagane:ancient", "https://kagane.to/series/ancient", 0) p.runOnce(context.Background()) if got := browser.callCount(); got != 6 { t.Fatalf("browser fetches with one ancient series = %d, want 6", got) } } // When one browser Lane loses the sidecar, the round's remaining browser // Lanes are skipped: every fetch would fail anyway, and their Series must not // burn their stamps on a dead Chrome (issue #100). func TestRunOnceUnreachableBrowserStopsBrowserLanes(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) seedForCheck(t, s, "kagane:k", "https://kagane.to/series/k", 0) seedForCheck(t, s, "novelfull:n", "https://novelfull.com/n.html", 0) interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) browser := &fakeFetcher{status: 200, err: interrupted} p := &Poller{ Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser, Now: func() time.Time { return now }, } p.runOnce(context.Background()) // Sorted lane order: comix, kagane, novelfull. Only comix attempted. if got := browser.callCount(); got != 1 { t.Fatalf("browser fetches = %d, want 1 (only the first browser lane)", got) } if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() { t.Fatalf("comix stamp = %d, want %d", got, now.UnixMilli()) } for _, key := range []string{"kagane:k", "novelfull:n"} { if got := readLatestCheckedAt(t, s, key); got != 0 { t.Fatalf("%s stamp = %d, want 0 (untried)", key, got) } } // The shared flag decays after RefuseBackoff: the next round probes // again. comix's Series is resting (stamped last round), so the probe // falls to kagane — the only Lane with something due — and its fresh // loss re-gates the Lanes behind it. p.Now = func() time.Time { return now.Add(16 * time.Minute) } p.runOnce(context.Background()) if got := browser.callCount(); got != 2 { t.Fatalf("browser fetches after backoff decay = %d, want 2 (kagane probes again)", got) } if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() { t.Fatalf("comix stamp after decay = %d, want %d (resting, untouched)", got, now.UnixMilli()) } if got := readLatestCheckedAt(t, s, "kagane:k"); got != now.Add(16*time.Minute).UnixMilli() { t.Fatalf("kagane stamp after decay = %d, want %d (the probe)", got, now.Add(16*time.Minute).UnixMilli()) } if got := readLatestCheckedAt(t, s, "novelfull:n"); got != 0 { t.Fatalf("novelfull stamp after decay = %d, want 0 (gated by kagane's fresh loss)", got) } } // A browser Lane that cannot keep up logs the backlog every pass, so the // decision to give browser Sites more pages is measured, not guessed. func TestRunOnceLogsBrowserLaneBehind(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) // Checked three rests ago: two rests' worth of wait past the due moment. seedForCheck(t, s, "kagane:old", "https://kagane.to/series/old", now.Add(-3*time.Hour).UnixMilli()) var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) p := &Poller{ Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, Now: func() time.Time { return now }, } p.runOnce(context.Background()) if got := logs.String(); !strings.Contains(got, "latest poll kagane: browser lane behind by 1h0m0s") { t.Fatalf("behind log = %q, want it to name kagane and the backlog", got) } } // gatedCoverFetcher blocks every Fetch on a gate, so a test can hold a cover // heal in flight and prove a Lane does not wait for it. type gatedCoverFetcher struct { inner *fakeBytesCoverFetcher gate chan struct{} started chan struct{} once sync.Once } func (g *gatedCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) { g.once.Do(func() { g.started <- struct{}{} }) <-g.gate return g.inner.Fetch(ctx, sourceURL) } // Cover heals run in the background: a heal stuck on a slow CDN must not // delay the Lane's next Series-page Poll, or a large import with many blanks // would make every Latest Chapter go stale (issue #100). func TestRunOnceCoverFetchDoesNotDelayNextPoll(t *testing.T) { s, dbURL := newTestStore(t) // Two Series whose legacy cover sources still need healing. for i := 0; i < 2; i++ { key := fmt.Sprintf("asura:cover-%d", i) seriesID := fmt.Sprintf("cover-%d", i) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: "https://asurascans.com/comics/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } seedCoverSource(t, dbURL, "asura", seriesID, fmt.Sprintf("https://cdn.example/covers/%d.jpg", i)) } pages := &fakeFetcher{body: asuraSeriesFixture, status: 200} gated := &gatedCoverFetcher{ inner: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}, gate: make(chan struct{}), started: make(chan struct{}, 1), } p := &Poller{ Store: s, Fetch: pages, CoverBytesFetch: gated, Now: func() time.Time { return time.UnixMilli(5_000_000) }, } done := make(chan struct{}) go func() { p.runOnce(context.Background()) close(done) }() <-gated.started // the first cover heal is now stuck on its CDN select { case <-done: // The Lane finished its page fetches without waiting for the cover. case <-time.After(5 * time.Second): t.Fatal("runOnce blocked on an in-flight cover fetch") } close(gated.gate) p.waitCovers() if got := pages.callCount(); got != 2 { t.Fatalf("series page fetches = %d, want 2", got) } if got := gated.inner.callCount(); got != 2 { t.Fatalf("cover fetches = %d, want 2", got) } } // At 3601 eligible Series the effective gap falls below the one-second floor; // the clamp warning must name the Site so the operator knows which Lane is // outrunning its plan. func TestRunOnceClampWarningNamesTheSite(t *testing.T) { s, dbURL := newTestStore(t) db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() if _, err := db.Exec(`INSERT INTO series (site, series_id, series_url, latest_checked_at) SELECT 'asura', 'bulk-' || g, 'https://asurascans.com/comics/bulk-' || g, 0 FROM generate_series(1, 3601) AS g`); err != nil { t.Fatalf("bulk seed series: %v", err) } if _, err := db.Exec(`INSERT INTO bookmarks (reader_id, site, series_id, updated_at) SELECT (SELECT id FROM readers ORDER BY id LIMIT 1), 'asura', 'bulk-' || g, 1000 FROM generate_series(1, 3601) AS g`); err != nil { t.Fatalf("bulk seed bookmarks: %v", err) } var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) newTestPoller(t, s, &fakeFetcher{body: "", status: 200}, time.UnixMilli(5_000_000)). runOnce(context.Background()) if got := logs.String(); !strings.Contains(got, "latest poll asura: gap clamped to 1s floor (eligible series=3601)") { t.Fatalf("clamp warning = %q, want it to name asura and 3601", got) } } // The Lanes page (issue #145) reads the durable pass log, so the poller's // only duty to it is that every return path writes its row; the snapshot it // used to mirror into memory is gone. A refusing pass still records why it // declined, and the latest row per Site is what the page renders — nothing // else is left to assert against here, because the page's seam moved into the // web layer (web_test.go, seeded-row tests). func TestPassLogIsTheLanesPagesOnlyWindow(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) // No pass has run: the log is empty, which the page renders as "none // observed" rather than confident zeroes. if _, ok, err := s.LatestLanePass("asura"); err != nil || ok { t.Fatalf("LatestLanePass before any pass = ok %v err %v, want no row", ok, err) } seedForCheck(t, s, "asura:chronicles", "https://asurascans.com/series/chronicles", 0) // Two refusals put kagane's Lane into backoff; asura sits on its own Lane. browser := &fakeFetcher{status: 403} p.BrowserFetch = browser for i := range 2 { key := fmt.Sprintf("kagane:s%d", i) seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0) } p.runOnce(context.Background()) // Both lanes wrote their rows: asura the pass it read; kagane the // mid-loop double-refusal exit, which records an empty skip by design // with its two refused counts — the durable row is the whole record, and // the page reads it rather than a snapshot. asura := latestPassFor(t, s, "asura") if asura.Due != 1 || asura.Checked != 1 || asura.GapMS == 0 { t.Fatalf("asura row = %+v, want due 1 checked 1 with the Lane's pace", asura) } kagane := latestPassFor(t, s, "kagane") if kagane.Skip != "" || kagane.Refused != 2 || kagane.Due != 2 || kagane.GapMS == 0 { t.Fatalf("kagane row = %+v, want an empty-skip double-refusal pass with 2 refused", kagane) } // With the refusal now durable, the next pass declines ahead of the loop: // it records the refusing skip and carries the previous pass's figures // forward rather than restating zeroes it never gathered (the carry logic // itself is driven in TestRunLanePassCarryForwardOnlyWhenGapZero). before := kagane p.Now = func() time.Time { return now.Add(time.Minute) } p.runOnce(context.Background()) again := latestPassFor(t, s, "kagane") if again.Skip != SkipRefusing { t.Fatalf("kagane refusing pass skip = %q, want %q", again.Skip, SkipRefusing) } if again.Due != before.Due || again.GapMS != before.GapMS || again.Checked != before.Checked { t.Fatalf("kagane after a skipped pass = due %d gap %d, want the previous pass's %d / %d", again.Due, again.GapMS, before.Due, before.GapMS) } } // latestPassFor reads a Site's newest durable pass row, failing rather than // returning a zero LanePass a caller would assert against by accident. func latestPassFor(t *testing.T, s *store.Store, site string) store.LanePass { t.Helper() pass, ok, err := s.LatestLanePass(site) if err != nil || !ok { t.Fatalf("LatestLanePass(%s): ok=%v err=%v", site, ok, err) } return pass } // countPassRows counts a Site's durable pass rows, for asserting that a pass // records exactly one. func countPassRows(t *testing.T, dbURL, site string) int { t.Helper() db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() var n int if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, site).Scan(&n); err != nil { t.Fatalf("count passes for %s: %v", site, err) } return n } // One durable pass row per exit, with the skip value naming the exit. The // mid-loop browser-unreachable return also writes one row — but with an empty // skip, so the row is stall-shaped (due > 0, checked 0, skip ”), matching the // deliberate absence of a tenth skip value. func TestRunLanePassRecordsEveryExit(t *testing.T) { now := time.UnixMilli(5_000_000) t.Run("paused", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { t.Fatalf("PauseLane: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} p := newTestPoller(t, s, f, now) if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { t.Fatalf("paused pace = %s, want 30m (sleep until the expiry)", pace) } if f.callCount() != 0 { t.Fatalf("fetches while paused = %d, want 0", f.callCount()) } if got := readLatestCheckedAt(t, s, "asura:x"); got != 0 { t.Fatalf("stamp while paused = %d, want 0 (Series stay due and unstamped)", got) } if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused { t.Fatalf("skip = %q, want %q", pass.Skip, SkipPaused) } if got := countPassRows(t, dbURL, "asura"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("refusing", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil { t.Fatalf("SetLaneRefusal: %v", err) } browser := &fakeFetcher{status: 200} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = browser if pace := p.runLanePass(context.Background(), "kagane", false); pace != 10*time.Minute { t.Fatalf("refusing pace = %s, want 10m", pace) } if browser.callCount() != 0 { t.Fatalf("fetches while refusing = %d, want 0", browser.callCount()) } if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipRefusing { t.Fatalf("skip = %q, want %q", pass.Skip, SkipRefusing) } if got := countPassRows(t, dbURL, "kagane"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("sidecar-down", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = browser p.setBrowserDown(now) // a sibling Lane lost Chrome within the backoff window if pace := p.runLanePass(context.Background(), "kagane", false); pace != RefuseBackoff { t.Fatalf("sidecar-down pace = %s, want %s", pace, RefuseBackoff) } if browser.callCount() != 0 { t.Fatalf("fetches with the sidecar down = %d, want 0", browser.callCount()) } if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipSidecarDown { t.Fatalf("skip = %q, want %q", pass.Skip, SkipSidecarDown) } if got := countPassRows(t, dbURL, "kagane"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("no-fetcher", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap { t.Fatalf("no-fetcher pace = %s, want %s", pace, defaultGap) } pass := latestPassFor(t, s, "comix") if pass.Skip != SkipNoFetcher || pass.GapMS != defaultGap.Milliseconds() { t.Fatalf("no-fetcher pass = %+v, want skip %q with its own gap %s", pass, SkipNoFetcher, defaultGap) } if got := countPassRows(t, dbURL, "comix"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("due-query", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) // The due query is the pass's first store read after the gates; making // it fail without touching poll_passes takes its tables away. db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } if _, err := db.Exec(`DROP TABLE bookmarks`); err != nil { t.Fatalf("drop bookmarks: %v", err) } db.Close() p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { t.Fatalf("due-query pace = %s, want %s", pace, defaultGap) } if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipDueQuery { t.Fatalf("skip = %q, want %q", pass.Skip, SkipDueQuery) } if got := countPassRows(t, dbURL, "asura"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("asleep", func(t *testing.T) { s, dbURL := newTestStore(t) for i := 0; i < 3; i++ { key := fmt.Sprintf("kagane:w%d", i) seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli()) } browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = browser if pace := p.runLanePass(context.Background(), "kagane", false); pace != defaultGap { t.Fatalf("asleep pace = %s, want %s", pace, defaultGap) } if browser.callCount() != 0 { t.Fatalf("fetches while Chrome is asleep = %d, want 0", browser.callCount()) } pass := latestPassFor(t, s, "kagane") if pass.Skip != SkipAsleep || pass.Due != 3 || pass.GapMS != defaultGap.Milliseconds() { t.Fatalf("asleep pass = %+v, want skip %q with 3 due and the default gap", pass, SkipAsleep) } if got := countPassRows(t, dbURL, "kagane"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("eligible-count", func(t *testing.T) { s, dbURL := newTestStore(t) // The eligible query shares the due query's tables, so no real store // failure reaches it after a successful due read; the seam is how the // path is driven at all (see Poller.eligibleCount). p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.eligibleCount = func(string) (int, error) { return 0, errors.New("count failed") } if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { t.Fatalf("eligible-count pace = %s, want %s", pace, defaultGap) } if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipEligibleCount { t.Fatalf("skip = %q, want %q", pass.Skip, SkipEligibleCount) } if got := countPassRows(t, dbURL, "asura"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("nothing-eligible", func(t *testing.T) { s, dbURL := newTestStore(t) p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) if pace := p.runLanePass(context.Background(), "asura", false); pace != sites["asura"].Rest { t.Fatalf("nothing-eligible pace = %s, want a full rest %s", pace, sites["asura"].Rest) } if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipNothingEligible { t.Fatalf("skip = %q, want %q", pass.Skip, SkipNothingEligible) } if got := countPassRows(t, dbURL, "asura"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("reached the loop", func(t *testing.T) { s, dbURL := newTestStore(t) const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0) p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { t.Fatalf("loop pace = %s, want %s", pace, defaultGap) } pass := latestPassFor(t, s, "asura") if pass.Skip != "" || pass.Due != 1 || pass.Checked != 1 { t.Fatalf("loop pass = %+v, want an empty skip with 1 due and 1 checked", pass) } if got := countPassRows(t, dbURL, "asura"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) t.Run("browser-unreachable mid-loop", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) browser := &fakeFetcher{status: 200, err: interrupted} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = browser if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap { t.Fatalf("mid-loop pace = %s, want %s", pace, defaultGap) } // One row was still written, and it is stall-shaped by construction: // empty skip with due > 0 and checked 0 because the return precedes // the checked counter. The assertion below pins that shape precisely. pass := latestPassFor(t, s, "comix") if pass.Skip != "" || pass.Unreachable != 1 || pass.Checked != 0 || pass.Due != 1 { t.Fatalf("mid-loop pass = %+v, want empty skip, unreachable 1, checked 0, due 1", pass) } if got := countPassRows(t, dbURL, "comix"); got != 1 { t.Fatalf("pass rows = %d, want exactly 1", got) } }) } // Carry-forward moves verbatim from the in-memory snapshot (issue #141): a // pass that never computed its own gap carries the previous pass's due, gap, // clamped and checked forward; a pass with a gap of its own records its own // figures, zeroes included, beside the skip reason that explains them. func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) { now := time.UnixMilli(5_000_000) t.Run("no gap of its own carries the previous pass's figures", func(t *testing.T) { s, _ := newTestStore(t) seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = &fakeFetcher{body: kaganeAPIFixture, status: 200} p.runLanePass(context.Background(), "kagane", false) first := latestPassFor(t, s, "kagane") if first.Due != 1 || first.Checked != 1 || first.GapMS == 0 { t.Fatalf("first pass = %+v, want a measured pass", first) } if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil { t.Fatalf("SetLaneRefusal: %v", err) } // The pass row is keyed (site, ran_at), so the second pass needs its // own timestamp: a minute later is still inside the refusal. p.Now = func() time.Time { return now.Add(time.Minute) } p.runLanePass(context.Background(), "kagane", false) second := latestPassFor(t, s, "kagane") if second.Skip != SkipRefusing { t.Fatalf("second pass skip = %q, want %q", second.Skip, SkipRefusing) } if second.Due != first.Due || second.Checked != first.Checked || second.GapMS != first.GapMS || second.Clamped != first.Clamped { t.Fatalf("carried pass = %+v, want the first pass's figures %+v", second, first) } }) t.Run("a gap of its own records its own figures", func(t *testing.T) { s, _ := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} p.runLanePass(context.Background(), "comix", false) first := latestPassFor(t, s, "comix") if first.Due != 1 || first.GapMS == 0 { t.Fatalf("first pass = %+v, want a measured pass", first) } // The no-fetcher exit sets its own gap, so no carry-forward: the due // count it never gathered records as zero beside its skip reason. A // minute later gives the second pass its own (site, ran_at) key. p.BrowserFetch = nil p.Now = func() time.Time { return now.Add(time.Minute) } p.runLanePass(context.Background(), "comix", false) second := latestPassFor(t, s, "comix") if second.Skip != SkipNoFetcher { t.Fatalf("second pass skip = %q, want %q", second.Skip, SkipNoFetcher) } if second.GapMS != defaultGap.Milliseconds() { t.Fatalf("second pass gap = %d, want its own %d (not carried)", second.GapMS, defaultGap.Milliseconds()) } if second.Due != 0 || second.Checked != 0 { t.Fatalf("second pass = %+v, want due 0 checked 0 (its own, not the previous pass's)", second) } }) } // The pass row's six outcome counts are the classification the Series read // already makes — never a second taxonomy (issue #141). Success is derived, // never stored: checked minus the five named failures, unreachable excluded // because its exit returns before the checked counter increments. func TestRunLanePassCountsOutcomes(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) const ( successKey = "asura:chronicles-of-the-demon-faction-f886a8af" successURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) seeds := map[string]string{ "asura:refused": "https://asurascans.com/series/refused", "asura:no-chapter": "https://asurascans.com/comics/no-chapter", "asura:unfetchable": "https://evil.example/x", "asura:transport": "https://asurascans.com/series/transport", successKey: successURL, } for key, url := range seeds { seedForCheck(t, s, key, url, 0) } f := &fakeFetcher{perURL: map[string]fakeResponse{ seeds["asura:refused"]: {status: 403}, seeds["asura:no-chapter"]: {body: "", status: 200}, seeds["asura:transport"]: {err: errors.New("dial tcp: refused")}, successURL: {body: asuraSeriesFixture, status: 200}, }} newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) pass := latestPassFor(t, s, "asura") if pass.Skip != "" || pass.Due != 5 || pass.Checked != 5 { t.Fatalf("pass = %+v, want a full pass over 5 due Series", pass) } if pass.Refused != 1 || pass.Unreachable != 0 || pass.NoChapter != 1 || pass.Unfetchable != 1 || pass.Errors != 1 { t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1", pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors) } if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 1 { t.Fatalf("derived success = %d, want 1", success) } // The one genuine read went through: the success Series carries the // fixture's newest chapter, and none of the four failures do. b, ok, err := s.Get(s.OwnerID(), successKey) if err != nil || !ok { t.Fatalf("Get: %v ok=%v", err, ok) } if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { t.Fatalf("success Series latest = %v, want 181 (the fixture's newest)", b.LatestChapterNum) } for _, key := range []string{"asura:refused", "asura:no-chapter", "asura:unfetchable", "asura:transport"} { b, _, err := s.Get(s.OwnerID(), key) if err != nil { t.Fatalf("Get %s: %v", key, err) } if b.LatestChapterNum != nil { t.Fatalf("%s latest = %v, want nil (no chapter survived a failed read)", key, *b.LatestChapterNum) } } } // A 4xx other than the 403 refusal means the page is gone — a sixth outcome // word (issue #164) — while a 5xx stays errors. Both land in the durable row, // so the owner can tell "correct the address" from "the Site is unwell" // without opening a log. func TestRunLanePassSplitsNotFoundFromErrors(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) seeds := map[string]string{ "asura:gone": "https://asurascans.com/series/gone", "asura:unwell": "https://asurascans.com/series/unwell", } for key, url := range seeds { seedForCheck(t, s, key, url, 0) } f := &fakeFetcher{perURL: map[string]fakeResponse{ seeds["asura:gone"]: {status: 404}, seeds["asura:unwell"]: {status: 503}, }} newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) pass := latestPassFor(t, s, "asura") if pass.NotFound != 1 || pass.Errors != 1 { t.Fatalf("not_found=%d errors=%d, want 1 1", pass.NotFound, pass.Errors) } if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 0 { t.Fatalf("derived success = %d, want 0 (both reads failed)", success) } } // A refusal is the Site's mood and outlives our process: the durable stamp a // pass writes is honoured by a freshly constructed poller, which must not // re-probe the Site inside its backoff (issue #141). func TestDurableRefusalSurvivesFreshPoller(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) // Four due Series: the Lane refuses twice, stamps those two, and leaves // the remaining two untried and due — the queue the fresh poller must // find intact once the durable backoff lifts. for i := 0; i < 4; i++ { key := fmt.Sprintf("kagane:s%d", i) seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0) } p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = &fakeFetcher{status: 403} p.runLanePass(context.Background(), "kagane", false) if got := p.BrowserFetch.(*fakeFetcher).callCount(); got != 2 { t.Fatalf("fetches on the refusing pass = %d, want 2 (refused twice)", got) } // A restart: a brand-new poller, no in-memory refusal, same store. The // durable stamp gates the pass. fresh := newTestPoller(t, s, &fakeFetcher{status: 200}, now.Add(14*time.Minute)) fresh.BrowserFetch = &fakeFetcher{status: 403} fresh.runLanePass(context.Background(), "kagane", false) if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 0 { t.Fatalf("fetches by a fresh poller inside the backoff = %d, want 0", got) } if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipRefusing { t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, SkipRefusing) } // Past the backoff the fresh poller probes again — the two Series the // original pass never reached, still due with their stamps untouched. fresh.Now = func() time.Time { return now.Add(16 * time.Minute) } fresh.runLanePass(context.Background(), "kagane", false) if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 2 { t.Fatalf("fetches after the backoff = %d, want 2", got) } for i := 2; i < 4; i++ { if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 { t.Fatalf("kagane:s%d still untried after the backoff", i) } } } // The pause is read ahead of the refusal check: a Lane that is both paused // and inside a refusal backoff records the paused skip value, not the // refusing one. The pause is the owner's order and outranks the Site's mood // (issue #147). func TestPauseGatePrecedesRefusalGate(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) if err := s.SetLaneRefusal("asura", now.Add(10*time.Minute).UnixMilli()); err != nil { t.Fatalf("SetLaneRefusal: %v", err) } if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { t.Fatalf("PauseLane: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} p := newTestPoller(t, s, f, now) if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { t.Fatalf("paused-while-refusing pace = %s, want 30m (the pause's expiry)", pace) } if f.callCount() != 0 { t.Fatalf("fetches while paused and refusing = %d, want 0", f.callCount()) } if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused { t.Fatalf("skip = %q, want %q (the pause outranks the refusal)", pass.Skip, SkipPaused) } } // A pause is a fact about the Site, not about the process: a freshly // constructed poller against a store holding a pause row stays paused until // the expiry, then runs the Lane normally. The restart criterion is the // whole point of writing a row instead of commanding a poller (issue #147). func TestDurablePauseSurvivesFreshPoller(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { t.Fatalf("PauseLane: %v", err) } // A restart: a brand-new poller, no in-memory state, same store. fresh := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) if pace := fresh.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { t.Fatalf("fresh poller's paused pace = %s, want 30m", pace) } if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused { t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, SkipPaused) } // Past the expiry the same fresh poller runs the Lane normally. fresh.Now = func() time.Time { return now.Add(31 * time.Minute) } fresh.runLanePass(context.Background(), "asura", false) if pass := latestPassFor(t, s, "asura"); pass.Skip != "" { t.Fatalf("pass after the expiry skip = %q, want the loop reached", pass.Skip) } if got := readLatestCheckedAt(t, s, "asura:x"); got == 0 { t.Fatal("the Series was not checked after the pause lifted") } } // ResumeLane zeroes the pause and the Lane's next pass finds its full queue // waiting: a pause delays work rather than discarding it, so the due Series // sit unstamped while paused and are all fetched once the pause lifts // (issue #147). func TestResumeLaneRestoresTheQueue(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) for i := 0; i < 3; i++ { seedForCheck(t, s, fmt.Sprintf("asura:s%d", i), "https://asurascans.com/series/x", 0) } if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { t.Fatalf("PauseLane: %v", err) } f := &fakeFetcher{body: asuraSeriesFixture, status: 200} p := newTestPoller(t, s, f, now) p.runLanePass(context.Background(), "asura", false) if f.callCount() != 0 { t.Fatalf("fetches while paused = %d, want 0", f.callCount()) } for i := 0; i < 3; i++ { if got := readLatestCheckedAt(t, s, fmt.Sprintf("asura:s%d", i)); got != 0 { t.Fatalf("asura:s%d stamp while paused = %d, want 0 (due and unstamped)", i, got) } } if err := s.ResumeLane("asura"); err != nil { t.Fatalf("ResumeLane: %v", err) } before := f.callCount() p.runLanePass(context.Background(), "asura", false) if got := f.callCount() - before; got != 3 { t.Fatalf("fetches after resume = %d, want 3 (the full due queue)", got) } for i := 0; i < 3; i++ { if got := readLatestCheckedAt(t, s, fmt.Sprintf("asura:s%d", i)); got == 0 { t.Fatalf("asura:s%d still untried after resume", i) } } } // RecordLanePass prunes in the same call that inserts, so the retention // cutoff the recorder passes is observable in what survives: a row just inside // 14 days behind the poller's clock is kept, one just outside is pruned // (issue #139, #141). func TestPassRetentionCutoffIsFourteenDays(t *testing.T) { s, dbURL := newTestStore(t) // A real-world clock: the seeded rows sit 14 days back, so they must be // positive timestamps or the seed's own prune (ran_at < 0) removes them. now := time.UnixMilli(1_800_000_000_000) kept := now.Add(-14*24*time.Hour + time.Minute).UnixMilli() pruned := now.Add(-14*24*time.Hour - time.Minute).UnixMilli() for _, ranAt := range []int64{kept, pruned} { if err := s.RecordLanePass(store.LanePass{Site: "asura", RanAt: ranAt}, 0); err != nil { t.Fatalf("seed pass at %d: %v", ranAt, err) } } seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now). runLanePass(context.Background(), "asura", false) db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() var n int if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1 AND ran_at = $2`, "asura", pruned).Scan(&n); err != nil { t.Fatalf("count pruned row: %v", err) } if n != 0 { t.Fatalf("row at %d survived, want it pruned (older than 14 days)", pruned) } if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&n); err != nil { t.Fatalf("count passes: %v", err) } if n != 2 { t.Fatalf("passes = %d, want 2 (this pass plus the kept row)", n) } } // stampRecordingFetcher is a fakeFetcher that records the Series' check stamp // at call time — the seam that proves the check stamp is written before the // fetch (issue #146). If the order were swapped, the recorded stamp would be // the pre-pass value and the ordering assertion would fail. type stampRecordingFetcher struct { fakeFetcher store *store.Store site string seriesID string stampAtCall int64 } func (f *stampRecordingFetcher) Get(ctx context.Context, url string) (string, int, error) { ts, err := f.store.LatestCheckedAt(f.site, f.seriesID) if err != nil { return "", 0, err } f.stampAtCall = ts return f.fakeFetcher.Get(ctx, url) } // The check stamp is written before the fetch is attempted: the fake fetcher // records the stamp it sees at call time, and it must already be the pass's // own stamp. The order is load-bearing — a forced request is pending while // force_poll_at > latest_checked_at, so stamping after the fetch would make a // failed forced request sticky — and the assertion fails if it is swapped. func TestCheckStampIsWrittenBeforeFetch(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) seedForCheck(t, s, key, seriesURL, 0) now := time.UnixMilli(7_000_000) rec := &stampRecordingFetcher{ fakeFetcher: fakeFetcher{body: asuraSeriesFixture, status: 200}, store: s, site: "asura", seriesID: "chronicles-of-the-demon-faction-f886a8af", } newTestPoller(t, s, rec, now).runOnce(context.Background()) if rec.stampAtCall != now.UnixMilli() { t.Fatalf("check stamp at fetch time = %d, want %d (the stamp must be written before the fetch)", rec.stampAtCall, now.UnixMilli()) } } // A forced Series whose attempt fails is no longer pending: the check stamp // is written before the fetch, so the first attempt ends the pending state // whatever it returns. A naive implementation (stamp only on success, or // after the fetch) leaves the request sticky and the row due next pass. func TestForcedSeriesSelfClearsOnFailedAttempt(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(7_000_000) const url = "https://asurascans.com/comics/x" // Freshly checked, so only the force flag makes it due. seedForCheck(t, s, "asura:x", url, now.Add(-30*time.Minute).UnixMilli()) if err := s.ForceSeriesPoll("asura", "x", now.UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } due, err := s.DueForLatestCheck("asura", now.Add(-time.Hour).UnixMilli(), -1) if err != nil { t.Fatalf("DueForLatestCheck: %v", err) } if len(due) != 1 || due[0].Key() != "asura:x" || !due[0].Forced { t.Fatalf("forced row not due and flagged before the attempt: %+v", due) } // The attempt fails, but the attempt still happened: the row is stamped // and no longer pending. f := &fakeFetcher{err: errors.New("dial tcp: refused")} newTestPoller(t, s, f, now).runOnce(context.Background()) if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) } due, err = s.DueForLatestCheck("asura", now.Add(-time.Hour).UnixMilli(), -1) if err != nil { t.Fatalf("DueForLatestCheck: %v", err) } if len(due) != 0 { t.Fatalf("failed attempt left the forced row due: %v", due) } } // The refusal backoff is a Lane gate, not a Series gate: a forced Series does // not override a Site that is actively refusing, because hand-forcing a // request into a refusal only makes it worse (issue #146). func TestForcedSeriesDoesNotOverrideRefusalBackoff(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) seedForCheck(t, s, "kagane:w", "https://kagane.to/series/w", 0) if err := s.ForceSeriesPoll("kagane", "w", now.UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } if err := s.SetLaneRefusal("kagane", now.Add(RefuseBackoff).UnixMilli()); err != nil { t.Fatalf("SetLaneRefusal: %v", err) } browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser, Now: func() time.Time { return now }, } p.runOnce(context.Background()) if got := browser.callCount(); got != 0 { t.Fatalf("browser fetches through a refusal backoff = %d, want 0", got) } } // A forced Series wakes a sleeping browser Lane: the wake thresholds exist to // stop the machine waking itself for one unattended check, and a human asking // is not that (issue #146). Below both thresholds the Lane still sleeps when // nothing is forced. func TestForcedSeriesWakesSleepingBrowser(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) // Freshly due: checked two minutes before the rest elapses, so the wait // is far below browserWakeAge and the count is under browserWakeCount. seedForCheck(t, s, "kagane:w1", "https://kagane.to/series/w1", now.Add(-62*time.Minute).UnixMilli()) seedForCheck(t, s, "kagane:w2", "https://kagane.to/series/w2", now.Add(-62*time.Minute).UnixMilli()) browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} p := &Poller{ Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser, Now: func() time.Time { return now }, } p.runOnce(context.Background()) if got := browser.callCount(); got != 0 { t.Fatalf("browser fetches without a forced series = %d, want 0 (Chrome stays asleep)", got) } if err := s.ForceSeriesPoll("kagane", "w1", now.UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } p.runOnce(context.Background()) if got := browser.callCount(); got != 2 { t.Fatalf("browser fetches with a forced series = %d, want 2 (the lane wakes)", got) } } // A forced pass accepts the page as it now stands, so it writes the Cover // through the replace path; an ordinary pass still only fills a blank one // (issue #135). func TestRunOnceForcedPassReplacesExistingCover(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } at := time.UnixMilli(5_000_000) public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"} t.Run("unforced pass leaves the cover alone", func(t *testing.T) { p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: public, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() if got := public.callCount(); got != 0 { t.Fatalf("cover fetch calls = %d, want 0", got) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want { t.Fatalf("Cover = %q, want the first one %q", got.Cover, want) } }) t.Run("forced pass replaces the cover", func(t *testing.T) { if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: public, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() if got := public.callCount(); got != 1 { t.Fatalf("cover fetch calls = %d, want 1", got) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("second")); got.Cover != want { t.Fatalf("Cover = %q, want the second one %q", got.Cover, want) } body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("second"))) if err != nil || !ok { t.Fatalf("CoverByAddress: %v found=%v", err, ok) } if string(body) != "second" { t.Fatalf("stored cover = %q, want second", body) } }) } // Identical artwork re-served is an honest no-op the caller can tell apart // from a replacement: the address comes from the bytes, so the row cannot // change in substance, and the replace call site reports the three outcomes // distinctly (issue #135). func TestRunOnceForcedPassIdenticalBytesLogsNoOp(t *testing.T) { s, _ := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("cover-bytes"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } at := time.UnixMilli(5_000_000) if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want { t.Fatalf("Cover = %q, want unchanged %q", got.Cover, want) } if body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || !ok || string(body) != "cover-bytes" { t.Fatalf("stored cover after no-op: found=%v err=%v", ok, err) } logged := logs.String() if !strings.Contains(logged, "cover unchanged") { t.Fatalf("no-op not reported as unchanged; log:\n%s", logged) } if strings.Contains(logged, "cover replaced") { t.Fatalf("no-op reported as a replacement; log:\n%s", logged) } } // A Series whose sharded Cover file was unlinked out from under it is // repaired by one forced pass: same bytes mean the same address and the file // re-linked (issue #135, story 32). func TestRunOnceForcedPassRelinksUnlinkedCoverFile(t *testing.T) { coverDir := t.TempDir() url := pgtest.URL(t) s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL) if err != nil { t.Fatalf("Open: %v", err) } t.Cleanup(func() { s.Close() }) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("cover-bytes"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } address := store.CoverAddressForBytes([]byte("cover-bytes")) coverPath := filepath.Join(coverDir, filepath.FromSlash(address[:2]+"/"+address[2:4]+"/"+address)) if err := os.Remove(coverPath); err != nil { t.Fatalf("unlink cover file: %v", err) } if _, _, ok, err := s.CoverByAddress(address); err != nil || ok { t.Fatalf("CoverByAddress after unlink = found %v err %v; want missing (file gone)", ok, err) } at := time.UnixMilli(5_000_000) if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() body, contentType, ok, err := s.CoverByAddress(address) if err != nil || !ok { t.Fatalf("CoverByAddress after forced pass: found=%v err=%v; want the file re-linked", ok, err) } if string(body) != "cover-bytes" || contentType != "image/jpeg" { t.Fatalf("re-linked cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType) } } // A forced pass degrades exactly like an ordinary one when the cover sidecar // is unreachable: the fetch is skipped and logged, the chapter poll is // untouched, and the stored Cover is not moved (issue #135, story 6). func TestRunOnceForcedPassWithoutCoverFetcherStillPolls(t *testing.T) { s, _ := newTestStore(t) const ( key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "kagane", SeriesID: seriesID, SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("kagane", seriesID, "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed", []byte("existing"), "image/webp"); err != nil { t.Fatalf("seed cover: %v", err) } at := time.UnixMilli(5_000_000) if err := s.ForceSeriesPoll("kagane", seriesID, at.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) p := &Poller{ Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixtureWithCover, status: 200}, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() got := readBookmark(t, s, key) if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 { t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum) } if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("existing")); got.Cover != want { t.Fatalf("Cover = %q, want the existing one %q untouched", got.Cover, want) } if checked := readLatestCheckedAt(t, s, key); checked != at.UnixMilli() { t.Fatalf("latest_checked_at = %d, want %d", checked, at.UnixMilli()) } if !strings.Contains(logs.String(), "fetch cover") { t.Fatalf("missing skipped-cover log; log:\n%s", logs.String()) } } // A forced replacement strands the previous address, and the poller reclaims // it from the stranded branch: the old sharded file and covers row are both // gone once the pass lands while the new bytes read back (issue #154). The // identical-bytes no-op that follows reclaims nothing: previous == current // there, and a reclamation would delete the Cover the pass just wrote. func TestRunOnceForcedPassReclaimsSupersededCover(t *testing.T) { coverDir := t.TempDir() url := pgtest.URL(t) s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL) if err != nil { t.Fatalf("Open: %v", err) } t.Cleanup(func() { s.Close() }) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } stale := store.CoverAddressForBytes([]byte("first")) stalePath := filepath.Join(coverDir, filepath.FromSlash(stale[:2]+"/"+stale[2:4]+"/"+stale)) at := time.UnixMilli(5_000_000) if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() // The stranded address is gone on disk and in SQL; the new one reads back. if _, err := os.Stat(stalePath); !errors.Is(err, fs.ErrNotExist) { t.Fatalf("stale sharded file after replacement = %v, want fs.ErrNotExist", err) } if _, _, ok, err := s.CoverByAddress(stale); err != nil || ok { t.Fatalf("stale bytes after replacement = found %v err %v, want reclaimed", ok, err) } if body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("second"))); err != nil || !ok || string(body) != "second" { t.Fatalf("new bytes after replacement = found %v err %v, want served", ok, err) } // The identical-bytes pass is an honest no-op and reclaims nothing. if err := s.ForceSeriesPoll("asura", seriesID, at.Add(2*time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } p.runOnce(context.Background()) p.waitCovers() current := store.CoverAddressForBytes([]byte("second")) currentPath := filepath.Join(coverDir, filepath.FromSlash(current[:2]+"/"+current[2:4]+"/"+current)) if _, err := os.Stat(currentPath); err != nil { t.Fatalf("live sharded file after no-op pass = %v, want present", err) } if body, _, ok, err := s.CoverByAddress(current); err != nil || !ok || string(body) != "second" { t.Fatalf("bytes after no-op pass = found %v err %v, want still served", ok, err) } } // A reclamation that fails must not fail the Poll: the failure is logged // against the Series and the replacement still lands, so the stranded bytes // stay reachable for a retry and the owner's act succeeded (issue #154). func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) { coverDir := t.TempDir() url := pgtest.URL(t) s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL) if err != nil { t.Fatalf("Open: %v", err) } t.Cleanup(func() { s.Close() }) const ( key = "asura:chronicles-of-the-fallen-f886a8af" seriesID = "chronicles-of-the-fallen-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-fallen-f886a8af" first = "https://cdn.example/covers/first.jpg" ) if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil { t.Fatalf("seed cover: %v", err) } // Make the stranded file unremovable: a non-empty directory in its place. stale := store.CoverAddressForBytes([]byte("first")) stalePath := filepath.Join(coverDir, filepath.FromSlash(stale[:2]+"/"+stale[2:4]+"/"+stale)) if err := os.Remove(stalePath); err != nil { t.Fatalf("clear file: %v", err) } if err := os.Mkdir(stalePath, 0o755); err != nil { t.Fatalf("replace file with dir: %v", err) } if err := os.WriteFile(filepath.Join(stalePath, "blob"), []byte("x"), 0o644); err != nil { t.Fatalf("fill dir: %v", err) } at := time.UnixMilli(5_000_000) if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } var logs strings.Builder prev := log.Writer() log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(prev) }) p := &Poller{ Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200}, CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}, Now: func() time.Time { return at }, } p.runOnce(context.Background()) p.waitCovers() logged := logs.String() if !strings.Contains(logged, "reclaim cover") { t.Fatalf("failed reclamation not logged; log:\n%s", logged) } if !strings.Contains(logged, "cover replaced") { t.Fatalf("replacement not reported after a failed reclaim; log:\n%s", logged) } got := readBookmark(t, s, key) if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("second")); got.Cover != want { t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want) } } // failureRow reads one Series' failure row as stored, for the poller tests // that must see the table the store API writes (ADR-0016). func failureRow(t *testing.T, dbURL, site, seriesID string) (word string, since int64, found bool) { t.Helper() db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() err = db.QueryRow( `SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`, site, seriesID).Scan(&word, &since) if errors.Is(err, sql.ErrNoRows) { return "", 0, false } if err != nil { t.Fatalf("read failure row %s:%s: %v", site, seriesID, err) } return word, since, true } // The failure row follows the read (ADR-0016): a 404 writes a row with the // outcome word and the pass's time; a second 404 an hour later leaves // failing_since alone — the age is the age of the run of failures, not of // the current word; a good read deletes the row. func TestRunLanePassFailureRowFollowsTheRead(t *testing.T) { s, dbURL := newTestStore(t) now := time.UnixMilli(5_000_000) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) seedForCheck(t, s, key, seriesURL, 0) f := &fakeFetcher{perURL: map[string]fakeResponse{seriesURL: {status: 404}}} p := newTestPoller(t, s, f, now) p.runLanePass(context.Background(), "asura", false) word, since, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af") if !found || word != "not_found" || since != now.UnixMilli() { t.Fatalf("row after 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli()) } // A repeated failure an hour later: the word is unchanged and the stamp // is untouched, so the age keeps meaning the run of failures. p.Now = func() time.Time { return now.Add(2 * time.Hour) } p.runLanePass(context.Background(), "asura", false) word, since, found = failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af") if !found || word != "not_found" || since != now.UnixMilli() { t.Fatalf("row after repeated 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli()) } // A good read ends the failure: the row is gone. f.perURL[seriesURL] = fakeResponse{body: asuraSeriesFixture, status: 200} p.Now = func() time.Time { return now.Add(4 * time.Hour) } p.runLanePass(context.Background(), "asura", false) if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found { t.Fatal("failure row after a good read: still present") } } // Every failure word the pass counts lands in the table under the wire // spelling the worklist left-joins on (C1): no_chapter, unfetchable, errors, // not_found — and a success writes no row. func TestRunLanePassStoresEachFailureWord(t *testing.T) { s, dbURL := newTestStore(t) now := time.UnixMilli(5_000_000) seeds := map[string]string{ "asura:no-chapter": "https://asurascans.com/comics/no-chapter", "asura:unfetchable": "https://evil.example/x", "asura:transport": "https://asurascans.com/series/transport", "asura:gone": "https://asurascans.com/series/gone", "asura:healthy": "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", } for key, url := range seeds { seedForCheck(t, s, key, url, 0) } f := &fakeFetcher{perURL: map[string]fakeResponse{ seeds["asura:no-chapter"]: {body: "", status: 200}, seeds["asura:transport"]: {err: errors.New("dial tcp: refused")}, seeds["asura:gone"]: {status: 404}, seeds["asura:healthy"]: {body: asuraSeriesFixture, status: 200}, }} newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) want := map[string]string{ "no-chapter": "no_chapter", "unfetchable": "unfetchable", "transport": "errors", "gone": "not_found", } for seriesID, word := range want { got, _, found := failureRow(t, dbURL, "asura", seriesID) if !found || got != word { t.Fatalf("failure word for %s = (%q, %v), want %q", seriesID, got, found, word) } } if _, _, found := failureRow(t, dbURL, "asura", "healthy"); found { t.Fatal("success Series gained a failure row") } } // refused and unreachable issue no statement at all (ADR-0016): a challenge // or a lost sidecar is no evidence about any particular Series, so a // pre-seeded row survives both untouched and a Series with no row gains none. func TestRunLanePassRefusalAndUnreachableWriteNothing(t *testing.T) { s, dbURL := newTestStore(t) now := time.UnixMilli(5_000_000) t.Run("refused", func(t *testing.T) { const ( seeded = "asura:held" neverFail = "asura:other" ) seedForCheck(t, s, seeded, "https://asurascans.com/series/held", 0) seedForCheck(t, s, neverFail, "https://asurascans.com/series/other", 0) if err := s.RecordSeriesFailure("asura", "held", "not_found", 7_000); err != nil { t.Fatalf("seed failure row: %v", err) } f := &fakeFetcher{perURL: map[string]fakeResponse{ "https://asurascans.com/series/held": {status: 403}, "https://asurascans.com/series/other": {status: 403}, }} newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) word, since, found := failureRow(t, dbURL, "asura", "held") if !found || word != "not_found" || since != 7_000 { t.Fatalf("pre-seeded row after refusal = (%q, %d, %v), want (not_found, 7000, true)", word, since, found) } if _, _, found := failureRow(t, dbURL, "asura", "other"); found { t.Fatal("row appeared for a refused Series with none: refusal must write nothing") } }) t.Run("unreachable mid-loop", func(t *testing.T) { const ( seeded = "comix:c" neverFail = "comix:d" ) seedForCheck(t, s, seeded, "https://comix.to/title/c", 0) seedForCheck(t, s, neverFail, "https://comix.to/title/d", 0) if err := s.RecordSeriesFailure("comix", "c", "errors", 7_000); err != nil { t.Fatalf("seed failure row: %v", err) } interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) browser := &fakeFetcher{status: 200, err: interrupted} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = browser p.runLanePass(context.Background(), "comix", false) word, since, found := failureRow(t, dbURL, "comix", "c") if !found || word != "errors" || since != 7_000 { t.Fatalf("pre-seeded row after unreachable = (%q, %d, %v), want (errors, 7000, true)", word, since, found) } if _, _, found := failureRow(t, dbURL, "comix", "d"); found { t.Fatal("row appeared for an unreachable Series with none: unreachable must write nothing") } }) } // A Forced Poll request clears nothing — it only stamps the request — and a // forced Poll that then reads the page clears the row through the ordinary // success path, with no forced branch in the code. func TestRunLanePassForcedPollClearsThroughSuccess(t *testing.T) { s, dbURL := newTestStore(t) now := time.UnixMilli(5_000_000) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) seedForCheck(t, s, key, seriesURL, 0) if err := s.RecordSeriesFailure("asura", "chronicles-of-the-demon-faction-f886a8af", "not_found", 7_000); err != nil { t.Fatalf("seed failure row: %v", err) } if err := s.ForceSeriesPoll("asura", "chronicles-of-the-demon-faction-f886a8af", now.Add(time.Hour).UnixMilli()); err != nil { t.Fatalf("ForceSeriesPoll: %v", err) } // The request alone cleared nothing. if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); !found { t.Fatal("failure row gone after only a Forced Poll request") } newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now). runLanePass(context.Background(), "asura", false) if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found { t.Fatal("failure row after a forced Poll that read the page: still present") } } // The due query does not join the failure table (AC7): a failing Series is // polled at the same pace as any other, so its failure age keeps meaning what // the worklist reads it as. func TestPollFailureDoesNotChangePacing(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) seedForCheck(t, s, "asura:failing", "https://asurascans.com/series/failing", 0) seedForCheck(t, s, "asura:healthy", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) f := &fakeFetcher{perURL: map[string]fakeResponse{ "https://asurascans.com/series/failing": {status: 404}, "https://asurascans.com/series/healthy": {body: asuraSeriesFixture, status: 200}, }} p := newTestPoller(t, s, f, now) p.runLanePass(context.Background(), "asura", false) if got := f.callCount(); got != 2 { t.Fatalf("first pass fetched %d series, want both (the failure row must not exclude the failing one)", got) } // After the rest both are due again: the row the first pass wrote changed // nothing about when the failing Series is polled. p.Now = func() time.Time { return now.Add(2 * time.Hour) } p.runLanePass(context.Background(), "asura", false) if got := f.callCount(); got != 4 { t.Fatalf("second pass fetched %d series, want both again", got) } } // readSiteCompletedAt reads the poller's site_completed_at column directly: // it is a poller fact with no client-visible getter, and #170 is the surface // that will read it. func readSiteCompletedAt(t *testing.T, dbURL, site, seriesID string) int64 { t.Helper() db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() var at int64 if err := db.QueryRow( `SELECT site_completed_at FROM series WHERE site = $1 AND series_id = $2`, site, seriesID).Scan(&at); err != nil { t.Fatalf("read site_completed_at %s:%s: %v", site, seriesID, err) } return at } // seedSiteCompletedAt writes the column directly, for the refused/unreachable // tests that need a pre-existing stamp. func seedSiteCompletedAt(t *testing.T, dbURL, site, seriesID string, at int64) { t.Helper() db, err := sql.Open("pgx", dbURL) if err != nil { t.Fatalf("open %s: %v", dbURL, err) } defer db.Close() if _, err := db.Exec( `UPDATE series SET site_completed_at = $3 WHERE site = $1 AND series_id = $2`, site, seriesID, at); err != nil { t.Fatalf("seed site_completed_at %s:%s: %v", site, seriesID, err) } } // A completed page stamps site_completed_at with the pass's own clock; the // same page an hour later is due again but must not move the stamp — the age // #170 prints is "since the Site first said so", not the age of the last // Poll. The transition is zero-versus-nonzero, so the exact value asserted // here is load-bearing. func TestRunOnceSiteCompletedStampsOnce(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) completed := asuraSeriesFixture + asuraCompletedFixture at := time.UnixMilli(5_000_000) seedForCheck(t, s, key, seriesURL, at.Add(-time.Hour).UnixMilli()) newTestPoller(t, s, &fakeFetcher{body: completed, status: 200}, at).runOnce(context.Background()) if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != at.UnixMilli() { t.Fatalf("site_completed_at after completed poll = %d, want %d", got, at.UnixMilli()) } hourLater := at.Add(time.Hour) newTestPoller(t, s, &fakeFetcher{body: completed, status: 200}, hourLater).runOnce(context.Background()) if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != at.UnixMilli() { t.Fatalf("site_completed_at after a second completed poll = %d, want the original stamp %d", got, at.UnixMilli()) } } // An ongoing page is the inverse transition: a never-hinted Series stays zero // (an ordinary Poll of an ongoing Series writes nothing), and a hinted one is // zeroed — the Site no longer says completed, so the claim must not outlive // the evidence. func TestRunOnceOngoingPageClearsOrSkipsSiteCompleted(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) ongoing := asuraSeriesFixture + asuraOngoingFixture at := time.UnixMilli(5_000_000) seedForCheck(t, s, key, seriesURL, at.Add(-time.Hour).UnixMilli()) newTestPoller(t, s, &fakeFetcher{body: ongoing, status: 200}, at).runOnce(context.Background()) if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != 0 { t.Fatalf("site_completed_at after ongoing poll of a never-hinted series = %d, want 0", got) } seedSiteCompletedAt(t, dbURL, "asura", seriesID, at.UnixMilli()) hourLater := at.Add(time.Hour) newTestPoller(t, s, &fakeFetcher{body: ongoing, status: 200}, hourLater).runOnce(context.Background()) if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != 0 { t.Fatalf("site_completed_at after ongoing poll of a hinted series = %d, want 0", got) } } // A refused or unreachable read reaches nothing after checkOne's error switch: // a challenge is not evidence about the work, so a pre-seeded stamp must // survive both. func TestRunOnceRefusedOrUnreachableLeavesSiteCompletedUntouched(t *testing.T) { now := time.UnixMilli(5_000_000) t.Run("refused", func(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) seedForCheck(t, s, key, seriesURL, 0) seedSiteCompletedAt(t, dbURL, "asura", seriesID, now.UnixMilli()) newTestPoller(t, s, &fakeFetcher{status: 403}, now).runOnce(context.Background()) if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != now.UnixMilli() { t.Fatalf("site_completed_at after refused poll = %d, want the pre-seeded %d", got, now.UnixMilli()) } }) t.Run("mid-loop unreachable", func(t *testing.T) { s, dbURL := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) seedSiteCompletedAt(t, dbURL, "comix", "c", now.UnixMilli()) interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) browser := &fakeFetcher{status: 200, err: interrupted} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = browser p.runLanePass(context.Background(), "comix", false) if got := readSiteCompletedAt(t, dbURL, "comix", "c"); got != now.UnixMilli() { t.Fatalf("site_completed_at after unreachable poll = %d, want the pre-seeded %d", got, now.UnixMilli()) } }) } // The regression AC2's second sentence exists to prevent: the completed write // must not ride the chapter setter, so a completed page whose chapter number // is unchanged — checkOne's equality early return — still stamps the column. func TestRunOnceSiteCompletedWritesDespiteUnchangedChapter(t *testing.T) { s, dbURL := newTestStore(t) const ( key = "asura:chronicles-of-the-demon-faction-f886a8af" seriesID = "chronicles-of-the-demon-faction-f886a8af" seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" ) completed := asuraSeriesFixture + asuraCompletedFixture at := time.UnixMilli(5_000_000) seedForCheck(t, s, key, seriesURL, 0) // The page parses Chapter 181 as the latest; store the same number so the // equality early return fires and only the completed write can run. if err := s.SetLatestChapter("asura", seriesID, "Chapter 181", 181); err != nil { t.Fatalf("seed latest chapter: %v", err) } newTestPoller(t, s, &fakeFetcher{body: completed, status: 200}, at).runOnce(context.Background()) if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != at.UnixMilli() { t.Fatalf("site_completed_at after unchanged-chapter completed poll = %d, want %d", got, at.UnixMilli()) } } // The stall judgement (issue #171) selects exactly the row the mid-loop // browser loss writes — due > 0, none checked, no skip value, and no refusal. // The no-refusal clause is AC6's amendment: the twice-refused break happens // inside the pass loop, not on an early return, so a newly-gated Site would // otherwise send two messages. A pause is excluded by Skip == "" (SkipPaused): // the owner's own act is not reported back (AC10). func TestFaultsFromStall(t *testing.T) { now := time.UnixMilli(5_000_000) tests := []struct { name string pass store.LanePass want int }{ { name: "stall-shaped pass is a stall", pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 2, Checked: 0, Skip: "", Refused: 0}, want: 1, }, { name: "nothing due is not a stall", pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 0, Checked: 0, Skip: "", Refused: 0}, want: 0, }, { name: "a pass that checked is not a stall", pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 1, Checked: 1, Skip: "", Refused: 0}, want: 0, }, { name: "paused is the owner's own act, not a stall", pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 1, Checked: 0, Skip: SkipPaused, Refused: 0}, want: 0, }, { name: "refusing has a skip value, not a stall", pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 1, Checked: 0, Skip: SkipRefusing, Refused: 0}, want: 0, }, { name: "stalled and refused fires nothing (AC6's collision)", pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 2, Checked: 0, Skip: "", Refused: 1}, want: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { faults := FaultsFrom(FaultInput{Passes: []store.LanePass{tt.pass}}, now) if got := len(faults); got != tt.want { t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want) } for _, f := range faults { if f.Condition != ConditionStall || f.Site != "comix" || f.Since != now.UnixMilli() { t.Fatalf("fault = %+v, want stall on comix since the pass time", f) } } }) } } // The stall fires exactly once while it holds, and again after it lifts and // returns: the suppression row is the whole state, so the second stall-shaped // pass is the same episode, a healthy pass in between clears the row, and the // next stall is a new episode that sends again. func TestOwnerNoticeStallFiresOncePerEpisode(t *testing.T) { now := time.UnixMilli(5_000_000) s, _ := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) notifier := &fakeNotifier{} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.Now = func() time.Time { return now } p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} p.Notify = notifier // First stall-shaped pass: the episode begins, the owner is told once. p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 1 { t.Fatalf("notices after first stall = %d, want 1", got) } if f := notifier.fault(0); f.Condition != ConditionStall || f.Site != "comix" || f.Since != now.UnixMilli() { t.Fatalf("fault = %+v, want stall on comix since the pass time", f) } if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || !sent { t.Fatalf("NoticeSent after first stall = %v, %v; want true, nil", sent, err) } // A second stall-shaped pass — the series was stamped, so re-seed it — is // the same episode: no second message. seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) now = now.Add(RefuseBackoff + time.Minute) p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 1 { t.Fatalf("notices after second stall = %d, want 1 (one per episode)", got) } // A healthy pass in between lifts the stall: the episode ends and the // suppression row is cleared. seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} now = now.Add(RefuseBackoff + time.Minute) p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 1 { t.Fatalf("notices after healthy pass = %d, want 1 (a healthy pass sends nothing)", got) } if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { t.Fatalf("NoticeSent after healthy pass = %v, %v; want false, nil (row cleared)", sent, err) } // The next stall is a new episode: the owner is told again. seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} now = now.Add(RefuseBackoff + time.Minute) p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 2 { t.Fatalf("notices after the next stall = %d, want 2 (again after it lifts and returns)", got) } } // A paused Lane sends nothing: the pause is the owner's own act, and the // skip value already keeps it out of the stall test (AC10). func TestOwnerNoticePausedSendsNothing(t *testing.T) { now := time.UnixMilli(5_000_000) s, _ := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) if err := s.PauseLane("comix", now.Add(30*time.Minute).UnixMilli()); err != nil { t.Fatalf("PauseLane: %v", err) } notifier := &fakeNotifier{} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.BrowserFetch = &fakeFetcher{status: 200} p.Notify = notifier p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 0 { t.Fatalf("notices while paused = %d, want 0", got) } if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { t.Fatalf("NoticeSent while paused = %v, %v; want false, nil", sent, err) } } // A failed POST is logged and the notified stamp left unset, so the next pass // retries while the condition holds. No queue, no backoff — the condition is // durable, and the suppression row is the only state. func TestOwnerNoticeFailedSendRetriesNextPass(t *testing.T) { now := time.UnixMilli(5_000_000) s, _ := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) notifier := &fakeNotifier{err: errors.New("webhook down")} p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.Now = func() time.Time { return now } p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} p.Notify = notifier p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 1 { t.Fatalf("send attempts = %d, want 1", got) } if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { t.Fatalf("NoticeSent after failed send = %v, %v; want false, nil (stamp left unset)", sent, err) } // The webhook recovers: the next stall-shaped pass delivers the one // message the episode was owed. seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) notifier.err = nil now = now.Add(RefuseBackoff + time.Minute) p.runLanePass(context.Background(), "comix", false) if got := notifier.callCount(); got != 2 { t.Fatalf("send attempts after recovery = %d, want 2 (retried next pass)", got) } if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || !sent { t.Fatalf("NoticeSent after recovery = %v, %v; want true, nil", sent, err) } } // Nil notifier means the whole path is off: a stall-shaped pass panics // nothing and stamps no row, yet the clear still runs, so a deployment that // turns the webhook off does not leave stale rows that suppress the first // real notice after it is turned back on. func TestOwnerNoticeNilNotifierStillClears(t *testing.T) { now := time.UnixMilli(5_000_000) s, _ := newTestStore(t) seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) // A stall-shaped pass with no notifier configured: nothing panics and no // row is stamped — a missing webhook is a silent, complete off switch. p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) p.Now = func() time.Time { return now } p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} p.runLanePass(context.Background(), "comix", false) if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { t.Fatalf("NoticeSent with nil notifier = %v, %v; want false, nil", sent, err) } // A stale row from before the webhook was turned off must not survive a // healthy pass: the clear runs even though no notifier is configured. if err := s.MarkNoticeSent(ConditionStall, "comix", now.UnixMilli()); err != nil { t.Fatalf("seed notice row: %v", err) } seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) now = now.Add(RefuseBackoff + time.Minute) p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} p.runLanePass(context.Background(), "comix", false) if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { t.Fatalf("NoticeSent after healthy pass with nil notifier = %v, %v; want false, nil (cleared)", sent, err) } }