sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: one Poll Lane per Site, replacing the shared pace (#100) (#106)

Browse Source 
Closes #100.

Each Site runs its own Poll Lane: an independent goroutine with its own rest
and pace from the registry (`backend/internal/latest/sites.go`), replacing the
shared cooldown/interval/stagger/batch configuration. Rest (1h, all six Sites
including the browser trio) is enforced by the due query's WHERE clause; the
Lane sleeps its effective gap between fetches — the registry 10s, or
rest/eligible when a Site holds enough Series, floored at 1s with a
Site-naming warning when the floor engages.

Lane-local failure handling:
- Two challenge-held results stop that Site's Lane for 15m; the probes keep
  their stamp, untried Series stay due.
- A lost browser sets a shared Poller flag: the other browser Lanes skip
  their passes for the same 15m (no stamp-per-pass-per-Lane on a dead tab),
  then decay and probe again.
- Browser wake gate preserved (5 due, or one waiting 15m, ADR-0005); one tab
  shared by the three browser Sites; "browser lane behind by X" logged every
  pass.
- Cover work (healing a stored source URL and filling a blank from the series
  page) runs in the background so a slow CDN cannot consume a Lane's gap.

Removed: `LATEST_CHAPTER_POLL_{COOLDOWN,BROWSER_COOLDOWN,INTERVAL,BATCH,STAGGER}`
and the 6h browser rest. Only `LATEST_CHAPTER_POLL_ENABLED` remains; DEPLOY.md
documents the exact `.env` edit. ADR-0010 records the decisions.

Reviewed-on: #106
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #106.
This commit is contained in:
Sulthan Zaki
2026-08-16 13:16:19 +07:00
committed by sulthan
parent ddbd57070d
commit 3303a55b20
22 changed files with 3645 additions and 1884 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/poller_test.go
+405 -114
View File
@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"database/sql"
"errors"
"fmt"
"log"
"os"
"strings"
@@ -156,19 +157,16 @@ func (f *fakeBytesCoverFetcher) callCount() int {
return len(f.calls)
}
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
// instantly and deterministically.
// newTestPoller wires a poller with a frozen clock. Rest and gap come from the
// Site registry, so tests seed checked_at relative to the one-hour rest; the
// round entry point (runOnce) runs every Lane back to back with no real
// pacing, so tests stay instant and deterministic.
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
t.Helper()
return &Poller{
Store: s,
Fetch: f,
Now: func() time.Time { return at },
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute,
Stagger: 0,
Batch: 14,
Store: s,
Fetch: f,
Now: func() time.Time { return at },
}
}
@@ -208,9 +206,10 @@ func TestRunOncePrefetchesPublicCover(t *testing.T) {
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got)
@@ -241,9 +240,10 @@ func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
@@ -350,23 +350,6 @@ func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
}
}
func TestRunOnceRespectsBatchLimit(t *testing.T) {
s, _ := newTestStore(t)
for i := 0; i < 20; i++ {
key := "asura:s" + string(rune('a'+i))
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
}
f := &fakeFetcher{body: "", status: 200}
p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
p.Batch = 5
p.runOnce(context.Background())
if got := f.callCount(); got != 5 {
t.Fatalf("fetched %d series, want 5 (batch limit)", got)
}
}
// The point of the split (ADR-0003): a series referenced by several bookmarks
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
// series when two readers track it (issue #22).
@@ -409,8 +392,8 @@ func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
}
}
// One unreachable series must not abandon the rest of the batch.
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
// One unreachable series must not abandon the rest of the Lane.
func TestRunOnceOneBadSeriesDoesNotStallLane(t *testing.T) {
s, _ := newTestStore(t)
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
for _, k := range keys {
@@ -436,7 +419,9 @@ func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
}
}
func TestRunLogsCooldowns(t *testing.T) {
// Pace now lives in the registry, not config: Run logs the lane defaults so a
// deployment can see what the poller is doing without reading the source.
func TestRunLogsLaneDefaults(t *testing.T) {
var logs strings.Builder
previous := log.Writer()
log.SetOutput(&logs)
@@ -444,21 +429,19 @@ func TestRunLogsCooldowns(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
(&Poller{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: time.Hour,
}).Run(ctx)
(&Poller{Now: func() time.Time { return time.Now() }}).Run(ctx)
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
!strings.Contains(got, "browser-cooldown=6h") {
t.Fatalf("startup log = %q, want both cooldowns", got)
got := logs.String()
for _, want := range []string{"6 lanes", "rest=1h0m0s", "gap=10s"} {
if !strings.Contains(got, want) {
t.Fatalf("startup log = %q, want %q", got, want)
}
}
}
// The cooldown is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the tick interval independent of it.
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
// The rest is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the Lane's sleep independent of it.
func TestRunOnceHonoursRestAcrossPasses(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/x"
seedForCheck(t, s, "asura:x", url, 0)
@@ -471,56 +454,18 @@ func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
if got := f.callCount(); got != 1 {
t.Fatalf("first pass fetched %d, want 1", got)
}
// Same instant, and again 59 minutes later: both inside the 1h cooldown.
// Same instant, and again 59 minutes later: both inside the 1h rest.
p.runOnce(context.Background())
p.Now = func() time.Time { return now.Add(59 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times inside the cooldown, want 1", got)
t.Fatalf("fetched %d times inside the rest, want 1", got)
}
// Past the cooldown, it is due again.
// Past the rest, it is due again.
p.Now = func() time.Time { return now.Add(61 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 2 {
t.Fatalf("fetched %d times after the cooldown, want 2", got)
}
}
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
s, _ := newTestStore(t)
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
hour := time.Hour
now := time.Unix(2*int64(hour/time.Second), 0)
tls := &fakeFetcher{status: 200}
browser := &fakeFetcher{status: 200}
p := &Poller{
Store: s,
Fetch: tls,
BrowserFetch: browser,
Now: func() time.Time { return now },
Cooldown: hour,
BrowserCooldown: 6 * hour,
Batch: 10,
}
p.runOnce(context.Background())
if got := tls.callCount(); got != 1 {
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
}
if got := browser.callCount(); got != 0 {
t.Fatalf("browser fetches after 2h = %d, want 0", got)
}
now = time.Unix(7*int64(hour/time.Second), 0)
p.runOnce(context.Background())
if got := tls.callCount(); got != 2 {
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
}
if got := browser.callCount(); got != 1 {
t.Fatalf("browser fetches after 7h = %d, want 1", got)
t.Fatalf("fetched %d times after the rest, want 2", got)
}
}
@@ -665,17 +610,18 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s,
Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
Store: s,
Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
if len(f.calls) != 0 {
t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
}
if got := readLatestCheckedAt(t, s, "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); got != 0 {
t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got)
}
}
// novelfull without a browser is not skipped outright: its challenge is a
@@ -699,11 +645,10 @@ func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if len(tlsF.calls) != 1 {
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
@@ -741,9 +686,7 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
@@ -789,11 +732,10 @@ func TestComixUsesBrowserFetcher(t *testing.T) {
p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF,
CoverFetch: covers, CoverBytesFetch: tlsCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if len(tlsF.calls) != 0 {
t.Errorf("TLS fetcher was called for comix: %v", tlsF.calls)
@@ -831,15 +773,16 @@ func TestComixSkippedWhenNoBrowserFetcher(t *testing.T) {
f := &fakeFetcher{body: comixSeriesFixture, status: 200}
p := &Poller{
Store: s, Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
if len(f.calls) != 0 {
t.Errorf("TLS fetcher was called for comix: %v", f.calls)
}
if got := readLatestCheckedAt(t, s, "comix:n8we-dungeons-and-crayons"); got != 0 {
t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got)
}
}
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
@@ -860,10 +803,10 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
if err != nil || !ok {
@@ -898,11 +841,13 @@ func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return at },
}
p.runOnce(context.Background())
p.waitCovers()
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
p.waitCovers()
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
@@ -927,9 +872,10 @@ func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
p.waitCovers()
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
@@ -960,9 +906,10 @@ func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
@@ -985,9 +932,10 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
@@ -1010,9 +958,10 @@ func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
CoverBytesFetch: publicCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
if got := publicCovers.callCount(); got != 1 {
t.Fatalf("public cover fetch calls = %d, want 1", got)
@@ -1128,10 +1077,10 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
p := &Poller{
Store: s, Fetch: page, BrowserFetch: page,
CoverBytesFetch: public, CoverFetch: browser,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
p.runOnce(context.Background())
p.waitCovers()
got := readBookmark(t, s, tc.key)
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
@@ -1181,7 +1130,7 @@ func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: public,
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
Now: func() time.Time { return at },
}
p.runOnce(context.Background())
at = at.Add(2 * time.Hour)
@@ -1216,9 +1165,10 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: public,
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
Now: func() time.Time { return at },
}
p.runOnce(context.Background())
p.waitCovers()
if got := readBookmark(t, s, key); got.Cover != "" {
t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover)
}
@@ -1231,6 +1181,7 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
public.contentType = "image/jpeg"
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
p.waitCovers()
if got := public.callCount(); got != 2 {
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
@@ -1264,9 +1215,10 @@ func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")},
Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
p.waitCovers()
got := readBookmark(t, s, key)
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
@@ -1289,3 +1241,342 @@ const kaganeAPIFixtureWithCover = `
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
`
func TestEffectiveGap(t *testing.T) {
asura := sites["asura"]
tests := []struct {
eligible int
want time.Duration
clamped bool
}{
{0, 10 * time.Second, false},
{5, 10 * time.Second, false},
{360, 10 * time.Second, false},
{720, 5 * time.Second, false},
{3600, time.Second, false},
{4000, time.Second, true},
}
for _, tt := range tests {
got, clamped := effectiveGap(asura, tt.eligible)
if got != tt.want || clamped != tt.clamped {
t.Errorf("effectiveGap(asura, %d) = (%s, %v), want (%s, %v)",
tt.eligible, got, clamped, tt.want, tt.clamped)
}
}
}
// The due query orders by sharedness first, then age: a series two readers
// track is polled before a single-reader series that has waited far longer
// (ADR-0003, issue #100).
func TestRunOnceOrdersBySharednessThenAge(t *testing.T) {
s, url := newTestStore(t)
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
if err != nil {
t.Fatalf("Open second reader: %v", err)
}
t.Cleanup(func() { other.Close() })
now := time.UnixMilli(5_000_000)
// popular: two readers, due for 2 minutes. loner: one reader, due for 23
// minutes. Popularity must win — the loner waited far longer.
const (
popularKey = "asura:popular"
lonerKey = "asura:loner"
popularURL = "https://asurascans.com/comics/popular"
lonerURL = "https://asurascans.com/comics/loner"
)
seedForCheck(t, s, popularKey, popularURL, now.Add(-62*time.Minute).UnixMilli())
seedForCheck(t, s, lonerKey, lonerURL, 0)
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
Key: popularKey, Site: "asura", SeriesID: "popular", UpdatedAt: 2000,
}); err != nil {
t.Fatalf("seed second reader: %v", err)
}
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).runOnce(context.Background())
if len(f.calls) != 2 {
t.Fatalf("fetches = %d, want 2", len(f.calls))
}
if f.calls[0] != popularURL {
t.Fatalf("first fetch = %q, want the shared series %q", f.calls[0], popularURL)
}
}
// Two refusals in one pass stop the Lane: the remaining Series stay unstamped
// and due, and the Lane backs off for refuseBackoff before trying the Site
// again. One hostile Site burns only its own Lane's budget (issue #100).
func TestRunOnceSiteRefusalSkipsRestOfLaneAndBacksOff(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
const n = 4
for i := 0; i < n; i++ {
key := fmt.Sprintf("kagane:s%d", i)
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0)
}
browser := &fakeFetcher{status: 403}
tls := &fakeFetcher{status: 200}
// An asura Series sits on its own Lane: it must still be polled while
// kagane's Lane burns its budget on refusals (story 4).
seedForCheck(t, s, "asura:still-polls", "https://asurascans.com/series/still-polls", 0)
p := &Poller{
Store: s, Fetch: tls, BrowserFetch: browser,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
if got := browser.callCount(); got != 2 {
t.Fatalf("browser fetches after first pass = %d, want 2 (refused twice)", got)
}
if got := tls.callCount(); got != 1 {
t.Fatalf("asura fetches after first pass = %d, want 1 (its Lane is independent)", got)
}
stamped := 0
for i := 0; i < n; i++ {
if readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)) == now.UnixMilli() {
stamped++
}
}
if stamped != 2 {
t.Fatalf("stamped series = %d, want 2; the remaining two stay due", stamped)
}
// Inside the backoff window nothing is attempted.
p.Now = func() time.Time { return now.Add(14 * time.Minute) }
p.runOnce(context.Background())
if got := browser.callCount(); got != 2 {
t.Fatalf("browser fetches inside backoff = %d, want still 2", got)
}
// Past the backoff the Lane resumes and the two untried Series are polled.
p.Now = func() time.Time { return now.Add(16 * time.Minute) }
p.runOnce(context.Background())
if got := browser.callCount(); got != 4 {
t.Fatalf("browser fetches after backoff = %d, want 4", got)
}
for i := 0; i < n; i++ {
if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 {
t.Fatalf("kagane:s%d still untried after backoff", i)
}
}
}
// Below five due Series with none waiting long, a browser Lane leaves Chrome
// asleep; five due, or one waiting browserWakeAge, wakes it (ADR-0005).
func TestBrowserLaneWakeThresholds(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
// Freshly due: checked two minutes before the rest elapses, so the wait
// is far below browserWakeAge.
seed := func(i int) {
key := fmt.Sprintf("kagane:w%d", i)
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli())
}
for i := 0; i < 3; i++ {
seed(i)
}
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
if got := browser.callCount(); got != 0 {
t.Fatalf("browser fetches with 3 freshly-due series = %d, want 0 (Chrome stays asleep)", got)
}
// 5 due crosses the count threshold.
for i := 3; i < 5; i++ {
seed(i)
}
p.runOnce(context.Background())
if got := browser.callCount(); got != 5 {
t.Fatalf("browser fetches with 5 due series = %d, want 5", got)
}
// A single long-neglected series wakes the browser by age alone.
seedForCheck(t, s, "kagane:ancient", "https://kagane.to/series/ancient", 0)
p.runOnce(context.Background())
if got := browser.callCount(); got != 6 {
t.Fatalf("browser fetches with one ancient series = %d, want 6", got)
}
}
// When one browser Lane loses the sidecar, the round's remaining browser
// Lanes are skipped: every fetch would fail anyway, and their Series must not
// burn their stamps on a dead Chrome (issue #100).
func TestRunOnceUnreachableBrowserStopsBrowserLanes(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0)
seedForCheck(t, s, "kagane:k", "https://kagane.to/series/k", 0)
seedForCheck(t, s, "novelfull:n", "https://novelfull.com/n.html", 0)
interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart"))
browser := &fakeFetcher{status: 200, err: interrupted}
p := &Poller{
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
// Sorted lane order: comix, kagane, novelfull. Only comix attempted.
if got := browser.callCount(); got != 1 {
t.Fatalf("browser fetches = %d, want 1 (only the first browser lane)", got)
}
if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() {
t.Fatalf("comix stamp = %d, want %d", got, now.UnixMilli())
}
for _, key := range []string{"kagane:k", "novelfull:n"} {
if got := readLatestCheckedAt(t, s, key); got != 0 {
t.Fatalf("%s stamp = %d, want 0 (untried)", key, got)
}
}
// The shared flag decays after refuseBackoff: the next round probes
// again. comix's Series is resting (stamped last round), so the probe
// falls to kagane — the only Lane with something due — and its fresh
// loss re-gates the Lanes behind it.
p.Now = func() time.Time { return now.Add(16 * time.Minute) }
p.runOnce(context.Background())
if got := browser.callCount(); got != 2 {
t.Fatalf("browser fetches after backoff decay = %d, want 2 (kagane probes again)", got)
}
if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() {
t.Fatalf("comix stamp after decay = %d, want %d (resting, untouched)", got, now.UnixMilli())
}
if got := readLatestCheckedAt(t, s, "kagane:k"); got != now.Add(16*time.Minute).UnixMilli() {
t.Fatalf("kagane stamp after decay = %d, want %d (the probe)", got, now.Add(16*time.Minute).UnixMilli())
}
if got := readLatestCheckedAt(t, s, "novelfull:n"); got != 0 {
t.Fatalf("novelfull stamp after decay = %d, want 0 (gated by kagane's fresh loss)", got)
}
}
// A browser Lane that cannot keep up logs the backlog every pass, so the
// decision to give browser Sites more pages is measured, not guessed.
func TestRunOnceLogsBrowserLaneBehind(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
// Checked three rests ago: two rests' worth of wait past the due moment.
seedForCheck(t, s, "kagane:old", "https://kagane.to/series/old", now.Add(-3*time.Hour).UnixMilli())
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
p := &Poller{
Store: s, Fetch: &fakeFetcher{status: 200},
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
if got := logs.String(); !strings.Contains(got, "latest poll kagane: browser lane behind by 1h0m0s") {
t.Fatalf("behind log = %q, want it to name kagane and the backlog", got)
}
}
// gatedCoverFetcher blocks every Fetch on a gate, so a test can hold a cover
// heal in flight and prove a Lane does not wait for it.
type gatedCoverFetcher struct {
inner *fakeBytesCoverFetcher
gate chan struct{}
started chan struct{}
once sync.Once
}
func (g *gatedCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
g.once.Do(func() { g.started <- struct{}{} })
<-g.gate
return g.inner.Fetch(ctx, sourceURL)
}
// Cover heals run in the background: a heal stuck on a slow CDN must not
// delay the Lane's next Series-page Poll, or a large import with many blanks
// would make every Latest Chapter go stale (issue #100).
func TestRunOnceCoverFetchDoesNotDelayNextPoll(t *testing.T) {
s, dbURL := newTestStore(t)
// Two Series whose legacy cover sources still need healing.
for i := 0; i < 2; i++ {
key := fmt.Sprintf("asura:cover-%d", i)
seriesID := fmt.Sprintf("cover-%d", i)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: seriesID,
SeriesURL: "https://asurascans.com/comics/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "asura", seriesID, fmt.Sprintf("https://cdn.example/covers/%d.jpg", i))
}
pages := &fakeFetcher{body: asuraSeriesFixture, status: 200}
gated := &gatedCoverFetcher{
inner: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
gate: make(chan struct{}),
started: make(chan struct{}, 1),
}
p := &Poller{
Store: s, Fetch: pages, CoverBytesFetch: gated,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
done := make(chan struct{})
go func() {
p.runOnce(context.Background())
close(done)
}()
<-gated.started // the first cover heal is now stuck on its CDN
select {
case <-done:
// The Lane finished its page fetches without waiting for the cover.
case <-time.After(5 * time.Second):
t.Fatal("runOnce blocked on an in-flight cover fetch")
}
close(gated.gate)
p.waitCovers()
if got := pages.callCount(); got != 2 {
t.Fatalf("series page fetches = %d, want 2", got)
}
if got := gated.inner.callCount(); got != 2 {
t.Fatalf("cover fetches = %d, want 2", got)
}
}
// At 3601 eligible Series the effective gap falls below the one-second floor;
// the clamp warning must name the Site so the operator knows which Lane is
// outrunning its plan.
func TestRunOnceClampWarningNamesTheSite(t *testing.T) {
s, dbURL := newTestStore(t)
db, err := sql.Open("pgx", dbURL)
if err != nil {
t.Fatalf("open %s: %v", dbURL, err)
}
defer db.Close()
if _, err := db.Exec(`INSERT INTO series (site, series_id, series_url, latest_checked_at)
SELECT 'asura', 'bulk-' || g, 'https://asurascans.com/comics/bulk-' || g, 0
FROM generate_series(1, 3601) AS g`); err != nil {
t.Fatalf("bulk seed series: %v", err)
}
if _, err := db.Exec(`INSERT INTO bookmarks (reader_id, site, series_id, updated_at)
SELECT (SELECT id FROM readers ORDER BY id LIMIT 1), 'asura', 'bulk-' || g, 1000
FROM generate_series(1, 3601) AS g`); err != nil {
t.Fatalf("bulk seed bookmarks: %v", err)
}
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
newTestPoller(t, s, &fakeFetcher{body: "<html></html>", status: 200}, time.UnixMilli(5_000_000)).
runOnce(context.Background())
if got := logs.String(); !strings.Contains(got, "latest poll asura: gap clamped to 1s floor (eligible series=3601)") {
t.Fatalf("clamp warning = %q, want it to name asura and 3601", got)
}
}