From 503fb49d0a15873e8e01e22bd0826c5c15f5e81e Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Fri, 21 Aug 2026 18:48:30 +0700 Subject: [PATCH] feat(latest): record one poll pass per exit with skip reason and outcome counts Every way a Lane pass can end now writes exactly one durable row: a skip value naming the exit (paused, refusing, sidecar-down, no-fetcher, due-query, asleep, eligible-count, nothing-eligible, or empty for the loop), five outcome counts from the classification the Series read already makes, and carry-forward of the previous pass's figures exactly when the pass's own gap is zero. Refusal is durable through the poll_lanes row, so a restart does not re-probe a Site inside its backoff. Retention is 14 days. The mid-loop browser-unreachable return writes an empty skip by design: a tenth value is not invented here. (#141) --- backend/internal/latest/poller.go | 224 ++++++++++-- backend/internal/latest/poller_test.go | 460 ++++++++++++++++++++++++- backend/internal/store/store.go | 18 +- backend/internal/store/store_test.go | 27 +- 4 files changed, 686 insertions(+), 43 deletions(-) diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 50741a8..dac582b 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -48,6 +48,12 @@ type Poller struct { // same failure-isolated prefetch path. CoverBytesFetch CoverBytesFetcher Now func() time.Time // injected so tests can freeze it + // eligibleCount reports how many of a Site's Series are eligible for + // polling, defaulting to Store.EligibleSeriesCount. Injected so tests can + // fail the count alone: the eligible query shares the due query's tables, + // so no real store failure can reach this path without breaking the due + // query first (issue #141). + eligibleCount func(site string) (int, error) // refuseUntil gates a Site's Lane after it refused twice in one run: no // Series of that Site is attempted again before this time (issue #100). @@ -216,6 +222,74 @@ func (p *Poller) runOnce(ctx context.Context) { } } +// One skip value per way a Lane Pass can return before its loop (issue #141); +// empty means the pass reached the loop. The values are wire strings — stored +// in poll_passes and read by the Lanes page — so they are stable, not prose. +const ( + skipPaused = "paused" // the pause row was read at the top + skipRefusing = "refusing" // refusal backoff + skipSidecarDown = "sidecar-down" // a sibling browser Lane lost Chrome + skipNoFetcher = "no-fetcher" // browser Site, no browser configured, no fallback + skipDueQuery = "due-query" // the due query failed + skipAsleep = "asleep" // under both browser wake thresholds + skipEligibleCount = "eligible-count" // the eligible count failed + skipNothingEligible = "nothing-eligible" // nothing eligible; sleeps a full rest +) + +// readOutcome classifies one Series read for the pass row's outcome counts +// (issue #141). The classification the read already makes is counted, never a +// second taxonomy: refused is the Site holding a challenge, unreachable the +// browser interrupting, noChapter a 200 with real HTML but no chapter links, +// unfetchable the host pin or a missing fetcher, and errors everything else. +type readOutcome int + +const ( + outcomeSuccess readOutcome = iota + outcomeRefused + outcomeUnreachable + outcomeNoChapter + outcomeUnfetchable + outcomeError +) + +// outcomeCounts are the five named outcome counts of one pass. A success +// count is derived, never stored: checked minus the four, with unreachable +// excluded because the sidecar-loss path returns before the checked counter +// increments (issue #141). +type outcomeCounts struct { + refused, unreachable, noChapter, unfetchable, errors int +} + +func (c *outcomeCounts) add(o readOutcome) { + switch o { + case outcomeRefused: + c.refused++ + case outcomeUnreachable: + c.unreachable++ + case outcomeNoChapter: + c.noChapter++ + case outcomeUnfetchable: + c.unfetchable++ + case outcomeError: + c.errors++ + } +} + +// passRecord is what one pass's durable row will be: the skip value and +// outcome counts filled in along the pass's return path. recordPass assembles +// the row, so every exit records exactly once. +type passRecord struct { + site string + ranAt int64 + skip string + counts outcomeCounts +} + +// lanePassRetention is how far back a Lane's pass log is kept. It is not the +// display window: retention is how far back a question can reach, and the +// window is what the owner is shown (issue #139). +const lanePassRetention = 14 * 24 * time.Hour + // runLanePass processes one pass of one Site's Lane: select the due Series, // pace through them, and report how long the Lane should wait before its next // pass. paced spaces consecutive fetches by the Site's effective gap — the @@ -226,9 +300,32 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. // return path, with the figures filled in where the pass computes them. st := LaneState{Site: name, LastRun: now, Browser: isBrowserSite(name)} defer func() { p.recordLaneState(st) }() - if until := p.refusalBackoff(name); now.Before(until) { + // Durable pass log (issue #141): one row per exit, written from the same + // snapshot so the two recordings cannot disagree. + rec := passRecord{site: name, ranAt: now.UnixMilli()} + defer func() { p.recordPass(rec, st) }() + + // One Lane row read at the top of a pass, serving two gates (issue #139). + // Both stamps outlive our process, so the gates read the durable row + // rather than memory: a refusal is the Site's mood and a pause the + // owner's order, and neither is lost to a restart. + pausedUntil, refuseUntil, err := p.Store.LaneGates(name) + if err != nil { + // Fail open: a store that cannot answer the gate cannot record the + // pass either, and one Lane must not stall on its own gate read. + log.Printf("latest poll %s: lane gates: %v", name, err) + } + if pausedUntil > now.UnixMilli() { + // Paused ahead of the refusal check: no Series is touched, so the + // queue stays intact for when the pause lifts (issue #141, #147). + rec.skip = skipPaused + log.Printf("latest poll %s: paused until %s, skipping pass", name, time.UnixMilli(pausedUntil).Format(time.RFC3339)) + return time.Duration(pausedUntil-now.UnixMilli()) * time.Millisecond + } + if refuseUntil > now.UnixMilli() { // Cooling down after a refusal: do not attempt this Site at all. - return until.Sub(now) + rec.skip = skipRefusing + return time.Duration(refuseUntil-now.UnixMilli()) * time.Millisecond } if isBrowserSite(name) { if downFor, down := p.browserDownFor(now); down && downFor < refuseBackoff { @@ -236,6 +333,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. // window: skip this pass, so a restarting Chrome does not stamp // this Site's Series one pass at a time. After refuseBackoff the // flag decays and the Lane probes again (issue #100, story 20). + rec.skip = skipSidecarDown log.Printf("latest poll %s: browser lane skipping pass (sidecar down %s ago)", name, downFor) return refuseBackoff - downFor } @@ -246,6 +344,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. // No fetcher at all right now (browser absent, no fallback): every // Series stays unstamped and due, so a browser that appears after a // restart finds its full queue waiting (issue #100). + rec.skip = skipNoFetcher st.Gap = defaultGap return defaultGap } @@ -253,6 +352,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. due, err := p.Store.DueForLatestCheck(name, now.Add(-s.Rest).UnixMilli(), now.Add(-sightingCeilingRests*s.Rest).UnixMilli()) if err != nil { + rec.skip = skipDueQuery log.Printf("latest poll %s: due query: %v", name, err) st.Gap = defaultGap return defaultGap @@ -263,6 +363,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. // browser): waking it for a single Poll would cost a challenge solve // per request. The Lane still paces at the default gap, which is what // the owner's page must show rather than a zero. + rec.skip = skipAsleep st.Gap, st.Asleep = defaultGap, true return defaultGap } @@ -276,8 +377,9 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. } } - eligible, err := p.Store.EligibleSeriesCount(name) + eligible, err := p.countEligible(name) if err != nil { + rec.skip = skipEligibleCount log.Printf("latest poll %s: eligible count: %v", name, err) st.Gap = defaultGap return defaultGap @@ -290,6 +392,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. if eligible == 0 { // Nothing to poll for the foreseeable future; sleep a full rest instead // of re-querying every gap. + rec.skip = skipNothingEligible return s.Rest } @@ -314,20 +417,24 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. break } } - if err := p.checkOne(ctx, sr); err != nil { - switch { - case errors.Is(err, errChallengeHeld): - refusals++ - case errors.Is(err, errBrowserInterrupted): - p.setBrowserDown(now) - log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, refuseBackoff) - return gap - default: - refusals = 0 - } + outcome := p.checkOne(ctx, sr) + if outcome == outcomeUnreachable { + // The mid-loop browser loss writes an empty skip on purpose: the + // pass returns before the checked counter increments, so its row + // is stall-shaped (due > 0, checked 0, skip ''), and a stall is + // the exact signal this exit produces. A tenth skip value would + // make it legible but is deliberately not invented here. + rec.counts.add(outcome) + p.setBrowserDown(now) + log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, refuseBackoff) + return gap + } + if outcome == outcomeRefused { + refusals++ } else { refusals = 0 } + rec.counts.add(outcome) st.Checked++ } if st.Checked > 0 { @@ -335,16 +442,65 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. } if refusals >= 2 { p.setRefusalBackoff(name, now.Add(refuseBackoff)) + // The refusal outlives the process: the durable stamp gates a restart, + // so a Site that just told us to back off is not re-probed. The + // in-memory twin is still written because the Lane status block reads + // it directly; the gate reads the durable stamp, so a restart does not + // forget the refusal. + if err := p.Store.SetLaneRefusal(name, now.Add(refuseBackoff).UnixMilli()); err != nil { + log.Printf("latest poll %s: persist refusal: %v", name, err) + } log.Printf("latest poll %s: refused twice this run, waiting %s", name, refuseBackoff) return refuseBackoff } return gap } -func (p *Poller) refusalBackoff(name string) time.Time { - p.mu.Lock() - defer p.mu.Unlock() - return p.refuseUntil[name] +// recordPass writes the durable row for one pass (issue #141). Called deferred +// from runLanePass so every return path records exactly one row. A pass that +// never computed its own figures — its gap is zero — carries the previous +// pass's due, gap, clamped and checked forward rather than stating zeroes it +// did not measure; the skip column says why it declined, so the zeroes that +// remain (due-query, no-fetcher) read as explanations rather than +// measurements. +func (p *Poller) recordPass(rec passRecord, st LaneState) { + row := store.LanePass{ + Site: rec.site, + RanAt: rec.ranAt, + Skip: rec.skip, + Due: st.Due, + Checked: st.Checked, + GapMS: st.Gap.Milliseconds(), + Clamped: st.Clamped, + Refused: rec.counts.refused, + Unreachable: rec.counts.unreachable, + NoChapter: rec.counts.noChapter, + Unfetchable: rec.counts.unfetchable, + Errors: rec.counts.errors, + } + if row.GapMS == 0 { + // The pass never computed a gap, so it has no figures of its own: + // carry the previous pass's, in one latest-per-Site read — the + // recorder needs one Site, not six (issue #139). + if prev, ok, err := p.Store.LatestLanePass(rec.site); err != nil { + log.Printf("latest poll %s: previous pass: %v", rec.site, err) + } else if ok { + row.Due, row.Checked = prev.Due, prev.Checked + row.GapMS, row.Clamped = prev.GapMS, prev.Clamped + } + } + if err := p.Store.RecordLanePass(row, rec.ranAt-lanePassRetention.Milliseconds()); err != nil { + log.Printf("latest poll %s: record lane pass: %v", rec.site, err) + } +} + +// countEligible routes the eligible count through the test seam when one is +// set, else the store. +func (p *Poller) countEligible(site string) (int, error) { + if p.eligibleCount != nil { + return p.eligibleCount(site) + } + return p.Store.EligibleSeriesCount(site) } func (p *Poller) setRefusalBackoff(name string, until time.Time) { @@ -409,13 +565,15 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D // checkOne re-checks one series. Every failure path here is "log and move on": // the poller is a best-effort enhancement, and no single bad series may stall a -// Lane or take down the process. The returned error is the page read's -// classified outcome so the Lane can tell a refusal from a loss of the -// browser; non-classified failures still return nil-equivalent behaviour. -func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { +// Lane or take down the process. The returned outcome classifies the read for +// the pass row (issue #141), so the Lane can count a refusal, a lost browser, +// a chapter-less page, an unfetchable address or a transport error without +// re-deriving the taxonomy. +func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) { defer func() { if r := recover(); r != nil { log.Printf("latest poll %q: recovered from panic: %v", sr.Key(), r) + outcome = outcomeError } }() @@ -427,7 +585,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // the stamp means "attempted", and an untried Series stays due. if err := p.Store.MarkLatestChecked(sr.Site, sr.SeriesID, p.Now().UnixMilli()); err != nil { log.Printf("latest poll %q: mark checked: %v", sr.Key(), err) - return nil + return outcomeError } facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, p.BrowserFetch, p.Fetch) @@ -438,17 +596,23 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // passes the gate is retried at rest pace rather than // hot-looping. log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL) - return err + return outcomeUnfetchable case errors.Is(err, errNoFetcher): log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site) - return err + return outcomeUnfetchable } // A legacy cover heals independently of the page read: its source may // answer — a CDN — while the origin does not, so a fetch failure does // not skip the heal, matching the order the shared read replaced. p.healCover(ctx, sr) log.Printf("latest poll %q: %v", sr.Key(), err) - return err + if errors.Is(err, errChallengeHeld) { + return outcomeRefused + } + if errors.Is(err, errBrowserInterrupted) { + return outcomeUnreachable + } + return outcomeError } // A legacy cover source is healed independently of the page read. p.healCover(ctx, sr) @@ -459,7 +623,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // Most likely a challenge page or a layout change. Either way the row is // already stamped, so this waits out a rest instead of hot-looping. log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen) - return nil + return outcomeNoChapter } // The Poll is the oracle for whatever Sighting last raised this Series @@ -472,7 +636,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // against the due-query snapshot; a concurrent write in between only costs // one redundant UPDATE of the same absolute value, never a wrong one. if sr.LatestChapterNum != nil && *sr.LatestChapterNum == facts.Latest.Num { - return nil + return outcomeSuccess } // Series-level write: the row is shared, so one update refreshes every @@ -481,10 +645,10 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // the list. if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil { log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err) - return nil + return outcomeError } log.Printf("latest poll %q: latest is now %s", sr.Key(), facts.Latest.Label) - return nil + return outcomeSuccess } // judgeSighting settles the Sighting the Series' stored Latest Chapter is owed diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 72736bb..0ae089e 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -427,9 +427,12 @@ func TestRunLogsLaneDefaults(t *testing.T) { log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(previous) }) + // The pass log writes through the store, so Run needs a real one — a + // poller without a Store is not a poller. + s, _ := newTestStore(t) ctx, cancel := context.WithCancel(context.Background()) cancel() - (&Poller{Now: func() time.Time { return time.Now() }}).Run(ctx) + (&Poller{Store: s, Now: func() time.Time { return time.Now() }}).Run(ctx) got := logs.String() for _, want := range []string{"6 lanes", "rest=1h0m0s", "gap=10s"} { @@ -1689,3 +1692,458 @@ func TestLaneStatus(t *testing.T) { t.Fatalf("browser after loss = configured=%v reachable=%v, want true/false", st.BrowserConfigured, st.BrowserReachable) } } + +// latestPassFor reads a Site's newest durable pass row, failing rather than +// returning a zero LanePass a caller would assert against by accident. +func latestPassFor(t *testing.T, s *store.Store, site string) store.LanePass { + t.Helper() + pass, ok, err := s.LatestLanePass(site) + if err != nil || !ok { + t.Fatalf("LatestLanePass(%s): ok=%v err=%v", site, ok, err) + } + return pass +} + +// countPassRows counts a Site's durable pass rows, for asserting that a pass +// records exactly one. +func countPassRows(t *testing.T, dbURL, site string) int { + t.Helper() + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, site).Scan(&n); err != nil { + t.Fatalf("count passes for %s: %v", site, err) + } + return n +} + +// One durable pass row per exit, with the skip value naming the exit. The +// mid-loop browser-unreachable return also writes one row — but with an empty +// skip, so the row is stall-shaped (due > 0, checked 0, skip ”), matching the +// deliberate absence of a tenth skip value. +func TestRunLanePassRecordsEveryExit(t *testing.T) { + now := time.UnixMilli(5_000_000) + + t.Run("paused", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + p := newTestPoller(t, s, f, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { + t.Fatalf("paused pace = %s, want 30m (sleep until the expiry)", pace) + } + if f.callCount() != 0 { + t.Fatalf("fetches while paused = %d, want 0", f.callCount()) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got != 0 { + t.Fatalf("stamp while paused = %d, want 0 (Series stay due and unstamped)", got) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != skipPaused { + t.Fatalf("skip = %q, want %q", pass.Skip, skipPaused) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("refusing", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) + if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + browser := &fakeFetcher{status: 200} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + if pace := p.runLanePass(context.Background(), "kagane", false); pace != 10*time.Minute { + t.Fatalf("refusing pace = %s, want 10m", pace) + } + if browser.callCount() != 0 { + t.Fatalf("fetches while refusing = %d, want 0", browser.callCount()) + } + if pass := latestPassFor(t, s, "kagane"); pass.Skip != skipRefusing { + t.Fatalf("skip = %q, want %q", pass.Skip, skipRefusing) + } + if got := countPassRows(t, dbURL, "kagane"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("sidecar-down", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) + browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + p.setBrowserDown(now) // a sibling Lane lost Chrome within the backoff window + if pace := p.runLanePass(context.Background(), "kagane", false); pace != refuseBackoff { + t.Fatalf("sidecar-down pace = %s, want %s", pace, refuseBackoff) + } + if browser.callCount() != 0 { + t.Fatalf("fetches with the sidecar down = %d, want 0", browser.callCount()) + } + if pass := latestPassFor(t, s, "kagane"); pass.Skip != skipSidecarDown { + t.Fatalf("skip = %q, want %q", pass.Skip, skipSidecarDown) + } + if got := countPassRows(t, dbURL, "kagane"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("no-fetcher", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap { + t.Fatalf("no-fetcher pace = %s, want %s", pace, defaultGap) + } + pass := latestPassFor(t, s, "comix") + if pass.Skip != skipNoFetcher || pass.GapMS != defaultGap.Milliseconds() { + t.Fatalf("no-fetcher pass = %+v, want skip %q with its own gap %s", pass, skipNoFetcher, defaultGap) + } + if got := countPassRows(t, dbURL, "comix"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("due-query", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + // The due query is the pass's first store read after the gates; making + // it fail without touching poll_passes takes its tables away. + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + if _, err := db.Exec(`DROP TABLE bookmarks`); err != nil { + t.Fatalf("drop bookmarks: %v", err) + } + db.Close() + p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { + t.Fatalf("due-query pace = %s, want %s", pace, defaultGap) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != skipDueQuery { + t.Fatalf("skip = %q, want %q", pass.Skip, skipDueQuery) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("asleep", func(t *testing.T) { + s, dbURL := newTestStore(t) + for i := 0; i < 3; i++ { + key := fmt.Sprintf("kagane:w%d", i) + seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli()) + } + browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + if pace := p.runLanePass(context.Background(), "kagane", false); pace != defaultGap { + t.Fatalf("asleep pace = %s, want %s", pace, defaultGap) + } + if browser.callCount() != 0 { + t.Fatalf("fetches while Chrome is asleep = %d, want 0", browser.callCount()) + } + pass := latestPassFor(t, s, "kagane") + if pass.Skip != skipAsleep || pass.Due != 3 || pass.GapMS != defaultGap.Milliseconds() { + t.Fatalf("asleep pass = %+v, want skip %q with 3 due and the default gap", pass, skipAsleep) + } + if got := countPassRows(t, dbURL, "kagane"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("eligible-count", func(t *testing.T) { + s, dbURL := newTestStore(t) + // The eligible query shares the due query's tables, so no real store + // failure reaches it after a successful due read; the seam is how the + // path is driven at all (see Poller.eligibleCount). + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.eligibleCount = func(string) (int, error) { return 0, errors.New("count failed") } + if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { + t.Fatalf("eligible-count pace = %s, want %s", pace, defaultGap) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != skipEligibleCount { + t.Fatalf("skip = %q, want %q", pass.Skip, skipEligibleCount) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("nothing-eligible", func(t *testing.T) { + s, dbURL := newTestStore(t) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != sites["asura"].Rest { + t.Fatalf("nothing-eligible pace = %s, want a full rest %s", pace, sites["asura"].Rest) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != skipNothingEligible { + t.Fatalf("skip = %q, want %q", pass.Skip, skipNothingEligible) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("reached the loop", func(t *testing.T) { + s, dbURL := newTestStore(t) + const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0) + p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { + t.Fatalf("loop pace = %s, want %s", pace, defaultGap) + } + pass := latestPassFor(t, s, "asura") + if pass.Skip != "" || pass.Due != 1 || pass.Checked != 1 { + t.Fatalf("loop pass = %+v, want an empty skip with 1 due and 1 checked", pass) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("browser-unreachable mid-loop", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + browser := &fakeFetcher{status: 200, err: interrupted} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap { + t.Fatalf("mid-loop pace = %s, want %s", pace, defaultGap) + } + // One row was still written, and it is stall-shaped by construction: + // empty skip with due > 0 and checked 0 because the return precedes + // the checked counter. The assertion below pins that shape precisely. + pass := latestPassFor(t, s, "comix") + if pass.Skip != "" || pass.Unreachable != 1 || pass.Checked != 0 || pass.Due != 1 { + t.Fatalf("mid-loop pass = %+v, want empty skip, unreachable 1, checked 0, due 1", pass) + } + if got := countPassRows(t, dbURL, "comix"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) +} + +// Carry-forward moves verbatim from the in-memory snapshot (issue #141): a +// pass that never computed its own gap carries the previous pass's due, gap, +// clamped and checked forward; a pass with a gap of its own records its own +// figures, zeroes included, beside the skip reason that explains them. +func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) { + now := time.UnixMilli(5_000_000) + + t.Run("no gap of its own carries the previous pass's figures", func(t *testing.T) { + s, _ := newTestStore(t) + seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{body: kaganeAPIFixture, status: 200} + p.runLanePass(context.Background(), "kagane", false) + first := latestPassFor(t, s, "kagane") + if first.Due != 1 || first.Checked != 1 || first.GapMS == 0 { + t.Fatalf("first pass = %+v, want a measured pass", first) + } + + if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + // The pass row is keyed (site, ran_at), so the second pass needs its + // own timestamp: a minute later is still inside the refusal. + p.Now = func() time.Time { return now.Add(time.Minute) } + p.runLanePass(context.Background(), "kagane", false) + second := latestPassFor(t, s, "kagane") + if second.Skip != skipRefusing { + t.Fatalf("second pass skip = %q, want %q", second.Skip, skipRefusing) + } + if second.Due != first.Due || second.Checked != first.Checked || + second.GapMS != first.GapMS || second.Clamped != first.Clamped { + t.Fatalf("carried pass = %+v, want the first pass's figures %+v", second, first) + } + }) + + t.Run("a gap of its own records its own figures", func(t *testing.T) { + s, _ := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} + p.runLanePass(context.Background(), "comix", false) + first := latestPassFor(t, s, "comix") + if first.Due != 1 || first.GapMS == 0 { + t.Fatalf("first pass = %+v, want a measured pass", first) + } + + // The no-fetcher exit sets its own gap, so no carry-forward: the due + // count it never gathered records as zero beside its skip reason. A + // minute later gives the second pass its own (site, ran_at) key. + p.BrowserFetch = nil + p.Now = func() time.Time { return now.Add(time.Minute) } + p.runLanePass(context.Background(), "comix", false) + second := latestPassFor(t, s, "comix") + if second.Skip != skipNoFetcher { + t.Fatalf("second pass skip = %q, want %q", second.Skip, skipNoFetcher) + } + if second.GapMS != defaultGap.Milliseconds() { + t.Fatalf("second pass gap = %d, want its own %d (not carried)", second.GapMS, defaultGap.Milliseconds()) + } + if second.Due != 0 || second.Checked != 0 { + t.Fatalf("second pass = %+v, want due 0 checked 0 (its own, not the previous pass's)", second) + } + }) +} + +// The pass row's five outcome counts are the classification the Series read +// already makes — never a second taxonomy (issue #141). Success is derived, +// never stored: checked minus the four named counts, unreachable excluded +// because its exit returns before the checked counter increments. +func TestRunLanePassCountsOutcomes(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + const ( + successKey = "asura:chronicles-of-the-demon-faction-f886a8af" + successURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seeds := map[string]string{ + "asura:refused": "https://asurascans.com/series/refused", + "asura:no-chapter": "https://asurascans.com/comics/no-chapter", + "asura:unfetchable": "https://evil.example/x", + "asura:transport": "https://asurascans.com/series/transport", + successKey: successURL, + } + for key, url := range seeds { + seedForCheck(t, s, key, url, 0) + } + + f := &fakeFetcher{perURL: map[string]fakeResponse{ + seeds["asura:refused"]: {status: 403}, + seeds["asura:no-chapter"]: {body: "", status: 200}, + seeds["asura:transport"]: {err: errors.New("dial tcp: refused")}, + successURL: {body: asuraSeriesFixture, status: 200}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + pass := latestPassFor(t, s, "asura") + if pass.Skip != "" || pass.Due != 5 || pass.Checked != 5 { + t.Fatalf("pass = %+v, want a full pass over 5 due Series", pass) + } + if pass.Refused != 1 || pass.Unreachable != 0 || pass.NoChapter != 1 || + pass.Unfetchable != 1 || pass.Errors != 1 { + t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1", + pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors) + } + if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 { + t.Fatalf("derived success = %d, want 1", success) + } + // The one genuine read went through: the success Series carries the + // fixture's newest chapter, and none of the four failures do. + b, ok, err := s.Get(s.OwnerID(), successKey) + if err != nil || !ok { + t.Fatalf("Get: %v ok=%v", err, ok) + } + if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { + t.Fatalf("success Series latest = %v, want 181 (the fixture's newest)", b.LatestChapterNum) + } + for _, key := range []string{"asura:refused", "asura:no-chapter", "asura:unfetchable", "asura:transport"} { + b, _, err := s.Get(s.OwnerID(), key) + if err != nil { + t.Fatalf("Get %s: %v", key, err) + } + if b.LatestChapterNum != nil { + t.Fatalf("%s latest = %v, want nil (no chapter survived a failed read)", key, *b.LatestChapterNum) + } + } +} + +// A refusal is the Site's mood and outlives our process: the durable stamp a +// pass writes is honoured by a freshly constructed poller, which must not +// re-probe the Site inside its backoff (issue #141). +func TestDurableRefusalSurvivesFreshPoller(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + // Four due Series: the Lane refuses twice, stamps those two, and leaves + // the remaining two untried and due — the queue the fresh poller must + // find intact once the durable backoff lifts. + for i := 0; i < 4; i++ { + key := fmt.Sprintf("kagane:s%d", i) + seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0) + } + + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{status: 403} + p.runLanePass(context.Background(), "kagane", false) + if got := p.BrowserFetch.(*fakeFetcher).callCount(); got != 2 { + t.Fatalf("fetches on the refusing pass = %d, want 2 (refused twice)", got) + } + + // A restart: a brand-new poller, no in-memory refusal, same store. The + // durable stamp gates the pass. + fresh := newTestPoller(t, s, &fakeFetcher{status: 200}, now.Add(14*time.Minute)) + fresh.BrowserFetch = &fakeFetcher{status: 403} + fresh.runLanePass(context.Background(), "kagane", false) + if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 0 { + t.Fatalf("fetches by a fresh poller inside the backoff = %d, want 0", got) + } + if pass := latestPassFor(t, s, "kagane"); pass.Skip != skipRefusing { + t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, skipRefusing) + } + + // Past the backoff the fresh poller probes again — the two Series the + // original pass never reached, still due with their stamps untouched. + fresh.Now = func() time.Time { return now.Add(16 * time.Minute) } + fresh.runLanePass(context.Background(), "kagane", false) + if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 2 { + t.Fatalf("fetches after the backoff = %d, want 2", got) + } + for i := 2; i < 4; i++ { + if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 { + t.Fatalf("kagane:s%d still untried after the backoff", i) + } + } +} + +// RecordLanePass prunes in the same call that inserts, so the retention +// cutoff the recorder passes is observable in what survives: a row just inside +// 14 days behind the poller's clock is kept, one just outside is pruned +// (issue #139, #141). +func TestPassRetentionCutoffIsFourteenDays(t *testing.T) { + s, dbURL := newTestStore(t) + // A real-world clock: the seeded rows sit 14 days back, so they must be + // positive timestamps or the seed's own prune (ran_at < 0) removes them. + now := time.UnixMilli(1_800_000_000_000) + kept := now.Add(-14*24*time.Hour + time.Minute).UnixMilli() + pruned := now.Add(-14*24*time.Hour - time.Minute).UnixMilli() + for _, ranAt := range []int64{kept, pruned} { + if err := s.RecordLanePass(store.LanePass{Site: "asura", RanAt: ranAt}, 0); err != nil { + t.Fatalf("seed pass at %d: %v", ranAt, err) + } + } + + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now). + runLanePass(context.Background(), "asura", false) + + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1 AND ran_at = $2`, "asura", pruned).Scan(&n); err != nil { + t.Fatalf("count pruned row: %v", err) + } + if n != 0 { + t.Fatalf("row at %d survived, want it pruned (older than 14 days)", pruned) + } + if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&n); err != nil { + t.Fatalf("count passes: %v", err) + } + if n != 2 { + t.Fatalf("passes = %d, want 2 (this pass plus the kept row)", n) + } +} diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 869f603..e7ee4b7 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1140,18 +1140,20 @@ func (s *Store) PausedLanes() ([]LanePause, error) { return out, rows.Err() } -// LanePausedUntil reads a Site's pause stamp. A missing state row is the -// default unpaused state. -func (s *Store) LanePausedUntil(site string) (int64, error) { - var until int64 - err := s.db.QueryRow(`SELECT paused_until FROM poll_lanes WHERE site = $1`, site).Scan(&until) +// LaneGates reads a Site's pause and refusal stamps in one row read — the +// top-of-pass gate the poller uses (issue #141). A missing state row is the +// default: unpaused and not refusing. +func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err error) { + err = s.db.QueryRow( + `SELECT paused_until, refuse_until FROM poll_lanes WHERE site = $1`, site). + Scan(&pausedUntil, &refuseUntil) if errors.Is(err, sql.ErrNoRows) { - return 0, nil + return 0, 0, nil } if err != nil { - return 0, fmt.Errorf("lane pause %s: %w", site, err) + return 0, 0, fmt.Errorf("lane gates %s: %w", site, err) } - return until, nil + return pausedUntil, refuseUntil, nil } // DueForLatestCheck returns one Site's series whose server-side diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index fac25e4..a663def 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -1681,6 +1681,24 @@ func TestLanePassOutcomesSumsWindow(t *testing.T) { } } +func TestLaneGatesReadsOneRow(t *testing.T) { + s := newTestStore(t) + paused, refused, err := s.LaneGates("asura") + if err != nil || paused != 0 || refused != 0 { + t.Fatalf("LaneGates on a missing state row = (%d, %d, %v), want (0, 0, nil)", paused, refused, err) + } + if err := s.PauseLane("asura", 2000); err != nil { + t.Fatalf("PauseLane: %v", err) + } + if err := s.SetLaneRefusal("asura", 3000); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + paused, refused, err = s.LaneGates("asura") + if err != nil || paused != 2000 || refused != 3000 { + t.Fatalf("LaneGates = (%d, %d, %v), want (2000, 3000, nil)", paused, refused, err) + } +} + func TestLaneStatePauseResumeAndRefusal(t *testing.T) { s := newTestStore(t) for _, until := range []int64{0, -1} { @@ -1697,8 +1715,9 @@ func TestLaneStatePauseResumeAndRefusal(t *testing.T) { if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 1}, -1); err != nil { t.Fatalf("RecordLanePass: %v", err) } - if got, err := s.LanePausedUntil("asura"); err != nil || got != 2000 { - t.Fatalf("LanePausedUntil = %d, %v; want 2000", got, err) + pausedUntil, refuseUntil, err := s.LaneGates("asura") + if err != nil || pausedUntil != 2000 || refuseUntil != 3000 { + t.Fatalf("LaneGates = %d/%d, %v; want 2000/3000", pausedUntil, refuseUntil, err) } paused, err := s.PausedLanes() if err != nil { @@ -1715,8 +1734,8 @@ func TestLaneStatePauseResumeAndRefusal(t *testing.T) { if err != nil || !ok || latest.RefuseUntil != 3000 { t.Fatalf("latest refusal after resume = %+v, ok=%v, err=%v; want 3000 preserved", latest, ok, err) } - if got, err := s.LanePausedUntil("asura"); err != nil || got != 0 { - t.Fatalf("LanePausedUntil after resume = %d, %v; want 0", got, err) + if got, got2, err := s.LaneGates("asura"); err != nil || got != 0 || got2 != 3000 { + t.Fatalf("LaneGates after resume = %d/%d, %v; want 0/3000", got, got2, err) } if paused, err := s.PausedLanes(); err != nil || len(paused) != 0 { t.Fatalf("PausedLanes after resume = %+v, %v; want empty", paused, err)