feat: one Poll Lane per Site, replacing the shared pace (#100)
Each Site runs its own poll goroutine, paced by Rest and Gap from the registry (sites.go) instead of the shared cooldown/interval/stagger/batch config. Rest is enforced by the due query's WHERE clause; the Lane sleeps its effective gap between fetches (hour/eligible, floored at 1s). Lane-local failures: two refusals stop that Site for 15m, a lost browser stops only the round's remaining browser Lanes, and cover heals moved to background goroutines so a slow CDN cannot consume a Lane's gap. The five LATEST_CHAPTER_POLL_* pace env vars are gone; only the kill switch remains.
This commit is contained in:
@@ -9,7 +9,7 @@ import (
|
||||
// seriesRead carries the two facts the poll and the acquirer both extract
|
||||
// from a series page. Persistence, stamps and scheduling stay with the
|
||||
// callers, so the policies that keep the two flows distinct (stamp order,
|
||||
// cooldowns) are not swallowed by the module.
|
||||
// rests) are not swallowed by the module.
|
||||
type seriesRead struct {
|
||||
Latest latestChapter
|
||||
HasLatest bool
|
||||
@@ -22,6 +22,9 @@ type seriesRead struct {
|
||||
|
||||
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
||||
// failures so each caller keeps its own distinct log line for all three.
|
||||
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
|
||||
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
|
||||
// is how a Lane tells a refusal from an ordinary failure (issue #100).
|
||||
var (
|
||||
errNotFetchable = errors.New("series url not fetchable")
|
||||
errNoFetcher = errors.New("no fetcher for site")
|
||||
@@ -49,9 +52,21 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
||||
if err != nil {
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: %w", seriesURL, err)
|
||||
}
|
||||
if status == 403 {
|
||||
// Cloudflare's challenge response for these Sites (cf-mitigated). The
|
||||
// browser fetcher returns exactly this on a held interstitial, and a
|
||||
// plain-TLS 403 means the same: the Site is refusing.
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
|
||||
}
|
||||
if status != 200 {
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
|
||||
}
|
||||
if isInterstitial(body) {
|
||||
// A 200 that is the challenge page, not the payload: the TLS route can
|
||||
// receive this where the browser would have kept re-reading. Same
|
||||
// refusal as the 403.
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: interstitial body", errChallengeHeld, seriesURL)
|
||||
}
|
||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||
|
||||
Reference in New Issue
Block a user