feat: one Poll Lane per Site, replacing the shared pace (#100)

Each Site runs its own poll goroutine, paced by Rest and Gap from the
registry (sites.go) instead of the shared cooldown/interval/stagger/batch
config. Rest is enforced by the due query's WHERE clause; the Lane sleeps
its effective gap between fetches (hour/eligible, floored at 1s).

Lane-local failures: two refusals stop that Site for 15m, a lost browser
stops only the round's remaining browser Lanes, and cover heals moved to
background goroutines so a slow CDN cannot consume a Lane's gap. The five
LATEST_CHAPTER_POLL_* pace env vars are gone; only the kill switch
remains.
This commit is contained in:
2026-08-16 12:51:01 +07:00
parent ddbd57070d
commit 90ce14aec6
22 changed files with 3645 additions and 1884 deletions
+65 -3
View File
@@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
"time"
"github.com/chromedp/chromedp"
)
@@ -32,6 +33,11 @@ type site struct {
LatestChapter func(seriesURL, body string) (latestChapter, bool)
// Cover finds the Cover address in a fetched body.
Cover func(seriesURL, body string) (string, bool)
// Rest is how long a Series of this Site rests between Polls.
Rest time.Duration
// Gap is the Lane's strictest pace: at least one second must pass between
// two consecutive Series-page Polls of this Site (issue #100).
Gap time.Duration
// Browser reads this Site's payload from a cleared browser tab; nil
// means the page is fetched over plain TLS.
Browser *browserRead
@@ -380,6 +386,50 @@ func publishedCoverURL(value string) string {
return strings.ReplaceAll(value, " ", "%20")
}
// Poll Lane constants (issue #100). The per-Site structure is deliberately
// uniform at first — every Site rests an hour and gaps ten seconds — but it
// exists so a single Site can be slowed if it turns hostile, and the numbers
// stay in the registry so the structure has a place to differ.
const (
// defaultRest is how long every Series rests between Polls.
defaultRest = time.Hour
// defaultGap is the strictest pace of every Lane unless the eligible
// Series count forces it tighter.
defaultGap = 10 * time.Second
// minGap floors the effective gap. One request per second is already an
// order of magnitude past the strictest rate rule a free-plan Site can
// express (docs/research/cloudflare-bot-scoring-and-poll-cadence.md);
// below it the Lane is outrunning its own plan and says so loudly.
minGap = time.Second
// refuseBackoff is how long a Lane waits after its Site refused twice in
// one run before attempting it again.
refuseBackoff = 15 * time.Minute
// browserWakeCount and browserWakeAge gate a browser Lane's run: five or
// more due Series, or any one of them waiting this long, or Chrome stays
// asleep (ADR-0005 on-demand browser).
browserWakeCount = 5
browserWakeAge = 15 * time.Minute
)
// effectiveGap is a Site's pace: the registry gap, or one rest divided by the
// eligible Series count when that is smaller, never below one second. The
// denominator follows defaultRest rather than a literal hour so a Site whose
// rest is ever changed keeps its per-Series pace in step. The second return is
// true when the one-second floor engaged (and the Lane logs a warning naming
// the Site, every round it does).
func effectiveGap(s site, eligible int) (time.Duration, bool) {
gap := s.Gap
if eligible > 0 {
if perSeries := defaultRest / time.Duration(eligible); perSeries < gap {
gap = perSeries
}
}
if gap < minGap {
return minGap, true
}
return gap, false
}
// sites is the registry: one entry per Site, keyed by the stored site string.
// Adding a Site means adding an entry here and nowhere else — the dispatch
// functions above and the poller's route list are lookups into this map. An
@@ -390,16 +440,22 @@ var sites = map[string]site{
Host: "asurascans.com",
LatestChapter: asuraLatestChapter,
Cover: ogImageCover,
Rest: defaultRest,
Gap: defaultGap,
},
"demonic": {
Host: "demonicscans.org",
LatestChapter: demonicLatestChapter,
Cover: ogImageCover,
Rest: defaultRest,
Gap: defaultGap,
},
"comix": {
Host: "comix.to",
LatestChapter: comixLatestChapter,
Cover: comixCoverEntry,
Rest: defaultRest,
Gap: defaultGap,
Browser: &browserRead{
Read: comixRead,
// The interstitial is served in place of the page, so "arrived"
@@ -414,6 +470,8 @@ var sites = map[string]site{
Host: "kagane.to",
LatestChapter: kaganeLatestChapter,
Cover: kaganeCoverEntry,
Rest: defaultRest,
Gap: defaultGap,
Browser: &browserRead{
Read: kaganeRead,
Done: func(body string) bool { return body != "" },
@@ -426,6 +484,8 @@ var sites = map[string]site{
Host: "novelfull.com",
LatestChapter: novelfullLatestChapter,
Cover: novelfullCoverEntry,
Rest: defaultRest,
Gap: defaultGap,
Browser: &browserRead{
Read: novelfullRead,
// The interstitial has a DOM too, so "the payload arrived" has to
@@ -438,13 +498,15 @@ var sites = map[string]site{
Host: "lightnovelworld.net",
LatestChapter: lnwLatestChapter,
Cover: ogImageCover,
Rest: defaultRest,
Gap: defaultGap,
},
}
// browserBackedSites is derived from the registry: the Sites whose pages are
// read through the browser sidecar, which are also the ones granted the longer
// cooldown. Sorted so callers that range it (the due query, the browser
// fetcher's dispatch) see a stable order instead of map-iteration noise.
// read through the browser sidecar. Sorted so callers that range it (the
// browser fetcher's dispatch) see a stable order instead of map-iteration
// noise.
func browserBackedSites() []string {
out := make([]string, 0, len(sites))
for name, s := range sites {