Closes #100. Each Site runs its own Poll Lane: an independent goroutine with its own rest and pace from the registry (`backend/internal/latest/sites.go`), replacing the shared cooldown/interval/stagger/batch configuration. Rest (1h, all six Sites including the browser trio) is enforced by the due query's WHERE clause; the Lane sleeps its effective gap between fetches — the registry 10s, or rest/eligible when a Site holds enough Series, floored at 1s with a Site-naming warning when the floor engages. Lane-local failure handling: - Two challenge-held results stop that Site's Lane for 15m; the probes keep their stamp, untried Series stay due. - A lost browser sets a shared Poller flag: the other browser Lanes skip their passes for the same 15m (no stamp-per-pass-per-Lane on a dead tab), then decay and probe again. - Browser wake gate preserved (5 due, or one waiting 15m, ADR-0005); one tab shared by the three browser Sites; "browser lane behind by X" logged every pass. - Cover work (healing a stored source URL and filling a blank from the series page) runs in the background so a slow CDN cannot consume a Lane's gap. Removed: `LATEST_CHAPTER_POLL_{COOLDOWN,BROWSER_COOLDOWN,INTERVAL,BATCH,STAGGER}` and the 6h browser rest. Only `LATEST_CHAPTER_POLL_ENABLED` remains; DEPLOY.md documents the exact `.env` edit. ADR-0010 records the decisions. Reviewed-on: #106 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #106.
This commit is contained in:
+12
-87
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -61,30 +60,15 @@ type Config struct {
|
||||
|
||||
// LatestPoll configures the background latest-chapter poller.
|
||||
//
|
||||
// Sizing: batch x (cooldown / interval) is how many series hold a true cooldown
|
||||
// cadence — 14 x (1h / 10m) = 84 with these defaults, which covers this
|
||||
// deployment. Past that nothing breaks; the effective cadence stretches to
|
||||
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
|
||||
// Only the kill switch lives here. Pace is per Site — rest time and gap are
|
||||
// registry properties (internal/latest/sites.go, issue #100), because each
|
||||
// Lane has to be able to differ from the others. The five environment
|
||||
// settings that used to size a shared pace (cooldown, browser cooldown,
|
||||
// interval, stagger, batch) are gone with it: no deployed .env may carry them.
|
||||
type LatestPoll struct {
|
||||
Enabled bool
|
||||
Cooldown time.Duration
|
||||
BrowserCooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
const (
|
||||
defaultPollCooldown = time.Hour
|
||||
defaultBrowserPollCooldown = 6 * time.Hour
|
||||
defaultPollInterval = 10 * time.Minute
|
||||
defaultPollStagger = 20 * time.Second
|
||||
defaultPollBatch = 14
|
||||
// minPollCooldown keeps a typo from turning a polite background check into
|
||||
// a hammer against sites that are already bot-scoring us.
|
||||
minPollCooldown = 15 * time.Minute
|
||||
)
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
@@ -107,66 +91,11 @@ func envBool(key string, def bool) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// envDuration reads a duration env var. An unparseable or non-positive value
|
||||
// falls back to def and logs rather than failing startup: the poller is an
|
||||
// enhancement, and a typo in one of its knobs must not stop bookmark sync.
|
||||
func envDuration(key string, def time.Duration) time.Duration {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
d, err := time.ParseDuration(raw)
|
||||
if err != nil || d <= 0 {
|
||||
log.Printf("config: %s=%q is not a positive duration, using %s", key, raw, def)
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// envInt reads a positive integer env var, with the same fallback policy.
|
||||
func envInt(key string, def int) int {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
log.Printf("config: %s=%q is not a positive integer, using %d", key, raw, def)
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func clampPollCooldown(name string, d time.Duration) time.Duration {
|
||||
if d < minPollCooldown {
|
||||
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
|
||||
return minPollCooldown
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// loadLatestPoll reads the poller's settings, clamping anything that would make
|
||||
// it antisocial.
|
||||
// loadLatestPoll reads the poller's settings. The pace knobs that used to be
|
||||
// clamped here are registry properties now (issue #100), so there is nothing
|
||||
// left to clamp.
|
||||
func loadLatestPoll() LatestPoll {
|
||||
p := LatestPoll{
|
||||
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
|
||||
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
|
||||
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
|
||||
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
|
||||
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
|
||||
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
|
||||
}
|
||||
p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
|
||||
p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
|
||||
// batch x stagger has to fit inside one tick or a batch is still running
|
||||
// when the next one is due. Run() serialises them, so this degrades to a
|
||||
// slower cadence rather than to overlapping fetches — worth a warning, not
|
||||
// a failure.
|
||||
if span := time.Duration(p.Batch) * p.Stagger; span > p.Interval {
|
||||
log.Printf("config: batch(%d) x stagger(%s) = %s exceeds interval %s; batches will overrun their tick",
|
||||
p.Batch, p.Stagger, span, p.Interval)
|
||||
}
|
||||
return p
|
||||
return LatestPoll{Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true)}
|
||||
}
|
||||
|
||||
func loadConfig() Config {
|
||||
@@ -377,7 +306,8 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
|
||||
// newLatestPoller wires the fetcher seams into the poller. Pace is registry
|
||||
// property, not config (issue #100), so there are no knobs to pass through.
|
||||
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
|
||||
var covers latest.BrowserCoverFetcher
|
||||
if f, ok := browser.(latest.BrowserCoverFetcher); ok {
|
||||
@@ -390,11 +320,6 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch
|
||||
CoverFetch: covers,
|
||||
CoverBytesFetch: latest.NewCoverFetcher(),
|
||||
Now: time.Now,
|
||||
Cooldown: cfg.Cooldown,
|
||||
BrowserCooldown: cfg.BrowserCooldown,
|
||||
Interval: cfg.Interval,
|
||||
Stagger: cfg.Stagger,
|
||||
Batch: cfg.Batch,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user