feat(latest): split browser poll cooldown (#48)

## Summary

Split latest-chapter polling cooldowns by fetch cost. Browser-backed kagane and novelfull series now rest longer without changing the cadence of plain-TLS sites.

## Behavior

- Plain-TLS series keep the 1h default cooldown.
- Browser-backed series use `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, defaulting to 6h.
- Both cooldowns share the existing 15m minimum floor; invalid values retain the existing fallback behavior.
- The poller still selects both classes in one due query per cycle.
- Existing ordering and exclusions remain unchanged: reader-count precedence, least-recently-checked ordering, finished exclusion, archived polling, and orphan exclusion.

## Implementation

- Added the browser cooldown to backend configuration and passed it through production poller construction.
- Added the browser-site list as the single routing source used for both due-query cutoff selection and fetcher choice.
- Kept all query values parameterized; the site list is passed as a bound PostgreSQL array parameter.
- Updated startup logging to report interval, plain cooldown, browser cooldown, batch, and stagger.
- Documented the variable, default, and floor in `README.md`, `.env.example`, `backend/AGENTS.md`, and `docker-compose.yml`.

## Review findings addressed

The first review found that configuration parsing was correct but `startLatestPoller` did not pass `BrowserCooldown` into `latest.Poller`; every browser-backed row would therefore have been due immediately. Production construction now goes through `newLatestPoller`, with a regression test covering both cooldown fields.

The review also identified duplicated browser-site knowledge in fetch routing. `slices.Contains(browserBackedSites, site)` now reuses the same list already supplied to the store query.

## Verification

- Focused backend tests pass: `go test ./internal/latest ./internal/store .`.
- Full suite passes: `go test ./...`.
- `graphify update .` completed.
- Issue #42 was updated and closed.

Reviewed-on: #48
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #48.
This commit is contained in:
2026-08-09 06:25:48 +07:00
committed by sulthan
parent 741b23322b
commit cd3a7e3d01
10 changed files with 218 additions and 89 deletions
+43 -28
View File
@@ -60,18 +60,20 @@ type Config struct {
// deployment. Past that nothing breaks; the effective cadence stretches to
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
type LatestPoll struct {
Enabled bool
Cooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
Enabled bool
Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
const (
defaultPollCooldown = time.Hour
defaultPollInterval = 10 * time.Minute
defaultPollStagger = 20 * time.Second
defaultPollBatch = 14
defaultPollCooldown = time.Hour
defaultBrowserPollCooldown = 6 * time.Hour
defaultPollInterval = 10 * time.Minute
defaultPollStagger = 20 * time.Second
defaultPollBatch = 14
// minPollCooldown keeps a typo from turning a polite background check into
// a hammer against sites that are already bot-scoring us.
minPollCooldown = 15 * time.Minute
@@ -129,20 +131,27 @@ func envInt(key string, def int) int {
return n
}
func clampPollCooldown(name string, d time.Duration) time.Duration {
if d < minPollCooldown {
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
return minPollCooldown
}
return d
}
// loadLatestPoll reads the poller's settings, clamping anything that would make
// it antisocial.
func loadLatestPoll() LatestPoll {
p := LatestPoll{
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
}
if p.Cooldown < minPollCooldown {
log.Printf("config: cooldown %s is below the %s floor, clamping", p.Cooldown, minPollCooldown)
p.Cooldown = minPollCooldown
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
}
p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
// batch x stagger has to fit inside one tick or a batch is still running
// when the next one is due. Run() serialises them, so this degrades to a
// slower cadence rather than to overlapping fetches — worth a warning, not
@@ -324,6 +333,21 @@ func main() {
}
}
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
return &latest.Poller{
Store: s,
Fetch: fetch,
BrowserFetch: browser,
Now: time.Now,
Cooldown: cfg.Cooldown,
BrowserCooldown: cfg.BrowserCooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
}
// startLatestPoller launches the background poller unless it is disabled or its
// HTTP client cannot be built. Any problem here is logged and skipped: this
// feature going missing degrades the service to userscript-only latest-chapter
@@ -341,16 +365,7 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, brow
// Nil browser: sites behind a JavaScript challenge are simply not polled,
// and their latest_chapter comes from the userscript alone — which is how
// the service behaved before the sidecar existed.
p := &latest.Poller{
Store: s,
Fetch: f,
BrowserFetch: browser,
Now: time.Now,
Cooldown: cfg.Cooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
p := newLatestPoller(s, cfg, f, browser)
go p.Run(ctx)
}