feat(latest): split browser poll cooldown (#48)
## Summary Split latest-chapter polling cooldowns by fetch cost. Browser-backed kagane and novelfull series now rest longer without changing the cadence of plain-TLS sites. ## Behavior - Plain-TLS series keep the 1h default cooldown. - Browser-backed series use `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, defaulting to 6h. - Both cooldowns share the existing 15m minimum floor; invalid values retain the existing fallback behavior. - The poller still selects both classes in one due query per cycle. - Existing ordering and exclusions remain unchanged: reader-count precedence, least-recently-checked ordering, finished exclusion, archived polling, and orphan exclusion. ## Implementation - Added the browser cooldown to backend configuration and passed it through production poller construction. - Added the browser-site list as the single routing source used for both due-query cutoff selection and fetcher choice. - Kept all query values parameterized; the site list is passed as a bound PostgreSQL array parameter. - Updated startup logging to report interval, plain cooldown, browser cooldown, batch, and stagger. - Documented the variable, default, and floor in `README.md`, `.env.example`, `backend/AGENTS.md`, and `docker-compose.yml`. ## Review findings addressed The first review found that configuration parsing was correct but `startLatestPoller` did not pass `BrowserCooldown` into `latest.Poller`; every browser-backed row would therefore have been due immediately. Production construction now goes through `newLatestPoller`, with a regression test covering both cooldown fields. The review also identified duplicated browser-site knowledge in fetch routing. `slices.Contains(browserBackedSites, site)` now reuses the same list already supplied to the store query. ## Verification - Focused backend tests pass: `go test ./internal/latest ./internal/store .`. - Full suite passes: `go test ./...`. - `graphify update .` completed. - Issue #42 was updated and closed. Reviewed-on: #48 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #48.
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
@@ -23,34 +24,36 @@ type Fetcher interface {
|
||||
// Two clocks, deliberately independent:
|
||||
//
|
||||
// - Interval is how often this goroutine wakes up and looks.
|
||||
// - Cooldown is how long one series rests since its own last check.
|
||||
// - Cooldowns are how long a series rests since its own last check. Browser-
|
||||
// backed sites use the longer BrowserCooldown.
|
||||
//
|
||||
// Only the cooldown is per series, and it is enforced by the WHERE clause in
|
||||
// DueForLatestCheck rather than by any timer. Shortening Interval therefore
|
||||
// cannot shorten anyone's cooldown; it only makes the poller wake up and find
|
||||
// nothing due more often.
|
||||
// Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
|
||||
// by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
|
||||
// it only makes the poller wake up and find nothing due more often.
|
||||
type Poller struct {
|
||||
Store *store.Store
|
||||
Fetch Fetcher
|
||||
// BrowserFetch handles sites behind a JavaScript challenge that Fetch
|
||||
// cannot clear. Nil disables those sites entirely rather than falling back
|
||||
// to Fetch, which would only ever retrieve a challenge page.
|
||||
BrowserFetch Fetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
Cooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
BrowserFetch Fetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
Cooldown time.Duration
|
||||
BrowserCooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
}
|
||||
|
||||
var browserBackedSites = []string{"kagane", "novelfull"}
|
||||
|
||||
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
|
||||
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
|
||||
// JavaScript challenge that no TLS fingerprint clears — kagane verified
|
||||
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
|
||||
// profile TLSFetcher uses — so they are browser-only or nothing.
|
||||
func (p *Poller) fetcherFor(site string) Fetcher {
|
||||
switch site {
|
||||
case "kagane", "novelfull":
|
||||
if slices.Contains(browserBackedSites, site) {
|
||||
return p.BrowserFetch
|
||||
}
|
||||
return p.Fetch
|
||||
@@ -63,8 +66,8 @@ func (p *Poller) fetcherFor(site string) Fetcher {
|
||||
// failure mode for a misconfigured batch x stagger: a slower cadence, never
|
||||
// concurrent fetch storms.
|
||||
func (p *Poller) Run(ctx context.Context) {
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s",
|
||||
p.Interval, p.Cooldown, p.Batch, p.Stagger)
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
|
||||
p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
|
||||
t := time.NewTicker(p.Interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
@@ -80,8 +83,10 @@ func (p *Poller) Run(ctx context.Context) {
|
||||
|
||||
// runOnce processes one batch of due series.
|
||||
func (p *Poller) runOnce(ctx context.Context) {
|
||||
cutoff := p.Now().Add(-p.Cooldown).UnixMilli()
|
||||
due, err := p.Store.DueForLatestCheck(cutoff, p.Batch)
|
||||
now := p.Now()
|
||||
cutoff := now.Add(-p.Cooldown).UnixMilli()
|
||||
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
|
||||
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
|
||||
if err != nil {
|
||||
log.Printf("latest poll: due query: %v", err)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user