feat(latest): split browser poll cooldown (#48)
## Summary Split latest-chapter polling cooldowns by fetch cost. Browser-backed kagane and novelfull series now rest longer without changing the cadence of plain-TLS sites. ## Behavior - Plain-TLS series keep the 1h default cooldown. - Browser-backed series use `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, defaulting to 6h. - Both cooldowns share the existing 15m minimum floor; invalid values retain the existing fallback behavior. - The poller still selects both classes in one due query per cycle. - Existing ordering and exclusions remain unchanged: reader-count precedence, least-recently-checked ordering, finished exclusion, archived polling, and orphan exclusion. ## Implementation - Added the browser cooldown to backend configuration and passed it through production poller construction. - Added the browser-site list as the single routing source used for both due-query cutoff selection and fetcher choice. - Kept all query values parameterized; the site list is passed as a bound PostgreSQL array parameter. - Updated startup logging to report interval, plain cooldown, browser cooldown, batch, and stagger. - Documented the variable, default, and floor in `README.md`, `.env.example`, `backend/AGENTS.md`, and `docker-compose.yml`. ## Review findings addressed The first review found that configuration parsing was correct but `startLatestPoller` did not pass `BrowserCooldown` into `latest.Poller`; every browser-backed row would therefore have been due immediately. Production construction now goes through `newLatestPoller`, with a regression test covering both cooldown fields. The review also identified duplicated browser-site knowledge in fetch routing. `slices.Contains(browserBackedSites, site)` now reuses the same list already supplied to the store query. ## Verification - Focused backend tests pass: `go test ./internal/latest ./internal/store .`. - Full suite passes: `go test ./...`. - `graphify update .` completed. - Issue #42 was updated and closed. Reviewed-on: #48 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #48.
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -108,13 +109,14 @@ func (f *fakeFetcher) callCount() int {
|
||||
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
|
||||
t.Helper()
|
||||
return &Poller{
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: func() time.Time { return at },
|
||||
Cooldown: time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 0,
|
||||
Batch: 14,
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: func() time.Time { return at },
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 0,
|
||||
Batch: 14,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,6 +306,26 @@ func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLogsCooldowns(t *testing.T) {
|
||||
var logs strings.Builder
|
||||
previous := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(previous) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
(&Poller{
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
Interval: time.Hour,
|
||||
}).Run(ctx)
|
||||
|
||||
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
|
||||
!strings.Contains(got, "browser-cooldown=6h") {
|
||||
t.Fatalf("startup log = %q, want both cooldowns", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The cooldown is enforced by the due query, so a second immediate pass must do
|
||||
// nothing at all — this is what makes the tick interval independent of it.
|
||||
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
||||
@@ -334,6 +356,44 @@ func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
|
||||
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
|
||||
|
||||
hour := time.Hour
|
||||
now := time.Unix(2*int64(hour/time.Second), 0)
|
||||
tls := &fakeFetcher{status: 200}
|
||||
browser := &fakeFetcher{status: 200}
|
||||
p := &Poller{
|
||||
Store: s,
|
||||
Fetch: tls,
|
||||
BrowserFetch: browser,
|
||||
Now: func() time.Time { return now },
|
||||
Cooldown: hour,
|
||||
BrowserCooldown: 6 * hour,
|
||||
Batch: 10,
|
||||
}
|
||||
|
||||
p.runOnce(context.Background())
|
||||
if got := tls.callCount(); got != 1 {
|
||||
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
|
||||
}
|
||||
if got := browser.callCount(); got != 0 {
|
||||
t.Fatalf("browser fetches after 2h = %d, want 0", got)
|
||||
}
|
||||
|
||||
now = time.Unix(7*int64(hour/time.Second), 0)
|
||||
p.runOnce(context.Background())
|
||||
if got := tls.callCount(); got != 2 {
|
||||
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
|
||||
}
|
||||
if got := browser.callCount(); got != 1 {
|
||||
t.Fatalf("browser fetches after 7h = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A site that retracts a chapter should correct the stored number downward,
|
||||
// mirroring the userscript's equality check (L427) rather than a >.
|
||||
func TestRunOnceCorrectsDownward(t *testing.T) {
|
||||
@@ -465,12 +525,14 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
||||
|
||||
p := &Poller{
|
||||
Store: s, Fetch: f,
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
||||
Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
|
||||
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
||||
Interval: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
@@ -498,7 +560,8 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
||||
p := &Poller{
|
||||
Store: s, Fetch: tlsF, BrowserFetch: browserF,
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
||||
Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
|
||||
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
||||
Interval: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user