sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(latest): split browser poll cooldown

Browse Source
This commit is contained in:
Sulthan Zaki
2026-08-09 06:12:11 +07:00
parent 741b23322b
commit a214076756
10 changed files with 218 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/poller.go
+22 -17
View File
@@ -4,6 +4,7 @@ import (
"context"
"log"
"net/url"
"slices"
"time"
"bookmarkmanager/backend/internal/store"
@@ -23,34 +24,36 @@ type Fetcher interface {
// Two clocks, deliberately independent:
//
// - Interval is how often this goroutine wakes up and looks.
// - Cooldown is how long one series rests since its own last check.
// - Cooldowns are how long a series rests since its own last check. Browser-
// backed sites use the longer BrowserCooldown.
//
// Only the cooldown is per series, and it is enforced by the WHERE clause in
// DueForLatestCheck rather than by any timer. Shortening Interval therefore
// cannot shorten anyone's cooldown; it only makes the poller wake up and find
// nothing due more often.
// Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
// by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
// it only makes the poller wake up and find nothing due more often.
type Poller struct {
Store *store.Store
Fetch Fetcher
// BrowserFetch handles sites behind a JavaScript challenge that Fetch
// cannot clear. Nil disables those sites entirely rather than falling back
// to Fetch, which would only ever retrieve a challenge page.
BrowserFetch Fetcher
Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
BrowserFetch Fetcher
Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
var browserBackedSites = []string{"kagane", "novelfull"}
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
// JavaScript challenge that no TLS fingerprint clears — kagane verified
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
// profile TLSFetcher uses — so they are browser-only or nothing.
func (p *Poller) fetcherFor(site string) Fetcher {
switch site {
case "kagane", "novelfull":
if slices.Contains(browserBackedSites, site) {
return p.BrowserFetch
}
return p.Fetch
@@ -63,8 +66,8 @@ func (p *Poller) fetcherFor(site string) Fetcher {
// failure mode for a misconfigured batch x stagger: a slower cadence, never
// concurrent fetch storms.
func (p *Poller) Run(ctx context.Context) {
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s",
p.Interval, p.Cooldown, p.Batch, p.Stagger)
log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
t := time.NewTicker(p.Interval)
defer t.Stop()
for {
@@ -80,8 +83,10 @@ func (p *Poller) Run(ctx context.Context) {
// runOnce processes one batch of due series.
func (p *Poller) runOnce(ctx context.Context) {
cutoff := p.Now().Add(-p.Cooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, p.Batch)
now := p.Now()
cutoff := now.Add(-p.Cooldown).UnixMilli()
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
if err != nil {
log.Printf("latest poll: due query: %v", err)
return