feat(latest): split browser poll cooldown #48

Merged
sulthan merged 1 commits from issue-42-browser-cooldown into main 2026-08-09 06:25:49 +07:00
10 changed files with 218 additions and 89 deletions
Showing only changes of commit a214076756 - Show all commits
+9 -7
View File
@@ -67,13 +67,15 @@ DISCORD_REDIRECT_URI=
# Set to 0 to turn it off entirely. # Set to 0 to turn it off entirely.
# LATEST_CHAPTER_POLL_ENABLED=1 # LATEST_CHAPTER_POLL_ENABLED=1
# #
# Two independent clocks. COOLDOWN is how long one series rests between checks; # Two independent clocks. COOLDOWN is how long a plain-TLS series rests between
# INTERVAL is how often the poller wakes up and looks for series past that # checks; BROWSER_COOLDOWN is the longer rest for kagane and novelfull. INTERVAL
# cooldown. Shortening INTERVAL cannot shorten a COOLDOWN. # is how often the poller wakes up and looks for series past their cooldowns.
# LATEST_CHAPTER_POLL_COOLDOWN=1h # per series, floor 15m # Shortening INTERVAL cannot shorten either cooldown.
# LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake LATEST_CHAPTER_POLL_COOLDOWN=1h # plain-TLS per series, floor 15m
# LATEST_CHAPTER_POLL_BATCH=14 # series per wake LATEST_CHAPTER_POLL_BROWSER_COOLDOWN=6h # browser-backed per series, floor 15m
# LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
LATEST_CHAPTER_POLL_BATCH=14 # series per wake
LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
# #
# Uses a ticker, not an immediate first run: the first poll happens one # Uses a ticker, not an immediate first run: the first poll happens one
# INTERVAL after startup, not at startup. A container restarting more often # INTERVAL after startup, not at startup. A container restarting more often
+3 -2
View File
@@ -40,8 +40,9 @@ Bromite userscript (isolated world, Shadow DOM UI, localStorage cache)
| `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. | | `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. |
| `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. | | `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. |
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. | | `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. |
| `LATEST_CHAPTER_POLL_COOLDOWN` | `1h` | Rest between checks of one series; floor `15m`. | | `LATEST_CHAPTER_POLL_COOLDOWN` | `1h` | Rest between checks of one plain-TLS series; floor `15m`. |
| `LATEST_CHAPTER_POLL_INTERVAL` | `10m` | How often the poller wakes. Cannot shorten a cooldown. | | `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN` | `6h` | Rest between checks of one browser-backed series; floor `15m`. |
| `LATEST_CHAPTER_POLL_INTERVAL` | `10m` | How often the poller wakes. Cannot shorten either cooldown. |
| `LATEST_CHAPTER_POLL_BATCH` | `14` | Series per wake. Keep `BATCH × STAGGER` under `INTERVAL`. | | `LATEST_CHAPTER_POLL_BATCH` | `14` | Series per wake. Keep `BATCH × STAGGER` under `INTERVAL`. |
| `LATEST_CHAPTER_POLL_STAGGER` | `20s` | Delay between fetches in a batch — this is the outbound request rate. | | `LATEST_CHAPTER_POLL_STAGGER` | `20s` | Delay between fetches in a batch — this is the outbound request rate. |
+4 -2
View File
@@ -118,8 +118,10 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
`_REDIRECT_URI` (required; Discord OAuth for the browser UI), `_REDIRECT_URI` (required; Discord OAuth for the browser UI),
`DISCORD_REQUIRED_ROLE` (optional role gate, empty by default), `DISCORD_REQUIRED_ROLE` (optional role gate, empty by default),
`DISCORD_API_BASE` (default `https://discord.com/api/v10`), `DISCORD_API_BASE` (default `https://discord.com/api/v10`),
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER` `LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/
(background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`). `_BATCH`/`_STAGGER` (background latest-chapter poller; defaults on,
`1h` plain-TLS cooldown, `6h` browser cooldown, `10m`/`14`/`20s`; both
cooldowns have a `15m` floor).
`USERSCRIPT_PATH` and `NOVEL_USERSCRIPT_PATH` (files served at `USERSCRIPT_PATH` and `NOVEL_USERSCRIPT_PATH` (files served at
`/u/{token}/manga-bookmark.user.js` and `/u/{token}/novel-bookmark.user.js`, `/u/{token}/manga-bookmark.user.js` and `/u/{token}/novel-bookmark.user.js`,
defaults `/userscript/manga-bookmark.user.js` and defaults `/userscript/manga-bookmark.user.js` and
+16 -11
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"log" "log"
"net/url" "net/url"
"slices"
"time" "time"
"bookmarkmanager/backend/internal/store" "bookmarkmanager/backend/internal/store"
@@ -23,12 +24,12 @@ type Fetcher interface {
// Two clocks, deliberately independent: // Two clocks, deliberately independent:
// //
// - Interval is how often this goroutine wakes up and looks. // - Interval is how often this goroutine wakes up and looks.
// - Cooldown is how long one series rests since its own last check. // - Cooldowns are how long a series rests since its own last check. Browser-
// backed sites use the longer BrowserCooldown.
// //
// Only the cooldown is per series, and it is enforced by the WHERE clause in // Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
// DueForLatestCheck rather than by any timer. Shortening Interval therefore // by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
// cannot shorten anyone's cooldown; it only makes the poller wake up and find // it only makes the poller wake up and find nothing due more often.
// nothing due more often.
type Poller struct { type Poller struct {
Store *store.Store Store *store.Store
Fetch Fetcher Fetch Fetcher
@@ -38,19 +39,21 @@ type Poller struct {
BrowserFetch Fetcher BrowserFetch Fetcher
Now func() time.Time // injected so tests can freeze it Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration Interval time.Duration
Stagger time.Duration Stagger time.Duration
Batch int Batch int
} }
var browserBackedSites = []string{"kagane", "novelfull"}
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be // fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare // fetched at all right now. kagane and novelfull both sit behind a Cloudflare
// JavaScript challenge that no TLS fingerprint clears — kagane verified // JavaScript challenge that no TLS fingerprint clears — kagane verified
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133 // 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
// profile TLSFetcher uses — so they are browser-only or nothing. // profile TLSFetcher uses — so they are browser-only or nothing.
func (p *Poller) fetcherFor(site string) Fetcher { func (p *Poller) fetcherFor(site string) Fetcher {
switch site { if slices.Contains(browserBackedSites, site) {
case "kagane", "novelfull":
return p.BrowserFetch return p.BrowserFetch
} }
return p.Fetch return p.Fetch
@@ -63,8 +66,8 @@ func (p *Poller) fetcherFor(site string) Fetcher {
// failure mode for a misconfigured batch x stagger: a slower cadence, never // failure mode for a misconfigured batch x stagger: a slower cadence, never
// concurrent fetch storms. // concurrent fetch storms.
func (p *Poller) Run(ctx context.Context) { func (p *Poller) Run(ctx context.Context) {
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s", log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
p.Interval, p.Cooldown, p.Batch, p.Stagger) p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
t := time.NewTicker(p.Interval) t := time.NewTicker(p.Interval)
defer t.Stop() defer t.Stop()
for { for {
@@ -80,8 +83,10 @@ func (p *Poller) Run(ctx context.Context) {
// runOnce processes one batch of due series. // runOnce processes one batch of due series.
func (p *Poller) runOnce(ctx context.Context) { func (p *Poller) runOnce(ctx context.Context) {
cutoff := p.Now().Add(-p.Cooldown).UnixMilli() now := p.Now()
due, err := p.Store.DueForLatestCheck(cutoff, p.Batch) cutoff := now.Add(-p.Cooldown).UnixMilli()
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
if err != nil { if err != nil {
log.Printf("latest poll: due query: %v", err) log.Printf("latest poll: due query: %v", err)
return return
+67 -4
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"errors" "errors"
"log"
"os" "os"
"strings" "strings"
"sync" "sync"
@@ -112,6 +113,7 @@ func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Polle
Fetch: f, Fetch: f,
Now: func() time.Time { return at }, Now: func() time.Time { return at },
Cooldown: time.Hour, Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute, Interval: 10 * time.Minute,
Stagger: 0, Stagger: 0,
Batch: 14, Batch: 14,
@@ -304,6 +306,26 @@ func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
} }
} }
func TestRunLogsCooldowns(t *testing.T) {
var logs strings.Builder
previous := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(previous) })
ctx, cancel := context.WithCancel(context.Background())
cancel()
(&Poller{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: time.Hour,
}).Run(ctx)
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
!strings.Contains(got, "browser-cooldown=6h") {
t.Fatalf("startup log = %q, want both cooldowns", got)
}
}
// The cooldown is enforced by the due query, so a second immediate pass must do // The cooldown is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the tick interval independent of it. // nothing at all — this is what makes the tick interval independent of it.
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) { func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
@@ -334,6 +356,44 @@ func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
} }
} }
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
s, _ := newTestStore(t)
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
hour := time.Hour
now := time.Unix(2*int64(hour/time.Second), 0)
tls := &fakeFetcher{status: 200}
browser := &fakeFetcher{status: 200}
p := &Poller{
Store: s,
Fetch: tls,
BrowserFetch: browser,
Now: func() time.Time { return now },
Cooldown: hour,
BrowserCooldown: 6 * hour,
Batch: 10,
}
p.runOnce(context.Background())
if got := tls.callCount(); got != 1 {
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
}
if got := browser.callCount(); got != 0 {
t.Fatalf("browser fetches after 2h = %d, want 0", got)
}
now = time.Unix(7*int64(hour/time.Second), 0)
p.runOnce(context.Background())
if got := tls.callCount(); got != 2 {
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
}
if got := browser.callCount(); got != 1 {
t.Fatalf("browser fetches after 7h = %d, want 1", got)
}
}
// A site that retracts a chapter should correct the stored number downward, // A site that retracts a chapter should correct the stored number downward,
// mirroring the userscript's equality check (L427) rather than a >. // mirroring the userscript's equality check (L427) rather than a >.
func TestRunOnceCorrectsDownward(t *testing.T) { func TestRunOnceCorrectsDownward(t *testing.T) {
@@ -465,12 +525,14 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
}); err != nil { }); err != nil {
t.Fatalf("seed: %v", err) t.Fatalf("seed: %v", err)
} }
f := &fakeFetcher{body: kaganeAPIFixture, status: 200} f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{ p := &Poller{
Store: s, Fetch: f, Store: s,
Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, Interval: time.Hour, Batch: 10, Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
} }
p.runOnce(context.Background()) p.runOnce(context.Background())
@@ -498,7 +560,8 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
p := &Poller{ p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF, Store: s, Fetch: tlsF, BrowserFetch: browserF,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, Interval: time.Hour, Batch: 10, Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
} }
p.runOnce(context.Background()) p.runOnce(context.Background())
+13 -8
View File
@@ -12,6 +12,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/jackc/pgx/v5/pgtype"
_ "github.com/jackc/pgx/v5/stdlib" _ "github.com/jackc/pgx/v5/stdlib"
) )
@@ -689,16 +690,17 @@ func (s *Store) Delete(readerID int64, key string) error {
} }
// DueForLatestCheck returns series whose server-side latest-chapter check has // DueForLatestCheck returns series whose server-side latest-chapter check has
// aged past cutoffMs, ordered by how many bookmarks reference them (descending) // aged past the appropriate cutoff, ordered by how many bookmarks reference
// then least-recently-checked first, at most limit of them. // them (descending) then least-recently-checked first, at most limit of them.
// Browser-backed sites use browserCutoffMs; every other site uses cutoffMs.
// //
// The reader_count ordering is the point of the split (ADR-0003): a series // The reader_count ordering is the point of the split (ADR-0003): a series
// shared by several readers is fetched once per due cycle, and the popular // shared by several readers is fetched once per due cycle, and the popular
// ones stay freshest while the long tail absorbs any shortfall. Within one // ones stay freshest while the long tail absorbs any shortfall. Within one
// reader count, oldest-first keeps the poll fair when the backlog outgrows its // reader count, oldest-first keeps the poll fair when the backlog outgrows
// throughput: the most neglected series is always next, so a large collection // throughput: the most neglected series is always next, so a large collection
// refreshes uniformly slower rather than leaving a tail that never refreshes at // refreshes uniformly slower rather than leaving a tail that never refreshes
// all. The userscript sorts its own queue the same way (L453). // at all. The userscript sorts its own queue the same way (L453).
// //
// Series with no series_url are skipped — there is nothing to fetch, which is // Series with no series_url are skipped — there is nothing to fetch, which is
// the same filter the userscript applies at L452. Series whose only bookmarks // the same filter the userscript applies at L452. Series whose only bookmarks
@@ -706,17 +708,20 @@ func (s *Store) Delete(readerID int64, key string) error {
// burns requests. Archived bookmarks still count — knowing what a shelved // burns requests. Archived bookmarks still count — knowing what a shelved
// series is up to is the whole reason for archiving instead of deleting. // series is up to is the whole reason for archiving instead of deleting.
// A series with no bookmarks at all never appears: the join excludes it. // A series with no bookmarks at all never appears: the join excludes it.
func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Series, error) { func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites []string, limit int) ([]Series, error) {
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
FROM series s FROM series s
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
WHERE s.series_url <> '' WHERE s.series_url <> ''
AND s.latest_checked_at <= $1 AND s.latest_checked_at <= CASE
WHEN s.site = ANY($3::text[]) THEN $2::bigint
ELSE $1::bigint
END
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover, GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0 HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
ORDER BY reader_count DESC, s.latest_checked_at ASC ORDER BY reader_count DESC, s.latest_checked_at ASC
LIMIT $2`, cutoffMs, limit) LIMIT $4`, cutoffMs, browserCutoffMs, pgtype.FlatArray[string](browserSites), limit)
if err != nil { if err != nil {
return nil, fmt.Errorf("query due series: %w", err) return nil, fmt.Errorf("query due series: %w", err)
} }
+7 -7
View File
@@ -291,7 +291,7 @@ func TestDueForLatestCheck(t *testing.T) {
s := newTestStore(t) s := newTestStore(t)
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt) seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
due, err := s.DueForLatestCheck(now-hour, 10) due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck: %v", err) t.Fatalf("DueForLatestCheck: %v", err)
} }
@@ -309,7 +309,7 @@ func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200) seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100) seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
due, err := s.DueForLatestCheck(1000, 2) due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck: %v", err) t.Fatalf("DueForLatestCheck: %v", err)
} }
@@ -489,7 +489,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
} }
} }
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), 10) due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck: %v", err) t.Fatalf("DueForLatestCheck: %v", err)
} }
@@ -844,7 +844,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001) seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100) seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
due, err := s.DueForLatestCheck(1000, 10) due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck: %v", err) t.Fatalf("DueForLatestCheck: %v", err)
} }
@@ -870,7 +870,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
t.Fatalf("seed orphan series: %v", err) t.Fatalf("seed orphan series: %v", err)
} }
due, err := s.DueForLatestCheck(1000, 10) due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck: %v", err) t.Fatalf("DueForLatestCheck: %v", err)
} }
@@ -1207,7 +1207,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series) t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
} }
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), 10) due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck: %v", err) t.Fatalf("DueForLatestCheck: %v", err)
} }
@@ -1223,7 +1223,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 { if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err) t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
} }
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), 10) due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
if err != nil { if err != nil {
t.Fatalf("DueForLatestCheck after delete: %v", err) t.Fatalf("DueForLatestCheck after delete: %v", err)
} }
+29 -14
View File
@@ -62,6 +62,7 @@ type Config struct {
type LatestPoll struct { type LatestPoll struct {
Enabled bool Enabled bool
Cooldown time.Duration Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration Interval time.Duration
Stagger time.Duration Stagger time.Duration
Batch int Batch int
@@ -69,6 +70,7 @@ type LatestPoll struct {
const ( const (
defaultPollCooldown = time.Hour defaultPollCooldown = time.Hour
defaultBrowserPollCooldown = 6 * time.Hour
defaultPollInterval = 10 * time.Minute defaultPollInterval = 10 * time.Minute
defaultPollStagger = 20 * time.Second defaultPollStagger = 20 * time.Second
defaultPollBatch = 14 defaultPollBatch = 14
@@ -129,20 +131,27 @@ func envInt(key string, def int) int {
return n return n
} }
func clampPollCooldown(name string, d time.Duration) time.Duration {
if d < minPollCooldown {
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
return minPollCooldown
}
return d
}
// loadLatestPoll reads the poller's settings, clamping anything that would make // loadLatestPoll reads the poller's settings, clamping anything that would make
// it antisocial. // it antisocial.
func loadLatestPoll() LatestPoll { func loadLatestPoll() LatestPoll {
p := LatestPoll{ p := LatestPoll{
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true), Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown), Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval), Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger), Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch), Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
} }
if p.Cooldown < minPollCooldown { p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
log.Printf("config: cooldown %s is below the %s floor, clamping", p.Cooldown, minPollCooldown) p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
p.Cooldown = minPollCooldown
}
// batch x stagger has to fit inside one tick or a batch is still running // batch x stagger has to fit inside one tick or a batch is still running
// when the next one is due. Run() serialises them, so this degrades to a // when the next one is due. Run() serialises them, so this degrades to a
// slower cadence rather than to overlapping fetches — worth a warning, not // slower cadence rather than to overlapping fetches — worth a warning, not
@@ -324,6 +333,21 @@ func main() {
} }
} }
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
return &latest.Poller{
Store: s,
Fetch: fetch,
BrowserFetch: browser,
Now: time.Now,
Cooldown: cfg.Cooldown,
BrowserCooldown: cfg.BrowserCooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
}
// startLatestPoller launches the background poller unless it is disabled or its // startLatestPoller launches the background poller unless it is disabled or its
// HTTP client cannot be built. Any problem here is logged and skipped: this // HTTP client cannot be built. Any problem here is logged and skipped: this
// feature going missing degrades the service to userscript-only latest-chapter // feature going missing degrades the service to userscript-only latest-chapter
@@ -341,16 +365,7 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, brow
// Nil browser: sites behind a JavaScript challenge are simply not polled, // Nil browser: sites behind a JavaScript challenge are simply not polled,
// and their latest_chapter comes from the userscript alone — which is how // and their latest_chapter comes from the userscript alone — which is how
// the service behaved before the sidecar existed. // the service behaved before the sidecar existed.
p := &latest.Poller{ p := newLatestPoller(s, cfg, f, browser)
Store: s,
Fetch: f,
BrowserFetch: browser,
Now: time.Now,
Cooldown: cfg.Cooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
go p.Run(ctx) go p.Run(ctx)
} }
+37 -2
View File
@@ -17,8 +17,8 @@ import (
func TestLoadLatestPollDefaults(t *testing.T) { func TestLoadLatestPollDefaults(t *testing.T) {
for _, k := range []string{ for _, k := range []string{
"LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN", "LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN",
"LATEST_CHAPTER_POLL_INTERVAL", "LATEST_CHAPTER_POLL_STAGGER", "LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", "LATEST_CHAPTER_POLL_INTERVAL",
"LATEST_CHAPTER_POLL_BATCH", "LATEST_CHAPTER_POLL_STAGGER", "LATEST_CHAPTER_POLL_BATCH",
} { } {
t.Setenv(k, "") t.Setenv(k, "")
} }
@@ -27,6 +27,7 @@ func TestLoadLatestPollDefaults(t *testing.T) {
want := LatestPoll{ want := LatestPoll{
Enabled: true, Enabled: true,
Cooldown: time.Hour, Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute, Interval: 10 * time.Minute,
Stagger: 20 * time.Second, Stagger: 20 * time.Second,
Batch: 14, Batch: 14,
@@ -74,6 +75,30 @@ func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
wantFrom: func(p LatestPoll) any { return p.Cooldown }, wantFrom: func(p LatestPoll) any { return p.Cooldown },
want: 15 * time.Minute, want: 15 * time.Minute,
}, },
{
name: "browser cooldown below the floor is clamped up",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "1m"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown at the floor is kept",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "15m"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown override is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "8h"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 8 * time.Hour,
},
{
name: "browser cooldown unparseable value falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "six hours"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 6 * time.Hour,
},
{ {
name: "a valid override is honoured", name: "a valid override is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"}, env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"},
@@ -123,6 +148,16 @@ func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
} }
} }
func TestNewLatestPollerWiresCooldowns(t *testing.T) {
p := newLatestPoller(nil, LatestPoll{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
}, nil, nil)
if p.Cooldown != time.Hour || p.BrowserCooldown != 6*time.Hour {
t.Fatalf("poller cooldowns = %s/%s, want 1h/6h", p.Cooldown, p.BrowserCooldown)
}
}
func TestPutStatusValidation(t *testing.T) { func TestPutStatusValidation(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
+1
View File
@@ -49,6 +49,7 @@ services:
# switch; it only takes effect because these are listed here. # switch; it only takes effect because these are listed here.
LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1} LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1}
LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h} LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h}
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN: ${LATEST_CHAPTER_POLL_BROWSER_COOLDOWN:-6h}
LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m} LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m}
LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14} LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14}
LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s} LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s}