diff --git a/.env.example b/.env.example index 119c07f..3ea053b 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,26 @@ WEB_PASSWORD= # publish the UI router on a domain you do not own. The same container also # answers on MANGA_API_HOST for the userscript's API. # MANGA_WEB_HOST=manga.example.com + +# --- Latest-chapter poller --- +# The backend re-checks each bookmarked series' newest published chapter on its +# own schedule, so latest_chapter stays fresh even when you never open the manga +# sites. This runs in parallel with the userscript's own in-browser check. +# Set to 0 to turn it off entirely. +# LATEST_CHAPTER_POLL_ENABLED=1 +# +# Two independent clocks. COOLDOWN is how long one series rests between checks; +# INTERVAL is how often the poller wakes up and looks for series past that +# cooldown. Shortening INTERVAL cannot shorten a COOLDOWN. +# LATEST_CHAPTER_POLL_COOLDOWN=1h # per series, floor 15m +# LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake +# LATEST_CHAPTER_POLL_BATCH=14 # series per wake +# LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch +# +# Uses a ticker, not an immediate first run: the first poll happens one +# INTERVAL after startup, not at startup. A container restarting more often +# than INTERVAL never polls. +# +# BATCH x (COOLDOWN / INTERVAL) series hold the cooldown cadence — 84 with these +# defaults. Beyond that the cadence stretches uniformly rather than breaking; +# raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL. diff --git a/CLAUDE.md b/CLAUDE.md index 9405fd0..ab83a1a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ Greenfield. Only `plans/mangaBookmark.md` exists — no code yet. That plan is t ## What this is -A manga read-progress tracker for a user reading on **asurascans.com** (the current domain; asuracomic.net is the older one) and **demonicscans.org** from **Bromite** (mobile Chromium). A userscript injects on-page UI (floating button + slide-in panel) and syncs progress to a self-hosted Go backend so bookmarks unify across both sites and across devices. +A manga read-progress tracker for a user reading on **asurascans.com** (the current domain; asuracomic.net 301s here) and **demonicscans.org** from **Bromite** (mobile Chromium). A userscript injects on-page UI (floating button + slide-in panel) and syncs progress to a self-hosted Go backend so bookmarks unify across both sites and across devices. ## Hard constraints (these drive the design — do not violate) @@ -17,7 +17,7 @@ Bromite uses Chromium's **native** userscript engine, not Tampermonkey: - Cross-origin `fetch()` works **only** against a CORS-enabled backend. Manga sites are `https://`, so backend **must be HTTPS** (mixed-content block otherwise). - Asura and Demonic are **separate origins with separate `localStorage`** — a shared remote store is the only way to unify bookmarks. Cloud sync is required, not optional. - Userscript runs in an **isolated world**, so the embedded API token is safe from the site's JS. -- Cloudflare blocks server-side fetch of the manga sites — **verify URL regex + `og:` selectors against live pages** (Playwright MCP or on-device devtools) before finalizing adapters, not by curling. +- Cloudflare's block on fetching the manga sites is **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both the CGNAT dev machine *and* the deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. This contradicts an earlier, untested assumption that the CGNAT dev IP would be blocked; it was not, at least on this date. Treat "does curl work right now" as a live, time-varying fact to re-check, not a fixed property of a given machine — Cloudflare's bot scoring can flip a previously-clean IP without notice. Any backend fetcher still needs a graceful-degrade path for when it does get challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, or a direct probe) before finalizing, not assumed from a single earlier test. ## Architecture @@ -38,8 +38,30 @@ Bromite userscript (isolated world, per-site adapters, localStorage cache) the web routes are not registered at all. UI mutations read-modify-write through `Store.Get` + `Store.Upsert` so the `updated_at` rule stays in one place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`. +- **Latest-chapter poller:** a ticker goroutine in the same binary re-checks + each bookmarked series' newest published chapter from the backend's own + network access, so `latest_chapter` stays fresh when the user is not + browsing. It is a *second, parallel* signal — the userscript keeps its own + `maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged. + Two independent clocks: a per-bookmark cooldown (`latest_checked_at` column, + enforced by `Store.DueForLatestCheck`'s WHERE clause) and a wake interval. + The row is stamped *before* the fetch so a broken series waits out a full + cooldown instead of retrying every tick, and writes go through + `Store.Get` + `Store.Upsert` so a new chapter never reorders the list. + Fetches use `bogdanfinn/tls-client` with a Chrome profile as defence in depth + against fingerprint-based blocking; any failure logs and skips. See + `docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`. + The poller's `Store.Get` + `Store.Upsert` is not wrapped in a transaction, so + a userscript `PUT` that commits between the two can be overwritten by the + poller's stale re-read — reverting that read progress and, since the stored + value now differs, moving `updated_at` and reordering the list. This is a + known, accepted limitation for a single-user deployment, not a bug to fix. - **`updated_at` drives list order, so it moves only on real reading progress:** the server applies its timestamp when the row is new or `last_chapter_num` changes, and otherwise keeps the stored value — favouriting a series or recording a newly published chapter must not reorder the list. `PUT` therefore returns the row **as stored**, and clients must adopt that response rather than their own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4. -- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list), `DB_PATH` (default `/data/bookmarks.db`), `PORT` (default `8080`), `WEB_PASSWORD` (gates the browser UI; unset disables it). +- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list), `DB_PATH` + (default `/data/bookmarks.db`), `PORT` (default `8080`), `WEB_PASSWORD` + (gates the browser UI; unset disables it), + `LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER` + (background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`). ### Userscript structure (single IIFE, `manga-bookmark.user.js`) @@ -47,7 +69,12 @@ Bromite userscript (isolated world, per-site adapters, localStorage cache) 2. **API client** — `apiGet/apiPut/apiDelete` with bearer header; `localStorage` key `mangabm:cache` for instant render + offline fallback. 3. **Progress logic** — auto-upsert `last_chapter` only when `chapterNum >= stored last_chapter_num` (re-reading old chapters must not regress progress; unparseable -> set current). Manual panel override forces any value. 4. **UI** — rendered inside a **Shadow DOM** root to isolate from site CSS (critical on mobile). -5. **SPA navigation** — Asura is Next.js client-routed: patch `history.pushState`/`replaceState` + listen `popstate`, re-run `detect()` on URL change so auto-update fires without reload. Demonic uses classic reloads (initial `document-idle` run suffices). +5. **SPA navigation** — Asura is Astro, client-routed on the comic/chapter pages: patch `history.pushState`/`replaceState` + listen `popstate`, re-run `detect()` on URL change so auto-update fires without reload. Demonic uses classic reloads (initial `document-idle` run suffices). + +### Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trusting) + +- **asurascans.com**: series `/comics/` (slug carries a trailing hash-like suffix, e.g. `-f886a8af`), chapter `/comics//chapter/`. Astro-rendered; chapter links are present in raw server HTML (no client-side-only render blocking a server fetch). +- **demonicscans.org**: series `/manga/` (slug may URL-encode punctuation, e.g. `%2527` for `'`), chapter `/title//chapter//` (the older `chaptered.php?manga=&chapter=` form still exists as a redirect and is what series-page chapter-list anchors link through). ## Commands (once code exists) diff --git a/backend/Dockerfile b/backend/Dockerfile index c4b45cd..fb494da 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # --- build stage: compile a static, CGO-free binary --- -FROM golang:1.23-alpine AS build +FROM golang:1.24-alpine AS build WORKDIR /src # Dependencies first for layer caching (changes rarely). diff --git a/backend/go.mod b/backend/go.mod index 3076eb2..56a710f 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,17 +1,33 @@ module mangabm/backend -go 1.23 - -require modernc.org/sqlite v1.34.4 +go 1.24.1 require ( + github.com/bogdanfinn/fhttp v0.6.8 + github.com/bogdanfinn/tls-client v1.15.1 + modernc.org/sqlite v1.34.4 +) + +require ( + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/bdandy/go-errors v1.2.2 // indirect + github.com/bdandy/go-socks4 v1.2.3 // indirect + github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect + github.com/bogdanfinn/utls v1.7.7-barnius // indirect + github.com/bogdanfinn/websocket v1.5.5-barnius // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/quic-go/qpack v0.6.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect + github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index c337282..cc6daad 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,3 +1,21 @@ +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/bdandy/go-errors v1.2.2 h1:WdFv/oukjTJCLa79UfkGmwX7ZxONAihKu4V0mLIs11Q= +github.com/bdandy/go-errors v1.2.2/go.mod h1:NkYHl4Fey9oRRdbB1CoC6e84tuqQHiqrOcZpqFEkBxM= +github.com/bdandy/go-socks4 v1.2.3 h1:Q6Y2heY1GRjCtHbmlKfnwrKVU/k81LS8mRGLRlmDlic= +github.com/bdandy/go-socks4 v1.2.3/go.mod h1:98kiVFgpdogR8aIGLWLvjDVZ8XcKPsSI/ypGrO+bqHI= +github.com/bogdanfinn/fhttp v0.6.8 h1:LiQyHOY3i0QoxxNB7nq27/nGNNbtPj0fuBPozhR7Ws4= +github.com/bogdanfinn/fhttp v0.6.8/go.mod h1:A+EKDzMx2hb4IUbMx4TlkoHnaJEiLl8r/1Ss1Y+5e5M= +github.com/bogdanfinn/quic-go-utls v1.0.9-utls h1:tV6eDEiRbRCcepALSzxR94JUVD3N3ACIiRLgyc2Ep8s= +github.com/bogdanfinn/quic-go-utls v1.0.9-utls/go.mod h1:aHph9B9H9yPOt5xnhWKSOum27DJAqpiHzwX+gjvaXcg= +github.com/bogdanfinn/tls-client v1.15.1 h1:KiFAlED55DJ8Fcocn+/1nX6PrDFcttIHAf/GDkV6KN8= +github.com/bogdanfinn/tls-client v1.15.1/go.mod h1:LsU6mXVn8MOFDwTkyRfI7V1BZM1p0wf2ZfZsICW/1fM= +github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5ZlrCR7fOLU= +github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg= +github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI= +github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= @@ -6,21 +24,49 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.0.0-20211104170005-ce137452f963/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= diff --git a/backend/latest.go b/backend/latest.go new file mode 100644 index 0000000..4d08d0d --- /dev/null +++ b/backend/latest.go @@ -0,0 +1,194 @@ +package main + +import ( + "context" + "log" + "net/url" + "time" +) + +// fetcher retrieves a series page. It exists as an interface so tests can inject +// a fake: nothing in the test suite may touch the network or the TLS client. +type fetcher interface { + Get(ctx context.Context, url string) (body string, status int, err error) +} + +// latestPoller re-checks each bookmarked series' newest published chapter on a +// schedule, independent of the userscript's own in-browser checks. The two run +// in parallel and report the same observable fact, so whichever writes last wins +// and neither needs to know about the other. +// +// Two clocks, deliberately independent: +// +// - interval is how often this goroutine wakes up and looks. +// - cooldown is how long one bookmark rests since its own last check. +// +// Only the cooldown is per bookmark, and it is enforced by the WHERE clause in +// DueForLatestCheck rather than by any timer. Shortening interval therefore +// cannot shorten anyone's cooldown; it only makes the poller wake up and find +// nothing due more often. +type latestPoller struct { + store *Store + fetch fetcher + now func() time.Time // injected so tests can freeze it + cooldown time.Duration + interval time.Duration + stagger time.Duration + batch int +} + +// Run polls until ctx is cancelled. +// +// runOnce is called synchronously, so a batch that overruns the tick delays the +// next one instead of stacking a second batch on top of it. That is the intended +// failure mode for a misconfigured batch x stagger: a slower cadence, never +// concurrent fetch storms. +func (p *latestPoller) Run(ctx context.Context) { + log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s", + p.interval, p.cooldown, p.batch, p.stagger) + t := time.NewTicker(p.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + log.Println("latest-chapter poller: stopped") + return + case <-t.C: + p.runOnce(ctx) + } + } +} + +// runOnce processes one batch of due bookmarks. +func (p *latestPoller) runOnce(ctx context.Context) { + cutoff := p.now().Add(-p.cooldown).UnixMilli() + due, err := p.store.DueForLatestCheck(cutoff, p.batch) + if err != nil { + log.Printf("latest poll: due query: %v", err) + return + } + + checked := 0 + for i, b := range due { + if ctx.Err() != nil { + break + } + // Staggered rather than fired together: a burst of simultaneous requests + // from one server IP is the traffic shape most likely to move that IP's + // bot score. This is the server-side analogue of the userscript's "one + // series per navigation ... indistinguishable from browsing" (L455-456). + stopped := false + if i > 0 && p.stagger > 0 { + select { + case <-ctx.Done(): + stopped = true + case <-time.After(p.stagger): + } + } + if stopped { + break + } + p.checkOne(ctx, b) + checked++ + } + // due vs checked is how you tell which constraint is binding: ticks that + // report due=0 mean the cooldown is the limit, ticks that report due==batch + // every time mean throughput is. + log.Printf("latest poll: due=%d checked=%d", len(due), checked) +} + +// checkOne re-checks one series. Every failure path here is "log and move on": +// the poller is a best-effort enhancement, and no single bad series may stall a +// batch or take down the process. +func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) { + defer func() { + if r := recover(); r != nil { + log.Printf("latest poll %q: recovered from panic: %v", b.Key, r) + } + }() + + // Stamped before the fetch, not after, so an error, a timeout, or a shutdown + // mid-request still consumes the cooldown. Otherwise a renamed or deleted + // series would be retried on every single tick forever. The userscript + // stamps in the same order and for the same reason (L471-473). + if err := p.store.MarkLatestChecked(b.Key, p.now().UnixMilli()); err != nil { + log.Printf("latest poll %q: mark checked: %v", b.Key, err) + return + } + + // series_url is client-supplied (PUT /bookmarks/{key} accepts any string), + // so this is not just an optimisation against burning a request on an + // unknown site: without it, the server would issue a GET from its own + // network position to whatever URL a token-holder writes, including + // link-local/internal addresses or non-https schemes. The cooldown above + // is already consumed, so a row that never passes this check is retried at + // cooldown pace rather than hot-looping. + if !fetchableSeriesURL(b.Site, b.SeriesURL) { + log.Printf("latest poll %q: not fetchable: site=%q url=%q", b.Key, b.Site, b.SeriesURL) + return + } + + body, status, err := p.fetch.Get(ctx, b.SeriesURL) + if err != nil { + log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err) + return + } + if status != 200 { + log.Printf("latest poll %q: fetch %s: status %d", b.Key, b.SeriesURL, status) + return + } + + latest, ok := latestChapterFrom(b.Site, b.SeriesURL, body) + if !ok { + // Most likely a challenge page or a layout change. Either way the row is + // already stamped, so this waits out a cooldown instead of hot-looping. + log.Printf("latest poll %q: no chapter links in %d bytes", b.Key, len(body)) + return + } + + // Re-read: the row may have been updated or deleted while the fetch was in + // flight, and writing b back wholesale would undo that. + cur, found, err := p.store.Get(b.Key) + if err != nil { + log.Printf("latest poll %q: reread: %v", b.Key, err) + return + } + if !found { + return + } + // Equality, not >, mirroring the userscript (L427): a site that retracts a + // chapter should correct the stored number downward. + if cur.LatestChapterNum != nil && *cur.LatestChapterNum == latest.Num { + return + } + + num := latest.Num + cur.LatestChapter = latest.Label + cur.LatestChapterNum = &num + // A candidate only. last_chapter_num is untouched, so the CASE in Upsert + // keeps the stored updated_at and the bookmark list does not reorder. + cur.UpdatedAt = p.now().UnixMilli() + if _, err := p.store.Upsert(cur); err != nil { + log.Printf("latest poll %q: upsert: %v", b.Key, err) + return + } + log.Printf("latest poll %q: latest is now %s", b.Key, latest.Label) +} + +// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how +// to parse and seriesURL is safe to hand to the fetcher: an https URL with a +// non-empty host. series_url comes from client-supplied PUT bodies, so this is +// a defence against the poller being used to probe arbitrary hosts from the +// server's own network position, not just a check against wasted requests. +func fetchableSeriesURL(site, seriesURL string) bool { + switch site { + case "asura", "demonic": + default: + return false + } + u, err := url.Parse(seriesURL) + if err != nil { + return false + } + return u.Scheme == "https" && u.Host != "" +} diff --git a/backend/latest_http.go b/backend/latest_http.go new file mode 100644 index 0000000..d6a34ec --- /dev/null +++ b/backend/latest_http.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "fmt" + "io" + + fhttp "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" + "github.com/bogdanfinn/tls-client/profiles" +) + +// maxBodyBytes caps what a single series page can cost in memory. Real pages +// measured 100-400 KB on 2026-07-26, so this is roughly 10x headroom and mostly +// guards against a proxy handing back something enormous. +const maxBodyBytes = 4 << 20 + +// chromeUA matches the client profile below. A Chrome fingerprint paired with a +// non-Chrome user agent is itself a signal. +const chromeUA = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36" + +// tlsFetcher fetches series pages with a Chrome TLS fingerprint. +// +// Plain net/http was verified working against both sites on 2026-07-26, so this +// is not fixing an observed block — it is deliberate defence-in-depth against a +// future fingerprint-based one, chosen up front rather than reacted to later. +// The library is pure Go, so CGO_ENABLED=0, the static binary, and the +// distroless image are all unaffected. +type tlsFetcher struct { + client tls_client.HttpClient +} + +var _ fetcher = (*tlsFetcher)(nil) + +func newTLSFetcher() (*tlsFetcher, error) { + c, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), + tls_client.WithTimeoutSeconds(30), + tls_client.WithClientProfile(profiles.Chrome_133), + ) + if err != nil { + return nil, fmt.Errorf("new tls client: %w", err) + } + return &tlsFetcher{client: c}, nil +} + +// Get fetches url and returns the body and status. Redirects are followed: the +// demonic chapter anchors are a redirect form, and asura has moved domains +// before. +func (f *tlsFetcher) Get(ctx context.Context, url string) (string, int, error) { + req, err := fhttp.NewRequest(fhttp.MethodGet, url, nil) + if err != nil { + return "", 0, fmt.Errorf("build request %q: %w", url, err) + } + req = req.WithContext(ctx) + // Header order is part of what is being fingerprinted, so it is stated + // explicitly instead of left to Go's map iteration order. + req.Header = fhttp.Header{ + "user-agent": {chromeUA}, + "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "accept-language": {"en-US,en;q=0.9"}, + fhttp.HeaderOrderKey: {"user-agent", "accept", "accept-language"}, + } + + resp, err := f.client.Do(req) + if err != nil { + return "", 0, fmt.Errorf("get %q: %w", url, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + if err != nil { + return "", resp.StatusCode, fmt.Errorf("read %q: %w", url, err) + } + return string(body), resp.StatusCode, nil +} diff --git a/backend/latest_sites.go b/backend/latest_sites.go new file mode 100644 index 0000000..5a4feb3 --- /dev/null +++ b/backend/latest_sites.go @@ -0,0 +1,75 @@ +package main + +import ( + "regexp" + "strconv" + "strings" +) + +// latestChapter is the newest chapter a series page advertises. +type latestChapter struct { + Num float64 + Label string +} + +// asuraSlugRe pulls the series slug out of a stored series_url. +// Shape verified live 2026-07-26: https://asurascans.com/comics/, where +// the slug carries a trailing hash-like suffix (e.g. "-f886a8af"). +var asuraSlugRe = regexp.MustCompile(`/comics/([^/?#]+)`) + +// demonicChapterRe matches the pre-redirect anchors demonic series pages link +// through. Both the raw "&" and the HTML-escaped "&" forms occur. +var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?chapter=([0-9.]+)`) + +// latestChapterFrom returns the highest chapter number body advertises for this +// series. ok is false when the body yields nothing usable — an unknown site, an +// empty body, a Cloudflare challenge page, and a site redesign all land here, +// and the caller treats all four identically. +// +// Ported from the userscript's latestChapterFromAnchors (asura L123-133, +// demonic L183-193), including its reason for taking a maximum rather than a +// first or last: neither site lists chapters in a dependable order. +// +// The userscript's asura rule additionally requires the anchor text to match +// /Chapter\s+[\d.]+/i. That check exists only to skip the "First Chapter" +// shortcut, which points at chapter/1 and therefore can never win a maximum, so +// it is redundant here. For asura, scoping the pattern to this series' own slug +// replaces it with a stronger guarantee: a chapter link belonging to some other +// series cannot contribute even if the page starts carrying them. demonic has no +// such guarantee — demonicChapterRe matches any chaptered.php?manga= anchor +// with no per-series scoping, because the stored series_id for demonic is a +// slug, not the numeric id the URL carries, so it cannot easily be scoped. +func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) { + var re *regexp.Regexp + switch site { + case "asura": + m := asuraSlugRe.FindStringSubmatch(seriesURL) + if m == nil { + return latestChapter{}, false + } + // Compiled per call rather than cached: this runs once per fetch, which + // is at most a few times a minute, and the slug varies per series. + re = regexp.MustCompile(`/comics/` + regexp.QuoteMeta(m[1]) + `/chapter/([0-9.]+)`) + case "demonic": + re = demonicChapterRe + default: + return latestChapter{}, false + } + + var best latestChapter + found := false + for _, m := range re.FindAllStringSubmatch(body, -1) { + // [0-9.]+ can swallow a trailing separator, e.g. "chapter/12." in a + // sentence; ParseFloat would reject the whole match. + raw := strings.Trim(m[1], ".") + num, err := strconv.ParseFloat(raw, 64) + if err != nil { + continue + } + if !found || num > best.Num { + best = latestChapter{Num: num, Label: "Chapter " + raw} + found = true + } + } + return best, found +} diff --git a/backend/latest_sites_test.go b/backend/latest_sites_test.go new file mode 100644 index 0000000..a65b346 --- /dev/null +++ b/backend/latest_sites_test.go @@ -0,0 +1,115 @@ +package main + +import "testing" + +// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af +// fetched 2026-07-26. The first anchor is the "First Chapter" shortcut: it is a +// real chapter link with no "Chapter N" text, and it must not be mistaken for +// the latest just because it parses. +const asuraSeriesFixture = ` +First Chapter +Chapter 179 +Chapter 181 +Chapter 180 +` + +// A chapter link belonging to a different series, of the kind a "you might also +// like" strip would introduce. Slug scoping must exclude it. +const asuraCrossSeriesFixture = asuraSeriesFixture + ` +Chapter 999 +` + +// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer fetched +// 2026-07-26. Note the raw "&", the doubled space after Chapter 0.5 +Chapter 294 +Chapter 296 +Chapter 295 +` + +// What Cloudflare serves instead of the page when an IP's bot score flips. +const challengeFixture = `Just a moment... + +
Checking your browser
` + +func TestLatestChapterFrom(t *testing.T) { + const asuraURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + const demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer" + + tests := []struct { + name string + site string + seriesURL string + body string + wantOK bool + wantNum float64 + wantLabel string + }{ + { + name: "asura takes the max, not the last listed", + site: "asura", seriesURL: asuraURL, body: asuraSeriesFixture, + wantOK: true, wantNum: 181, wantLabel: "Chapter 181", + }, + { + name: "asura ignores another series' chapter links", + site: "asura", seriesURL: asuraURL, body: asuraCrossSeriesFixture, + wantOK: true, wantNum: 181, wantLabel: "Chapter 181", + }, + { + name: "asura with an unparseable series url", + site: "asura", seriesURL: "https://asurascans.com/", body: asuraSeriesFixture, + wantOK: false, + }, + { + name: "demonic takes the max across raw and escaped ampersands", + site: "demonic", seriesURL: demonicURL, body: demonicSeriesFixture, + wantOK: true, wantNum: 296, wantLabel: "Chapter 296", + }, + { + name: "demonic keeps decimal chapters parseable", + site: "demonic", seriesURL: demonicURL, + body: `Chapter 0.5`, + wantOK: true, wantNum: 0.5, wantLabel: "Chapter 0.5", + }, + { + name: "empty body", + site: "asura", seriesURL: asuraURL, body: "", + wantOK: false, + }, + { + name: "cloudflare challenge page", + site: "asura", seriesURL: asuraURL, body: challengeFixture, + wantOK: false, + }, + { + name: "demonic markup handed to the asura rule", + site: "asura", seriesURL: asuraURL, body: demonicSeriesFixture, + wantOK: false, + }, + { + name: "unknown site", + site: "mangadex", seriesURL: "https://example.com/x", body: asuraSeriesFixture, + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := latestChapterFrom(tt.site, tt.seriesURL, tt.body) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v (got %+v)", ok, tt.wantOK, got) + } + if !tt.wantOK { + return + } + if got.Num != tt.wantNum { + t.Errorf("Num = %v, want %v", got.Num, tt.wantNum) + } + if got.Label != tt.wantLabel { + t.Errorf("Label = %q, want %q", got.Label, tt.wantLabel) + } + }) + } +} diff --git a/backend/latest_test.go b/backend/latest_test.go new file mode 100644 index 0000000..04502e0 --- /dev/null +++ b/backend/latest_test.go @@ -0,0 +1,320 @@ +package main + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// fakeFetcher stands in for the network. Every poller test uses it, so nothing +// in this file can reach tls-client or a real site. +type fakeFetcher struct { + mu sync.Mutex + calls []string + body string + status int + err error + // perURL overrides body/status/err for specific URLs. + perURL map[string]fakeResponse +} + +type fakeResponse struct { + body string + status int + err error +} + +func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) { + f.mu.Lock() + f.calls = append(f.calls, url) + f.mu.Unlock() + if r, ok := f.perURL[url]; ok { + return r.body, r.status, r.err + } + return f.body, f.status, f.err +} + +func (f *fakeFetcher) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +// newTestPoller wires a poller with a frozen clock and no stagger, so tests run +// instantly and deterministically. +func newTestPoller(t *testing.T, s *Store, f fetcher, at time.Time) *latestPoller { + t.Helper() + return &latestPoller{ + store: s, + fetch: f, + now: func() time.Time { return at }, + cooldown: time.Hour, + interval: 10 * time.Minute, + stagger: 0, + batch: 14, + } +} + +func TestRunOnceRecordsLatestChapter(t *testing.T) { + s := newTestStore(t) + const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0) + + now := time.UnixMilli(5_000_000) + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + newTestPoller(t, s, f, now).runOnce(context.Background()) + + b, ok, err := s.Get("asura:chronicles-of-the-demon-faction-f886a8af") + if err != nil || !ok { + t.Fatalf("Get: %v ok=%v", err, ok) + } + if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { + t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum) + } + if b.LatestChapter != "Chapter 181" { + t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181") + } + if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() { + t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) + } +} + +// The whole point of the updated_at CASE in Upsert: a newly published chapter is +// not reading progress and must not move the series up the list. +func TestRunOnceDoesNotReorderList(t *testing.T) { + s := newTestStore(t) + const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + const key = "asura:chronicles-of-the-demon-faction-f886a8af" + + // "other" is the most recently read, so it must stay at the top of List(). + if _, err := s.Upsert(Bookmark{ + Key: "asura:other", Site: "asura", SeriesID: "other", + SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000, + }); err != nil { + t.Fatalf("seed other: %v", err) + } + seedForCheck(t, s, key, url, 0) + before, _, err := s.Get(key) + if err != nil { + t.Fatalf("Get before: %v", err) + } + + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background()) + + after, _, err := s.Get(key) + if err != nil { + t.Fatalf("Get after: %v", err) + } + if after.UpdatedAt != before.UpdatedAt { + t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump", + before.UpdatedAt, after.UpdatedAt) + } + list, err := s.List() + if err != nil { + t.Fatalf("List: %v", err) + } + if list[0].Key != "asura:other" { + t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key) + } +} + +// A failed fetch must still consume the cooldown, or a renamed series gets +// retried on every tick forever. +func TestRunOnceMarksCheckedOnFailure(t *testing.T) { + tests := []struct { + name string + resp fakeResponse + }{ + {"network error", fakeResponse{err: errors.New("dial tcp: refused")}}, + {"non-200", fakeResponse{body: "nope", status: 503}}, + {"challenge page", fakeResponse{body: challengeFixture, status: 200}}, + {"empty body", fakeResponse{body: "", status: 200}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestStore(t) + const url = "https://asurascans.com/comics/x" + seedForCheck(t, s, "asura:x", url, 0) + + now := time.UnixMilli(7_000_000) + f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}} + newTestPoller(t, s, f, now).runOnce(context.Background()) + + if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() { + t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) + } + b, _, err := s.Get("asura:x") + if err != nil { + t.Fatalf("Get: %v", err) + } + if b.LatestChapterNum != nil { + t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum) + } + }) + } +} + +func TestRunOnceRespectsBatchLimit(t *testing.T) { + s := newTestStore(t) + for i := 0; i < 20; i++ { + key := "asura:s" + string(rune('a'+i)) + seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0) + } + + f := &fakeFetcher{body: "", status: 200} + p := newTestPoller(t, s, f, time.UnixMilli(5_000_000)) + p.batch = 5 + p.runOnce(context.Background()) + + if got := f.callCount(); got != 5 { + t.Fatalf("fetched %d series, want 5 (batch limit)", got) + } +} + +// One unreachable series must not abandon the rest of the batch. +func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) { + s := newTestStore(t) + keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"} + for _, k := range keys { + seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0) + } + + now := time.UnixMilli(6_000_000) + f := &fakeFetcher{ + body: "", status: 200, + perURL: map[string]fakeResponse{ + "https://asurascans.com/comics/asura:b": {err: errors.New("boom")}, + }, + } + newTestPoller(t, s, f, now).runOnce(context.Background()) + + if got := f.callCount(); got != 5 { + t.Fatalf("fetched %d series, want all 5 attempted", got) + } + for _, k := range keys { + if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() { + t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli()) + } + } +} + +// The cooldown is enforced by the due query, so a second immediate pass must do +// nothing at all — this is what makes the tick interval independent of it. +func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) { + s := newTestStore(t) + const url = "https://asurascans.com/comics/x" + seedForCheck(t, s, "asura:x", url, 0) + + now := time.UnixMilli(8_000_000) + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + p := newTestPoller(t, s, f, now) + + p.runOnce(context.Background()) + if got := f.callCount(); got != 1 { + t.Fatalf("first pass fetched %d, want 1", got) + } + // Same instant, and again 59 minutes later: both inside the 1h cooldown. + p.runOnce(context.Background()) + p.now = func() time.Time { return now.Add(59 * time.Minute) } + p.runOnce(context.Background()) + if got := f.callCount(); got != 1 { + t.Fatalf("fetched %d times inside the cooldown, want 1", got) + } + // Past the cooldown, it is due again. + p.now = func() time.Time { return now.Add(61 * time.Minute) } + p.runOnce(context.Background()) + if got := f.callCount(); got != 2 { + t.Fatalf("fetched %d times after the cooldown, want 2", got) + } +} + +// A site that retracts a chapter should correct the stored number downward, +// mirroring the userscript's equality check (L427) rather than a >. +func TestRunOnceCorrectsDownward(t *testing.T) { + s := newTestStore(t) + const url = "https://demonicscans.org/manga/Catastrophic-Necromancer" + const key = "demonic:Catastrophic-Necromancer" + + high := 400.0 + if _, err := s.Upsert(Bookmark{ + Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer", + SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high, + UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + f := &fakeFetcher{body: demonicSeriesFixture, status: 200} + newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background()) + + b, _, err := s.Get(key) + if err != nil { + t.Fatalf("Get: %v", err) + } + if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 { + t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum) + } +} + +// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must +// reject anything that is not a known site with an https URL before spending a +// request on it — the cooldown still gets consumed either way. +func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) { + tests := []struct { + name string + site string + seriesURL string + wantCalls int + }{ + {"unknown site", "mangadex", "https://mangadex.org/title/x", 0}, + {"http scheme", "asura", "http://asurascans.com/comics/x", 0}, + {"unparseable url", "asura", "http://[::1", 0}, + {"valid https asura", "asura", "https://asurascans.com/comics/x", 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestStore(t) + key := tt.site + ":x" + if _, err := s.Upsert(Bookmark{ + Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL, + UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + now := time.UnixMilli(4_000_000) + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + newTestPoller(t, s, f, now).checkOne(context.Background(), Bookmark{ + Key: key, Site: tt.site, SeriesURL: tt.seriesURL, + }) + + if got := f.callCount(); got != tt.wantCalls { + t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls) + } + if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() { + t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli()) + } + }) + } +} + +// A cancelled context must abandon the batch rather than run it to completion. +func TestRunOnceStopsOnCancelledContext(t *testing.T) { + s := newTestStore(t) + for _, k := range []string{"asura:a", "asura:b", "asura:c"} { + seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + f := &fakeFetcher{body: "", status: 200} + newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx) + + if got := f.callCount(); got != 0 { + t.Fatalf("fetched %d series with a cancelled context, want 0", got) + } +} diff --git a/backend/main.go b/backend/main.go index 117028b..2706b78 100644 --- a/backend/main.go +++ b/backend/main.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "syscall" "time" @@ -20,8 +21,34 @@ type Config struct { Port string // WebPassword gates the browser UI. Empty disables the web routes entirely. WebPassword string + // LatestPoll configures the background latest-chapter fetcher. + LatestPoll LatestPoll } +// LatestPoll configures the background latest-chapter poller. +// +// Sizing: batch x (cooldown / interval) is how many series hold a true cooldown +// cadence — 14 x (1h / 10m) = 84 with these defaults, which covers this +// deployment. Past that nothing breaks; the effective cadence stretches to +// N x interval / batch and the oldest-checked-first ordering keeps it uniform. +type LatestPoll struct { + Enabled bool + Cooldown time.Duration + Interval time.Duration + Stagger time.Duration + Batch int +} + +const ( + defaultPollCooldown = time.Hour + defaultPollInterval = 10 * time.Minute + defaultPollStagger = 20 * time.Second + defaultPollBatch = 14 + // minPollCooldown keeps a typo from turning a polite background check into + // a hammer against sites that are already bot-scoring us. + minPollCooldown = 15 * time.Minute +) + func envOr(key, def string) string { if v := os.Getenv(key); v != "" { return v @@ -29,12 +56,83 @@ func envOr(key, def string) string { return def } +// envBool reads a boolean env var. Anything unrecognised falls back to def. +func envBool(key string, def bool) bool { + switch v := strings.ToLower(strings.TrimSpace(os.Getenv(key))); v { + case "": + return def + case "0", "false", "no", "off": + return false + case "1", "true", "yes", "on": + return true + default: + log.Printf("config: %s=%q is not a boolean, using %v", key, v, def) + return def + } +} + +// envDuration reads a duration env var. An unparseable or non-positive value +// falls back to def and logs rather than failing startup: the poller is an +// enhancement, and a typo in one of its knobs must not stop bookmark sync. +func envDuration(key string, def time.Duration) time.Duration { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return def + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + log.Printf("config: %s=%q is not a positive duration, using %s", key, raw, def) + return def + } + return d +} + +// envInt reads a positive integer env var, with the same fallback policy. +func envInt(key string, def int) int { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return def + } + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + log.Printf("config: %s=%q is not a positive integer, using %d", key, raw, def) + return def + } + return n +} + +// loadLatestPoll reads the poller's settings, clamping anything that would make +// it antisocial. +func loadLatestPoll() LatestPoll { + p := LatestPoll{ + Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true), + Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown), + Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval), + Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger), + Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch), + } + if p.Cooldown < minPollCooldown { + log.Printf("config: cooldown %s is below the %s floor, clamping", p.Cooldown, minPollCooldown) + p.Cooldown = minPollCooldown + } + // batch x stagger has to fit inside one tick or a batch is still running + // when the next one is due. Run() serialises them, so this degrades to a + // slower cadence rather than to overlapping fetches — worth a warning, not + // a failure. + if span := time.Duration(p.Batch) * p.Stagger; span > p.Interval { + log.Printf("config: batch(%d) x stagger(%s) = %s exceeds interval %s; batches will overrun their tick", + p.Batch, p.Stagger, span, p.Interval) + } + return p +} + func loadConfig() Config { c := Config{ Token: os.Getenv("API_TOKEN"), DBPath: envOr("DB_PATH", "/data/bookmarks.db"), Port: envOr("PORT", "8080"), WebPassword: os.Getenv("WEB_PASSWORD"), + LatestPoll: loadLatestPoll(), } for _, o := range strings.Split(os.Getenv("ALLOWED_ORIGINS"), ",") { if o = strings.TrimSpace(o); o != "" { @@ -87,6 +185,13 @@ func main() { } defer store.Close() + // The poller is off the request path entirely: if it cannot start, the + // service still serves bookmarks and the userscript still captures latest + // chapters on its own. + pollCtx, stopPoll := context.WithCancel(context.Background()) + defer stopPoll() + startLatestPoller(pollCtx, store, cfg.LatestPoll) + srv := &http.Server{ Addr: ":" + cfg.Port, Handler: newRouter(store, cfg), @@ -105,9 +210,38 @@ func main() { <-stop log.Println("shutting down") + // Stop polling before draining requests, so an in-flight series fetch does + // not hold the process open past the shutdown deadline. + stopPoll() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := srv.Shutdown(ctx); err != nil { log.Printf("shutdown: %v", err) } } + +// startLatestPoller launches the background poller unless it is disabled or its +// HTTP client cannot be built. Any problem here is logged and skipped: this +// feature going missing degrades the service to userscript-only latest-chapter +// tracking, which is exactly how it behaved before. +func startLatestPoller(ctx context.Context, store *Store, cfg LatestPoll) { + if !cfg.Enabled { + log.Println("latest-chapter poller: disabled by config") + return + } + f, err := newTLSFetcher() + if err != nil { + log.Printf("latest-chapter poller: disabled, cannot build client: %v", err) + return + } + p := &latestPoller{ + store: store, + fetch: f, + now: time.Now, + cooldown: cfg.Cooldown, + interval: cfg.Interval, + stagger: cfg.Stagger, + batch: cfg.Batch, + } + go p.Run(ctx) +} diff --git a/backend/main_test.go b/backend/main_test.go new file mode 100644 index 0000000..90145bb --- /dev/null +++ b/backend/main_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "testing" + "time" +) + +func TestLoadLatestPollDefaults(t *testing.T) { + for _, k := range []string{ + "LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN", + "LATEST_CHAPTER_POLL_INTERVAL", "LATEST_CHAPTER_POLL_STAGGER", + "LATEST_CHAPTER_POLL_BATCH", + } { + t.Setenv(k, "") + } + + got := loadLatestPoll() + want := LatestPoll{ + Enabled: true, + Cooldown: time.Hour, + Interval: 10 * time.Minute, + Stagger: 20 * time.Second, + Batch: 14, + } + if got != want { + t.Fatalf("loadLatestPoll() = %+v, want %+v", got, want) + } +} + +func TestLoadLatestPollEnabledParsing(t *testing.T) { + tests := []struct { + raw string + want bool + }{ + {"", true}, {"1", true}, {"true", true}, {"TRUE", true}, {"yes", true}, {"on", true}, + {"0", false}, {"false", false}, {"False", false}, {"no", false}, {"off", false}, + {"maybe", true}, // unparseable falls back to the default + } + for _, tt := range tests { + t.Run("raw="+tt.raw, func(t *testing.T) { + t.Setenv("LATEST_CHAPTER_POLL_ENABLED", tt.raw) + if got := loadLatestPoll().Enabled; got != tt.want { + t.Fatalf("Enabled = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoadLatestPollClampsAndFallsBack(t *testing.T) { + tests := []struct { + name string + env map[string]string + wantFrom func(LatestPoll) any + want any + }{ + { + name: "cooldown below the floor is clamped up", + env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "1m"}, + wantFrom: func(p LatestPoll) any { return p.Cooldown }, + want: 15 * time.Minute, + }, + { + name: "cooldown at the floor is kept", + env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "15m"}, + wantFrom: func(p LatestPoll) any { return p.Cooldown }, + want: 15 * time.Minute, + }, + { + name: "a valid override is honoured", + env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"}, + wantFrom: func(p LatestPoll) any { return p.Interval }, + want: 5 * time.Minute, + }, + { + name: "an unparseable duration falls back", + env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "ten minutes"}, + wantFrom: func(p LatestPoll) any { return p.Interval }, + want: 10 * time.Minute, + }, + { + name: "a zero duration falls back", + env: map[string]string{"LATEST_CHAPTER_POLL_STAGGER": "0s"}, + wantFrom: func(p LatestPoll) any { return p.Stagger }, + want: 20 * time.Second, + }, + { + name: "a valid batch is honoured", + env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "30"}, + wantFrom: func(p LatestPoll) any { return p.Batch }, + want: 30, + }, + { + name: "a negative batch falls back", + env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "-5"}, + wantFrom: func(p LatestPoll) any { return p.Batch }, + want: 14, + }, + { + name: "a non-numeric batch falls back", + env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "lots"}, + wantFrom: func(p LatestPoll) any { return p.Batch }, + want: 14, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.env { + t.Setenv(k, v) + } + if got := tt.wantFrom(loadLatestPoll()); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/store.go b/backend/store.go index 7588509..d75916c 100644 --- a/backend/store.go +++ b/backend/store.go @@ -59,6 +59,7 @@ CREATE TABLE IF NOT EXISTS bookmarks ( favorite INTEGER NOT NULL DEFAULT 0, latest_chapter TEXT NOT NULL DEFAULT '', latest_chapter_num REAL, + latest_checked_at INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL );` @@ -68,6 +69,10 @@ var addedColumns = []struct{ name, ddl string }{ {"favorite", `ALTER TABLE bookmarks ADD COLUMN favorite INTEGER NOT NULL DEFAULT 0`}, {"latest_chapter", `ALTER TABLE bookmarks ADD COLUMN latest_chapter TEXT NOT NULL DEFAULT ''`}, {"latest_chapter_num", `ALTER TABLE bookmarks ADD COLUMN latest_chapter_num REAL`}, + // When the server last looked at this series, unix ms; 0 means never, and + // sorts first so a new bookmark is picked up on the next tick with no + // special case. Deliberately NOT in bookmarkColumns — see MarkLatestChecked. + {"latest_checked_at", `ALTER TABLE bookmarks ADD COLUMN latest_checked_at INTEGER NOT NULL DEFAULT 0`}, } const bookmarkColumns = `key, site, series_id, title, series_url, cover, @@ -277,3 +282,54 @@ func (s *Store) Delete(key string) error { } return nil } + +// DueForLatestCheck returns bookmarks whose server-side latest-chapter check has +// aged past cutoffMs, least-recently-checked first, at most limit of them. +// +// Oldest-first is what keeps the poller fair when the backlog outgrows its +// throughput: the most neglected series is always next, so a large collection +// refreshes uniformly slower rather than leaving a tail that never refreshes at +// all. The userscript sorts its own queue the same way (L453). +// +// Bookmarks with no series_url are skipped — there is nothing to fetch, which +// is the same filter the userscript applies at L452. +func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Bookmark, error) { + rows, err := s.db.Query(`SELECT `+bookmarkColumns+` + FROM bookmarks + WHERE series_url IS NOT NULL AND series_url <> '' + AND latest_checked_at <= ? + ORDER BY latest_checked_at ASC + LIMIT ?`, cutoffMs, limit) + if err != nil { + return nil, fmt.Errorf("query due bookmarks: %w", err) + } + defer rows.Close() + + out := []Bookmark{} + for rows.Next() { + b, err := scanBookmark(rows.Scan) + if err != nil { + return nil, fmt.Errorf("scan due bookmark: %w", err) + } + out = append(out, b) + } + return out, rows.Err() +} + +// MarkLatestChecked records that the server looked at key at ts, whatever the +// look turned up. Marking a missing key is not an error: the row may have been +// deleted while a fetch was in flight. +// +// This is the one write that does not go through Upsert, and the column is kept +// out of bookmarkColumns on purpose. PUT /bookmarks/{key} decodes a whole +// Bookmark from the client and Upsert writes every column it knows about, so a +// userscript PUT — which has no idea this field exists — would write a zero and +// reset the cooldown, making the poller re-fetch that series every tick for as +// long as the user kept reading it. +func (s *Store) MarkLatestChecked(key string, ts int64) error { + if _, err := s.db.Exec( + `UPDATE bookmarks SET latest_checked_at = ? WHERE key = ?`, ts, key); err != nil { + return fmt.Errorf("mark checked %q: %w", key, err) + } + return nil +} diff --git a/backend/store_test.go b/backend/store_test.go index 1a442ee..6102db9 100644 --- a/backend/store_test.go +++ b/backend/store_test.go @@ -528,3 +528,197 @@ func TestLoadConfigWebPassword(t *testing.T) { t.Fatalf("WebPassword = %q with the variable unset, want empty", got) } } + +// readLatestCheckedAt reads the column directly. It is deliberately absent from +// Bookmark (see Store.Upsert), so tests cannot assert on it any other way. +func readLatestCheckedAt(t *testing.T, s *Store, key string) int64 { + t.Helper() + var ts int64 + if err := s.db.QueryRow( + `SELECT latest_checked_at FROM bookmarks WHERE key = ?`, key).Scan(&ts); err != nil { + t.Fatalf("read latest_checked_at %q: %v", key, err) + } + return ts +} + +// seedForCheck inserts a bookmark and forces its latest_checked_at. +func seedForCheck(t *testing.T, s *Store, key, seriesURL string, checkedAt int64) { + t.Helper() + if _, err := s.Upsert(Bookmark{ + Key: key, + Site: "asura", + SeriesID: key, + SeriesURL: seriesURL, + UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed %q: %v", key, err) + } + if err := s.MarkLatestChecked(key, checkedAt); err != nil { + t.Fatalf("seed mark %q: %v", key, err) + } +} + +func TestDueForLatestCheck(t *testing.T) { + const hour = int64(3600_000) + now := 10 * hour + + tests := []struct { + name string + checkedAt int64 + seriesURL string + wantDue bool + }{ + {"never checked", 0, "https://asurascans.com/comics/a", true}, + {"checked 30m ago", now - hour/2, "https://asurascans.com/comics/a", false}, + {"checked exactly at cutoff", now - hour, "https://asurascans.com/comics/a", true}, + {"checked 2h ago", now - 2*hour, "https://asurascans.com/comics/a", true}, + {"no series url", 0, "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt) + + due, err := s.DueForLatestCheck(now-hour, 10) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if got := len(due) == 1; got != tt.wantDue { + t.Fatalf("due=%v, want %v (got %d rows)", got, tt.wantDue, len(due)) + } + }) + } +} + +func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) { + s := newTestStore(t) + // Insert newest-checked first so a correct ORDER BY has to reverse it. + seedForCheck(t, s, "asura:c", "https://asurascans.com/comics/c", 300) + seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200) + seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100) + + due, err := s.DueForLatestCheck(1000, 2) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 2 { + t.Fatalf("got %d rows, want 2 (limit)", len(due)) + } + if due[0].Key != "asura:a" || due[1].Key != "asura:b" { + t.Fatalf("got %q,%q; want asura:a,asura:b (oldest first)", due[0].Key, due[1].Key) + } +} + +func TestMarkLatestChecked(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 0) + + if err := s.MarkLatestChecked("asura:x", 4242); err != nil { + t.Fatalf("MarkLatestChecked: %v", err) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got != 4242 { + t.Fatalf("latest_checked_at = %d, want 4242", got) + } + // A missing key is not an error: the row may have been deleted mid-fetch. + if err := s.MarkLatestChecked("asura:gone", 1); err != nil { + t.Fatalf("MarkLatestChecked on missing key: %v", err) + } +} + +// Upsert must not touch latest_checked_at. If the column ever migrates into +// bookmarkColumns, this fails and the cooldown is silently dead. +func TestUpsertPreservesLatestCheckedAt(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 999) + + b, ok, err := s.Get("asura:x") + if err != nil || !ok { + t.Fatalf("Get: %v ok=%v", err, ok) + } + b.Title = "changed" + if _, err := s.Upsert(b); err != nil { + t.Fatalf("Upsert: %v", err) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got != 999 { + t.Fatalf("latest_checked_at = %d after Upsert, want 999", got) + } +} + +// migrateColumns must be able to bring a database created before this column up +// to date, not just create it fresh. +func TestMigrateAddsLatestCheckedAt(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + + old, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open: %v", err) + } + // A pre-latest_checked_at table, matching the schema as it shipped before. + if _, err := old.Exec(`CREATE TABLE bookmarks ( + key TEXT PRIMARY KEY, site TEXT NOT NULL, series_id TEXT NOT NULL, + title TEXT, series_url TEXT, cover TEXT, + last_chapter TEXT, last_chapter_num REAL, last_chapter_url TEXT, + favorite INTEGER NOT NULL DEFAULT 0, + latest_chapter TEXT NOT NULL DEFAULT '', latest_chapter_num REAL, + updated_at INTEGER NOT NULL)`); err != nil { + t.Fatalf("create old table: %v", err) + } + if _, err := old.Exec( + `INSERT INTO bookmarks (key, site, series_id, series_url, updated_at) + VALUES ('asura:x', 'asura', 'x', 'https://asurascans.com/comics/x', 5)`); err != nil { + t.Fatalf("seed old row: %v", err) + } + if err := old.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + s, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore on pre-existing db: %v", err) + } + t.Cleanup(func() { s.Close() }) + + // The migrated row must default to 0 (never checked) and so be due. + if got := readLatestCheckedAt(t, s, "asura:x"); got != 0 { + t.Fatalf("migrated latest_checked_at = %d, want 0", got) + } + due, err := s.DueForLatestCheck(1000, 10) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 1 { + t.Fatalf("got %d due rows after migration, want 1", len(due)) + } +} + +// A userscript PUT body has no latest_checked_at field. If the column is ever +// moved into bookmarkColumns, this test catches it: the PUT would reset the +// cooldown and the poller would re-fetch that series on every single tick. +func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + store, err := OpenStore(dbPath) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + srv := newRouter(store, testConfig()) + + seedForCheck(t, store, "asura:x", "https://asurascans.com/comics/x", 777) + + // Exactly what the userscript sends: no latest_checked_at key at all. + body := `{"key":"asura:x","site":"asura","series_id":"x", + "series_url":"https://asurascans.com/comics/x", + "last_chapter":"Chapter 5","last_chapter_num":5}` + req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+testToken) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + if got := readLatestCheckedAt(t, store, "asura:x"); got != 777 { + t.Fatalf("latest_checked_at = %d after client PUT, want 777 preserved", got) + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 8436895..dacce1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,13 @@ services: PORT: "8080" # Gates the browser UI. Unset means the web routes are not served at all. WEB_PASSWORD: ${WEB_PASSWORD:-} + # Latest-chapter poller. LATEST_CHAPTER_POLL_ENABLED=0 in .env is the kill + # switch; it only takes effect because these are listed here. + LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1} + LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h} + LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m} + LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14} + LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s} volumes: - bookmarks-data:/data # Bound to loopback only: the proxy (or curl during smoke test) reaches it,