From a66491acd27e39c996899fae30e855be0e26ae3e Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Mon, 10 Aug 2026 10:56:34 +0700 Subject: [PATCH] Unify page routing and harden acquire wiring after review (#62) --- AGENTS.md | 2 +- backend/AGENTS.md | 12 +++-- backend/internal/latest/acquire.go | 32 ++---------- backend/internal/latest/acquire_test.go | 24 +++++++++ backend/internal/latest/poller.go | 32 ++++++++---- backend/internal/latest/poller_test.go | 65 ++++++++++++++++++++++--- backend/main.go | 18 ++++--- 7 files changed, 127 insertions(+), 58 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6ab3941..7e12d90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free - Every site is its **own origin with its own `localStorage`** — a shared remote store is the only way to unify bookmarks. Cloud sync required, not optional. - Userscript run in **isolated world**, so embedded API token safe from site's JS. - Cloudflare's block on manga sites **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare's bot scoring can flip previously-clean IP without notice. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test. -- **kagane.to and novelfull.com are the exception to the above** — both sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`) and skips them entirely when that's unset. The four other sites poll fine over plain TLS. +- **kagane.to and novelfull.com are the exception to the above** — both sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`). When that's unset, kagane is skipped entirely (a plain fetch would only retrieve a challenge page) while novelfull pages are still attempted over plain TLS — its challenge is a live time-varying fact and its cover bytes never need the browser. The four other sites poll fine over plain TLS. - **The CDP browser must look like a real browser, and stock headless images don't.** Measured 2026-08-08 against kagane.to, all from the same IP: `chromedp/headless-shell:stable` never cleared the challenge in 90s (`navigator.webdriver` true, empty plugin list, Chromium-branded client hints — suppressing `webdriver` alone changed nothing); `zenika/alpine-chrome` ships Chrome 124, refused outright; real Chrome with the default `--headless=new` UA never cleared, because the UA says `HeadlessChrome`; real Chrome with a stock UA **and** a non-UTC clock zone cleared in ~4s. Hence `chrome/` — a Debian image with `google-chrome-stable`, a version-derived UA, and `TZ`/`BROWSER_TZ`. Chrome reads the zone *name* through ICU from `/etc/localtime`'s symlink target, ignoring the file's contents, so mounting the host's `/etc/localtime` does **not** work; `/etc/timezone` is mounted instead. - **The browser is not in the API stack and must not be put back.** It's its own compose unit (`chrome/docker-compose.yml`) on a second machine, reached over the tailnet — it held 471 MiB on a 1974 MiB swapless VPS, and a residential egress scores better with Cloudflare anyway (ADR-0006). Consequences that constrain code: `BROWSER_WS_URL` must be a tailnet **IP** (a MagicDNS name 500s at `/json/version`, same trap as the old Docker service name); the CDP port binds to the tailnet address only, since CDP authenticates nothing and that host has a real LAN; and the browser is on-demand (ADR-0005), so an unreachable or asleep one must degrade exactly as an unset `BROWSER_WS_URL` — plain-TLS libraries unaffected, kagane/novelfull logged and skipped, stored covers still served. Never add `chromedp.NoModifyURL`: discovery per fetch is what makes a restarted Chrome invisible. - **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, so Cloudflare scores it as one; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index b846fe3..1781651 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -91,8 +91,10 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN Fetches use `bogdanfinn/tls-client` with Chrome profile as defence in depth against fingerprint-based blocking; any failure log and skip. kagane and novelfull sit behind Cloudflare JavaScript challenges the TLS client can't - clear, so they are browser-only: fetched over CDP via `BROWSER_WS_URL`, and - simply not polled when that's unset. See + clear, so they are fetched over CDP via `BROWSER_WS_URL`; kagane is simply + not polled when that's unset, while novelfull falls back to a plain-TLS + attempt — its challenge is a live time-varying fact, and its cover bytes + never need the browser. See `docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`. The poller's series write is a single-column UPDATE (`Store.SetLatestChapter`), not a read-modify-write of the whole bookmark: @@ -115,9 +117,9 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN fetch, which would only retrieve a challenge page), while novelfull needs the browser only for its HTML — the cover URL comes out of the browser-fetched page and the bytes go over plain TLS. With no browser - configured, kagane Covers are simply absent; novelfull still acquires one - when its page body happens to answer a plain request (the challenge is a - live time-varying fact). + configured, kagane Covers are simply absent; novelfull still gets one — at + creation and on the poll — when its page body happens to answer a plain + request (the challenge is a live time-varying fact). - **`updated_at` drives list order, so moves only on real reading progress:** server apply its timestamp when row new or `last_chapter_num` changes, else keep stored value — favouriting series or recording newly published chapter must not reorder list. `PUT` therefore returns row **as stored**, clients must adopt that response rather than own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4. - **Lifecycle buckets:** `status` on each bookmark is `reading` | `archived` | `finished`, orthogonal to `favorite`. Archived and finished appear only in diff --git a/backend/internal/latest/acquire.go b/backend/internal/latest/acquire.go index dd83de6..0551a87 100644 --- a/backend/internal/latest/acquire.go +++ b/backend/internal/latest/acquire.go @@ -3,7 +3,6 @@ package latest import ( "context" "log" - "slices" "sync" "time" @@ -36,11 +35,9 @@ type Acquirer struct { // BrowserFetch disables acquisition entirely. Fetch Fetcher // BrowserFetch retrieves kagane and novelfull pages through the browser - // sidecar, which is the only thing that clears their Cloudflare - // challenge. Nil leaves those Sites unacquired; kagane never falls back - // to Fetch (a plain request only retrieves a challenge page), while - // novelfull does, because its challenge is a live time-varying fact and - // its cover bytes never need the browser. + // sidecar, the only thing that clears their Cloudflare challenge. The + // per-site fallback policy lives in fetcherFor. Nil leaves those Sites + // unacquired when no fallback applies. BrowserFetch Fetcher // Covers retrieves the cover bytes. Nil leaves the Cover blank and the // chapter half working. @@ -108,7 +105,7 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) { return } - f := a.fetcherFor(sr.Site) + f := fetcherFor(sr.Site, a.BrowserFetch, a.Fetch) if f == nil { log.Printf("acquire %q: no fetcher for site %q", sr.Key(), sr.Site) return @@ -148,24 +145,3 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) { log.Printf("acquire %q: persist cover: %v", sr.Key(), err) } } - -// fetcherFor returns the fetcher a site's page needs, or nil when the site -// cannot be fetched at all right now. kagane and novelfull pages sit behind a -// Cloudflare JavaScript challenge, so they prefer the browser; novelfull alone -// falls back to the plain-TLS fetcher when no browser is configured, because -// its challenge is a live time-varying fact (AGENTS.md) and its cover bytes -// never need the browser. kagane never falls back: a plain fetch of a kagane -// page or cover would only ever retrieve a challenge page. -func (a *Acquirer) fetcherFor(site string) Fetcher { - switch { - case site == "kagane": - return a.BrowserFetch - case slices.Contains(browserBackedSites, site): // novelfull - if a.BrowserFetch != nil { - return a.BrowserFetch - } - return a.Fetch - default: - return a.Fetch - } -} diff --git a/backend/internal/latest/acquire_test.go b/backend/internal/latest/acquire_test.go index 47e2be1..499a970 100644 --- a/backend/internal/latest/acquire_test.go +++ b/backend/internal/latest/acquire_test.go @@ -382,6 +382,30 @@ func TestAcquireKaganeSkippedWithoutBrowser(t *testing.T) { } } +// The byte half of "nothing falls back to a plain fetch": with a browser for +// the page but none for the bytes, a kagane Cover stays absent and the TLS +// cover fetcher is never consulted. +func TestAcquireKaganeBytesNeverFallBackToPlainTLS(t *testing.T) { + s, _ := newTestStore(t) + browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200} + tlsCovers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} + acq := &Acquirer{ + Store: s, Fetch: &fakeFetcher{body: "", status: 403}, + BrowserFetch: browserPage, Covers: tlsCovers, + } + s.OnSeriesCreated = acq.Acquire + + bookmarkNewKaganeSeries(t, s) + acq.Wait() + + if got := tlsCovers.callCount(); got != 0 { + t.Fatalf("plain-TLS cover fetches = %d, want 0 — kagane bytes are browser-only", got) + } + if got := readBookmark(t, s, kaganeKey); got.Cover != "" { + t.Fatalf("Cover = %q, want empty without a browser cover fetcher", got.Cover) + } +} + // novelfull's no-browser degradation differs from kagane's: only its HTML // needs the sidecar, so when the page body is available — the challenge is a // live time-varying fact that sometimes answers a plain request — the Cover diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 246138e..cfe7e3a 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -116,16 +116,28 @@ func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL stri } } -// fetcherFor returns the fetcher a site needs, or nil when the site cannot be -// fetched at all right now. kagane and novelfull both sit behind a Cloudflare -// JavaScript challenge that no TLS fingerprint clears — kagane verified -// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133 -// profile TLSFetcher uses — so they are browser-only or nothing. -func (p *Poller) fetcherFor(site string) Fetcher { - if slices.Contains(browserBackedSites, site) { - return p.BrowserFetch +// fetcherFor returns the fetcher a site's page needs, or nil when the site +// cannot be fetched at all right now. kagane and novelfull pages sit behind a +// Cloudflare JavaScript challenge that no TLS fingerprint clears (kagane +// verified 2026-08-03, novelfull verified 2026-08-05, both against the same +// Chrome_133 profile TLSFetcher uses), so both prefer the browser; novelfull +// alone falls back to the plain-TLS fetcher when no browser is configured, +// because its challenge is a live time-varying fact (AGENTS.md) and its cover +// bytes never need the browser. kagane never falls back: a plain fetch of a +// kagane page or cover would only ever retrieve a challenge page. One routing +// rule for the poll and the acquirer, so the two cannot drift apart. +func fetcherFor(site string, browser, tls Fetcher) Fetcher { + switch { + case site == "kagane": + return browser + case slices.Contains(browserBackedSites, site): // novelfull + if browser != nil { + return browser + } + return tls + default: + return tls } - return p.Fetch } // Run polls until ctx is cancelled. @@ -221,7 +233,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) { return } - f := p.fetcherFor(sr.Site) + f := fetcherFor(sr.Site, p.BrowserFetch, p.Fetch) if f == nil { log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site) return diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index a8d5353..faf9df8 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -671,6 +671,51 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) { } } +// novelfull without a browser is not skipped outright: its challenge is a +// live time-varying fact, so the plain-TLS page fetch is attempted and — when +// the body answers — fills both the chapter and the Cover, exactly the +// client-scraped rows #62 wants healed. +func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) { + s, _ := newTestStore(t) + key := "novelfull:reverend-insanity" + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, + Site: "novelfull", + SeriesID: "reverend-insanity", + SeriesURL: "https://novelfull.com/reverend-insanity.html", + UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200} + covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} + p := &Poller{ + Store: s, Fetch: tlsF, CoverBytesFetch: covers, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, + Cooldown: time.Hour, BrowserCooldown: time.Hour, + Interval: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + if len(tlsF.calls) != 1 { + t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls)) + } + if got := covers.callCount(); got != 1 { + t.Fatalf("cover fetches = %d, want 1", got) + } + got, found, err := s.Get(s.OwnerID(), key) + if err != nil || !found { + t.Fatalf("Get: %v found=%v", err, found) + } + if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want { + t.Fatalf("Cover = %q, want %q", got.Cover, want) + } + if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 { + t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum) + } +} + // With a browser fetcher wired up, kagane goes to it and not to the TLS one. func TestKaganeUsesBrowserFetcher(t *testing.T) { s, _ := newTestStore(t) @@ -893,20 +938,24 @@ func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) { func TestFetcherForRoutesNovelSites(t *testing.T) { tls := &fakeFetcher{} browser := &fakeFetcher{} - p := &Poller{Fetch: tls, BrowserFetch: browser} cases := []struct { - site string - want Fetcher + site string + browser Fetcher + tls Fetcher + want Fetcher }{ - {"asura", tls}, - {"lightnovelworld", tls}, - {"kagane", browser}, - {"novelfull", browser}, + {"asura", browser, tls, tls}, + {"lightnovelworld", browser, tls, tls}, + {"kagane", browser, tls, browser}, + {"novelfull", browser, tls, browser}, + // browser-less deployment: kagane is nothing, novelfull degrades to TLS + {"kagane", nil, tls, nil}, + {"novelfull", nil, tls, tls}, } for _, tc := range cases { t.Run(tc.site, func(t *testing.T) { - if got := p.fetcherFor(tc.site); got != tc.want { + if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want { t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want) } }) diff --git a/backend/main.go b/backend/main.go index c7eb1ca..7c47ed7 100644 --- a/backend/main.go +++ b/backend/main.go @@ -328,16 +328,22 @@ func main() { // Cover from one fetch, at creation, instead of waiting out a poll queue // ordered by Reader count. Off the write path: the hook returns as soon // as the goroutine is started. + var tlsFetch latest.Fetcher if f, err := latest.NewTLSFetcher(); err != nil { - log.Printf("creation-time acquisition disabled, cannot build client: %v", err) + log.Printf("creation-time acquisition: plain-TLS Sites disabled, cannot build client: %v", err) } else { - var browserCover latest.BrowserCoverFetcher - if b, ok := browser.(latest.BrowserCoverFetcher); ok { - browserCover = b - } + tlsFetch = f + } + var browserCover latest.BrowserCoverFetcher + if b, ok := browser.(latest.BrowserCoverFetcher); ok { + browserCover = b + } + // The Acquirer must survive a TLS client failure: kagane needs only the + // sidecar, and novelfull degrades to whatever is left. + if tlsFetch != nil || browser != nil { acq := &latest.Acquirer{ Store: s, - Fetch: f, + Fetch: tlsFetch, BrowserFetch: browser, BrowserCoverFetch: browserCover, Covers: latest.NewCoverFetcher(),