diff --git a/.env.example b/.env.example index 7f1e213..889d7c4 100644 --- a/.env.example +++ b/.env.example @@ -90,3 +90,23 @@ DISCORD_REDIRECT_URI= # HTTP handler 500s any /json/version request whose Host header isn't an IP or # "localhost", which silently breaks every kagane poll. # BROWSER_WS_URL=ws://172.28.0.10:9222 + +# Clock zone the headless browser reports. A UTC clock is itself the bot +# signal — Cloudflare treats it as the datacenter default — and kagane's +# challenge then never clears. Measured 2026-08-08, identical container, one +# Indonesian egress IP: UTC never cleared in 60s (twice); Asia/Jakarta and +# America/New_York both cleared in 4s. So any real zone works; it does not +# have to match the IP's country, it just must not be UTC. +# +# Unset falls back to the host's /etc/timezone, which is a real zone whenever +# the host clock is set to local time. Set this when the host runs UTC — a UTC +# server is exactly the case that fails. Only the browser sidecar reads it — +# the backend's own zone is API_TZ below, and is cosmetic. +# BROWSER_TZ=Asia/Jakarta + +# Zone the backend stamps its log lines in. Cosmetic only — it exists so the +# API's logs read on the same clock as the browser sidecar's. Nothing else in +# the service has a zone: bookmark timestamps are unix ms, and the two real +# time columns are timestamptz. Defaults to Asia/Jakarta; set to UTC for the +# conventional server default. +# API_TZ=Asia/Jakarta diff --git a/AGENTS.md b/AGENTS.md index 2447414..98d94bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,9 @@ Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free - Userscript run in **isolated world**, so embedded API token safe from site's JS. - Cloudflare's block on manga sites **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare's bot scoring can flip previously-clean IP without notice. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test. - **kagane.to and novelfull.com are the exception to the above** — both sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`) and skips them entirely when that's unset. The four other sites poll fine over plain TLS. +- **The CDP sidecar must look like a real browser, and stock headless images don't.** Measured 2026-08-08 against kagane.to, all from the same IP: `chromedp/headless-shell:stable` never cleared the challenge in 90s (`navigator.webdriver` true, empty plugin list, Chromium-branded client hints — suppressing `webdriver` alone changed nothing); `zenika/alpine-chrome` ships Chrome 124, refused outright; real Chrome with the default `--headless=new` UA never cleared, because the UA says `HeadlessChrome`; real Chrome with a stock UA **and** a non-UTC clock zone cleared in ~4s. Hence `chrome/` — a Debian image with `google-chrome-stable`, a version-derived UA, and `TZ`/`BROWSER_TZ`. Chrome reads the zone *name* through ICU from `/etc/localtime`'s symlink target, ignoring the file's contents, so mounting the host's `/etc/localtime` does **not** work; `/etc/timezone` is mounted instead. +- **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, so Cloudflare scores it as one; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one. +- **A challenged page needs the tab kept open.** The interstitial takes seconds to solve and only then writes clearance into the browser's shared cookie jar. Navigate-read-close never clears anything; `BrowserFetcher.run` holds one tab and re-reads until the payload arrives. ## Architecture @@ -37,7 +40,12 @@ Backend (`cd backend`): - Single test: `go test -run TestName ./...` - Build static binary: `CGO_ENABLED=0 go build` -Local stack: `docker compose up` (bookmark-api + postgres + headless-shell; `postgres-data` named volume, `restart: unless-stopped`). +Local stack: `docker compose up` (bookmark-api + postgres + headless-shell; `postgres-data` named volume, `restart: unless-stopped`). The `headless-shell` service keeps its name but now builds `chrome/` — real Google Chrome, for the reason in the hard constraints above. + +Live CDP proof (needs a sidecar and network, skipped otherwise): +`SMOKE_BROWSER_WS_URL=ws://: go test -run TestSmokeKagane ./internal/latest` +— fetches a real kagane cover and chapter list. A red run means the challenge is +not clearing from this IP, which is a live fact to re-check, not necessarily a defect. Smoke test: `curl` endpoints with `Authorization: Bearer `; confirm `OPTIONS` preflight return CORS headers and `/healthz` return 200. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 736710a..ed46a1c 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -126,9 +126,20 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN `/userscript/novel-bookmark.user.js`, both supplied by bindmount; the `__API_TOKEN__` placeholder inside them is substituted with the requesting Reader's credential at serve time). - `BROWSER_WS_URL` (headless-shell CDP endpoint for kagane and novelfull; - unset disables browser polling and leaves those sites to the userscript - alone). + `BROWSER_WS_URL` (CDP endpoint of the `chrome/` sidecar, used by the poller + for kagane and novelfull *and* by the web UI's kagane cover proxy; unset + disables browser polling and serves 404 from the proxy, leaving those sites + to the userscript alone). +- **kagane covers are proxied, not hot-linked:** kagane serves cover images + behind the same challenge as its pages and with + `cross-origin-resource-policy: same-origin`, so no `` on the web UI's + origin can load one — not even from a browser holding the clearance cookie + (verified 2026-08-08). `Bookmark.CoverURL` rewrites a stored kagane + `og:image` to `/img/kagane/{id}`, served by `internal/web/cover.go` through + `latest.BrowserFetcher.Image` and memoised in-process. The templates render + `.CoverURL`, never `.Cover`. The id is matched against a UUID regex before it + reaches the browser: the stored value is client-supplied, so an unchecked one + is an SSRF primitive pointed at the deployment's own network. - **Web UI also owns:** session-gated `GET /install/{manga,novel}-bookmark.user.js` (renders the bindmounted script with the acting Reader's derived credential substituted in — the credential never appears in page markup, the address diff --git a/backend/cover_test.go b/backend/cover_test.go new file mode 100644 index 0000000..bf9b44f --- /dev/null +++ b/backend/cover_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +// fakeCovers stands in for the headless browser. It counts calls so the test +// can prove the cache spares the browser a second navigation. +type fakeCovers struct { + body []byte + contentType string + err error + calls atomic.Int32 + lastID atomic.Value +} + +func (f *fakeCovers) Image(_ context.Context, imageID string) ([]byte, string, error) { + f.calls.Add(1) + f.lastID.Store(imageID) + if f.err != nil { + return nil, "", f.err + } + return f.body, f.contentType, nil +} + +const testCoverID = "019fe11a-84c3-7fc3-a84b-88787374b617" + +func getCover(t *testing.T, srv http.Handler, path string, cookie *http.Cookie) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + if cookie != nil { + req.AddCookie(cookie) + } + rr := httptest.NewRecorder() + srv.ServeHTTP(rr, req) + return rr +} + +// kagane serves its covers behind a Cloudflare challenge and with +// cross-origin-resource-policy: same-origin, so the UI can only show one by +// re-serving the bytes from its own origin. +func TestKaganeCoverProxiesAndCaches(t *testing.T) { + cf := &fakeCovers{body: []byte("\x00webp-bytes"), contentType: "image/webp"} + cfg := testConfig() + cfg.Covers = cf + srv, st := newWebTestServer(t, cfg) + cookie := sessionCookie(t, st) + + for i := range 2 { + rr := getCover(t, srv, "/img/kagane/"+testCoverID, cookie) + if rr.Code != http.StatusOK { + t.Fatalf("request %d: status = %d, want 200", i, rr.Code) + } + if got := rr.Body.String(); got != string(cf.body) { + t.Fatalf("request %d: body = %q, want %q", i, got, cf.body) + } + if got := rr.Header().Get("Content-Type"); got != "image/webp" { + t.Fatalf("request %d: Content-Type = %q, want image/webp", i, got) + } + } + if got := cf.calls.Load(); got != 1 { + t.Fatalf("fetcher called %d times, want 1 — the second read must come from the cache", got) + } + if got := cf.lastID.Load(); got != testCoverID { + t.Fatalf("fetched image id = %v, want %s", got, testCoverID) + } +} + +// The proxy reaches a headless browser, so it is not open to the internet. +func TestKaganeCoverRequiresSession(t *testing.T) { + cf := &fakeCovers{body: []byte("x"), contentType: "image/webp"} + cfg := testConfig() + cfg.Covers = cf + srv, _ := newWebTestServer(t, cfg) + + rr := getCover(t, srv, "/img/kagane/"+testCoverID, nil) + if rr.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", rr.Code) + } + if got := cf.calls.Load(); got != 0 { + t.Fatalf("fetcher called %d times for an unauthenticated request, want 0", got) + } +} + +func TestKaganeCoverRejectsBadInput(t *testing.T) { + cases := []struct { + name string + id string + fetch *fakeCovers + }{ + { + "an id that is not a uuid never reaches the browser", + "solo-leveling", + &fakeCovers{body: []byte("x"), contentType: "image/webp"}, + }, + { + "a uuid-shaped id with a trailing segment is rejected whole", + testCoverID + "x", + &fakeCovers{body: []byte("x"), contentType: "image/webp"}, + }, + { + "a challenged fetch is a missing cover", + testCoverID, + &fakeCovers{err: errors.New("challenge held")}, + }, + { + "a content type outside the image set is not echoed back", + testCoverID, + &fakeCovers{body: []byte("