Compare commits

..

14 Commits

Author SHA1 Message Date
sulthan c545b1b92e chore: address final review must-fix items 2026-08-06 03:34:41 +07:00
sulthan 50e4d6a6e2 feat: serve novel-bookmark.user.js and allow the novel sites' origins 2026-08-06 03:29:24 +07:00
sulthan c9a2fd4614 feat(userscript): add novel-bookmark.user.js for novelfull and lightnovelworld 2026-08-06 03:26:03 +07:00
sulthan 51fcd8732b feat(userscript): stamp and filter by kind in the manga script 2026-08-06 03:22:28 +07:00
sulthan d465028443 feat(web): split the UI into manga and novel libraries 2026-08-06 03:19:49 +07:00
sulthan f24924031e feat(web): apply design snapshot — libswitch, novel site colours, login art 2026-08-06 03:15:36 +07:00
sulthan 92f1fbf6ec feat(latest): poll novelfull via headless browser, lightnovelworld via TLS 2026-08-06 03:10:50 +07:00
sulthan c81e50c7f1 feat(latest): parse newest chapter for novelfull and lightnovelworld 2026-08-06 03:07:22 +07:00
sulthan d9d7a1c00d feat(api): validate kind on PUT /bookmarks 2026-08-06 03:05:14 +07:00
sulthan dcec12ae72 feat(store): add kind column splitting manga and novel libraries 2026-08-06 03:03:43 +07:00
sulthan 44a8df43e1 refactor: rebrand compose, env and docs to BookmarkManager 2026-08-06 02:55:46 +07:00
sulthan d41a1d2c0b refactor: rebrand userscript storage keys, hosts and wordmark 2026-08-06 02:47:33 +07:00
sulthan f30d4cc7cb refactor: rename Go module to bookmarkmanager 2026-08-06 02:44:43 +07:00
sulthan 3149dc0c26 chore: pre-flight baseline for novel-support plan
- .gitignore: drop .superpowers/ and local go.work
- AGENTS.md, docs/design-system.md, backend/AGENTS.md, userscript/AGENTS.md
- login-art.png: 2.8MB design asset, picked up by //go:embed
2026-08-06 02:43:29 +07:00
85 changed files with 2131 additions and 10007 deletions
@@ -14,7 +14,7 @@ parsers, helpers. UI, network, and storage behaviour are verified on-device.
```bash
node --check userscript/manga-bookmark.user.js # parse check, silent on success
node --test userscript/test/logic.test.js # 35 tests as of 2026-08-10
node --test userscript/test/logic.test.js # 14 tests as of 2026-07-28
```
Run both before every commit that touches the userscript.
@@ -31,7 +31,7 @@ The test file installs four globals **before** requiring the userscript:
|---|---|---|
| `localStorage` | `Map`-backed stub | `loadCache`, `loadQueue`, and the key-migration IIFE touch it at module scope |
| `location` | `{href, hostname, pathname, origin}` | read during boot |
| `document` | `querySelector` for `meta[property="…"]` only, plus a no-op `addEventListener` | adapters read `og:title` (covers are the backend's, never scraped) |
| `document` | `querySelector` for `meta[property="…"]` only, plus a no-op `addEventListener` | adapters read `og:title`/`og:image` |
| `document.body` | **left `undefined`** | this is the whole trick |
`document.body === undefined` sends the userscript's boot block down its `else`
+26 -84
View File
@@ -1,39 +1,14 @@
# Copy to .env and fill in. Never commit the real .env.
# Secret every Reader's userscript credential is derived from (issue #24):
# the backend rebuilds install URLs from it, and only SHA-256 hashes of the
# credentials ever touch the database. Generate one:
# Long random secret shared with the userscript's API_TOKEN. Generate one:
# openssl rand -hex 32
TOKEN_KEY=changeme-generate-a-long-random-token
# The owner's Discord user ID — seeded at startup as the first Reader, the
# administrator (the only one who can revoke another Reader's sessions), and
# the owner of every bookmark that predates registration. Discord snowflake,
# e.g. 1046923170000000000.
OWNER_DISCORD_ID=changeme-your-discord-user-id
API_TOKEN=changeme-generate-a-long-random-token
# Comma-separated origins allowed to call the API (CORS). Both Asura domains
# plus Demonic, Comix, Kagane, and the two novel sites. Add/remove as the
# sites' hostnames change.
ALLOWED_ORIGINS=https://asuracomic.net,https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to,https://novelfull.com,https://lightnovelworld.net
# Password for the bundled Postgres container, and therefore half of the
# DATABASE_URL compose builds for the backend. Generate one:
# openssl rand -hex 24
POSTGRES_PASSWORD=changeme-generate-a-long-random-password
# Override only to point the backend at a Postgres compose does not run.
# DATABASE_URL=postgres://user:pass@host:5432/bookmarks?sslmode=require
# Directory inside bookmark-api for immutable, content-addressed Cover bytes.
# Compose builds the image and mounts its named volume at this path.
COVER_DIR=/covers
# Public origin this deployment answers on, no trailing slash. Required: Cover
# URLs go out absolute, because the userscript renders them on a Site's own
# origin where a relative path would resolve against the Site (ADR-0007).
PUBLIC_BASE_URL=https://bookmark-api.example.com
# --- Prod override (Traefik) only ---
# Subdomain Traefik routes to this service (required by the prod override).
# BOOKMARK_API_HOST=bookmark-api.example.com
@@ -43,30 +18,17 @@ PUBLIC_BASE_URL=https://bookmark-api.example.com
# TRAEFIK_ENTRYPOINT=websecure
# TRAEFIK_CERTRESOLVER=le
# --- Web UI (Discord OAuth) ---
# Sign-in is a Discord authorization code grant (ADR-0002), and it is also
# registration: any member of the configured guild becomes a Reader on their
# first successful login, with their own empty library. Create the application
# at https://discord.com/developers/applications and register the exact
# callback URL ($BOOKMARK_WEB_HOST/auth/discord/callback) as an OAuth2
# redirect.
DISCORD_CLIENT_ID=
DISCORD_CLIENT_SECRET=
# The guild whose membership gates sign-in (Developer Mode -> right-click the
# server -> Copy Server ID).
DISCORD_GUILD_ID=
# Exact callback URL, e.g. https://bookmark.example.com/auth/discord/callback.
# Discord matches it verbatim, so it must equal the registered redirect.
DISCORD_REDIRECT_URI=
# Optional: a role snowflake members must hold on top of guild membership.
# Empty (the default) means membership alone suffices.
# DISCORD_REQUIRED_ROLE=
# --- Web UI ---
# Password for the browser UI at https://$BOOKMARK_WEB_HOST. Leave unset to
# disable the web UI entirely (the routes are not registered at all).
# Generate one: openssl rand -base64 18
WEB_PASSWORD=
# Subdomain Traefik routes to the browser UI (required by the prod override).
# Left commented on purpose: an example value here would be a silent
# wrong-hostname fallback, and Traefik would publish the UI router on a domain
# you do not own. The same container also answers on BOOKMARK_API_HOST for the
# userscript's API.
# Subdomain Traefik routes to the browser UI (required by the prod override,
# whether or not WEB_PASSWORD is set). Left commented on purpose: an example
# value here would be a silent wrong-hostname fallback, and Traefik would
# publish the UI router on a domain you do not own. The same container also
# answers on BOOKMARK_API_HOST for the userscript's API.
# BOOKMARK_WEB_HOST=bookmark.example.com
# --- Latest-chapter poller ---
@@ -76,15 +38,13 @@ DISCORD_REDIRECT_URI=
# Set to 0 to turn it off entirely.
# LATEST_CHAPTER_POLL_ENABLED=1
#
# Two independent clocks. COOLDOWN is how long a plain-TLS series rests between
# checks; BROWSER_COOLDOWN is the longer rest for kagane and novelfull. INTERVAL
# is how often the poller wakes up and looks for series past their cooldowns.
# Shortening INTERVAL cannot shorten either cooldown.
LATEST_CHAPTER_POLL_COOLDOWN=1h # plain-TLS per series, floor 15m
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN=6h # browser-backed per series, floor 15m
LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
LATEST_CHAPTER_POLL_BATCH=14 # series per wake
LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
# Two independent clocks. COOLDOWN is how long one series rests between checks;
# INTERVAL is how often the poller wakes up and looks for series past that
# cooldown. Shortening INTERVAL cannot shorten a COOLDOWN.
# LATEST_CHAPTER_POLL_COOLDOWN=1h # per series, floor 15m
# LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
# LATEST_CHAPTER_POLL_BATCH=14 # series per wake
# LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
#
# Uses a ticker, not an immediate first run: the first poll happens one
# INTERVAL after startup, not at startup. A container restarting more often
@@ -94,28 +54,10 @@ LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
# defaults. Beyond that the cadence stretches uniformly rather than breaking;
# raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL.
# CDP endpoint of the browser, used for the two sites behind a Cloudflare
# JavaScript challenge (kagane, novelfull) and by the web UI's kagane cover
# proxy. Unset disables browser polling and serves 404 for covers not already
# stored; those sites then rely on the userscript alone. That is also exactly
# how an unreachable browser degrades, so a home machine that is off costs
# chapter freshness and nothing else.
#
# The browser does NOT run in this stack. It is its own compose unit on the
# home machine (chrome/docker-compose.yml, chrome/.env.example) and is reached
# over the tailnet, so set this to that machine's tailnet address:
#
# BROWSER_WS_URL=ws://100.x.y.z:9222
#
# It must be the tailnet **IP**, never a MagicDNS hostname and never the old
# Docker service name: Chrome's DevTools HTTP handler 500s any /json/version
# request whose Host header isn't an IP or "localhost", which silently breaks
# every kagane poll. Left unset here on purpose — a wrong default would poll a
# stranger's address, and "no browser" is a safe, self-announcing state.
# BROWSER_WS_URL=ws://100.x.y.z:9222
# Zone the backend stamps its log lines in. Cosmetic only. Nothing else in
# the service has a zone: bookmark timestamps are unix ms, and the two real
# time columns are timestamptz. Defaults to Asia/Jakarta; set to UTC for the
# conventional server default.
# API_TZ=Asia/Jakarta
# Headless-shell CDP endpoint for sites behind a JavaScript challenge (kagane).
# Unset disables browser polling; those sites then rely on the userscript alone.
# Leave commented — the compose files' own default (ws://172.28.0.10:9222) is
# correct. Do NOT set this to the "headless-shell" DNS name: Chrome's DevTools
# HTTP handler 500s any /json/version request whose Host header isn't an IP or
# "localhost", which silently breaks every kagane poll.
# BROWSER_WS_URL=ws://172.28.0.10:9222
+8 -75
View File
@@ -4,53 +4,34 @@ Guidance for OpenCode (and Claude Code) working in this repo.
## What this is
Read-progress tracker for two libraries — manga and novels — behind one self-hosted Go backend. Two separate Violentmonkey userscripts inject on-page UI (floating button + slide-in panel) and sync progress, so bookmarks unify across sites and devices:
- `manga-bookmark.user.js` — **asurascans.com** (current domain; asuracomic.net 301s here), **demonicscans.org**, **comix.to**, **kagane.to**.
- `novel-bookmark.user.js` — **novelfull.com**, **lightnovelworld.net**.
One backend, one `bookmarks` table: a `kind` column (`manga`|`novel`) splits the libraries and the web UI switches between them. Rows are keyed `<site>:<series_id>`.
Manga read-progress tracker, user read on **asurascans.com** (current domain; asuracomic.net 301s here) and **demonicscans.org** via **Violentmonkey**. Userscript inject on-page UI (floating button + slide-in panel), sync progress to self-hosted Go backend so bookmarks unify across both sites and devices.
## Hard constraints (drive design — don't violate)
Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free where plain web APIs suffice — keeps portability across engines:
- **Avoid `GM_*` unless needed.** Prefer page `localStorage` over `GM_setValue`/`GM_getValue`, on-page UI over `GM_registerMenuCommand`, plain `fetch()` over `GM_xmlhttpRequest` for cross-origin.
- Cross-origin `fetch()` work **only** against CORS-enabled backend. Manga sites `https://`, so backend **must be HTTPS** (else mixed-content block).
- Every site is its **own origin with its own `localStorage`** — a shared remote store is the only way to unify bookmarks. Cloud sync required, not optional.
- Asura and Demonic are **separate origins with separate `localStorage`** — shared remote store only way to unify bookmarks. Cloud sync required, not optional.
- Userscript run in **isolated world**, so embedded API token safe from site's JS.
- Cloudflare's block on manga sites **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare's bot scoring can flip previously-clean IP without notice. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test.
- **kagane.to and novelfull.com are the exception to the above** — both sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`). When that's unset, kagane is skipped entirely (a plain fetch would only retrieve a challenge page) while novelfull pages are still attempted over plain TLS — its challenge is a live time-varying fact and its cover bytes never need the browser. The four other sites poll fine over plain TLS.
- **The CDP browser must look like a real browser, and stock headless images don't.** Measured 2026-08-08 against kagane.to, all from the same IP: `chromedp/headless-shell:stable` never cleared the challenge in 90s (`navigator.webdriver` true, empty plugin list, Chromium-branded client hints — suppressing `webdriver` alone changed nothing); `zenika/alpine-chrome` ships Chrome 124, refused outright; real Chrome with the default `--headless=new` UA never cleared, because the UA says `HeadlessChrome`; real Chrome with a stock UA **and** a non-UTC clock zone cleared in ~4s. Hence `chrome/` — a Debian image with `google-chrome-stable`, a version-derived UA, and `TZ`/`BROWSER_TZ`. Chrome reads the zone *name* through ICU from `/etc/localtime`'s symlink target, ignoring the file's contents, so mounting the host's `/etc/localtime` does **not** work; `/etc/timezone` is mounted instead.
- **The browser is not in the API stack and must not be put back.** It's its own compose unit (`chrome/docker-compose.yml`) on a second machine, reached over the tailnet — it held 471 MiB on a 1974 MiB swapless VPS, and a residential egress scores better with Cloudflare anyway (ADR-0006). Consequences that constrain code: `BROWSER_WS_URL` must be a tailnet **IP** (a MagicDNS name 500s at `/json/version`, same trap as the old Docker service name); the CDP port binds to the tailnet address only, since CDP authenticates nothing and that host has a real LAN; and the browser is on-demand (ADR-0005), so an unreachable or asleep one must degrade exactly as an unset `BROWSER_WS_URL` — plain-TLS libraries unaffected, kagane/novelfull logged and skipped, stored covers still served. Never add `chromedp.NoModifyURL`: discovery per fetch is what makes a restarted Chrome invisible.
- **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, so Cloudflare scores it as one; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one.
- **A challenged page needs the tab kept open.** The interstitial takes seconds to solve and only then writes clearance into the browser's shared cookie jar. Navigate-read-close never clears anything; `BrowserFetcher.run` holds one tab and re-reads until the payload arrives.
## Architecture
```
Two Violentmonkey userscripts (isolated world, per-site adapters, localStorage cache)
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> Postgres (volume)
|
| CDP over tailnet
v
on-demand Chrome, separate machine (chrome/)
Violentmonkey userscript (isolated world, per-site adapters, localStorage cache)
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> SQLite (volume)
```
Two deployable units on two machines: the API stack (`docker-compose.yml` + `docker-compose.prod.yml`, on the VPS) and the browser (`chrome/docker-compose.yml`, on the home machine). They share nothing but `BROWSER_WS_URL` and update independently. Backend-specific architecture (packages, endpoints, poller, config env vars) lives in `backend/AGENTS.md`. Userscript-specific structure (adapters, retry queue, UI, live URL shapes) lives in `userscript/AGENTS.md`. Deploy order `DEPLOY.md` (§7 for the browser), redeploy `REDEPLOY.md` (§8 for the browser).
Backend-specific architecture (packages, endpoints, poller, config env vars) lives in `backend/AGENTS.md`. Userscript-specific structure (adapters, retry queue, UI, live URL shapes) lives in `userscript/AGENTS.md`.
## Commands
Backend (`cd backend`):
- Test all: `go test ./...` — **needs Docker.** Each test package starts a throwaway `postgres:17-alpine` container (`internal/pgtest`).
- Test all: `go test ./...`
- Single test: `go test -run TestName ./...`
- Build static binary: `CGO_ENABLED=0 go build`
Local stack: `docker compose up` (bookmark-api + postgres only; `postgres-data` named volume, `restart: unless-stopped`). No browser — without `BROWSER_WS_URL` the poller logs and skips kagane and novelfull. To run one: `cd chrome && BROWSER_BIND_ADDR=172.17.0.1 docker compose up -d --build`, then `BROWSER_WS_URL=ws://172.17.0.1:9222` in the root `.env` (bridge gateway, so the API container can name it by IP).
Live CDP proof (needs that browser and network, skipped otherwise):
`SMOKE_BROWSER_WS_URL=ws://<ip>:<port> go test -run TestSmokeKagane ./internal/latest`
— fetches a real kagane cover and chapter list. A red run means the challenge is
not clearing from this IP, which is a live fact to re-check, not necessarily a defect.
Local stack: `docker compose up` (named volume mounted at `/data`, `restart: unless-stopped`).
Smoke test: `curl` endpoints with `Authorization: Bearer <token>`; confirm `OPTIONS` preflight return CORS headers and `/healthz` return 200.
@@ -81,41 +62,9 @@ instantly.
## Security invariants
Existing guarantees — don't regress:
- Auth on `/bookmarks*`: require `Authorization: Bearer <credential>` — the acting Reader's credential, matched by SHA-256 against `readers.token_sha256` — **constant-time compare** (via the hash, never the secret itself), 401 otherwise.
- Auth on `/bookmarks*`: require `Authorization: Bearer <API_TOKEN>`, **constant-time compare**, 401 otherwise.
- CORS: reflect `Origin` only when in `ALLOWED_ORIGINS`; allow `GET,PUT,DELETE,OPTIONS` + headers `Authorization,Content-Type`; answer preflight `OPTIONS` with `204`.
## Secure coding rules (code you write here)
Anchored to OWASP Top 10 / ASVS. Every rule below already has a working example in-tree — match it, don't start a second convention. AI-written backends fail on exactly these: broken access control, injection, weak session/error handling, invented dependencies.
Go backend:
- SQL always parameterized (`$N`). Only compile-time constants (`bookmarkColumns`) may be concatenated into query text — never a request value, not even a validated one.
- `html/template` only for anything a browser parses, never `text/template`. Never wrap stored or fetched strings in `template.HTML`/`JS`/`URL`; that switches off the escaping every template depends on.
- Any outbound fetch of a client-supplied URL passes `fetchableSeriesURL` (site + `https` + host check) first. `series_url` arrives in a PUT body, so without the gate the poller will probe arbitrary hosts from the server's own network position. New fetch path reuses the gate rather than re-deriving one.
- Cap every remote body with `io.LimitReader` (`maxBodyBytes`). An unbounded read is an OOM handed to whatever is on the other end.
- Compare secrets with `hmac.Equal` / `subtle.ConstantTimeCompare`, never `==`. A credential is matched by the SHA-256 the `readers` table holds, which is already a fixed-width equality — a new secret comparison must not regress to `==`.
- Errors: generic text to the client (`http.Error(w, "internal error", 500)`), detail to `log.Printf`. Never log `TOKEN_KEY`, a Reader's credential, `DISCORD_CLIENT_SECRET`, a session id, or a whole `Authorization` header.
- Proxy headers are trusted only where they already are: `X-Forwarded-Proto` for the Secure cookie flag, **rightmost** `X-Forwarded-For` for client IP (leftmost is attacker-supplied). Don't read either anywhere else.
- Session cookies keep `HttpOnly`, `SameSite`, `Secure`-when-HTTPS; expiry is enforced by the `sessions` table lookup, not a signature.
- Stdlib crypto only. No hand-rolled hashing, no MD5/SHA-1 anywhere security-bearing.
- Validate at the handler boundary before storing: body capped by `http.MaxBytesReader` (64 KB), empty `key` and unknown `status`/`kind` rejected with `400`. A bad value that reaches the store becomes every later reader's problem.
Userscript:
- Site-derived and stored strings render via `el(..., {text})` / `textContent`. `{html}` and `innerHTML` are for author-written literal markup only (`TEMPLATE`, `CSS`) — never a title, chapter label, or API response field. The page DOM belongs to a third-party site; treat it as attacker-controlled.
- Isolated world protects the credential from the site's JS. It does not protect anything from an `innerHTML` sink you add yourself.
- The userscripts carry `__API_TOKEN__` placeholders, substituted at serve time with the requesting Reader's credential (`internal/userscript`). Never put a real credential in the repo, docs, commit messages, or issues. Rotation is a web-UI action (epoch bump, `internal/token`); `TOKEN_KEY` in backend env is what derives every credential — never log it.
- `fetch()` targets `API_BASE` only — no dynamic origin, no site-supplied URL. `authHeaders()` goes nowhere but the backend.
- `localStorage` is shared with the site's own JS: cache and queue live there, credentials never do.
- Wrap every `localStorage` read/write and `JSON.parse` in try/catch (quota, private mode, corrupt entry), as the existing helpers do.
Dependencies: stdlib first; a new module needs a stated reason. Confirm a package actually exists before adding it — a plausible name may be fiction (~20% of LLM-proposed packages don't resolve, which is how slopsquatting lands). Pin exact versions.
Review gate: auth, CORS, session, crypto, and the fetch gate are security-critical. Editing one is not a drive-by change — say which invariant you preserved and run `go test ./...` before calling it done.
## Comments
Comment only if code alone can't carry info. Cost per read — must earn spot.
@@ -146,22 +95,6 @@ Test: "competent reader get this from code in few sec?" Yes → skip. Needs deto
`golang-code-style`, `golang-error-handling`, `golang-performance`, `golang-testing` for backend Go work.
## Agent skills
`AGENTS.md` is the single source of truth for agent guidance; every `CLAUDE.md` in this repo is a symlink to the `AGENTS.md` beside it. Edit `AGENTS.md`.
### Issue tracker
Issues live as Gitea issues on `gitea.violetcrown.my.id` (`sulthan/mangaBookmark`), driven by the `tea` CLI — not `gh`. See `docs/agents/issue-tracker.md`.
### Triage labels
Default five-role vocabulary, label strings unchanged (`needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`). See `docs/agents/triage-labels.md`.
### Domain docs
Single-context: one root `CONTEXT.md` plus `docs/adr/`, both created lazily. See `docs/agents/domain.md`.
## graphify
Project has knowledge graph at graphify-out/ with god nodes, community structure, cross-file relationships.
-1
View File
@@ -1 +0,0 @@
AGENTS.md
+106
View File
@@ -0,0 +1,106 @@
# CLAUDE.md
Guidance for Claude Code (claude.ai/code) working in this repo.
## What this is
Manga read-progress tracker, user read on **asurascans.com** (current domain; asuracomic.net 301s here) and **demonicscans.org** via **Violentmonkey**. Userscript inject on-page UI (floating button + slide-in panel), sync progress to self-hosted Go backend so bookmarks unify across both sites and devices.
## Hard constraints (drive design — don't violate)
Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free where plain web APIs suffice — keeps portability across engines:
- **Avoid `GM_*` unless needed.** Prefer page `localStorage` over `GM_setValue`/`GM_getValue`, on-page UI over `GM_registerMenuCommand`, plain `fetch()` over `GM_xmlhttpRequest` for cross-origin.
- Cross-origin `fetch()` work **only** against CORS-enabled backend. Manga sites `https://`, so backend **must be HTTPS** (else mixed-content block).
- Asura and Demonic are **separate origins with separate `localStorage`** — shared remote store only way to unify bookmarks. Cloud sync required, not optional.
- Userscript run in **isolated world**, so embedded API token safe from site's JS.
- Cloudflare's block on manga sites **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare's bot scoring can flip previously-clean IP without notice. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, or direct probe) before finalize, not assumed from single earlier test.
## Architecture
```
Violentmonkey userscript (isolated world, per-site adapters, localStorage cache)
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> SQLite (volume)
```
Backend-specific architecture (packages, endpoints, poller, config env vars) lives in `backend/CLAUDE.md`. Userscript-specific structure (adapters, retry queue, UI, live URL shapes) lives in `userscript/CLAUDE.md`.
## Commands
Backend (`cd backend`):
- Test all: `go test ./...`
- Single test: `go test -run TestName ./...`
- Build static binary: `CGO_ENABLED=0 go build`
Local stack: `docker compose up` (named volume mounted at `/data`, `restart: unless-stopped`).
Smoke test: `curl` endpoints with `Authorization: Bearer <token>`; confirm `OPTIONS` preflight return CORS headers and `/healthz` return 200.
## Forge: Gitea, not GitHub
`origin` is self-hosted Gitea instance (`gitea.violetcrown.my.id`), so **`gh` don't work here — use `tea` (Gitea CLI) for anything past plain git.** Common ones:
- Open PR: `tea pr create --head <branch> --base main --title "..." --description "..."`
- List / view / check out: `tea pr list`, `tea pr <n>`, `tea pr checkout <n>`
- Issues: `tea issue create`, `tea issue list`
- Auth lives in `tea login`, not `GH_TOKEN` env var.
`tea` print output as rendered boxes rather than plain text; PR URL lands on last line.
## Design system
Web UI + userscript panel follow **Cinder**, rules in `docs/design-system.md`
— source of truth Claude Design project `BookmarkManager Web UI`
(`969ac210-fe02-4c01-ae1b-9a271dcc779a`). Read it before touching
`backend/internal/web/static/style.css`, `backend/internal/web/templates/*`, or userscript
`TEMPLATE`/`CSS`. Core law: **ember means new chapter only** — no other
state (busy, error, destruction) may use `--ember`; destruction gets
`--danger`. No cards/corners/shadows, one `--measure: 760px` column, tokens
only (never hardcode hex outside `:root`), both colour branches touched
together. Any move that pulls series out of list (archive/finish/remove)
must be confirm-gated via its own `.confirm-row`; only restore fires
instantly.
## Security invariants
- Auth on `/bookmarks*`: require `Authorization: Bearer <API_TOKEN>`, **constant-time compare**, 401 otherwise.
- CORS: reflect `Origin` only when in `ALLOWED_ORIGINS`; allow `GET,PUT,DELETE,OPTIONS` + headers `Authorization,Content-Type`; answer preflight `OPTIONS` with `204`.
## Comments
Comment only if code alone can't carry info. Cost per read — must earn spot.
Write for:
- Why not what. Tradeoffs, non-obvious decisions.
- Load-bearing detail looking incidental — say so if "simplify" breaks it.
- Non-local consequence, invisible from function alone.
- Wire format / encoding / interface contract — save callers re-deriving.
- Gotcha/workaround, with ref if exists.
- Domain/business rule not derivable from code.
Skip:
- Restating code (no `// increment i` above `i++`).
- Trivial getter/setter/pass-through.
- Banners, dividers, `// helpers`.
- Change narration (`// fix bug`, `// as requested`, `// new impl`) — git's job.
- Commented-out code — delete.
- TODO without concrete action.
Style: one dense comment over function beats one per line inside. Tight, no worked example unless bug subtle. Wrong comment worse than none — update/delete on change. Default fewer — sparse+high-signal beats comprehensive.
Test: "competent reader get this from code in few sec?" Yes → skip. Needs detour through another file/spec/git-blame → write it.
## Relevant skills
`multi-stage-dockerfile` and `docker-compose-orchestration` for container work (referenced in plan).
`golang-code-style`, `golang-error-handling`, `golang-performance`, `golang-testing` for backend Go work.
## graphify
Project has knowledge graph at graphify-out/ with god nodes, community structure, cross-file relationships.
Rules:
- For codebase questions, first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. Return scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
- If graphify-out/wiki/index.md exists, use for broad navigation instead of raw source browsing.
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain don't surface enough context.
- After modifying code, run `graphify update .` to keep graph current (AST-only, no API cost).
-76
View File
@@ -1,76 +0,0 @@
# Bookmark Manager
Read-progress tracker for serialised fiction. A reader browses third-party manga and
novel sites; userscripts capture where they got to and sync it to a self-hosted backend,
so progress survives across sites and devices.
## Language
**Series**:
One ongoing work — a manga or a novel — as published by a Site. Identified by its
stable slug on that Site, never by its title. A Series exists once and is shared by
every Reader who bookmarks it; it owns the facts that are true regardless of who is
reading — title, cover, Latest Chapter. A Reader cannot change them; they describe the
Series, not anyone's relationship to it.
_Avoid_: manga, title, book, comic
**Site**:
One third-party source a Series is published on. A Series on two Sites is two Series.
_Avoid_: source, host, provider, domain
**Cover**:
The image that stands for a Series wherever it is listed. A fact about the Series like
its title — one Cover per Series, shared by every Reader, never per-Reader. Defined by
what a Reader's browser can display, not by where the Site keeps the picture: an address
no client can load is not a Cover, it is a missing one.
_Avoid_: thumbnail, poster, image URL, artwork
**Reader**:
A person with their own Progress. Exactly one per set of credentials, so there is no
separate "account" concept to model — the credential belongs to the Reader.
_Avoid_: user, account, member, subscriber
**Bookmark**:
One Reader's tracked relationship with one Series, holding only what differs between
Readers: Progress, Favourite, Lifecycle bucket. Facts about the Series itself belong
to the Series, not here.
_Avoid_: entry, item, record, subscription
**Library**:
One of the two halves of the collection — manga or novel — selected by a Bookmark's
`kind`. The web UI and the userscripts each address exactly one Library at a time.
Not a per-person concept: "everything one person has bookmarked" is a different idea
and must not be called a Library.
_Avoid_: section, tab, category
**Progress**:
The furthest chapter a reader has actually read in a Series. Only a change in Progress
is real activity, so only Progress reorders the list.
_Avoid_: position, bookmark (the noun is taken), last read
**Latest Chapter**:
The newest chapter a Site has published for a Series, discovered without the reader
present. Distinct from Progress in every way that matters: it is a fact about the Site,
not about the reader, and it must never reorder the list.
_Avoid_: newest, current chapter, update
**Poll**:
The backend's own check of a Site for a Series's Latest Chapter, made without the
Reader present. Performed once per Series no matter how many Readers bookmarked it —
a Poll is work done on behalf of the Series, never on behalf of a Reader.
_Avoid_: scrape, refresh, check, sync
**New Chapter**:
The state where Latest Chapter is ahead of Progress. The single condition the ember
accent is permitted to signal.
_Avoid_: unread, update available
**Lifecycle bucket**:
Which of three mutually exclusive states a Bookmark sits in — reading, archived, or
finished. A Bookmark is in exactly one. Orthogonal to being a favourite.
_Avoid_: state, status (as a domain word), list
**Favourite**:
A reader's manual pin on a Bookmark. Orthogonal to the Lifecycle bucket, and never a
reason to reorder the list.
_Avoid_: starred, pinned, priority
-299
View File
@@ -1,299 +0,0 @@
# SQLite → Postgres cutover runbook
One-way, one-time. Moves the owner's reading history out of the retired SQLite
volume (`<compose project>_bookmarks-data`, holding `/data/bookmarks.db`) and into
the Postgres schema the migration runner builds. There is no dual-write period:
the old database is read once, at cutover, from a **fresh export** — anything
written to SQLite after the export is lost, so the old API must already be down.
Routine deploys are `REDEPLOY.md`; first-time setup is `DEPLOY.md`. This file is
run once and then only ever read for reference.
Proven end to end on 2026-08-08 against a copy of `bookmarks-20260807-213515.db`
into a scratch Postgres: 29 Bookmarks (18 reading, 11 archived, 7 favourites),
29 Series, all owned by the seeded Reader, and every field of every row matching
the source exactly. Production was not touched.
---
## 0. The generator is throwaway
It is written at cutover, run once, and deleted. It is deliberately **not** in
this repository and never will be:
- Its output is the owner's personal reading history. That does not enter
version control.
- It reads SQLite. The backend module dropped `modernc.org/sqlite` (ADR-0001);
a committed generator would drag the dependency back in through the side door.
So §3 specifies the transformation rather than shipping a script. It is a
twenty-line program against a sixteen-column table (fifteen after `key`, which
is dropped) — writing it from the spec below costs less than maintaining it
would.
Beyond `DEPLOY.md`'s prerequisites (Docker and Compose), this runbook needs
`python3`: its stdlib `sqlite3` module is the whole SQLite dependency, and §5's
read-path check uses it in place of `jq`, which the server does not have. It
does not have to run on the server — §3 only reads the snapshot copy, so it can
run on a laptop and the resulting `import.sql` be copied over.
---
## 1. Stop the old API and take a fresh export
**Order matters.** Export after the API stops, or you migrate a snapshot that is
already stale.
```bash
cd ~/mangaBookmark # wherever the checkout lives
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
BACKUP_DIR="$(cd .. && pwd)/$(basename "$PWD")-backups"; mkdir -p "$BACKUP_DIR"
STAMP=$(date -u +%Y%m%d-%H%M%S)
# The volume is <compose project>_bookmarks-data, and the project name defaults
# to the lowercased *directory* name, not the repo name — on this host the
# checkout is ~/mangaBookmark, so the volume is mangabookmark_bookmarks-data.
# Derive it exactly rather than with a `--filter name=` substring match, which
# would return every volume whose name merely contains the string.
VOL="$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_bookmarks-data"
docker volume inspect "$VOL" >/dev/null && echo "$VOL"
$COMPOSE stop bookmark-api
# A clean SIGTERM closes the store, which checkpoints and unlinks the -wal, so
# bookmarks.db alone is then the whole database. But `compose stop` SIGKILLs
# after 10s, and a surviving -wal holds writes the main file does not — assert
# it is gone rather than assuming the shutdown was clean.
docker run --rm -v "$VOL":/d:ro alpine ls -l /d # -> bookmarks.db, alone
docker run --rm -v "$VOL":/from:ro -v "$BACKUP_DIR":/to \
alpine cp /from/bookmarks.db "/to/bookmarks-$STAMP.db"
ls -lh "$BACKUP_DIR/bookmarks-$STAMP.db"
```
If `-wal` and `-shm` are still there, the container was killed mid-write. Copy
all three under the same basename and let SQLite replay the log when §3 opens
it — copying only `bookmarks.db` silently drops whatever the log still holds.
Work on a **copy** of that file for the rest of this runbook. The export is the
last line of retreat; nothing below should be able to write to it.
```bash
mkdir -p /tmp/cutover && cp "$BACKUP_DIR/bookmarks-$STAMP.db" /tmp/cutover/snapshot.db
chmod 444 /tmp/cutover/snapshot.db
```
---
## 2. Bring up Postgres with the schema and the owner Reader
The new stack builds its own schema and seeds exactly one Reader from
`OWNER_DISCORD_ID` — do not hand-write either. Pull the Postgres-era commit
first: on a server that has only ever run the SQLite build, `--build` without a
pull silently rebuilds the old image and the checks below fail with
"relation readers does not exist".
```bash
git pull --ff-only
git log --oneline -1
# .env needs the new required vars (DATABASE_URL is built from
# POSTGRES_PASSWORD; TOKEN_KEY, OWNER_DISCORD_ID and the DISCORD_* set are
# required). Compose fails at start for a missing one.
git diff HEAD@{1} HEAD -- .env.example docker-compose.yml docker-compose.prod.yml
$COMPOSE up -d --build
docker logs bookmark-api --tail 20 # -> "listening on :8080"
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c '\dt'
# -> bookmarks, readers, schema_migrations, series, sessions
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks \
-c 'select id, discord_id from readers'
# -> exactly one row, and discord_id is the owner's
```
Two rows in `readers`, or zero, means `OWNER_DISCORD_ID` is wrong or the seed
failed. Stop here — the import attaches history to "the oldest reader row", and
that is only unambiguous while there is one.
`bookmarks` and `series` are empty at this point. That is what makes the import
a plain sequence of `INSERT`s with no conflict handling.
---
## 3. Generate the import SQL
Read `/tmp/cutover/snapshot.db` and emit plain SQL on stdout. The old table is
flat and its columns map one-for-one onto the split schema — no transformation
beyond the split itself:
| SQLite `bookmarks` column | lands in | notes |
|---|---|---|
| `site`, `series_id` | both tables | the Series key; the wire `key` column is dropped, it is re-derived as `site:series_id` on read |
| `title`, `series_url`, `cover`, `kind` | `series` | shared facts (ADR-0003) |
| `latest_chapter`, `latest_chapter_num`, `latest_checked_at` | `series` | `latest_chapter_num` is nullable on **both** sides and `NULL` is meaningful — never coerce it to `0` |
| `last_chapter`, `last_chapter_num`, `last_chapter_url` | `bookmarks` | Progress |
| `favorite`, `status`, `updated_at` | `bookmarks` | `favorite` is `0`/`1` in SQLite and a real `boolean` in Postgres — emit `true`/`false` |
| — | `bookmarks.reader_id` | the seeded owner |
**`latest_chapter_num` is the only column where `NULL` survives.** The SQLite
table declares `title`, `series_url`, `cover`, `last_chapter`,
`last_chapter_url` as bare `TEXT` and `last_chapter_num` as bare `REAL` — all
six nullable — while their Postgres targets are `NOT NULL DEFAULT ''` /
`NOT NULL DEFAULT 0`. One `NULL` in any of them aborts the whole import on a
not-null violation. Coalesce them in the `SELECT` (`ifnull(title,'')`,
`ifnull(last_chapter_num,0)`, …) rather than discovering it at §5. The
2026-08-07 export happened to have none; a fresh export is not promised the
same.
Rules the generator must follow:
- **Series first, Bookmarks second.** `bookmarks` has a foreign key onto
`series (site, series_id)`; the reverse order fails on the first row.
- **`SELECT DISTINCT` the Series.** The old key's uniqueness already makes
`(site, series_id)` unique, so this is belt and braces — but if it ever
collapses two rows, the count check in §5 catches it.
- **Never hardcode the reader id.** Emit
`INSERT INTO bookmarks (reader_id, …) SELECT id, … FROM owner`, where `owner`
is a temp table built once at the top:
`CREATE TEMP TABLE owner ON COMMIT DROP AS SELECT id FROM readers ORDER BY id LIMIT 1;`
A literal id is a number nobody verifies; this one cannot be wrong.
- **Wrap the whole file in `BEGIN; … COMMIT;`, temp table included.** Postgres
has transactional DDL and DML: a failure half way leaves an empty database
rather than half a library. The ordering is load-bearing —
`ON COMMIT DROP` outside the transaction means the temp table drops itself
the instant it is created (psql autocommits) and every
`SELECT … FROM owner` then fails.
- **Quote strings by doubling `'`.** Titles contain apostrophes and the URLs
contain `%5C%27` escapes. Emit standard SQL literals only — no `E''` strings,
no backslash escaping (`standard_conforming_strings` is on, so a backslash is
a literal backslash and the URLs survive verbatim).
```bash
python3 gen_import.py /tmp/cutover/snapshot.db > /tmp/cutover/import.sql
wc -l /tmp/cutover/import.sql # -> 2 header + 29 series + 29 bookmarks + framing
```
---
## 4. Review it by eye
29 rows is small enough to actually read, and this is the last point at which a
mistake is free:
```bash
less /tmp/cutover/import.sql
grep -c '^INSERT INTO series' /tmp/cutover/import.sql # -> 29
grep -c '^INSERT INTO bookmarks' /tmp/cutover/import.sql # -> 29
```
Look for: a title whose apostrophe is not doubled, a `favorite` that is still
`0`/`1`, a `latest_chapter_num` that turned into `0`, and any `reader_id`
written as a bare number.
---
## 5. Apply it
```bash
docker cp /tmp/cutover/import.sql "$($COMPOSE ps -q postgres)":/tmp/import.sql
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -v ON_ERROR_STOP=1 \
-f /tmp/import.sql
```
`ON_ERROR_STOP=1` is not optional: without it `psql` reports the error, keeps
going, and exits `0` on a half-imported database.
Then the checklist. Every number here is asserted, not eyeballed:
```bash
OWNER=$(grep -E '^OWNER_DISCORD_ID=' .env | cut -d= -f2)
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -x -c "
SELECT (SELECT count(*) FROM bookmarks) AS bookmarks_total,
(SELECT count(*) FROM bookmarks WHERE status='reading') AS reading,
(SELECT count(*) FROM bookmarks WHERE status='archived')AS archived,
(SELECT count(*) FROM series) AS series_total,
(SELECT count(*) FROM readers) AS readers_total,
(SELECT count(*) FROM bookmarks
WHERE reader_id <> (SELECT id FROM readers WHERE discord_id='$OWNER'))
AS not_owned_by_owner;"
```
`not_owned_by_owner` resolves the Reader by **Discord id**, not by
`ORDER BY id LIMIT 1`. The second form is the expression §3 tells the generator
to import with, so comparing against it is true by construction and could never
fail; resolving by Discord id is an independent check that the rows landed on
the identity the owner will actually log in as. If that subquery returns NULL
the whole count comes back `0` for the wrong reason — hence `readers_total`
beside it.
Expected, for the 2026-08-07 export: `29`, `18`, `11`, `29`, `1`, `0`. Against a
different export, the invariants rather than the literals are what hold:
- `bookmarks_total` equals the SQLite row count.
- `reading + archived` equals `bookmarks_total` (nothing was `finished`).
- `series_total` equals `SELECT count(*) FROM (SELECT DISTINCT site, series_id FROM bookmarks)`
in the source.
- `readers_total` is `1` and `not_owned_by_owner` is `0`.
Then spot-check the values themselves against the source — read position,
favourite flag and latest chapter. Take the sample from each bucket explicitly:
`ORDER BY updated_at DESC LIMIT 5` alone returns the most recently *progressed*
rows, which are the ones least likely to be archived.
```bash
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
s.latest_chapter, b.status
FROM bookmarks b JOIN series s USING (site, series_id)
WHERE b.status='reading' ORDER BY b.updated_at DESC LIMIT 3;"
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
s.latest_chapter, b.status
FROM bookmarks b JOIN series s USING (site, series_id)
WHERE b.status='archived' ORDER BY b.updated_at DESC LIMIT 2;"
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
s.latest_chapter, b.status
FROM bookmarks b JOIN series s USING (site, series_id)
WHERE b.favorite ORDER BY b.updated_at DESC LIMIT 2;"
```
Compare each against the same row in the snapshot — the generator's own source
is the reference, so read it back with the same `python3` you used in §3.
Finally, prove the **read path**, not just the tables — this is the check that
would catch a correct import behind a broken join:
```bash
API=https://bookmark-api.violetcrown.my.id
# Your own Reader credential: sign in to the web UI and take it from the
# Userscripts panel's install link, or read the API_TOKEN constant out of an
# already-installed script. There is no credential in .env to grep.
TOKEN=<your Reader credential>
curl -s -H "Authorization: Bearer $TOKEN" $API/bookmarks |
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' # -> 29
```
---
## 6. Afterwards
- **Keep the old SQLite volume for a month.** It is already undeclared in
compose, so `docker compose down -v` cannot take it. Remove it by hand once
the Postgres data has been trusted for a while. That happens in a shell where
`$VOL` from §1 is long gone, so re-derive it:
`docker volume rm "$(basename ~/mangaBookmark | tr '[:upper:]' '[:lower:]')_bookmarks-data"`
(see `REDEPLOY.md` §1).
- **Delete the generator and the working copies:** `rm -rf /tmp/cutover`. The
timestamped export in `$BACKUP_DIR` is the copy that is kept.
- **Take the first Postgres dump immediately** — `REDEPLOY.md` §1. Until that
exists, the only backup of the migrated data is the SQLite file it came from.
If the import is wrong, there is nothing to unpick: drop the rows and start
again from §3 — `TRUNCATE bookmarks, series;` leaves the seeded Reader and the
schema in place.
+63 -346
View File
@@ -11,7 +11,7 @@ ACME/cert resolver, and control a domain.
- Docker + Docker Compose on the server.
- A Traefik instance watching a Docker network (default name assumed: `proxy`).
- DNS: an `A`/`AAAA` record for `bookmark-api.<yourdomain>` pointing at the server.
- The repo copied to the server, e.g. `~/mangaBookmark/` (needs `backend/`,
- The repo copied to the server, e.g. `/opt/bookmarkmanager/` (needs `backend/`,
`docker-compose.yml`, `docker-compose.prod.yml`, `.env.example`).
Confirm the Traefik network exists (create if not):
@@ -25,46 +25,22 @@ docker network ls | grep proxy || docker network create proxy
## 1. Configure `.env`
```bash
cd ~/mangaBookmark
cd /opt/bookmarkmanager
cp .env.example .env
```
Edit `.env`:
```ini
# Required — secret every Reader's userscript credential is derived from.
# Only SHA-256 hashes of credentials are stored.
TOKEN_KEY=<paste output of: openssl rand -hex 32>
# Required — the owner's Discord user ID. Seeds the first Reader: the
# administrator, and the owner of every bookmark that predates registration.
# The value is the snowflake in your Discord profile (Settings →
# Advanced → Developer Mode → right-click your name → Copy User ID).
OWNER_DISCORD_ID=<discord user id>
# Required — long random secret, also goes in the userscript.
API_TOKEN=<paste output of: openssl rand -hex 32>
# CORS allowlist — leave as-is unless a site changes hostname.
ALLOWED_ORIGINS=https://asuracomic.net,https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to
# Required — password for the bundled Postgres container. Compose builds the
# backend's DATABASE_URL out of it and has no fallback for either.
POSTGRES_PASSWORD=<paste output of: openssl rand -hex 24>
# Leave unset. Only set this to point the backend at a Postgres compose does
# not run; it then replaces the URL built from POSTGRES_PASSWORD above.
# DATABASE_URL=postgres://user:pass@host:5432/bookmarks?sslmode=require
# Required path inside bookmark-api. Compose builds the image and mounts the
# named cover-data volume at this path.
COVER_DIR=/covers
# Required — the origin this deployment answers on, no trailing slash. Cover
# URLs on the wire are absolute, because the userscript renders them on a
# Site's own origin (ADR-0007). Same host as BOOKMARK_API_HOST below.
PUBLIC_BASE_URL=https://bookmark-api.violetcrown.my.id
# Required for the Traefik override. Both have no fallback — compose refuses
# to start without them. BOOKMARK_WEB_HOST is required even if the web UI
# were unused; see 1b.
# to start without them. BOOKMARK_WEB_HOST is required even if you never set
# WEB_PASSWORD; see 1b.
BOOKMARK_API_HOST=bookmark-api.violetcrown.my.id
BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
@@ -74,25 +50,13 @@ BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
# TRAEFIK_CERTRESOLVER=le
```
Generate + insert the two secrets in three lines:
Generate + insert the token in one line:
```bash
sed -i "s|^TOKEN_KEY=.*|TOKEN_KEY=$(openssl rand -hex 32)|" .env
sed -i "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$(openssl rand -hex 24)|" .env
grep -E '^TOKEN_KEY=' .env
sed -i "s|^API_TOKEN=.*|API_TOKEN=$(openssl rand -hex 32)|" .env
grep -E '^API_TOKEN=' .env # copy this — the userscript needs the same value
```
`TOKEN_KEY` derives every Reader's userscript credential (issue #24); only
SHA-256 hashes of the credentials are stored, so this secret is what a
database leak alone cannot recover. Changing it invalidates every installed
script at once.
`POSTGRES_PASSWORD` is read **only while the `postgres-data` volume is empty**,
which in practice means at first boot. Changing it afterwards changes the URL
the backend dials but not the password the database expects, and `bookmark-api`
crash-loops on `password authentication failed`. Set it before §2 and leave it
alone.
> Match `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER` to your Traefik's actual
> names (check your Traefik static config — common alternatives: `https`,
> `myresolver`, `cloudflare`). Wrong names = no certificate issued.
@@ -101,58 +65,44 @@ alone.
## 1b. Web UI
The browser UI is served by the same container on a second hostname. Sign-in
is a Discord authorization code grant (ADR-0002): the owner's Discord account,
gated by membership in one configured guild.
The browser UI is served by the same container on a second hostname.
1. Add a DNS `A`/`AAAA` record for `bookmark.<yourdomain>` pointing at the
server — the same address as `bookmark-api.<yourdomain>`.
1. Add a DNS `A`/`AAAA` record for `bookmark.<yourdomain>` pointing at the server —
the same address as `bookmark-api.<yourdomain>`.
2. Create the Discord application at <https://discord.com/developers/applications>:
- **OAuth2 → Redirects:** add the exact callback URL
`https://bookmark.violetcrown.my.id/auth/discord/callback`. Discord
matches it verbatim — a trailing slash or different hostname breaks
sign-in.
- **OAuth2 → General:** note the Client ID, and generate a Client Secret.
- No scopes or bot setup are needed in the dashboard; the service requests
`identify` and `guilds.members.read` itself, and checks the *user's*
membership of the guild, not the application's.
3. Set the variables in `.env`:
2. Set both variables in `.env`:
```ini
BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
DISCORD_CLIENT_ID=<client id>
DISCORD_CLIENT_SECRET=<client secret>
DISCORD_GUILD_ID=<guild snowflake>
DISCORD_REDIRECT_URI=https://bookmark.violetcrown.my.id/auth/discord/callback
# Optional: only members holding this role may sign in.
# DISCORD_REQUIRED_ROLE=<role snowflake>
WEB_PASSWORD=<paste output of: openssl rand -base64 18>
```
The guild id is in Discord's client with Developer Mode on: right-click the
server name → Copy Server ID. The four uncommented variables are required —
the backend refuses to start without them. Guild membership *is*
registration: any member of `DISCORD_GUILD_ID` becomes a Reader with their
own library on their first sign-in. `OWNER_DISCORD_ID` from §1 is only the
administrator — the Reader who can revoke another Reader's sessions.
Generate and insert in one line:
4. Redeploy and check:
```bash
sed -i "s|^WEB_PASSWORD=.*|WEB_PASSWORD=$(openssl rand -base64 18)|" .env
grep -E '^WEB_PASSWORD=' .env # this is what you type into the site
```
3. Redeploy and check:
```bash
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build
curl -s -o /dev/null -w '%{http_code}\n' https://bookmark.violetcrown.my.id/
```
Expected `200`, serving the login page with the Discord button. Signing in
lands on the library; an account outside the guild is refused with a message
that names neither the guild nor its id.
Expected `200`, serving the login page.
Sessions are rows in the database: the cookie carries only an opaque id, and
every request looks the row up and checks its expiry. Deleting a session row —
or the whole `sessions` table — logs the browser out immediately; nothing is
signed, so rotating a credential does not affect browser sessions. Sessions
last 60 days.
Leaving `WEB_PASSWORD` unset is safe: the web routes are not registered and `/`
returns 404. The userscript's API on `BOOKMARK_API_HOST` is unaffected either way.
`BOOKMARK_WEB_HOST` itself is required by the prod override regardless — like
`BOOKMARK_API_HOST`, its Traefik label has no fallback, so `docker compose up`
refuses to start without it even if `WEB_PASSWORD` is unset and the web UI is
otherwise dormant.
Sessions are signed with a key derived from `API_TOKEN` and `WEB_PASSWORD`, so
rotating either one logs every browser out. The session cookie lasts 60 days.
---
@@ -166,23 +116,16 @@ This merges the base file (build/image/env/volume) with the prod override
(no host port, Traefik network + router labels). Always pass **both** `-f`
flags — the prod file is not standalone.
Two services come up: `bookmark-api` (the backend) and `postgres` (its
database, `postgres:17-alpine`). Postgres publishes no port — it sits alone
with `bookmark-api` on an `internal: true` network — and stops everything if it
is missing: `bookmark-api` waits for `pg_isready` to pass, then applies its
embedded migrations, and only then listens. The schema is created that way;
there is nothing to import by hand.
There is deliberately no browser here. Kagane and novelfull need one, and it
runs on a **separate machine** over the tailnet — §7. Until you do that step,
`BROWSER_WS_URL` is unset, the poller logs and skips those two sites, and
everything else works normally.
Two services come up: `bookmark-api` (the backend) and `headless-shell`, a CDP
sidecar the poller uses to fetch kagane (behind a Cloudflare JS challenge).
It has no published port — only `bookmark-api` can reach it, over
`BROWSER_WS_URL`. Missing or unreachable, the poller just skips kagane and
logs it; nothing else is affected.
Check it's up and healthy:
```bash
docker compose -f docker-compose.yml -f docker-compose.prod.yml ps
# bookmark-api Up; postgres Up (healthy)
docker logs bookmark-api --tail 20 # expect: "listening on :8080 ..."
```
@@ -200,10 +143,7 @@ curl -s https://bookmark-api.violetcrown.my.id/healthz # -> ok
curl -s -o /dev/null -w '%{http_code}\n' \
https://bookmark-api.violetcrown.my.id/bookmarks # -> 401
# A Reader's own credential. It is derived, never stored in .env — take it from
# the Userscripts panel's install link after signing in, or from an installed
# script's API_TOKEN constant.
TOKEN=<your Reader credential>
TOKEN=$(grep -E '^API_TOKEN=' .env | cut -d= -f2)
curl -s -H "Authorization: Bearer $TOKEN" \
https://bookmark-api.violetcrown.my.id/bookmarks # -> []
@@ -222,30 +162,29 @@ a bad cert makes the browser block the userscript's `fetch()` (mixed content).
## 4. Configure the userscript
The bindmounted `userscript/*.user.js` files carry `__API_TOKEN__` placeholders
and the deployment's `@downloadURL`/`@updateURL` lines. Check the metadata
block — it ships hardcoded to this deployment's domain, so a deployer who
copies the repo to another domain must edit the two lines or the script
auto-updates from someone else's backend:
Edit the config block at the top of `userscript/manga-bookmark.user.js`:
```js
// @downloadURL https://bookmark-api.yourdomain.com/u/__API_TOKEN__/manga-bookmark.user.js
// @updateURL https://bookmark-api.yourdomain.com/u/__API_TOKEN__/manga-bookmark.user.js
const API_BASE = "https://bookmark-api.yourdomain.com"; // no trailing slash
const API_TOKEN = "<same token as .env>";
```
The backend substitutes `__API_TOKEN__` with the requesting Reader's derived
credential at serve time (issue #24), so no real credential ever sits in the
file. Only the `API_BASE` constant and the metadata hostname are deployer
edits; do not put a credential in this file.
The token sits in the userscript's isolated world — the manga sites' JS can't
read it.
Also edit the `@downloadURL`/`@updateURL` metadata lines near the top of the
file — they ship hardcoded to this deployment's domain and token, so a
deployer who skips them ends up auto-updating from someone else's backend.
See "Installing / updating the userscript" below for how those two lines are
used.
---
## 5. Install on Bromite
1. Bromite → **Settings → User scripts** → enable (accept the permission prompt).
2. Sign in to the web UI, open the **Userscripts** panel, and open the install
link — Bromite detects `.user.js` and offers to install. The script already
carries your credential; you never see or type one.
2. Put the edited `manga-bookmark.user.js` on the device (save the file, or open
its raw URL). Bromite detects `.user.js` and offers to install.
3. Confirm install — the `@match` list covers both sites.
4. Open a series on asurascans.com or demonicscans.org → a 📑 button appears
bottom-right → tap → **+ Bookmark this**.
@@ -253,9 +192,6 @@ edits; do not put a credential in this file.
Optional desktop test: the script is `GM_*`-free, so the same file installs in
Tampermonkey/Violentmonkey for quick checks before going mobile.
Rotating the credential in the same web-UI panel invalidates every installed
copy immediately — reinstall on all devices, or they silently stop syncing.
---
## 6. Smoke-test the full loop
@@ -270,204 +206,6 @@ copy immediately — reinstall on all devices, or they silently stop syncing.
---
## 7. The browser, on the home machine
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
fingerprint clears, so the poller reaches them through a real Chrome over CDP.
That browser does **not** run on the VPS: it held 471 MiB of a 1974 MiB box
with no swap, and it scores better from a residential IP anyway (ADR-0006). It
is its own compose unit, deployed and updated independently of everything
above.
Do this after §2, on the second machine. Both machines must already be on the
same tailnet.
First, on the VPS, record what you are reclaiming — this is the whole point of
the move and there is no way to measure it afterwards:
```bash
free -m | awk '/^Mem:/ {print "available before:", $NF, "MiB"}'
```
Take it again after §7 is finished and the old sidecar is gone. Expect roughly
the sidecar's former footprint back (measured at 471 MiB working set, 595 MiB
cgroup).
**On the home machine:**
```bash
git clone <this repo> ~/mangaBookmark && cd ~/mangaBookmark/chrome
tailscale ip -4 # -> 100.x.y.z, this machine's tailnet IP
cp .env.example .env
echo "BROWSER_BIND_ADDR=$(tailscale ip -4)" >> .env
docker compose up -d --build
```
The clone is only for `chrome/`; nothing else on this machine reads the rest of
the repo. The unit is its own compose project (`bookmark-browser`), so it shares
no volume, network or lifecycle with an API stack that happens to sit beside it.
`BROWSER_BIND_ADDR` has no default on purpose. CDP authenticates nothing —
whatever reaches port 9222 drives the browser and, through it, this host — so
the bind address *is* the access control, backed by Tailscale device identity.
On the VPS that job was done by Docker network membership; this machine has a
real LAN, so `0.0.0.0` would be a hole punched into your home network. Compose
refuses to start rather than guess.
**Narrow it to the one device that needs it.** The bind address keeps CDP off
your LAN; it still leaves port 9222 open to every device on the tailnet, and
CDP has no login — a compromised phone is enough to drive this host. A new
tailnet's policy is allow-all, so this is the step that makes "Tailscale
identity is the access control" true rather than aspirational.
Tailscale has no `deny`, so a restriction is expressed by removing the blanket
grant and enumerating what is left. That only works if the browser machine can
be *excluded* from a selector that still covers your own devices — which is
what tagging buys: a tagged device has no user, so `autogroup:member` and
`autogroup:self` stop matching it. Tagging is the mechanism, not decoration.
In the admin console, under **Access controls**, the shipped policy grants
`{"src": ["*"], "dst": ["*"], "ip": ["*"]}`. Replace it:
```jsonc
{
"tagOwners": {
// Empty list: implicitly owned by the tailnet Owner/Admins, which is you.
"tag:bookmark-api": [],
"tag:bookmark-browser": [],
},
"grants": [
// The only thing on the tailnet that may drive the browser.
{
"src": ["tag:bookmark-api"],
"dst": ["tag:bookmark-browser"],
"ip": ["tcp:9222"],
},
// Your own devices reach your own devices, and the VPS, in full.
{
"src": ["autogroup:member"],
"dst": ["autogroup:self", "tag:bookmark-api"],
"ip": ["*"],
},
// On the browser machine you get SSH and nothing else. Widen this to `*`
// and the restriction above is void; delete it and you are locked out.
{
"src": ["autogroup:member"],
"dst": ["tag:bookmark-browser"],
"ip": ["tcp:22"],
},
// Uncomment if you route traffic through an exit node — dropping the
// blanket grant takes exit-node access with it.
// {"src": ["autogroup:member"], "dst": ["autogroup:internet"], "ip": ["*"]},
],
// Tagged devices left `autogroup:self`, so Tailscale SSH needs them named.
// Irrelevant if you reach these boxes with ordinary sshd over the tailnet —
// that is the `tcp:22` grant above.
"ssh": [
{
"action": "check",
"src": ["autogroup:member"],
"dst": ["autogroup:self", "tag:bookmark-api", "tag:bookmark-browser"],
"users": ["autogroup:nonroot", "root"],
},
],
// Run on every save, so a later edit that reopens 9222 is rejected outright.
"tests": [
{ "src": "tag:bookmark-api", "accept": ["tag:bookmark-browser:9222"] },
{
"src": "you@example.com",
"accept": ["tag:bookmark-browser:22"],
"deny": ["tag:bookmark-browser:9222"],
},
],
}
```
Then apply the tags — on the VPS and the home machine respectively:
```bash
sudo tailscale up --advertise-tags=tag:bookmark-api
sudo tailscale up --advertise-tags=tag:bookmark-browser
```
Each re-authenticates in a browser and issues a new node key; the tailnet IP is
unchanged, so `BROWSER_WS_URL` and `BROWSER_BIND_ADDR` still hold. Key expiry is
disabled once a device is tagged, which is what you want for a server — an
expired key would otherwise take the poller down every few months.
**Tagging replaces the device's user identity**, so do this only to machines
that exist to run these services. If your "home machine" is also your daily
driver, tag it anyway and reach it through the `:22` rule above, or skip the
tag and accept that any device of yours can reach CDP.
Enforcement is by the destination's packet filter, so the check below is real,
not advisory.
Prove the bind is tight, from the home machine itself:
```bash
curl -s -m 3 http://$(tailscale ip -4):9222/json/version # -> JSON
curl -s -m 3 http://<this machine's LAN IP>:9222/json/version
# -> curl: (7) Failed to connect ... Connection refused
```
The first call is also what wakes Chrome: it is not running until something
connects, and it is reaped again after five idle minutes. A cold first response
takes a few seconds; that is the browser starting, not a fault.
That check proves the *bind*, not the ACL — traffic that starts on the node is
not filtered. Prove the ACL from somewhere else: on your laptop or phone the
same URL must now time out, and from the VPS it must answer.
```bash
# on any other device of yours -> hangs until timeout
curl -s -m 5 http://<home machine tailnet IP>:9222/json/version
# on the VPS -> JSON
curl -s -m 20 http://<home machine tailnet IP>:9222/json/version
```
**On the VPS:**
```bash
cd ~/mangaBookmark
echo 'BROWSER_WS_URL=ws://100.x.y.z:9222' >> .env # the home machine's tailnet IP
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
```
It must be the tailnet **IP**. A MagicDNS hostname fails: Chrome's DevTools HTTP
handler answers `/json/version` with a 500 for any `Host` header that is not an
IP or `localhost`, and the failure looks like a broken site rather than a broken
hostname.
**Prove it end to end.** This is the only check that says the challenge actually
clears from that machine's egress — it fetches a real kagane cover and a real
chapter list:
```bash
cd backend
SMOKE_BROWSER_WS_URL=ws://100.x.y.z:9222 go test -run TestSmokeKagane ./internal/latest
```
A red run means "not clearing from this address right now", which is a live
fact to re-check before it is a defect — Cloudflare's scoring moves. Then, from
the web UI, open a bookmarked kagane series and confirm the cover renders. Once
a cover is stored it is served from Postgres forever after, so the browser being
asleep, unreachable, or mid-power-outage costs chapter freshness and nothing
visible.
Finally, take the VPS `free -m` reading again and compare it against the one
from the top of this section.
**Updating the browser** is independent of the API stack and has its own
runbook — `REDEPLOY.md` §8.
---
## Updating
Pull new code, then rebuild:
@@ -476,14 +214,7 @@ Pull new code, then rebuild:
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build
```
Data persists in the named volume `postgres-data` across rebuilds. (If this
server predates the Postgres migration, the old SQLite volume `bookmarks-data`
is still on disk and deliberately undeclared in compose so `down -v` cannot take
it; see `REDEPLOY.md` §1 for when to remove it.)
The browser is a separate unit on a separate machine with its own update
command — §7. Nothing above touches it, and it needs no coordination: the API
picks up a restarted Chrome's new debugger UUID by itself.
SQLite data persists in the named volume `bookmarks-data` across rebuilds.
---
@@ -494,18 +225,9 @@ picks up a restarted Chrome's new debugger UUID by itself.
| No cert / TLS error at the domain | `TRAEFIK_ENTRYPOINT` or `TRAEFIK_CERTRESOLVER` name wrong; or DNS not resolving yet. Check `docker logs <traefik>`. |
| 404 from Traefik | Service not on the `proxy` network, or `BOOKMARK_API_HOST` mismatch. Confirm `docker network inspect proxy` lists `bookmark-api`. |
| `fetch` fails in the userscript, `curl` works | Origin missing from `ALLOWED_ORIGINS`, or mixed content (backend not HTTPS). |
| 401 with the right credential | The script's credential no longer matches the stored hash — most likely a rotation happened and the device was not reinstalled. Reinstall from the web UI. |
| 401 after rotation, even right after reinstalling | `TOKEN_KEY` changed between the rotation and the reinstall; credentials are derived from it, so changing it invalidates every credential. Keep it stable. |
| 401 with the right token | Trailing space/newline in `API_TOKEN`; regenerate and restart. |
| Panel button absent | URL didn't match an adapter, or user scripts disabled in Bromite. |
| `compose ... config` errors about `TOKEN_KEY`, `OWNER_DISCORD_ID` or `POSTGRES_PASSWORD` | Run compose from the dir with `.env`, or export the vars. All three are required and none has a fallback. |
| `bookmark-api` restarts in a loop, `password authentication failed for user "bookmarks"` | `POSTGRES_PASSWORD` was changed after first boot; Postgres only applies it to an empty `postgres-data`. Restore the old value, or reset the role (`REDEPLOY.md` troubleshooting). |
| `bookmark-api` never logs `listening on :8080` | It is blocked on `postgres` passing `pg_isready`, or a migration failed. `docker compose -f docker-compose.yml -f docker-compose.prod.yml logs postgres`. |
| kagane rows never get a `latest_chapter`; log says `browser fetcher disabled` or nothing at all | `BROWSER_WS_URL` unset. Expected before §7 is done. |
| kagane polls all fail; log shows a 500 from `/json/version` | `BROWSER_WS_URL` names a MagicDNS hostname (or any name). Chrome's DevTools handler only accepts an IP or `localhost` — use the tailnet IP. |
| kagane polls fail with a connection error | Home machine off, off the tailnet, or the unit is down. `tailscale ping <machine>`, then `docker compose ps` in its `chrome/`. Costs freshness only; stored covers keep serving. |
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series (up to `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, default 6h). |
| `compose` in `chrome/` errors `set BROWSER_BIND_ADDR to this machine's tailnet IP` | No `chrome/.env`, or the variable is empty. Deliberate — it has no default so an unset value cannot publish CDP to the LAN. |
| browser container restarts, or is OOM-killed | `docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'`. The 512 MiB cap is sized against a measured 645 MiB untuned peak; a real breach is a Chrome regression worth reading `docker logs` for, not a number to raise reflexively. |
| `compose ... config` errors about `API_TOKEN` | Run compose from the dir with `.env`, or export the vars. |
Backend config reference and endpoint list: see `README.md`.
@@ -514,25 +236,20 @@ Backend config reference and endpoint list: see `README.md`.
## Installing / updating the userscript
The backend serves the script itself, so Violentmonkey can auto-update it.
Complements §4 above — the `@downloadURL`/`@updateURL` lines point at the
credential-bearing path, so auto-updates come from the same place as the
install.
Complements §4 above — that step points `API_BASE`/`API_TOKEN` at your
backend; this one points `@downloadURL`/`@updateURL` at the same place so
auto-updates come from it too.
Install once, on the phone (Cromite + Violentmonkey): sign in to the web UI,
open the **Userscripts** panel, and open the install link for the library —
the script is served with your credential already inside it. Its
`@downloadURL`/`@updateURL` point at the same credential-bearing path for
updates:
Install once, on the phone (Cromite + Violentmonkey):
```
https://bookmark-api.<your-domain>/u/<your credential>/manga-bookmark.user.js
https://bookmark-api.<your-domain>/u/<API_TOKEN>/manga-bookmark.user.js
```
Violentmonkey offers to install it. The credential is in the path because
Violentmonkey's update poll sends no `Authorization` header, and the script
embeds the credential in plain text — an open URL would leak it. A wrong
credential answers 404. The credential is derived from `TOKEN_KEY` and never
appears anywhere but this URL and the rendered script.
Open that URL in Cromite; Violentmonkey offers to install it. The token is in
the path because Violentmonkey's update poll sends no `Authorization` header,
and the script embeds `API_TOKEN` in plain text — an open URL would leak it. A
wrong token answers 404.
Updating, without a redeploy:
+14 -20
View File
@@ -8,53 +8,47 @@ web
## Users
Members of one private Discord guild, each with their own library. Accounts exist and are created by signing in — there is no signup form, no invite code and no approval step: any member of the configured guild becomes a Reader on their first Discord login. The person running the deployment is the owner, seeded at startup, and the only Reader with an administrative capability (revoking another Reader's sessions).
Reading happens on **asurascans.com**, **demonicscans.org**, **comix.to** and **kagane.to** for manga and **novelfull.com** and **lightnovelworld.net** for novels, primarily via Bromite on mobile, with checks and corrections from a desktop browser. The web UI is the cross-device view into progress the userscripts capture while reading.
Single user (self-hosted, no accounts, no multi-user planned). Reads manga on **asurascans.com** and **demonicscans.org** primarily via Bromite on mobile, also checks/updates from a desktop browser. The web UI is the cross-device view into progress captured by the userscript while reading.
## Product Purpose
Tracks read-progress ("last chapter read") per series across sites that each have their own separate `localStorage`. A Go backend unifies bookmarks into one store; the web UI is a Discord-gated browser view of one Reader's own bookmarks, for reviewing, favouriting, correcting, shelving or removing them, and jumping back into a series to continue reading. A background poller refreshes each series' latest-published-chapter so the list can flag "NEW" without the Reader visiting the site.
Tracks read-progress ("last chapter read") per manga series across two otherwise-unrelated manga sites that each have their own separate `localStorage`. A Go backend unifies bookmarks into one store; the web UI is a password-gated browser view of that store for reviewing, favouriting, correcting, or removing bookmarks, and jumping back into a series to continue reading. A background poller also refreshes each series' latest-published-chapter so the list can flag "NEW" without the user visiting the site.
## Positioning
Not a public reading tracker or social app — a private, self-hosted sync layer for one Discord community, purpose-built for a fixed set of scraped sites. Multi-Reader, not multi-tenant: libraries are isolated, but the deployment belongs to one group and its membership is the whole access model.
Not a public reading tracker or social app — a private, self-hosted sync layer purpose-built for two specific scraped sites, with no server-side account system (single bearer token + one password-gated session).
## Operating Context
- Primary reading device: Bromite (mobile Chromium), where a userscript captures progress automatically. Each Reader installs their own copy, rendered with their own credential.
- Primary reading device: Bromite (mobile Chromium), where a userscript captures progress automatically.
- Web UI is a secondary surface: checking list state, correcting a wrong chapter number, removing dead bookmarks, jumping to "continue reading."
- Cover art and titles come from the source sites' `og:image`/`og:title` — real content, not placeholders. They are facts about the series, so they are shared between Readers who track it; progress is not.
- List order is driven by `updated_at`, which moves only on real reading progress (not favouriting, not a newly detected chapter) — a UI constraint the design must not break.
- Manga cover art and titles come from the source sites' `og:image`/`og:title` — real content, not placeholders.
- List order is driven by `updated_at`, which moves only on real reading progress (not favouriting, not a newly detected chapter) — a UI constraint the redesign must not break.
## Capabilities and Constraints
- Two libraries (manga, novels) with lifecycle tabs: All / Updated / Favourites / Archived / Finished. Search-filter by title (client-side, `filter.js`).
- Card actions: continue (opens source site), toggle favourite, manual chapter override, archive, finish, remove — each move out of the list confirm-gated.
- "Continue reading" horizontal strip for series with an unread chapter.
- A Reader with no bookmarks at all sees a deliberate empty library offering both userscript install links, not an error and not a blank page.
- Isolation is the load-bearing invariant: two Readers cannot see or change each other's bookmarks. A series both track is one shared row polled once, with independent progress on each side.
- The owner can revoke a specific Reader's sessions; nothing else in the UI differs by Reader.
- htmx-driven partial updates, no client-side framework or build step — templates are Go `html/template`, `go:embed`-ed.
- Two tabs: All / Favourites. Search-filter by title (client-side, `filter.js`).
- Card actions: continue (opens source site), toggle favourite, manual chapter override, delete (with confirm).
- "Continue reading" horizontal strip for recently-progressed series.
- htmx-driven partial updates (card re-render on favourite/chapter/delete), no client-side framework/build step — templates are Go `html/template`, `go:embed`-ed.
- Mobile-first is a hard functional constraint (primary device is a phone), not just a starting breakpoint.
## Brand Commitments
- Name: **BookmarkManager**.
- **Dark-first is binding**: dark-by-default / light-follows-system-preference must be preserved as a design constraint, not just a starting default, because reading happens at night.
- **Dark-first is binding**: current dark-by-default / light-follows-system-preference behavior must be preserved as a design constraint, not just a starting default, because reading happens at night.
## Evidence on Hand
- Live templates/CSS at `backend/internal/web/templates/*.html`, `backend/internal/web/static/style.css`, governed by the Cinder design system (`docs/design-system.md`).
- No logo beyond the wordmark, no screenshots, no marketing copy; none should be fabricated.
- Live templates/CSS at `backend/templates/*.html`, `backend/static/style.css` — current implemented UI, functional but not yet treated as an intentional design system.
- No logo, screenshots, or marketing copy exist; none should be fabricated.
## Product Principles
- Dark-first, night-reading-optimized — never regress to a light-default or high-glare surface.
- Mobile is the primary target; desktop is an enhancement, not the design center.
- Progress data integrity over visual flourish: `updated_at`/list-ordering behavior is a correctness constraint the UI must respect, not decorate over.
- A leak between Readers fails silently and looks like working software — isolation is asserted from both directions, never inferred from counting one Reader's rows.
- No roles, no org chrome: the owner's Readers panel is one list with one button (revoke someone's sessions), not an admin console, and otherwise every Reader's view is the same.
- No accounts, no multi-tenant chrome — the whole product is for one reader.
- Prefer native platform affordances (system dark/light, native touch targets) over custom widgetry — this is a lean self-hosted tool, not a product to demo.
## Accessibility & Inclusion
+22 -90
View File
@@ -7,29 +7,16 @@ all four sites and all devices.
Two parts:
- **`backend/`** — tiny Go (`net/http` + Postgres via pure-Go `pgx`) sync service. 4 routes,
- **`backend/`** — tiny Go (`net/http` + pure-Go SQLite) sync service. 4 routes,
static binary, distroless container.
- **`userscript/manga-bookmark.user.js`** — single Bromite-compatible userscript
(no `GM_*` APIs) that injects an on-page bookmark UI and syncs via `fetch()`.
```
Bromite userscript (isolated world, Shadow DOM UI, localStorage cache)
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> Postgres (volume)
|
| CDP over the tailnet
v
headless Chrome, on-demand,
on a separate machine
(chrome/, ADR-0006)
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> SQLite (volume)
```
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
fingerprint clears, so the poller reaches those two through a real Chrome over
CDP. That browser is **not** part of the API stack: it is its own compose unit
on a second machine, spawned on the first connection and reaped when idle. The
API needs it only to discover new chapters and to fetch a kagane cover once —
covers are stored, so the library renders in full with the browser switched off.
---
## 1. Backend
@@ -38,47 +25,21 @@ covers are stored, so the library renders in full with the browser switched off.
| Var | Default | Notes |
|-----|---------|-------|
| `TOKEN_KEY` | *(required)* | Secret every Reader's userscript credential is derived from (issue #24); only SHA-256 hashes of credentials are stored. |
| `OWNER_DISCORD_ID` | *(required)* | Discord user ID of the owner: seeded as the first Reader, owns every pre-registration bookmark, and is the only Reader who can revoke another's sessions. |
| `API_TOKEN` | *(required)* | Bearer token shared with the userscript. |
| `ALLOWED_ORIGINS` | Asura + Demonic + Comix + Kagane origins | Comma-separated CORS allowlist. |
| `DATABASE_URL` | *(required)* | Postgres connection URL, e.g. `postgres://bookmarks:…@postgres:5432/bookmarks?sslmode=disable`. Compose builds it from `POSTGRES_PASSWORD`. |
| `COVER_DIR` | *(required)* | Filesystem volume for immutable, content-addressed Cover bytes. Compose builds the image and mounts `cover-data` at this path; standalone runs may choose another writable durable path. |
| `DB_PATH` | `/data/bookmarks.db` | SQLite file location. |
| `PORT` | `8080` | Plain HTTP; TLS terminated by the proxy. |
| `BROWSER_WS_URL` | empty | CDP endpoint of the remote browser (`ws://<tailnet IP>:9222`), used to poll Kagane/Novelfull past their JS challenge and to fetch uncached Kagane covers. Must be an IP or `localhost` — Chrome's DevTools handler 500s any other Host header, MagicDNS names included. Unset disables both; stored covers still serve. |
| `DISCORD_CLIENT_ID` | *(required)* | Discord application credentials for the browser sign-in (ADR-0002). |
| `DISCORD_CLIENT_SECRET` | *(required)* | As above. Never logged, never echoed in an error. |
| `DISCORD_GUILD_ID` | *(required)* | The one guild whose membership gates sign-in, checked at login only. Membership *is* registration: any member becomes a Reader on first login. |
| `DISCORD_REDIRECT_URI` | *(required)* | Exact callback URL; Discord matches it verbatim against the registered redirect. |
| `DISCORD_REQUIRED_ROLE` | empty | Role snowflake a member must additionally hold. Empty means guild membership alone suffices. |
| `DISCORD_API_BASE` | `https://discord.com/api/v10` | Test seam — tests point it at a local stub so the real token exchange runs. |
| `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. |
| `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. |
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. |
| `LATEST_CHAPTER_POLL_COOLDOWN` | `1h` | Rest between checks of one plain-TLS series; floor `15m`. |
| `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN` | `6h` | Rest between checks of one browser-backed series; floor `15m`. |
| `LATEST_CHAPTER_POLL_INTERVAL` | `10m` | How often the poller wakes. Cannot shorten either cooldown. |
| `LATEST_CHAPTER_POLL_BATCH` | `14` | Series per wake. Keep `BATCH × STAGGER` under `INTERVAL`. |
| `LATEST_CHAPTER_POLL_STAGGER` | `20s` | Delay between fetches in a batch — this is the outbound request rate. |
Compose reads a few more from the same `.env` that the backend never sees:
`POSTGRES_PASSWORD` (required — `DATABASE_URL` is built from it, and Postgres
only applies it while `postgres-data` is empty), `BOOKMARK_API_HOST` and
`BOOKMARK_WEB_HOST` (required by the prod override), and the optional
`PROXY_NETWORK` / `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER`. The browser
unit has its own `chrome/.env` on its own machine — `BROWSER_BIND_ADDR`
(required, the tailnet IP the CDP port is published on) and the optional
`BROWSER_TZ`. Full commentary is in `.env.example` and `chrome/.env.example`;
deployment order is `DEPLOY.md`.
| `BROWSER_WS_URL` | `ws://172.28.0.10:9222` | Headless-shell CDP endpoint used to poll Kagane past its JS challenge. Must be an IP or `localhost` — Chrome's DevTools handler 500s any other Host header. |
### Endpoints
| Method | Path | Auth | Description |
|--------|------|------|-------------|
| `GET` | `/bookmarks` | Bearer | All bookmarks of the acting Reader. |
| `GET` | `/bookmarks` | Bearer | All bookmarks (single-user). |
| `PUT` | `/bookmarks/{key}` | Bearer | Upsert one series; returns the row as stored. |
| `DELETE` | `/bookmarks/{key}` | Bearer | Remove one. |
| `GET` | `/healthz` | none | `200 ok`. |
| `GET` | `/u/{token}/manga-bookmark.user.js` | credential in path | Serves the userscript with the requesting Reader's credential substituted in and an mtime-derived `@version`. |
| `GET` | `/u/{token}/manga-bookmark.user.js` | token in path | Serves the userscript with an mtime-derived `@version`. |
`key` is `<site>:<series_id>` — e.g. `asura:trash-of-the-counts-family-f886a8af`,
`demonic:Infinite-Level-Up-in-Murim`, `comix:12345`, or
@@ -99,45 +60,19 @@ go test ./... # unit + handler tests
CGO_ENABLED=0 go build # static binary
```
**`go test ./...` requires Docker.** The store talks to a real Postgres, so
each test package starts a throwaway `postgres:17-alpine` container and gives
every test its own database inside it (`internal/pgtest`). Nothing is stubbed
and nothing reaches the network beyond the local Docker daemon.
### Run the stack
```bash
cp .env.example .env
# edit .env: set TOKEN_KEY (openssl rand -hex 32) and
# POSTGRES_PASSWORD (openssl rand -hex 24)
# edit .env: set API_TOKEN (openssl rand -hex 32)
docker compose up -d --build # binds 127.0.0.1:8080
```
That brings up two services — the API and Postgres. The browser is deliberately
not one of them; without `BROWSER_WS_URL` the poller logs and skips kagane and
novelfull, and everything else works. To run one locally, publish it on the
Docker bridge gateway so the API container can name it by IP:
```bash
cd chrome
echo 'BROWSER_BIND_ADDR=172.17.0.1' > .env
docker compose up -d --build
# then in the repo's own .env: BROWSER_WS_URL=ws://172.17.0.1:9222
```
Bind it to `127.0.0.1` instead if you only want to drive it from the host, e.g.
`SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeKagane ./internal/latest`.
In production that address is the home machine's tailnet IP and nothing else —
see `DEPLOY.md` §7 and ADR-0006.
Smoke test:
```bash
# The credential is per Reader and derived, so there is no token in .env to
# grep. Take yours from the Userscripts panel's install link after signing in,
# or read it out of an installed script's API_TOKEN constant.
TOKEN=<your Reader credential>
TOKEN=$(grep '^API_TOKEN=' .env | cut -d= -f2)
curl -s localhost:8080/healthz # ok
curl -s localhost:8080/bookmarks # 401
curl -s -H "Authorization: Bearer $TOKEN" localhost:8080/bookmarks # []
@@ -172,18 +107,17 @@ CORS headers.
## 2. Userscript
### Install
### Configure
Sign in to the web UI and open the **Userscripts** panel: it offers one
install link per library. Each link serves a script rendered with your own
credential already inside it — you never see, type or copy a credential. The
served script carries `@downloadURL`/`@updateURL` pointing at its
credential-bearing path, so Violentmonkey keeps auto-updating it.
Edit the config block at the top of `userscript/manga-bookmark.user.js`:
The bindmounted files carry `__API_TOKEN__` placeholders; the backend
substitutes the requesting Reader's credential at serve time, so no real
credential is ever committed. Rotating the credential (same panel) invalidates
every installed copy immediately — reinstall on all devices.
```js
const API_BASE = "https://bookmark-api.<domain>"; // no trailing slash
const API_TOKEN = "<same token as backend>";
```
The token lives in the userscript's **isolated world** — the manga sites' own
JS cannot read it.
### Install on Bromite (mobile)
@@ -191,8 +125,8 @@ Bromite runs Chromium's native userscript engine (no Tampermonkey needed):
1. Bromite → **Settings → User scripts** → enable user scripts (allow the
permission prompt).
2. Open the install link from the web UI — Bromite detects the `.user.js` and
offers to install it.
2. Save the configured `manga-bookmark.user.js` to the device (or open its raw
URL). Bromite detects the `.user.js` and offers to install it.
3. Confirm the install; the `@match` list covers both sites.
4. Open a series on either site — a 📑 button appears bottom-right.
@@ -261,10 +195,8 @@ an API.
## Adapter reference (verified live 2026-07-24)
The site adapters key everything off URL regex, with `title` from `og:title`
(or the page's own heading where a site ships none). No adapter reads a cover:
the backend acquires, stores and serves every Cover from its own origin
(ADR-0007). Confirmed against live pages via Playwright:
The site adapters key everything off URL regex, with `title`/`cover` from
`og:title` / `og:image`. Confirmed against live pages via Playwright:
| Site | Series URL | Chapter URL | `series_id` |
|------|-----------|-------------|-------------|
+92 -218
View File
@@ -9,16 +9,16 @@ Whole thing is ~5 minutes, most of it waiting on `docker build`. Order matters:
**back up before you pull.** A backup taken after a bad migration is a backup of
the damage.
Paths below assume the checkout is at `~/mangaBookmark`, which is where it lives
on this deployment; substitute your own. The one absolute rule about paths:
**backups live in a `-backups` sibling of the checkout**, never inside it. It
Paths below assume the checkout is at `/opt/bookmarkmanager`; substitute your own. The
one absolute rule about paths: **backups live in `../bookmarkmanager-backups/`**, a
sibling of the project directory (`/opt/bookmarkmanager-backups`), never inside it. It
sits outside the repo so `git pull`, `git clean -fd` and a bad `rm -rf` inside
the checkout cannot take the backups with them.
```
~/
├── mangaBookmark/ <- the checkout (this repo)
└── mangaBookmark-backups/ <- bookmarks-YYYYmmdd-HHMMSS.dump
/opt/
├── bookmarkmanager/ <- the checkout (this repo)
└── bookmarkmanager-backups/ <- bookmarks-YYYYmmdd-HHMMSS.db
```
---
@@ -26,7 +26,7 @@ the checkout cannot take the backups with them.
## 0. Preflight
```bash
cd ~/mangaBookmark
cd /opt/bookmarkmanager
# Both -f flags, every time. The prod override is not standalone.
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
@@ -44,117 +44,87 @@ dirty tree fails halfway and leaves you in a worse spot than either.
Create the backup directory once, and make sure it is a sibling, not a child:
```bash
BACKUP_DIR="$(cd .. && pwd)/$(basename "$PWD")-backups" # absolute — Docker needs it
mkdir -p "$BACKUP_DIR"
echo "$BACKUP_DIR" # -> /home/sulthan/mangaBookmark-backups
mkdir -p ../bookmarkmanager-backups
BACKUP_DIR="$(cd .. && pwd)/bookmarkmanager-backups" # absolute — Docker needs it
echo "$BACKUP_DIR" # -> /opt/bookmarkmanager-backups
```
---
## 1. Back up the database
The database is Postgres, running as the `postgres` service on the named volume
`postgres-data`. It has **no published port** — nothing outside the internal `db`
network can reach it — so every command below goes in through the container:
The database is a single SQLite file in the named Docker volume, at
`/data/bookmarks.db` inside the container. Find the volume's real name — Compose
prefixes it with the project directory:
```bash
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c '\dt'
# -> bookmarks, covers, readers, schema_migrations, series, sessions
docker volume ls --filter name=bookmarks-data
# -> local bookmarkmanager_bookmarks-data
VOL=$(docker volume ls --filter name=bookmarks-data -q | head -1)
```
The `covers` table is metadata only after the filesystem cutover: bytes live in
the separate `cover-data` volume. Back that volume up with the database dump;
restoring only Postgres leaves stored Cover addresses without files.
### Preferred: hot backup, no downtime
Inside the container that connects over the local socket as the `bookmarks`
superuser, so no password is needed anywhere in this section. `-T` is not
optional: without it Compose allocates a TTY, which rewrites `\n` to `\r\n` and
silently corrupts any binary stream flowing back out — see the dump below.
### Preferred: hot dump, no downtime
`pg_dump` runs in a single repeatable-read transaction, so it writes one
point-in-time-consistent snapshot while the API keeps serving. No stopping, no
WAL to worry about — that is the server's problem, not yours.
The store runs in **WAL mode**, so recent writes may still be sitting in
`bookmarks.db-wal`. Copying `bookmarks.db` alone while the container runs can
therefore silently drop the newest bookmarks. `VACUUM INTO` folds the WAL in and
writes one consistent file, safe to run against a live database:
```bash
STAMP=$(date -u +%Y%m%d-%H%M%S) # UTC, sorts chronologically as text
$COMPOSE exec -T postgres pg_dump -U bookmarks -d bookmarks -Fc \
> "$BACKUP_DIR/bookmarks-$STAMP.dump"
docker run --rm \
-v "$VOL":/data \
-v "$BACKUP_DIR":/backup \
alpine sh -c "apk add -q sqlite &&
sqlite3 /data/bookmarks.db \"VACUUM INTO '/backup/bookmarks-$STAMP.db'\""
ls -lh "$BACKUP_DIR"/bookmarks-$STAMP.dump
ls -lh "$BACKUP_DIR"/bookmarks-$STAMP.db
```
`-Fc` is the custom archive format rather than plain SQL: it is compressed, and
`pg_restore` can inspect and replay it selectively — list its table of contents,
restore one table, restore schema without data, reorder. A plain `.sql` dump can
only be piped into `psql` whole, and gives you no way to check what is in it
short of reading it.
`$STAMP` is the "time in the name" — `bookmarks-20260730-014233.db`. UTC, so the
files sort in real order and never collide across a DST shift.
`$STAMP` is the "time in the name" — `bookmarks-20260730-014233.dump`. UTC, so
the files sort in real order and never collide across a DST shift.
Note the source volume is mounted **read-write**, which looks wrong for a backup
and is not. Opening a WAL database requires creating the `-shm` shared-memory
file; with `:ro` the command fails with `unable to open database file` and no
backup is produced. `VACUUM INTO` never writes to the source itself.
Verify it before you trust it. An unreadable backup is worse than none, because
you will act as though you have one:
```bash
# 1. The dump parses and contains the tables. Uses the same image compose
# already pulls, so nothing new to install.
docker run --rm -v "$BACKUP_DIR":/backup postgres:17-alpine \
pg_restore --list "/backup/bookmarks-$STAMP.dump" | grep 'TABLE DATA'
# -> 1234; 0 0 TABLE DATA public bookmarks bookmarks
# -> 1235; 0 0 TABLE DATA public covers bookmarks
# -> 1236; 0 0 TABLE DATA public readers bookmarks
# -> 1237; 0 0 TABLE DATA public schema_migrations bookmarks
# -> 1238; 0 0 TABLE DATA public series bookmarks
# -> 1239; 0 0 TABLE DATA public sessions bookmarks
# 2. Sanity-check the live row count you just captured.
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks \
-c 'select count(*) from bookmarks'
docker run --rm -v "$BACKUP_DIR":/backup alpine sh -c "apk add -q sqlite &&
sqlite3 /backup/bookmarks-$STAMP.db 'PRAGMA integrity_check;' &&
sqlite3 /backup/bookmarks-$STAMP.db 'SELECT count(*) FROM bookmarks;'"
# -> ok
# -> 37
```
A custom-format archive stores row counts nowhere, so step 1 proves the file is
a readable archive with the right tables in it, not that the rows are there;
step 2 is the number those rows should be. It should match what the web UI
shows. Zero on a server you know has bookmarks means the API and your `psql`
are looking at different databases — check `DATABASE_URL`.
The count should match what the web UI shows. Zero rows on a server you know has
bookmarks means you backed up the wrong volume.
### Fallback: cold volume archive
### Fallback: cold copy (no network for `apk add sqlite`)
Use this when you want the whole data directory rather than a logical dump — a
like-for-like restore of the same Postgres major version onto the same host.
**The stack must be stopped first.** A running Postgres has dirty pages in
shared buffers and WAL that has not been replayed into the data files, and `tar`
walks the directory over several seconds while the server keeps writing to it.
The archive you get is torn: files from different instants, possibly a
half-written page. It may restore, start, and be quietly wrong. Online
filesystem-level backup is `pg_basebackup`'s job, not `tar`'s; with the
container stopped the shutdown checkpoint has already flushed everything and a
plain archive of the volume is consistent.
Stop the service first, then copy the database **and its sidecars** — the `-wal`
is not optional, it is where the newest writes are:
```bash
# Derived exactly, not with a `--filter name=` substring match plus `head -1`:
# that quietly picks the first of however many volumes happen to contain the
# string, and archiving the wrong data directory is not a visible failure.
VOL="$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_postgres-data"
docker volume inspect "$VOL" >/dev/null && echo "$VOL" # -> mangabookmark_postgres-data
$COMPOSE stop
docker run --rm -v "$VOL":/from:ro -v "$BACKUP_DIR":/to alpine \
tar czf "/to/postgres-data-$STAMP.tgz" -C /from .
docker run --rm -v "$VOL":/data:ro -v "$BACKUP_DIR":/backup alpine sh -c "
cp /data/bookmarks.db /backup/bookmarks-$STAMP.db
[ -f /data/bookmarks.db-wal ] && cp /data/bookmarks.db-wal /backup/bookmarks-$STAMP.db-wal
[ -f /data/bookmarks.db-shm ] && cp /data/bookmarks.db-shm /backup/bookmarks-$STAMP.db-shm
ls -1 /backup"
$COMPOSE start
ls -lh "$BACKUP_DIR"/postgres-data-$STAMP.tgz
```
Costs ~15 seconds of downtime. Read-only on the source is safe here precisely
because nothing is running against it. Restoring this variant means untarring it
back into an *empty* `postgres-data` volume with the stack down — it is a whole
data directory, not a file you can drop next to the live one, and it will only
start under `postgres:17`.
Costs ~10 seconds of downtime. A clean shutdown usually checkpoints the WAL away,
so seeing only the `.db` file is normal and fine — the `[ -f ]` guards exist for
the case where it did not. Restoring this variant means putting whichever files
you got back together, under their original names.
Read-only is safe here precisely because nothing opens the database: it is a file
copy, not a SQLite connection.
### Retention
@@ -162,21 +132,7 @@ Keep a month, drop the rest — a bookmark database this small compresses the
decision to "disk is free, but not infinite":
```bash
ls -1t "$BACKUP_DIR"/bookmarks-*.dump | tail -n +31 | xargs -r rm -v
```
### A note on the old `bookmarks-data` volume
`bookmarks-data` is the **pre-migration SQLite volume**. It is deliberately not
declared in `docker-compose.yml` any more, which is what keeps `docker compose
down -v` from taking it with the rest of the stack. It is not the live database
and nothing reads it — the one-way move out of it is `CUTOVER.md`. Once the
Postgres data has been trusted for a while, remove it by hand — nothing else will.
Its full name is `<compose project>_bookmarks-data`, and the project name is the
lowercased directory name of the checkout:
```bash
docker volume rm "$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_bookmarks-data"
ls -1t "$BACKUP_DIR"/bookmarks-*.db | tail -n +31 | xargs -r rm -v
```
---
@@ -215,15 +171,11 @@ rebuilt. The one exception is `userscript/manga-bookmark.user.js`, which is
bindmounted read-only and read fresh per request.
```bash
$COMPOSE ps # bookmark-api Up; postgres Up (healthy)
$COMPOSE ps # Up, and recently (re)created
docker logs bookmark-api --tail 20 # -> "listening on :8080 ..."
```
Nothing in the log about the database, the migrations or the poller failing.
`bookmark-api` waits on `postgres` reporting healthy before it starts and the
binary applies any pending migration before it listens, so an API that never
says "listening" is usually the database, not the code — `$COMPOSE logs
postgres` first. The image is tagged
Nothing in the log about the database or the poller failing. The image is tagged
`bookmarkmanager-backend:latest`, so the previous image is still on disk untagged —
that is what makes the rollback in §6 quick.
@@ -236,10 +188,7 @@ Same four API checks as `DEPLOY.md` §3, plus the web UI. Set the host names onc
```bash
API=https://bookmark-api.violetcrown.my.id
WEB=https://bookmark.violetcrown.my.id
# Your own Reader credential - derived, never stored in .env. Take it from the
# Userscripts panel's install link after signing in, or from an installed
# script's API_TOKEN constant.
TOKEN=<your Reader credential>
TOKEN=$(grep -E '^API_TOKEN=' .env | cut -d= -f2)
curl -s $API/healthz # -> ok
curl -s -o /dev/null -w '%{http_code}\n' $API/bookmarks # -> 401
@@ -250,16 +199,9 @@ curl -s -i -X OPTIONS -H 'Origin: https://asurascans.com' \
$API/bookmarks/x | grep -i access-control # -> allow-origin echoed
```
`[]` from the third call is the alarm that matters: you are talking to an empty
database, which means the API found a *different* Postgres than the one holding
your data — a renamed project directory, a fresh `postgres-data`, or a
`DATABASE_URL` override in `.env` pointing elsewhere. Stop and check, before
touching anything else:
```bash
$COMPOSE config --volumes # -> postgres-data
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c 'select count(*) from bookmarks'
```
`[]` from the third call is the alarm that matters: the volume is not attached
and you are looking at an empty database. Stop and check `$COMPOSE config
--volumes` before touching anything else.
Web UI and its assets:
@@ -325,41 +267,33 @@ git checkout <previous-hash>
$COMPOSE up -d --build
```
**Database damaged** — restore the dump from §1. Stop **only the API**, not the
whole stack: `pg_restore` needs the server up to restore into, and it needs
`bookmark-api`'s connection pool gone, because `--clean` cannot drop a table
other sessions are holding open.
**Database damaged** — restore the backup from §1. Stop first: the running
process holds the WAL, and dropping a file under a live SQLite connection
corrupts what you were trying to save.
```bash
$COMPOSE stop bookmark-api
$COMPOSE stop
$COMPOSE exec -T postgres pg_restore -U bookmarks -d bookmarks --clean --if-exists \
< "$BACKUP_DIR/bookmarks-<STAMP>.dump"
docker run --rm -v "$VOL":/data -v "$BACKUP_DIR":/backup alpine sh -c '
rm -f /data/bookmarks.db /data/bookmarks.db-wal /data/bookmarks.db-shm &&
cp /backup/bookmarks-<STAMP>.db /data/bookmarks.db &&
chown 65532:65532 /data/bookmarks.db &&
ls -l /data'
$COMPOSE start bookmark-api
$COMPOSE start
docker logs bookmark-api --tail 20
curl -s -H "Authorization: Bearer $TOKEN" $API/bookmarks | head -c 200
```
Three things here are easy to skip and all three bite:
Two steps here are easy to skip and both bite:
- **`--clean --if-exists`.** Without `--clean` the dump's rows land *on top of*
what is already there and you get primary-key collisions half way through, a
partially restored database, and a non-zero exit you may not notice.
`--if-exists` only suppresses the "does not exist" noise when the target is
already empty; it is not the part doing the work.
- **`-T` again.** Feeding a custom-format archive into a TTY-allocated `exec`
corrupts it in flight and `pg_restore` fails with a garbled-header error on a
file that is perfectly fine on disk.
- **Stop the API, not Postgres.** `$COMPOSE stop` (everything) leaves you with
nothing to restore into; leaving `bookmark-api` running leaves connections
that block the drops *and* lets the poller write into a half-restored table.
No ownership fixing is needed any more — the Postgres image owns `postgres-data`
itself and `pg_restore` writes through the server, not the filesystem.
`schema_migrations` is inside the dump, so the database comes back at whatever
schema version the backup was taken at; the migration runner applies anything
newer the next time `bookmark-api` starts.
- **Delete the stale `-wal` and `-shm`.** Leaving them beside a restored database
mixes two different histories; SQLite will either refuse to open it or quietly
reapply writes you meant to discard.
- **`chown 65532:65532`.** The image is `distroless/static:nonroot` and runs as
that uid, while the helper container above writes as root. A root-owned
database opens read-only-ish: reads work, so `/bookmarks` looks fine, and then
every write fails. That is the worst possible failure mode — it looks restored.
---
@@ -371,73 +305,21 @@ For a routine redeploy where nothing needs deciding:
cd /opt/bookmarkmanager
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
BACKUP_DIR="$(cd .. && pwd)/bookmarkmanager-backups"; mkdir -p "$BACKUP_DIR"
VOL=$(docker volume ls --filter name=bookmarks-data -q | head -1)
STAMP=$(date -u +%Y%m%d-%H%M%S)
$COMPOSE exec -T postgres pg_dump -U bookmarks -d bookmarks -Fc \
> "$BACKUP_DIR/bookmarks-$STAMP.dump" &&
docker run --rm -v "$BACKUP_DIR":/backup postgres:17-alpine \
pg_restore --list "/backup/bookmarks-$STAMP.dump" > /dev/null &&
docker run --rm -v "$VOL":/data -v "$BACKUP_DIR":/backup alpine sh -c \
"apk add -q sqlite && sqlite3 /data/bookmarks.db \"VACUUM INTO '/backup/bookmarks-$STAMP.db'\" &&
sqlite3 /backup/bookmarks-$STAMP.db 'PRAGMA integrity_check;'" &&
git pull --ff-only &&
$COMPOSE up -d --build &&
sleep 5 &&
curl -sf https://bookmark-api.violetcrown.my.id/healthz && echo " deploy ok"
```
The `&&` chain is deliberate: if the dump or its `pg_restore --list` check
fails, nothing is pulled and nothing is rebuilt. A failed dump still leaves a
short or empty `.dump` behind — the shell creates the file before `pg_dump`
runs — so delete it rather than letting it sit in the backup directory looking
like a backup. Then still do §5 by hand — no shell command can tell you the
panel works on the phone.
---
## 8. The browser unit (separate machine, separate cadence)
Everything above is the API stack on the VPS. The headless browser is its own
compose unit on the home machine (ADR-0006, `DEPLOY.md` §7) and is redeployed
on its own schedule — it holds no data you can lose, so there is nothing to
back up and no ordering constraint against the API.
```bash
cd ~/mangaBookmark/chrome
git pull --ff-only
docker compose up -d --build
```
Then confirm it answers, and that a stopped-and-restarted Chrome is invisible
to the API:
```bash
curl -s -m 15 http://$(tailscale ip -4):9222/json/version | head -c 120
# -> {"Browser":"Chrome/1xx...","webSocketDebuggerUrl":"ws://...<new uuid>"}
```
The first call takes a few seconds: Chrome is not running until something
connects, and it is reaped again after five idle minutes. The debugger UUID
changes on every start and the API does not care — chromedp re-runs
`/json/version` discovery per fetch, which is exactly why `chromedp.NoModifyURL`
must never be added to `browser.go`.
**Rebuild is the Chrome upgrade path.** The image installs
`google-chrome-stable` unpinned on purpose: a stale browser is what Cloudflare
turns away, and the pinned Chrome 124 in `zenika/alpine-chrome` is the worked
example. The `chrome-profile` volume survives `--build`, so clearance cookies
are reused rather than re-solved.
Two things worth a glance after several days, both from the acceptance criteria
of the move:
```bash
docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'
# -> 0 false
free -m # the Gitea runner should still have its headroom
```
Nothing here needs doing during an API redeploy. The API stack does not
`depends_on` the browser, and an unreachable one degrades exactly as an unset
`BROWSER_WS_URL`: plain-TLS libraries unaffected, kagane and novelfull logged
and skipped, stored covers still served.
The `&&` chain is deliberate: if the backup or its integrity check fails,
nothing is pulled and nothing is rebuilt. Then still do §5 by hand — no shell
command can tell you the panel works on the phone.
---
@@ -445,24 +327,16 @@ and skipped, stored covers still served.
| Symptom | Cause / fix |
|---|---|
| `/bookmarks` returns `[]` after redeploy | You are on an empty Postgres. Check `$COMPOSE config --volumes` lists `postgres-data`, that you passed both `-f` files, and that `.env` has no stray `DATABASE_URL` override. Do **not** re-bookmark; the data is still in the volume. |
| `/bookmarks` returns `[]` after redeploy | Volume not attached — check `$COMPOSE config --volumes` and that you passed both `-f` files. Do **not** re-bookmark; the data is still in the volume. |
| UI looks like plain Georgia / system sans | `static/fonts/` missing from the image, or the browser cached an old `style.css`. `/static/*` is served `max-age=3600`, so hard-reload or wait an hour. |
| CSS or template change did not appear | You restarted without `--build`. Assets are `//go:embed`ed. |
| Font answers `application/octet-stream` | Old binary — the `.woff2` MIME registration is in `web.go`. Rebuild. |
| Everyone logged out of the web UI | The `sessions` table was wiped; sessions are database rows, not signed cookies. Expected after a deliberate revoke. |
| Everyone logged out of the web UI | `API_TOKEN` or `WEB_PASSWORD` changed; sessions are derived from both. Expected, just log in again. |
| `compose` errors about `BOOKMARK_WEB_HOST` | Run from the directory holding `.env`. Both host vars are required even when the web UI is unused. |
| Userscript did not update on the phone | Violentmonkey polls on its own schedule; force a check. `@version` comes from the file's mtime, so confirm the pull actually touched it. |
| `bookmark-api` crash-loops, log says `password authentication failed for user "bookmarks"` | `POSTGRES_PASSWORD` in `.env` no longer matches the one burned into `postgres-data` at first init — Postgres reads that variable only when initialising an empty volume. Put the old value back, or reset the role: `$COMPOSE exec postgres psql -U bookmarks -d bookmarks -c '\password bookmarks'` (prompts, so nothing lands in shell history) and then match `.env` to it. |
| `compose` errors `set POSTGRES_PASSWORD in .env` | Unset. Compose builds the backend's `DATABASE_URL` out of it, so it is required even though you never write that URL yourself. Run from the directory holding `.env`. |
| `postgres` never leaves `starting`; `bookmark-api` never starts either | The healthcheck (`pg_isready`) is failing and `bookmark-api` waits on it. `$COMPOSE logs postgres` — usually `postgres-data` was initialised by a different major version ("database files are incompatible with server"), or the disk is full. |
| `pg_restore`: `cannot drop … other objects depend on it` / `being accessed by other users` | Live connections block `--clean`. `$COMPOSE stop bookmark-api` first (§6). If they persist: `$COMPOSE exec -T postgres psql -U bookmarks -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='bookmarks' and pid <> pg_backend_pid()"`. |
| Dump is 0 bytes, or `pg_restore`: `did not find magic string in file header` | You ran `exec` without `-T`. The allocated TTY rewrites newlines in the binary stream and corrupts the archive in flight (§1). |
| `git pull`: `could not read Username for 'https://…'` | The checkout's remote is the HTTPS clone URL and the server has no credential helper, so the pull prompts into a closed stdin. Switch it to SSH once — `git remote set-url origin ssh://git@gitea.violetcrown.my.id:2222/sulthan/mangaBookmark.git`. Gitea's SSH listens on **2222**, not 22; port 22 is the host's own sshd and answers `Permission denied (publickey)` no matter which key is registered. |
| kagane rows stopped updating after a redeploy | Check `BROWSER_WS_URL` survived the `.env` edit and still names the home machine's tailnet **IP**. A hostname 500s at `/json/version`; an empty value disables the browser silently. Plain-TLS sites keep working either way, which is why this is easy to miss. |
| kagane covers went blank in the web UI | Covers use the `cover-data` volume now. Restore/check that volume alongside Postgres; rows in `covers` are metadata only. If the database has rows but files are missing, the next browser-backed request refetches them; without a browser it remains a 404. |
| Browser unit will not start: `set BROWSER_BIND_ADDR to this machine's tailnet IP` | `chrome/.env` is missing or the variable is empty. It has no default on purpose — an unset value must fail the deploy rather than publish an unauthenticated CDP port to the LAN. |
| `bookmark-browser` shows `OOMKilled true` | The cap did its job. Read `docker logs bookmark-browser` before raising it — the sizing and what the cap protects are in ADR-0006. |
| `apk add sqlite` fails (no network) | Use the cold-copy fallback in §1 — and copy `bookmarks.db-wal` too. |
| Reads work but every write fails after a restore | Restored file is root-owned; the container is uid 65532. `chown 65532:65532` it (§6). |
| Backup command: `unable to open database file` | Source volume mounted `:ro`. WAL needs to create `-shm`; mount it read-write (§1). |
Full first-time setup: `DEPLOY.md`. The one-off SQLite→Postgres move:
`CUTOVER.md`. Config reference and endpoints: `README.md`.
Full first-time setup: `DEPLOY.md`. Config reference and endpoints: `README.md`.
UI conventions: `docs/design-system.md`.
+29 -133
View File
@@ -1,8 +1,8 @@
Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGENTS.md` for the project-wide architecture diagram, hard constraints, and design system.
- **Backend** (`backend/`): stdlib `net/http` (handful routes, no framework) + Postgres over `jackc/pgx/v5` (pure Go, `CGO_ENABLED=0` -> static binary -> distroless/scratch image). Reverse proxy terminates TLS; Go service listens plain `:8080`.
- **Backend** (`backend/`): stdlib `net/http` (handful routes, no framework) + `modernc.org/sqlite` (pure Go, `CGO_ENABLED=0` -> static binary -> distroless/scratch image). Reverse proxy terminates TLS; Go service listens plain `:8080`.
Single binary, split into packages under `backend/internal/`: `store`
(Bookmark type, Postgres persistence, migration runner), `latest` (background
(Bookmark type, SQLite persistence, migrations), `latest` (background
poller, site parsers, TLS fetcher), `session` (cookie signing, login
rate limiter), `httpmw` (Auth/Gzip/CORS middleware), `api` (JSON
bookmark handlers), `userscript` (userscript-serving handler), `web`
@@ -11,52 +11,16 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
packages together into `newRouter`. Root-level `*_test.go` hold
integration tests that exercise the full router; unit tests for a
package live beside it under `internal/`.
- **Schema is migration-owned.** `internal/store/migrations/*.sql` is
`go:embed`-ed and applied on every start by `store.migrate`: one numbered
file per change, one transaction each, versions recorded in
`schema_migrations`. Files are **append-only** — editing an applied one
changes nothing on a database that already ran it. No column probing, no
data-fixup migrations: both were SQLite-era machinery and are gone.
- **Tests need Docker.** `internal/pgtest` starts one `postgres:17-alpine`
container per test binary (`TestMain` -> `pgtest.Main`) and hands each test
its own database (`pgtest.URL(t)`). A package whose tests touch the store
must have that `TestMain`.
- **Reader-owned store, four tables.** `readers` is keyed by Discord user ID
and carries the SHA-256 of the Reader's userscript credential plus a
`token_epoch` (issue #24). Credentials are derived, never stored: `token.Token(TOKEN_KEY, discord_id, epoch)` (HMAC, `internal/token`), and only its SHA-256 sits in `readers.token_sha256`, so install URLs can be rebuilt after any restart while a database leak yields nothing but hashes. The seed creates the **owner** row at startup; its epoch-0 hash is refreshed on every start **only while the row has never been rotated**, so a restart can never resurrect a rotated-away credential. Every other row is created by that Reader's own first login (`Store.EnsureReader`, idempotent on `discord_id`, and it never rewrites an existing row's hash). Rotation is `Store.RotateToken` (epoch bump + hash rewrite in one transaction), driven by the web UI.
`series` keyed `(site, series_id)`
(`asura`|`demonic`|`comix`|`kagane`|`novelfull`|`lightnovelworld`) owns the
shared facts — title, cover, canonical URL, `kind` (`manga`|`novel`),
Latest Chapter, `latest_checked_at` — and `bookmarks` holds only what
differs between readers: progress, favourite, lifecycle bucket,
`updated_at`. A bookmark is keyed `(reader_id, site, series_id)` — no
surrogate id; the wire `key` is derived as `site:series_id` on read — and
every store read/write is scoped to the reader it names. Auth resolves the
acting Reader from the presented credential (`httpmw.Auth`) and nothing
else — there is no unauthenticated-by-Reader route and no global token; the
reader id travels in the request context. Sync **last-write-wins**; the wire format
stays flat (ADR-0004). `Store.Upsert` decomposes one flat body across two
tables and enforces the ownership rule: client `title`/`series_url`/`cover`
are written only when the series row is new (ADR-0003).
- **Single-user store.** One `bookmarks` table keyed `<site>:<series_id>` (`asura`|`demonic`|`comix`|`kagane`). Sync **last-write-wins**. Schema and endpoint list in plan.
- **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth).
- **Web UI:** same binary serve the browser UI on a second
- **Web UI:** same binary serve password-gated browser UI on second
hostname — `GET /` (list, or login page when no session),
`GET /auth/discord` + `GET /auth/discord/callback` (Discord OAuth,
ADR-0002), `POST /logout`, `GET /static/*`, htmx fragment endpoints
`POST /login`, `POST /logout`, `GET /static/*`, htmx fragment endpoints
under `/ui/*`. Templates + assets `go:embed`-ed under
`backend/internal/web/`, so `backend/Dockerfile` must copy the whole
`internal/` tree, not just `*.go`. Sessions are rows in the `sessions`
table: the cookie carries only an opaque id, looked up (and expiry-
checked) on every request, and deleting the row revokes the session.
Guild membership *is* registration (issue #27): `discordCallback` gates on
membership (and `DISCORD_REQUIRED_ROLE` when set) and then calls
`Store.EnsureReader`, so a refusal creates nothing and a returning Reader
reuses their row. The owner is the only Reader with administrative reach:
`POST /readers/{id}/revoke` (404 for anyone else) drops that Reader's
sessions, and the `readers` panel renders only on the owner's page.
A Reader with no bookmarks at all sees `listView.Fresh`, whose empty state
offers both install links instead of describing a filter.
UI mutations read-modify-write
`internal/` tree, not just `*.go`. Sessions stateless
HMAC cookies keyed off `API_TOKEN`; `WEB_PASSWORD` gates them, and when empty,
web routes not registered at all. UI mutations read-modify-write
through `Store.Get` + `Store.Upsert` so `updated_at` rule stays one
place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`.
**Design-tool caveat:** templates link `/static/style.css` root-absolutely
@@ -79,49 +43,22 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
network access, so `latest_chapter` stay fresh when user not
browsing. Second, parallel signal — userscript keep own
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged.
Two independent clocks: per-series cooldown (`series.latest_checked_at`,
Two independent clocks: per-bookmark cooldown (`latest_checked_at` column,
enforced by `Store.DueForLatestCheck`'s WHERE clause) and wake interval.
The poller walks **Series, not Bookmarks** — a series referenced by several
bookmarks is fetched once per cycle, and the due queue orders
`reader_count DESC, latest_checked_at ASC` (ADR-0003). Series row stamped
*before* fetch so broken series wait out full cooldown instead of retrying
every tick; found chapter written straight to the series row via
`Store.SetLatestChapter`, so a bookmark's `updated_at` — and the list
order — is never touched.
Row stamped *before* fetch so broken series wait out full
cooldown instead of retrying every tick, and writes go through
`Store.Get` + `Store.Upsert` so new chapter never reorders list.
Fetches use `bogdanfinn/tls-client` with Chrome profile as defence in depth
against fingerprint-based blocking; any failure log and skip. kagane and
novelfull sit behind Cloudflare JavaScript challenges the TLS client can't
clear, so they are fetched over CDP via `BROWSER_WS_URL`; kagane is simply
not polled when that's unset, while novelfull falls back to a plain-TLS
attempt — its challenge is a live time-varying fact, and its cover bytes
never need the browser. See
clear, so they are browser-only: fetched over CDP via `BROWSER_WS_URL`, and
simply not polled when that's unset. See
`docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`.
The poller's series write is a single-column UPDATE
(`Store.SetLatestChapter`), not a read-modify-write of the whole bookmark:
it cannot revert read progress or move `updated_at`, so the old
stale-re-read race is gone with the Get+Upsert flow.
- **Covers are acquired at creation, then served from our own origin
(ADR-0007):** the first Bookmark of a Series fires `Store.OnSeriesCreated`,
which `latest.Acquirer` turns into one series-page fetch yielding both the
Latest Chapter and the cover URL; the bytes then go through
`latest.CoverBytesFetcher` into `Store.SetSeriesCover`. It runs in a
goroutine — the Reader's PUT must neither block on a Site nor fail with one
— and every failure is logged and dropped, leaving the Bookmark intact. The
wire's `cover` is the absolute `PUBLIC_BASE_URL + /covers/{sha256}` once
bytes exist and `""` before, never an address that 404s. `GET /covers/{addr}`
is public and uncredentialed: the userscript renders it on a Site's origin,
where no cookie or token of ours travels. A client-sent `cover` is decoded
and discarded, permanently (ADR-0004 compatibility).
Browser-backed Sites join the same pipeline (issue #62): kagane pages *and*
cover bytes go through the browser sidecar (nothing falls back to a plain
fetch, which would only retrieve a challenge page), while novelfull needs
the browser only for its HTML — the cover URL comes out of the
browser-fetched page and the bytes go over plain TLS. With no browser
configured, kagane Covers are simply absent; novelfull still gets one — at
creation and on the poll — when its page body happens to answer a plain
request (the challenge is a live time-varying fact). The old kagane-only
serving path (`/img/kagane/{id}`, template rewrite, `CoverFetcher`) is gone
(issue #63): the one public route serves every Site.
Poller's `Store.Get` + `Store.Upsert` not wrapped in transaction, so
userscript `PUT` that commits between the two can get overwritten by
poller's stale re-read — reverting that read progress and, since stored
value now differs, moving `updated_at` and reordering list. Known,
accepted limitation for single-user deployment, not bug to fix.
- **`updated_at` drives list order, so moves only on real reading progress:** server apply its timestamp when row new or `last_chapter_num` changes, else keep stored value — favouriting series or recording newly published chapter must not reorder list. `PUT` therefore returns row **as stored**, clients must adopt that response rather than own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4.
- **Lifecycle buckets:** `status` on each bookmark is `reading` | `archived` |
`finished`, orthogonal to `favorite`. Archived and finished appear only in
@@ -133,54 +70,13 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
`excluded.*` is post-evaluation row and default applied there would
wipe bucket on every PUT from client that predates column. See
`docs/superpowers/specs/2026-07-27-status-buckets-design.md`.
- **Config via env:** `TOKEN_KEY` (derives every Reader's userscript credential;
required), `OWNER_DISCORD_ID` (seeds the owner Reader — the administrator and
the owner of every pre-registration bookmark; required),
`ALLOWED_ORIGINS` (comma list),
`DATABASE_URL` (Postgres connection URL, required — no default),
`COVER_DIR` (required filesystem volume for content-addressed Cover bytes),
`PUBLIC_BASE_URL` (required origin this deployment answers on, trailing
slash trimmed; every Cover URL on the wire is built from it, absolute
because the userscript renders on a Site's origin — ADR-0007),
`PORT` (default `8080`), `DISCORD_CLIENT_ID`/`_CLIENT_SECRET`/`_GUILD_ID`/
`_REDIRECT_URI` (required; Discord OAuth for the browser UI),
`DISCORD_REQUIRED_ROLE` (optional role gate, empty by default),
`DISCORD_API_BASE` (default `https://discord.com/api/v10`),
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/
`_BATCH`/`_STAGGER` (background latest-chapter poller; defaults on,
`1h` plain-TLS cooldown, `6h` browser cooldown, `10m`/`14`/`20s`; both
cooldowns have a `15m` floor).
`USERSCRIPT_PATH` and `NOVEL_USERSCRIPT_PATH` (files served at
`/u/{token}/manga-bookmark.user.js` and `/u/{token}/novel-bookmark.user.js`,
defaults `/userscript/manga-bookmark.user.js` and
`/userscript/novel-bookmark.user.js`, both supplied by bindmount; the
`__API_TOKEN__` placeholder inside them is substituted with the requesting
Reader's credential at serve time).
`BROWSER_WS_URL` (CDP endpoint of the browser, which runs on a **separate
machine** and is reached over the tailnet — ADR-0006, `chrome/docker-compose.yml`.
Used by the poller for kagane and novelfull page fetches and by the cover
pipeline for kagane's image bytes (the browser is the only route that clears
the challenge kagane serves its covers behind); unset — the default —
disables browser polling and leaves kagane Covers blank until stored bytes
exist. Must be a tailnet IP, never a hostname: Chrome's DevTools handler 500s
`/json/version` for any Host that isn't an IP or `localhost`).
- **No per-Site cover path (issue #63):** every Cover — all six Sites — is
served by the one public `GET /covers/{addr}` route from content-addressed
bytes. There is no proxy, no per-Site rewrite, no second place that decides
a Cover's renderable address: the wire `cover` is it. The only place a Site
name still appears in cover code is the extraction module (`latest`), where
kagane's image URLs are claimed by `browserOnlyCoverURL` — they answer a
plain fetch with a challenge and `cross-origin-resource-policy: same-origin`;
every other Site's CDN answers plain TLS. Templates render `.Cover` — the
wire value — never anything else.
- **Web UI also owns:** session-gated `GET /install/{manga,novel}-bookmark.user.js`
(renders the bindmounted script with the acting Reader's derived credential
substituted in — the credential never appears in page markup, the address
bar, or a redirect; `?download=1` adds `Content-Disposition: attachment` for
mobile Violentmonkey, which ignores a `.user.js` navigation) and
`POST /rotate-token` (atomic epoch bump + hash
rewrite; invalidates every installed copy, so the panel warns to reinstall
on all devices).
Owner-only `POST /readers/{id}/revoke` (drops one Reader's session rows and
re-renders the `readers` panel; 404 for any non-owner) is the only route that
reaches across Readers.
- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list), `DB_PATH`
(default `/data/bookmarks.db`), `PORT` (default `8080`), `WEB_PASSWORD`
(gates browser UI; unset disable it),
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER`
(background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`).
`USERSCRIPT_PATH` (file served at `/u/{token}/manga-bookmark.user.js`,
default `/userscript/manga-bookmark.user.js`, supplied by bindmount).
`BROWSER_WS_URL` (headless-shell CDP endpoint for kagane and novelfull;
unset disables browser polling and leaves those sites to the userscript
alone).
-1
View File
@@ -1 +0,0 @@
AGENTS.md
+81
View File
@@ -0,0 +1,81 @@
Guidance for Claude Code working under `backend/`. See root `CLAUDE.md` for the project-wide architecture diagram, hard constraints, and design system.
- **Backend** (`backend/`): stdlib `net/http` (handful routes, no framework) + `modernc.org/sqlite` (pure Go, `CGO_ENABLED=0` -> static binary -> distroless/scratch image). Reverse proxy terminates TLS; Go service listens plain `:8080`.
Single binary, split into packages under `backend/internal/`: `store`
(Bookmark type, SQLite persistence, migrations), `latest` (background
poller, site parsers, TLS fetcher), `session` (cookie signing, login
rate limiter), `httpmw` (Auth/Gzip/CORS middleware), `api` (JSON
bookmark handlers), `userscript` (userscript-serving handler), `web`
(browser UI handler + `templates/` + `static/`, `go:embed`-ed).
`backend/main.go` is the composition root — the only place that wires
packages together into `newRouter`. Root-level `*_test.go` hold
integration tests that exercise the full router; unit tests for a
package live beside it under `internal/`.
- **Single-user store.** One `bookmarks` table keyed `<site>:<series_id>` (`asura`|`demonic`|`comix`|`kagane`). Sync **last-write-wins**. Schema and endpoint list in plan.
- **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth).
- **Web UI:** same binary serve password-gated browser UI on second
hostname — `GET /` (list, or login page when no session),
`POST /login`, `POST /logout`, `GET /static/*`, htmx fragment endpoints
under `/ui/*`. Templates + assets `go:embed`-ed under
`backend/internal/web/`, so `backend/Dockerfile` must copy the whole
`internal/` tree, not just `*.go`. Sessions stateless
HMAC cookies keyed off `API_TOKEN`; `WEB_PASSWORD` gates them, and when empty,
web routes not registered at all. UI mutations read-modify-write
through `Store.Get` + `Store.Upsert` so `updated_at` rule stays one
place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`.
**Design-tool caveat:** templates link `/static/style.css` root-absolutely
(correct — served from `/`), but impeccable detector resolves
stylesheet href with `path.resolve(fileDir, href)`, drops directory
on leading `/` and silently skip file. Relative href don't help
either: template's directory isn't its served path. So
`detect.mjs backend/internal/web/templates` reports **false clean** —
always pass `backend/internal/web/static` too. One finding there,
`overused-font` on "Instrument Serif", deliberate identity choice, not debt.
- **Every action that moves series out of list is confirm-gated.**
Archive, finish, remove each open own `.confirm-row` disclosure
(`toggleConfirmRow(key, kind)` in `filter.js`, `kind` ∈
`archive|finish|remove`); restore fire instantly since it's the reversal.
Remove's row wear ember wash, two reversible ones wear `.calm` grey.
`--ember` stay reserved for new-chapter signal: busy bar and inline
error use `--mute`.
- **Latest-chapter poller:** ticker goroutine in same binary re-check
each bookmarked series' newest published chapter from backend's own
network access, so `latest_chapter` stay fresh when user not
browsing. Second, parallel signal — userscript keep own
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged.
Two independent clocks: per-bookmark cooldown (`latest_checked_at` column,
enforced by `Store.DueForLatestCheck`'s WHERE clause) and wake interval.
Row stamped *before* fetch so broken series wait out full
cooldown instead of retrying every tick, and writes go through
`Store.Get` + `Store.Upsert` so new chapter never reorders list.
Fetches use `bogdanfinn/tls-client` with Chrome profile as defence in depth
against fingerprint-based blocking; any failure log and skip. kagane sits
behind a Cloudflare JavaScript challenge the TLS client can't clear, so it is
browser-only: fetched over CDP via `BROWSER_WS_URL`, and simply not polled
when that's unset. See
`docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`.
Poller's `Store.Get` + `Store.Upsert` not wrapped in transaction, so
userscript `PUT` that commits between the two can get overwritten by
poller's stale re-read — reverting that read progress and, since stored
value now differs, moving `updated_at` and reordering list. Known,
accepted limitation for single-user deployment, not bug to fix.
- **`updated_at` drives list order, so moves only on real reading progress:** server apply its timestamp when row new or `last_chapter_num` changes, else keep stored value — favouriting series or recording newly published chapter must not reorder list. `PUT` therefore returns row **as stored**, clients must adopt that response rather than own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4.
- **Lifecycle buckets:** `status` on each bookmark is `reading` | `archived` |
`finished`, orthogonal to `favorite`. Archived and finished appear only in
own tab — not in All, Updated, Favourites, or recent strip. Poller keeps
checking archived series and skip finished ones. `finished` settable
only from web UI; `PUT /bookmarks/{key}` reject it with 400.
**Empty incoming status means "keep stored one"** — resolved on the
`VALUES` side of `Store.Upsert`, not conflict clause, since
`excluded.*` is post-evaluation row and default applied there would
wipe bucket on every PUT from client that predates column. See
`docs/superpowers/specs/2026-07-27-status-buckets-design.md`.
- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list), `DB_PATH`
(default `/data/bookmarks.db`), `PORT` (default `8080`), `WEB_PASSWORD`
(gates browser UI; unset disable it),
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER`
(background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`).
`USERSCRIPT_PATH` (file served at `/u/{token}/manga-bookmark.user.js`,
default `/userscript/manga-bookmark.user.js`, supplied by bindmount).
`BROWSER_WS_URL` (headless-shell CDP endpoint for kagane; unset disables
browser polling and leaves that site to the userscript alone).
+7 -8
View File
@@ -2,7 +2,6 @@
# --- build stage: compile a static, CGO-free binary ---
FROM golang:1.26-alpine AS build
ARG COVER_DIR=/covers
WORKDIR /src
# Dependencies first for layer caching (changes rarely).
@@ -15,21 +14,21 @@ RUN go mod download
COPY *.go ./
COPY internal/ ./internal/
# Static binary: the Postgres driver (jackc/pgx) is pure Go, so CGO_ENABLED=0
# leaves no libc dependency.
# Static binary: pure-Go sqlite means CGO_ENABLED=0 -> no libc dependency.
# -trimpath + -ldflags strip paths and debug info for a smaller image.
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/server .
# Create the source directory; runtime COPY sets ownership for the named volume.
RUN mkdir -p "$COVER_DIR"
# Data dir with the runtime user's ownership so the mounted volume inherits it.
RUN mkdir -p /out/data
# --- runtime stage: distroless static, non-root ---
FROM gcr.io/distroless/static:nonroot
ARG COVER_DIR=/covers
WORKDIR /
COPY --from=build --chown=65532:65532 ${COVER_DIR} ${COVER_DIR}
COPY --from=build /out/server /server
COPY --from=build --chown=65532:65532 /out/data /data
VOLUME ["/data"]
EXPOSE 8080
USER nonroot:nonroot
ENV PORT=8080
ENV DB_PATH=/data/bookmarks.db PORT=8080
ENTRYPOINT ["/server"]
+48 -219
View File
@@ -12,104 +12,57 @@ import (
"testing"
"time"
"bookmarkmanager/backend/internal/pgtest"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/token"
)
// testTokenKey derives every test Reader's credential; it must match the key
// newTestStoreURL seeds the owner with, or derived credentials authenticate
// nothing.
const testTokenKey = "test-token-key"
// testDiscordID is the owner row's discord_id (newTestStoreURL); the derived
// credential is a function of it.
const testDiscordID = "test-owner"
// testCoverBaseURL is the public origin cover URLs are built from, standing in
// for PUBLIC_BASE_URL.
const testCoverBaseURL = "https://bookmarks.test"
const testToken = "s3cret-token"
func testConfig() Config {
return Config{
TokenKey: testTokenKey,
Token: testToken,
AllowedOrigins: []string{"https://asuracomic.net", "https://demonicscans.org"},
Port: "8080",
}
}
// ownerCredential is the owner's epoch-0 derived credential: the string the
// install links carry and the userscript routes authenticate.
func ownerCredential() string {
return token.Token([]byte(testTokenKey), testDiscordID, 0)
}
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
func newTestServer(t *testing.T) http.Handler {
t.Helper()
return newRouter(newTestStore(t), testConfig())
}
func newTestStore(t *testing.T) *store.Store {
t.Helper()
s, _ := newTestStoreURL(t)
return s
}
// newTestStoreURL is newTestStore plus the database URL, for tests that need
// to reach the same database directly.
func newTestStoreURL(t *testing.T) (*store.Store, string) {
t.Helper()
url := pgtest.URL(t)
s, err := store.Open(url, store.Owner{
DiscordID: testDiscordID, TokenHash: token.Hash(ownerCredential()),
}, t.TempDir(), testCoverBaseURL)
dbPath := filepath.Join(t.TempDir(), "test.db")
s, err := store.Open(dbPath)
if err != nil {
t.Fatalf("store.Open: %v", err)
}
t.Cleanup(func() { s.Close() })
return s, url
return newRouter(s, testConfig())
}
// auth authenticates a request as the owner Reader, whose derived credential
// is the only thing the API accepts.
func auth(req *http.Request) *http.Request {
req.Header.Set("Authorization", "Bearer "+ownerCredential())
req.Header.Set("Authorization", "Bearer "+testToken)
return req
}
func floatPtr(f float64) *float64 { return &f }
// seedForCheck inserts a bookmark (and with it its series) and forces the
// series' latest_checked_at.
// seedForCheck inserts a bookmark and forces its latest_checked_at.
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
t.Helper()
site, seriesID, ok := strings.Cut(key, ":")
if !ok {
t.Fatalf("key %q: no ':' separator", key)
}
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
if _, err := s.Upsert(store.Bookmark{
Key: key,
Site: site,
SeriesID: seriesID,
Site: "asura",
SeriesID: key,
SeriesURL: seriesURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed %q: %v", key, err)
}
if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
if err := s.MarkLatestChecked(key, checkedAt); err != nil {
t.Fatalf("seed mark %q: %v", key, err)
}
}
func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
t.Helper()
site, seriesID, ok := strings.Cut(key, ":")
if !ok {
t.Fatalf("key %q: no ':' separator", key)
}
ts, err := s.LatestCheckedAt(site, seriesID)
ts, err := s.LatestCheckedAt(key)
if err != nil {
t.Fatalf("LatestCheckedAt %q: %v", key, err)
}
@@ -136,7 +89,7 @@ func TestAuthRequired(t *testing.T) {
}{
{"no header", ""},
{"bad token", "Bearer wrong"},
{"not bearer", "Basic " + ownerCredential()},
{"not bearer", "Basic " + testToken},
{"empty bearer", "Bearer "},
}
for _, tc := range cases {
@@ -269,127 +222,6 @@ func TestBookmarkRoundTrip(t *testing.T) {
}
}
// The wire contract (ADR-0004): GET and PUT speak exactly the flat field set
// they always did, with the series-owned fields as siblings of the bookmark
// fields, not nested. Asserted as a key set, not by inspection.
func TestFlatWireFieldSet(t *testing.T) {
srv := newTestServer(t)
key := "comix:some-title"
in := store.Bookmark{
Key: key,
Site: "comix",
SeriesID: "some-title",
Title: "Some Title",
SeriesURL: "https://comix.to/title/some-title",
Cover: "https://comix.to/covers/some-title.jpg",
LastChapter: "Chapter 7",
LastChapterNum: 7,
LastChapterURL: "https://comix.to/title/some-title/ch/7",
Favorite: true,
LatestChapter: "Chapter 8",
LatestChapterNum: floatPtr(8),
Status: store.StatusArchived,
Kind: store.KindManga,
}
body, _ := json.Marshal(in)
wantKeys := map[string]bool{
"key": true, "site": true, "series_id": true, "title": true,
"series_url": true, "cover": true, "last_chapter": true,
"last_chapter_num": true, "last_chapter_url": true, "favorite": true,
"latest_chapter": true, "latest_chapter_num": true, "updated_at": true,
"status": true, "kind": true,
}
checkFlat := func(t *testing.T, payload []byte) map[string]json.RawMessage {
t.Helper()
var obj map[string]json.RawMessage
if err := json.Unmarshal(payload, &obj); err != nil {
t.Fatalf("decode: %v", err)
}
if len(obj) != len(wantKeys) {
t.Fatalf("field count = %d, want %d (%s)", len(obj), len(wantKeys), payload)
}
for k := range obj {
if !wantKeys[k] {
t.Fatalf("unexpected field %q", k)
}
}
return obj
}
// PUT
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, "/bookmarks/"+key, bytes.NewReader(body))))
if rr.Code != http.StatusOK {
t.Fatalf("PUT status = %d, want 200", rr.Code)
}
checkFlat(t, rr.Body.Bytes())
// Every field round-trips with its value, and updated_at is server-stamped.
var stored store.Bookmark
if err := json.Unmarshal(rr.Body.Bytes(), &stored); err != nil {
t.Fatalf("decode PUT response: %v", err)
}
latestNum := floatPtr(8)
want := store.Bookmark{
Key: key, Site: "comix", SeriesID: "some-title",
Title: in.Title, SeriesURL: in.SeriesURL,
LastChapter: in.LastChapter, LastChapterNum: in.LastChapterNum,
LastChapterURL: in.LastChapterURL, Favorite: true,
LatestChapter: in.LatestChapter, LatestChapterNum: latestNum,
Status: store.StatusArchived, Kind: store.KindManga,
}
// Cover is deliberately absent above: the client's cover is discarded, and
// this wiring acquires none, so the field is present and empty (ADR-0007).
if stored.Title != want.Title || stored.SeriesURL != want.SeriesURL || stored.Cover != want.Cover ||
stored.LastChapter != want.LastChapter || stored.LastChapterNum != want.LastChapterNum ||
stored.LastChapterURL != want.LastChapterURL || stored.Favorite != want.Favorite ||
stored.LatestChapter != want.LatestChapter ||
stored.LatestChapterNum == nil || *stored.LatestChapterNum != *want.LatestChapterNum ||
stored.Status != want.Status || stored.Kind != want.Kind {
t.Fatalf("PUT response = %+v, want %+v", stored, want)
}
if stored.UpdatedAt == 0 {
t.Fatal("updated_at not server-stamped")
}
// GET reports the same flat shape.
list := getBookmarks(t, srv)
if len(list) != 1 {
t.Fatalf("list = %d items, want 1", len(list))
}
body2, _ := json.Marshal(list[0])
checkFlat(t, body2)
}
// A PUT naming an existing series must ignore client-supplied title, cover and
// URL — the security boundary from ADR-0003, where a hostile site's scraped
// values could otherwise land on a shared row — while progress still lands.
func TestPutExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
srv := newTestServer(t)
key := "asura:solo"
first := putBookmark(t, srv, key, store.Bookmark{
Title: "Solo Leveling",
SeriesURL: "https://asurascans.com/comics/solo",
Cover: "https://asurascans.com/covers/solo.jpg",
LastChapterNum: 10,
})
second := putBookmark(t, srv, key, store.Bookmark{
Title: "Scraped Rename",
SeriesURL: "https://evil.example/solo",
Cover: "https://evil.example/solo.jpg",
LastChapterNum: 11,
})
if second.Title != first.Title || second.SeriesURL != first.SeriesURL || second.Cover != first.Cover {
t.Fatalf("stored = %+v, want original title/url/cover kept", second)
}
if second.LastChapterNum != 11 {
t.Fatalf("LastChapterNum = %v, want 11 — progress must still land", second.LastChapterNum)
}
}
// putBookmark PUTs b at key and returns the bookmark the server echoes back,
// which is the row as actually stored (not the request payload).
func putBookmark(t *testing.T, srv http.Handler, key string, b store.Bookmark) store.Bookmark {
@@ -568,29 +400,16 @@ func TestLatestChapterNullable(t *testing.T) {
}
}
func TestLoadConfigDiscord(t *testing.T) {
t.Setenv("DISCORD_CLIENT_ID", "client-1")
t.Setenv("DISCORD_CLIENT_SECRET", "client-secret-1")
t.Setenv("DISCORD_GUILD_ID", "guild-1")
t.Setenv("DISCORD_REQUIRED_ROLE", "role-9")
t.Setenv("DISCORD_REDIRECT_URI", "https://bm.example.com/auth/discord/callback")
t.Setenv("DISCORD_API_BASE", "https://stub.example/api")
if got := loadConfig().Discord; got.ClientID != "client-1" || got.ClientSecret != "client-secret-1" ||
got.GuildID != "guild-1" || got.RequiredRole != "role-9" ||
got.RedirectURI != "https://bm.example.com/auth/discord/callback" ||
got.APIBase != "https://stub.example/api" {
t.Fatalf("Discord config = %+v, want every field set", got)
func TestLoadConfigWebPassword(t *testing.T) {
t.Setenv("API_TOKEN", "token-abc")
t.Setenv("WEB_PASSWORD", "hunter2")
if got := loadConfig().WebPassword; got != "hunter2" {
t.Fatalf("WebPassword = %q, want hunter2", got)
}
// API base falls back to the Discord default; the role is optional.
t.Setenv("DISCORD_REQUIRED_ROLE", "")
t.Setenv("DISCORD_API_BASE", "")
got := loadConfig().Discord
if got.RequiredRole != "" {
t.Fatalf("RequiredRole = %q, want empty by default", got.RequiredRole)
}
if got.APIBase != "https://discord.com/api/v10" {
t.Fatalf("APIBase = %q, want the Discord default", got.APIBase)
t.Setenv("WEB_PASSWORD", "")
if got := loadConfig().WebPassword; got != "" {
t.Fatalf("WebPassword = %q with the variable unset, want empty", got)
}
}
@@ -598,7 +417,12 @@ func TestLoadConfigDiscord(t *testing.T) {
// moved into bookmarkColumns, this test catches it: the PUT would reset the
// cooldown and the poller would re-fetch that series on every single tick.
func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
s := newTestStore(t)
dbPath := filepath.Join(t.TempDir(), "test.db")
s, err := store.Open(dbPath)
if err != nil {
t.Fatalf("store.Open: %v", err)
}
t.Cleanup(func() { s.Close() })
srv := newRouter(s, testConfig())
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 777)
@@ -608,7 +432,7 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
"series_url":"https://asurascans.com/comics/x",
"last_chapter":"Chapter 5","last_chapter_num":5}`
req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body))
req.Header.Set("Authorization", "Bearer "+ownerCredential())
req.Header.Set("Authorization", "Bearer "+testToken)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, req)
@@ -621,33 +445,34 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
}
}
// The userscript route is registered outside the web UI's Discord auth, so it
// must keep working whatever the web config — see internal/userscript for the
// handler's own behaviour. The credential in the path is the owner's derived
// one, and the served script carries it substituted in.
// The userscript route is registered outside the `if cfg.WebPassword != ""`
// block in newRouter, so it must keep working on a deployment that never set
// WEB_PASSWORD — see internal/userscript for the handler's own behaviour.
func TestUserscriptServedWithWebUIDisabled(t *testing.T) {
path := filepath.Join(t.TempDir(), "manga-bookmark.user.js")
if err := os.WriteFile(path, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
if err := os.WriteFile(path, []byte("console.log(1);\n"), 0o644); err != nil {
t.Fatalf("write script: %v", err)
}
s := newTestStore(t)
cfg := testConfig() // no Discord config needed for the userscript route
dbPath := filepath.Join(t.TempDir(), "nopass.db")
s, err := store.Open(dbPath)
if err != nil {
t.Fatalf("store.Open: %v", err)
}
t.Cleanup(func() { s.Close() })
cfg := testConfig() // WebPassword empty
cfg.UserscriptPath = path
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/u/"+ownerCredential()+"/manga-bookmark.user.js", nil)
req := httptest.NewRequest(http.MethodGet, "/u/"+testToken+"/manga-bookmark.user.js", nil)
newRouter(s, cfg).ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code)
}
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+ownerCredential()+`"`) {
t.Fatalf("served script does not carry the requesting Reader's credential:\n%s", got)
}
}
// Both scripts are served from the same handler, outside the web UI's auth —
// a wrong credential is a 404, never a 401.
// Both scripts are served from the same handler on the same token, outside the
// WEB_PASSWORD gate — a wrong token is a 404, never a 401.
func TestNovelUserscriptServed(t *testing.T) {
dir := t.TempDir()
novelPath := filepath.Join(dir, "novel-bookmark.user.js")
@@ -655,7 +480,11 @@ func TestNovelUserscriptServed(t *testing.T) {
t.Fatalf("write script: %v", err)
}
s := newTestStore(t)
s, err := store.Open(filepath.Join(dir, "test.db"))
if err != nil {
t.Fatalf("store.Open: %v", err)
}
t.Cleanup(func() { s.Close() })
cfg := testConfig()
cfg.NovelUserscriptPath = novelPath
@@ -663,7 +492,7 @@ func TestNovelUserscriptServed(t *testing.T) {
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet,
"/u/"+ownerCredential()+"/novel-bookmark.user.js", nil))
"/u/"+testToken+"/novel-bookmark.user.js", nil))
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code)
}
-134
View File
@@ -1,134 +0,0 @@
package main
import (
"database/sql"
"net/http"
"net/http/httptest"
"strings"
"testing"
"bookmarkmanager/backend/internal/store"
)
func getCover(t *testing.T, srv http.Handler, path string, cookie *http.Cookie) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodGet, path, nil)
if cookie != nil {
req.AddCookie(cookie)
}
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, req)
return rr
}
// The acquired Cover is served from this deployment's own origin, to any
// browser rendering a third-party page — no session, no credential (ADR-0007).
func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
const sourceURL = "https://cdn.asurascans.com/covers/solo.webp"
srv, st := newWebTestServer(t, testConfig())
if err := st.PutCover(sourceURL, []byte("\x00webp-bytes"), "image/webp"); err != nil {
t.Fatalf("PutCover: %v", err)
}
// The wire URL is what a client actually requests, so the path under test
// is taken from it rather than rebuilt by hand.
wire := st.CoverWireURL(store.CoverAddress(sourceURL))
path, ok := strings.CutPrefix(wire, testCoverBaseURL)
if !ok {
t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL)
}
rr := getCover(t, srv, path, nil)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200 without any credential", rr.Code)
}
if got := rr.Body.String(); got != "\x00webp-bytes" {
t.Fatalf("body = %q, want the stored bytes", got)
}
if got := rr.Header().Get("Content-Type"); got != "image/webp" {
t.Fatalf("Content-Type = %q, want the stored one", got)
}
// Content-addressed bytes never change, so a client that has them must
// never need to ask again.
if got := rr.Header().Get("Cache-Control"); !strings.Contains(got, "immutable") {
t.Fatalf("Cache-Control = %q, want an immutable cache directive", got)
}
}
func TestPublicCoverRejectsUnknownAddress(t *testing.T) {
srv, _ := newWebTestServer(t, testConfig())
cases := map[string]string{
"unknown": "/covers/" + store.CoverAddress("https://cdn.example/never-stored.jpg"),
"malformed": "/covers/not-an-address",
"traversal": "/covers/../../etc/passwd",
"empty": "/covers/",
}
for name, path := range cases {
t.Run(name, func(t *testing.T) {
if rr := getCover(t, srv, path, nil); rr.Code == http.StatusOK {
t.Fatalf("%s: status = 200, want anything but a served body", path)
}
})
}
}
// A content type outside the image set is never echoed back. The old kagane
// proxy could fetch text/html from a challenged fetch and had to refuse it;
// the general route's only input is the store, and the store refuses to
// record anything that is not an image — but the guarantee is pinned at the
// serving boundary, not the write gate, so a poisoned row (migrated data, a
// writer that skips the gate) is also never served.
func TestPublicCoverNeverEchoesNonImage(t *testing.T) {
const sourceURL = "https://cdn.example/cover"
st, dsn := newTestStoreURL(t)
// The write gate refuses non-image content types outright.
if err := st.PutCover(sourceURL, []byte("<script>"), "text/html"); err == nil {
t.Fatal("PutCover accepted a non-image content type")
}
// A legitimate row, then the content type flipped behind the store's back:
// the bytes exist at the address, so only the type is hostile.
address := store.CoverAddress(sourceURL)
if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil {
t.Fatalf("seed row: %v", err)
}
db, err := sql.Open("pgx", dsn)
if err != nil {
t.Fatalf("open %s: %v", dsn, err)
}
defer db.Close()
if _, err := db.Exec(`UPDATE covers SET content_type = 'text/html' WHERE address = $1`, address); err != nil {
t.Fatalf("poison row: %v", err)
}
rr := getCover(t, newRouter(st, testConfig()), "/covers/"+address, nil)
if rr.Code == http.StatusOK {
t.Fatalf("status = 200, want a refusal for a non-image row (body %q)", rr.Body.String())
}
}
// The whole point of acquiring bytes is that the UI shows them: the card's
// <img> must carry the public address, not a third-party URL and not a
// placeholder.
func TestListRendersAcquiredCover(t *testing.T) {
const sourceURL = "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg"
srv, st := newWebTestServer(t, testConfig())
if _, err := st.Upsert(st.OwnerID(), store.Bookmark{
Key: "comix:n8we", Site: "comix", SeriesID: "n8we", Title: "Dungeons and Crayons",
SeriesURL: "https://comix.to/title/n8we", UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
if err := st.SetSeriesCover("comix", "n8we", sourceURL, []byte("\xff\xd8jpeg"), "image/jpeg"); err != nil {
t.Fatalf("SetSeriesCover: %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/ui/list", nil)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code)
}
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddress(sourceURL) + `"`
if !strings.Contains(rr.Body.String(), want) {
t.Fatalf("rendered list does not contain %s", want)
}
}
+13 -5
View File
@@ -7,7 +7,7 @@ require (
github.com/bogdanfinn/tls-client v1.15.1
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f
github.com/chromedp/chromedp v0.16.0
github.com/jackc/pgx/v5 v5.10.0
modernc.org/sqlite v1.34.4
)
require (
@@ -18,19 +18,27 @@ require (
github.com/bogdanfinn/utls v1.7.7-barnius // indirect
github.com/bogdanfinn/websocket v1.5.5-barnius // indirect
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
golang.org/x/crypto v0.46.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.32.0 // indirect
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
modernc.org/libc v1.55.3 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.8.0 // indirect
modernc.org/strutil v1.2.0 // indirect
modernc.org/token v1.1.0 // indirect
)
+44 -14
View File
@@ -20,9 +20,10 @@ github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg=
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
@@ -31,27 +32,28 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
@@ -62,6 +64,8 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
golang.org/x/net v0.0.0-20211104170005-ce137452f963/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
@@ -77,7 +81,33 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
modernc.org/sqlite v1.34.4 h1:sjdARozcL5KJBvYQvLlZEmctRgW9xqIZc2ncN7PU0P8=
modernc.org/sqlite v1.34.4/go.mod h1:3QQFCG2SEMtc2nv+Wq4cQCH7Hjcg+p/RMlS1XK+zwbk=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+4 -45
View File
@@ -7,7 +7,6 @@ import (
"strings"
"time"
"bookmarkmanager/backend/internal/httpmw"
"bookmarkmanager/backend/internal/store"
)
@@ -26,9 +25,9 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
}
}
// List returns all bookmarks of the acting Reader. GET /bookmarks
// List returns all bookmarks. GET /bookmarks
func (h *Handler) List(w http.ResponseWriter, r *http.Request) {
items, err := h.Store.List(httpmw.ReaderID(r))
items, err := h.Store.List()
if err != nil {
log.Printf("list: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
@@ -50,13 +49,6 @@ func (h *Handler) Put(w http.ResponseWriter, r *http.Request) {
http.Error(w, "invalid JSON body", http.StatusBadRequest)
return
}
// A body may carry a cover, and it is discarded here rather than
// rejected: an older installed userscript may still send one, and
// ADR-0004's compatibility argument depends on those scripts continuing
// to work. The Cover is acquired server-side (ADR-0007), so the field is
// permanently inert - not pending removal, and not a value any later code
// should start reading.
b.Cover = ""
// Path key is authoritative; derive site/series_id from it when the body
// omits them so the stored row is always self-consistent.
@@ -99,7 +91,7 @@ func (h *Handler) Put(w http.ResponseWriter, r *http.Request) {
// reading progress actually moved. Any client value is ignored.
b.UpdatedAt = time.Now().UnixMilli()
stored, err := h.Store.Upsert(httpmw.ReaderID(r), b)
stored, err := h.Store.Upsert(b)
if err != nil {
log.Printf("upsert: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
@@ -117,7 +109,7 @@ func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
http.Error(w, "missing key", http.StatusBadRequest)
return
}
if err := h.Store.Delete(httpmw.ReaderID(r), key); err != nil {
if err := h.Store.Delete(key); err != nil {
log.Printf("delete: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
@@ -131,36 +123,3 @@ func Healthz(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}
// Cover serves stored cover bytes. GET /covers/{address}
//
// Public on purpose: the userscript renders these on Sites the deployment
// does not control, where no credential of ours may be sent, and the address
// is the SHA-256 of a URL the Site already publishes (ADR-0007). An unknown
// address is a 404 rather than an error - "no Cover yet" is a normal state,
// and the clients fall back to their placeholder.
func (h *Handler) Cover(w http.ResponseWriter, r *http.Request) {
body, contentType, ok, err := h.Store.CoverByAddress(r.PathValue("address"))
if err != nil {
log.Printf("cover: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
if !ok {
http.NotFound(w, r)
return
}
// Refuse anything the write gate would not have recorded: a poisoned row
// (migrated data, a writer that skips the gate) must never be echoed back
// as bytes of a type no Cover may have.
if _, ok := store.CoverContentType(contentType); !ok {
log.Printf("cover %s: refusing non-image content type %q", r.PathValue("address"), contentType)
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", contentType)
// Content-addressed, so the bytes at this URL can never change. Public
// rather than private: no credential gates the route.
w.Header().Set("Cache-Control", "public, max-age=604800, immutable")
_, _ = w.Write(body)
}
+7 -35
View File
@@ -2,56 +2,28 @@ package httpmw
import (
"compress/gzip"
"context"
"log"
"crypto/subtle"
"net/http"
"strings"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/token"
)
const bearerPrefix = "Bearer "
type ctxKey int
// readerCtxKey is where Auth stashes the authenticated Reader id.
const readerCtxKey ctxKey = iota
// ReaderID returns the Reader id Auth authenticated, for handlers that take
// the acting Reader from the request rather than from a fixed field.
func ReaderID(r *http.Request) int64 { return r.Context().Value(readerCtxKey).(int64) }
// ResolveReader maps a presented credential to a Reader. The credential is
// hashed and matched against readers.token_sha256 — an equality on 32-byte
// values, never a comparison of the credential itself. The same resolution
// backs the API bearer header and the userscript download path, so a Reader
// has exactly one credential with one blast radius.
func ResolveReader(s *store.Store, cred string) (int64, bool) {
readerID, ok, err := s.ReaderIDForTokenHash(token.Hash(cred))
if err != nil {
log.Printf("auth: reader lookup: %v", err)
return 0, false
}
return readerID, ok
}
// Auth guards a handler with a per-Reader bearer credential. The acting
// Reader travels in the request context, so a handler scopes every store call
// to exactly the Reader that authenticated.
func Auth(s *store.Store, next http.Handler) http.Handler {
// Auth guards a handler with a constant-time bearer-token check.
func Auth(token string, next http.Handler) http.Handler {
want := []byte(token)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h := r.Header.Get("Authorization")
if !strings.HasPrefix(h, bearerPrefix) {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
readerID, ok := ResolveReader(s, strings.TrimPrefix(h, bearerPrefix))
if !ok {
got := []byte(strings.TrimPrefix(h, bearerPrefix))
if subtle.ConstantTimeCompare(got, want) != 1 {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), readerCtxKey, readerID)))
next.ServeHTTP(w, r)
})
}
-147
View File
@@ -1,147 +0,0 @@
package latest
import (
"context"
"log"
"sync"
"time"
"bookmarkmanager/backend/internal/store"
)
// acquireTimeout bounds one creation-time acquisition end to end: the series
// page plus the cover bytes. Nothing is waiting on it — the Reader's write has
// already returned — so this only stops a stalled Site from holding a
// goroutine and a connection open forever.
const acquireTimeout = 45 * time.Second
// Acquirer gives a Series its Latest Chapter and its Cover the moment the
// first Bookmark creates it, instead of leaving the Reader to wait out the
// poll queue — which is ordered by Reader count, so a Series with one Reader
// sits behind every popular one (ADR-0007).
//
// Both facts come from a single series-page fetch, which is also why no
// client-supplied cover hint is worth accepting: the page has to be fetched
// for the chapter signal regardless, so a hint would save no request while
// adding a client-controlled input to a server-side fetch.
//
// Every failure path is "log and move on". The Bookmark, its progress and its
// Latest Chapter are already committed; a Site that is down or a Cover that
// cannot be produced must not disturb any of them, and the Series is simply
// left blank until the poll's own cover pass (#61) fills it.
type Acquirer struct {
Store *store.Store
// Fetch retrieves the series page over plain TLS. Nil with a nil
// BrowserFetch disables acquisition entirely.
Fetch Fetcher
// BrowserFetch retrieves kagane and novelfull pages through the browser
// sidecar, the only thing that clears their Cloudflare challenge. The
// per-site fallback policy lives in fetcherFor. Nil leaves those Sites
// unacquired when no fallback applies.
BrowserFetch Fetcher
// Covers retrieves the cover bytes. Nil leaves the Cover blank and the
// chapter half working.
Covers CoverBytesFetcher
// BrowserCoverFetch retrieves browser-claimed cover bytes through the
// sidecar. Nil leaves those Covers blank; nothing falls back to a plain
// fetch, which would only ever retrieve a challenge page.
BrowserCoverFetch BrowserCoverFetcher
// Ctx cancels in-flight acquisitions at shutdown. A hook signature has
// nowhere to pass one, so it lives here; nil means context.Background.
Ctx context.Context
inflight sync.WaitGroup
}
// acquireSlots caps how many creation-time fetches run at once. A Reader whose
// userscript bulk-syncs creates many Series at once, and a burst of
// simultaneous requests from one server IP is the traffic shape most likely to
// move that IP's bot score — the same reason the poller staggers its batch.
var acquireSlots = make(chan struct{}, 2)
// Acquire starts one acquisition and returns immediately: a Reader's bookmark
// action may not block on a third-party Site's latency, nor fail with it. It
// is the store's OnSeriesCreated hook, so it only ever runs for a Series no
// Reader had bookmarked before.
func (a *Acquirer) Acquire(sr store.Series) {
a.inflight.Add(1)
go func() {
defer a.inflight.Done()
defer func() {
if r := recover(); r != nil {
log.Printf("acquire %q: recovered from panic: %v", sr.Key(), r)
}
}()
parent := a.Ctx
if parent == nil {
parent = context.Background()
}
select {
case acquireSlots <- struct{}{}:
defer func() { <-acquireSlots }()
case <-parent.Done():
return
}
ctx, cancel := context.WithTimeout(parent, acquireTimeout)
defer cancel()
a.acquire(ctx, sr)
}()
}
// Wait blocks until every started acquisition has finished. It exists for
// tests: an asynchronous side effect is otherwise unobservable without
// polling for it.
func (a *Acquirer) Wait() { a.inflight.Wait() }
func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
if a.Fetch == nil && a.BrowserFetch == nil {
return
}
// series_url arrives in a client-supplied PUT body, so the same gate the
// poller uses applies here — without it a token-holder chooses what the
// server fetches from its own network position.
if !fetchableSeriesURL(sr.Site, sr.SeriesURL) {
log.Printf("acquire %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
return
}
f := fetcherFor(sr.Site, a.BrowserFetch, a.Fetch)
if f == nil {
log.Printf("acquire %q: no fetcher for site %q", sr.Key(), sr.Site)
return
}
body, status, err := f.Get(ctx, sr.SeriesURL)
if err != nil {
log.Printf("acquire %q: fetch %s: %v", sr.Key(), sr.SeriesURL, err)
return
}
if status != 200 {
log.Printf("acquire %q: fetch %s: status %d", sr.Key(), sr.SeriesURL, status)
return
}
// This page just served the same purpose a poll tick would have; without
// the stamp the row stays due and the poller refetches it immediately.
if err := a.Store.MarkLatestChecked(sr.Site, sr.SeriesID, time.Now().UnixMilli()); err != nil {
log.Printf("acquire %q: mark checked: %v", sr.Key(), err)
}
if latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body); ok {
if err := a.Store.SetLatestChapter(sr.Site, sr.SeriesID, latest.Label, latest.Num); err != nil {
log.Printf("acquire %q: set latest chapter: %v", sr.Key(), err)
}
}
cover, ok := coverFrom(sr.Site, sr.SeriesURL, body)
if !ok {
return
}
bytes, contentType, err := fetchCoverBytes(ctx, cover, a.BrowserCoverFetch, a.Covers)
if err != nil {
log.Printf("acquire %q: fetch cover %s: %v", sr.Key(), cover, err)
return
}
if err := a.Store.SetSeriesCover(sr.Site, sr.SeriesID, cover, bytes, contentType); err != nil {
log.Printf("acquire %q: persist cover: %v", sr.Key(), err)
}
}
-430
View File
@@ -1,430 +0,0 @@
package latest
import (
"context"
"errors"
"testing"
"time"
"bookmarkmanager/backend/internal/store"
)
// The series page carries both facts, which is the whole argument for taking
// them from one fetch.
const asuraSeriesAndCoverFixture = asuraSeriesFixture + asuraCoverFixture
const (
acquireKey = "asura:chronicles-of-the-demon-faction-f886a8af"
acquireSeriesID = "chronicles-of-the-demon-faction-f886a8af"
acquireSeriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
acquireCoverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
)
const (
kaganeKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
kaganeSeriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
kaganeSeriesURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
kaganeImageID = "019fe11a-84c3-7fc3-a84b-88787374b617"
kaganeCoverSrc = "https://kagane.to/api/v2/image/" + kaganeImageID + "/compressed"
)
// kagane's browser-fetched body is one JSON object carrying both the chapter
// list (series_books) and the cover image ids (series_covers), so the single
// acquisition fetch yields both facts.
const kaganeSeriesAndCoverFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",` +
`"series_books":[{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41}],` +
`"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en",` +
`"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}]}`
const (
novelfullKey = "novelfull:reverend-insanity"
novelfullSeriesID = "reverend-insanity"
novelfullSeriesURI = "https://novelfull.com/reverend-insanity.html"
novelfullCoverURL = "https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp"
)
// newAcquirer wires an acquirer onto the store's creation hook, which is how
// main wires it: the write path is what starts an acquisition.
func newAcquirer(s *store.Store, page *fakeFetcher, covers *fakeBytesCoverFetcher) *Acquirer {
a := &Acquirer{Store: s, Fetch: page, Covers: covers}
s.OnSeriesCreated = a.Acquire
return a
}
func bookmarkNewSeries(t *testing.T, s *store.Store, seriesURL string) store.Bookmark {
t.Helper()
stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: acquireKey, Site: "asura", SeriesID: acquireSeriesID,
Title: "Chronicles of the Demon Faction", SeriesURL: seriesURL,
Cover: "https://evil.example/client-supplied.jpg", UpdatedAt: 1000,
})
if err != nil {
t.Fatalf("Upsert: %v", err)
}
return stored
}
func readBookmark(t *testing.T, s *store.Store, key string) store.Bookmark {
t.Helper()
b, ok, err := s.Get(s.OwnerID(), key)
if err != nil || !ok {
t.Fatalf("Get %q = %v, %v", key, ok, err)
}
return b
}
func bookmarkNewKaganeSeries(t *testing.T, s *store.Store) store.Bookmark {
t.Helper()
stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: kaganeKey, Site: "kagane", SeriesID: kaganeSeriesID,
Title: "Infinite Decryption", SeriesURL: kaganeSeriesURL, UpdatedAt: 1000,
})
if err != nil {
t.Fatalf("Upsert: %v", err)
}
return stored
}
func bookmarkNewNovelfullSeries(t *testing.T, s *store.Store) store.Bookmark {
t.Helper()
stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: novelfullKey, Site: "novelfull", SeriesID: novelfullSeriesID,
Title: "Reverend Insanity", SeriesURL: novelfullSeriesURI, UpdatedAt: 1000,
})
if err != nil {
t.Fatalf("Upsert: %v", err)
}
return stored
}
// The reported bug: a Reader bookmarks a Series nobody holds and expects the
// Cover, not a broken image. Both facts come from the one series-page fetch.
func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
s, _ := newTestStore(t)
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
acq := newAcquirer(s, page, covers)
// The write itself must not carry the acquisition: it returns before the
// Cover exists, and the field is empty until the bytes land.
stored := bookmarkNewSeries(t, s, acquireSeriesURL)
if stored.Cover != "" {
t.Fatalf("Cover on the creating write = %q, want empty", stored.Cover)
}
acq.Wait()
if got := page.callCount(); got != 1 {
t.Fatalf("series page fetches = %d, want exactly 1", got)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetches = %d, want 1", got)
}
got := readBookmark(t, s, acquireKey)
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
}
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want)
}
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(acquireCoverURL))
if err != nil || !ok {
t.Fatalf("CoverByAddress = %v, %v", ok, err)
}
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
t.Fatalf("stored cover = (%q, %q), want the fetched bytes", body, contentType)
}
}
// A Series that already exists is not re-acquired: no fetch, and the Cover it
// already has is left alone.
func TestAcquireSkipsAnExistingSeries(t *testing.T) {
s, _ := newTestStore(t)
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
acq := newAcquirer(s, page, covers)
bookmarkNewSeries(t, s, acquireSeriesURL)
acq.Wait()
bookmarkNewSeries(t, s, acquireSeriesURL)
acq.Wait()
if got := page.callCount(); got != 1 {
t.Fatalf("series page fetches = %d, want 1 — an existing series is not re-acquired", got)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetches = %d, want 1", got)
}
got := readBookmark(t, s, acquireKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want)
}
}
// A Site that is down costs the Cover and nothing else.
func TestAcquireFailureLeavesTheBookmarkIntact(t *testing.T) {
cases := []struct {
name string
page *fakeFetcher
covers *fakeBytesCoverFetcher
// wantLatest is the chapter that still lands; 0 means none did.
wantLatest float64
}{
{
"the series page is unreachable",
&fakeFetcher{err: errors.New("connection reset")},
&fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
0,
},
{
"the series page answers with a challenge",
&fakeFetcher{body: challengeFixture, status: 200},
&fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
0,
},
{
"only the cover bytes fail",
&fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200},
&fakeBytesCoverFetcher{err: errors.New("403")},
181,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s, _ := newTestStore(t)
acq := newAcquirer(s, tc.page, tc.covers)
stored := bookmarkNewSeries(t, s, acquireSeriesURL)
acq.Wait()
got := readBookmark(t, s, acquireKey)
if got.Cover != "" {
t.Fatalf("Cover = %q, want empty rather than an address that 404s", got.Cover)
}
if got.Title != stored.Title || got.UpdatedAt != stored.UpdatedAt {
t.Fatalf("bookmark = %+v, want it untouched by the failed acquisition", got)
}
if tc.wantLatest == 0 {
if got.LatestChapterNum != nil {
t.Fatalf("LatestChapterNum = %v, want none captured", *got.LatestChapterNum)
}
return
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != tc.wantLatest {
t.Fatalf("LatestChapterNum = %v, want %v", got.LatestChapterNum, tc.wantLatest)
}
})
}
}
// series_url arrives in a client-supplied body, so the acquisition reuses the
// poller's gate rather than deriving a second one: a non-https scheme, a
// site the parsers do not know, or a host pinned to another site is refused
// before the server spends a request from its own network position.
func TestAcquireRefusesAnUnfetchableSeriesURL(t *testing.T) {
for _, seriesURL := range []string{
"http://asurascans.com/comics/x",
"file:///etc/passwd",
"",
} {
t.Run(seriesURL, func(t *testing.T) {
s, _ := newTestStore(t)
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
acq := newAcquirer(s, page, &fakeBytesCoverFetcher{})
bookmarkNewSeries(t, s, seriesURL)
acq.Wait()
if got := page.callCount(); got != 0 {
t.Fatalf("fetches for %q = %d, want 0", seriesURL, got)
}
})
}
}
// blockingFetcher stands in for a Site that never answers, so a synchronous
// acquisition would be visible as a stalled write rather than a slow one.
type blockingFetcher struct {
release <-chan struct{}
body string
}
func (f *blockingFetcher) Get(ctx context.Context, _ string) (string, int, error) {
select {
case <-f.release:
return f.body, 200, nil
case <-ctx.Done():
return "", 0, ctx.Err()
}
}
// The Reader's write may not wait on a third-party Site: with the acquisition
// wedged on an unanswering page, the PUT still returns.
func TestAcquireDoesNotBlockTheWrite(t *testing.T) {
s, _ := newTestStore(t)
release := make(chan struct{})
acq := &Acquirer{Store: s, Fetch: &blockingFetcher{release: release, body: asuraSeriesAndCoverFixture}}
s.OnSeriesCreated = acq.Acquire
upserted := make(chan error, 1)
go func() {
_, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: acquireKey, Site: "asura", SeriesID: acquireSeriesID,
Title: "Chronicles of the Demon Faction", SeriesURL: acquireSeriesURL, UpdatedAt: 1000,
})
upserted <- err
}()
select {
case err := <-upserted:
if err != nil {
t.Fatalf("Upsert: %v", err)
}
case <-time.After(10 * time.Second):
t.Fatal("the creating write blocked on the acquisition")
}
close(release)
acq.Wait()
}
// The second symptom of #47: a kagane Series bookmarked from a chapter page
// gets its Cover at creation, with the bytes fetched through the browser
// sidecar — the only path that clears the challenge — into the
// content-addressed store.
func TestAcquireKaganeCoverThroughBrowser(t *testing.T) {
s, _ := newTestStore(t)
tlsPage := &fakeFetcher{body: "", status: 403}
browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
acq := &Acquirer{
Store: s, Fetch: tlsPage, BrowserFetch: browserPage,
BrowserCoverFetch: covers, Covers: &fakeBytesCoverFetcher{},
}
s.OnSeriesCreated = acq.Acquire
bookmarkNewKaganeSeries(t, s)
acq.Wait()
if got := tlsPage.callCount(); got != 0 {
t.Fatalf("plain-TLS page fetches = %d, want 0 — kagane pages are browser-only", got)
}
if got := browserPage.callCount(); got != 1 {
t.Fatalf("browser page fetches = %d, want 1", got)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("browser cover fetches = %d, want 1", got)
}
if got := covers.calls[0]; got != kaganeCoverSrc {
t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc)
}
got := readBookmark(t, s, kaganeKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(kaganeCoverSrc); got.Cover != want {
t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
}
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(kaganeCoverSrc))
if err != nil || !ok {
t.Fatalf("CoverByAddress = %v, %v", ok, err)
}
if string(body) != "cover-bytes" || contentType != "image/webp" {
t.Fatalf("stored cover = (%q, %q), want the browser-fetched bytes", body, contentType)
}
}
// novelfull needs the browser only for its HTML: the cover URL comes out of
// the browser-fetched page, but the bytes go over plain TLS through the
// ordinary gated fetcher, never through the browser (issue #62).
func TestAcquireNovelfullCoverOverPlainTLS(t *testing.T) {
s, _ := newTestStore(t)
browserPage := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
acq := &Acquirer{
Store: s, Fetch: &fakeFetcher{body: "", status: 403},
BrowserFetch: browserPage, Covers: covers,
}
s.OnSeriesCreated = acq.Acquire
bookmarkNewNovelfullSeries(t, s)
acq.Wait()
if got := browserPage.callCount(); got != 1 {
t.Fatalf("browser page fetches = %d, want 1", got)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetches = %d, want 1 — novelfull bytes never touch the browser", got)
}
if got := covers.calls[0]; got != novelfullCoverURL {
t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
}
got := readBookmark(t, s, novelfullKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want)
}
}
// With no browser sidecar configured, kagane is simply not acquired: no
// request is spent on a page that could only ever answer with a challenge,
// and nothing falls back to a plain fetch.
func TestAcquireKaganeSkippedWithoutBrowser(t *testing.T) {
s, _ := newTestStore(t)
tlsPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
acq := &Acquirer{
Store: s, Fetch: tlsPage,
Covers: &fakeBytesCoverFetcher{body: []byte("x"), contentType: "image/webp"},
}
s.OnSeriesCreated = acq.Acquire
bookmarkNewKaganeSeries(t, s)
acq.Wait()
if got := tlsPage.callCount(); got != 0 {
t.Fatalf("plain-TLS fetches for kagane = %d, want 0", got)
}
if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
t.Fatalf("Cover = %q, want empty without a browser", got.Cover)
}
}
// The byte half of "nothing falls back to a plain fetch": with a browser for
// the page but none for the bytes, a kagane Cover stays absent and the TLS
// cover fetcher is never consulted.
func TestAcquireKaganeBytesNeverFallBackToPlainTLS(t *testing.T) {
s, _ := newTestStore(t)
browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
tlsCovers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
acq := &Acquirer{
Store: s, Fetch: &fakeFetcher{body: "", status: 403},
BrowserFetch: browserPage, Covers: tlsCovers,
}
s.OnSeriesCreated = acq.Acquire
bookmarkNewKaganeSeries(t, s)
acq.Wait()
if got := tlsCovers.callCount(); got != 0 {
t.Fatalf("plain-TLS cover fetches = %d, want 0 — kagane bytes are browser-only", got)
}
if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
t.Fatalf("Cover = %q, want empty without a browser cover fetcher", got.Cover)
}
}
// novelfull's no-browser degradation differs from kagane's: only its HTML
// needs the sidecar, so when the page body is available — the challenge is a
// live time-varying fact that sometimes answers a plain request — the Cover
// still lands, bytes over plain TLS.
func TestAcquireNovelfullCoverWithoutBrowser(t *testing.T) {
s, _ := newTestStore(t)
tlsPage := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
acq := &Acquirer{Store: s, Fetch: tlsPage, Covers: covers}
s.OnSeriesCreated = acq.Acquire
bookmarkNewNovelfullSeries(t, s)
acq.Wait()
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetches = %d, want 1", got)
}
got := readBookmark(t, s, novelfullKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want)
}
}
+40 -181
View File
@@ -2,9 +2,7 @@ package latest
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/url"
"regexp"
@@ -47,22 +45,19 @@ var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
type BrowserFetcher struct {
allocCtx context.Context
cancel context.CancelFunc
// One page at a time: caps the browser's memory — it runs under a hard
// cgroup cap on a shared machine — and keeps series from sharing page state.
// One page at a time: caps the sidecar's memory and keeps series from
// sharing page state.
mu sync.Mutex
}
var _ Fetcher = (*BrowserFetcher)(nil)
// NewBrowserFetcher connects to a Chrome over CDP. The browser is not a
// sidecar: it runs on a separate machine and is reached over the tailnet
// (ADR-0006), so wsURL is that machine's tailnet address, e.g.
// ws://100.64.0.5:9222.
//
// It must be an IP, never a hostname — not MagicDNS, not a Docker service
// name. Chrome's DevTools HTTP handler 500s any /json/version request whose
// Host header isn't an IP or "localhost" (confirmed 2026-08-03), so a name
// fails at discovery and surfaces as a dead site rather than a bad URL.
// NewBrowserFetcher connects to a headless-shell over CDP. wsURL must name the
// sidecar by IP, e.g. ws://172.28.0.10:9222 — not by Docker DNS name. Chrome's
// DevTools HTTP handler 500s any /json/version request whose Host header
// isn't an IP or "localhost" (confirmed 2026-08-03 against
// chromedp/headless-shell:stable), so the compose network pins the sidecar's
// address for this to resolve at all.
//
// Do not add chromedp.NoModifyURL here: that option skips the /json/version
// discovery request entirely and dials wsURL as if it were already the full
@@ -70,10 +65,8 @@ var _ Fetcher = (*BrowserFetcher)(nil)
// /devtools/browser/<uuid>, a path chosen fresh at every Chrome start — dialing
// the bare host:port 404s. The default (discovery) path works precisely
// because Chrome's /json/version response echoes back the Host header of the
// discovery request in webSocketDebuggerUrl, so as long as wsURL is an IP this
// process can reach, the URL chromedp gets back already points at it. That is
// also why a Chrome restarted behind a stable endpoint needs no reconnect
// here: the fresh UUID arrives with the next discovery.
// discovery request in webSocketDebuggerUrl, so as long as wsURL is a
// container-reachable IP, the URL chromedp gets back already points at it.
func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
if wsURL == "" {
return nil, fmt.Errorf("empty browser websocket url")
@@ -98,6 +91,23 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
}
f.mu.Lock()
defer f.mu.Unlock()
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
defer cancel()
// A fresh tab per fetch, closed on return, so one wedged page cannot
// poison later polls.
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
defer cancelTab()
// Bind the caller's deadline to the tab.
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
defer cancelDeadline()
go func() {
<-ctx.Done()
cancelDeadline()
}()
var body string
// kagane's chapter list is only in its JSON API, which must be called from
// inside the page so the request carries the clearance cookie. novelfull
@@ -113,175 +123,24 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int
)
}
// novelfull's payload is the DOM itself, and the interstitial has a DOM
// too, so "we have an answer" has to exclude it explicitly. kagane's
// in-page fetch just fails while challenged, which is already the signal.
done := func() bool { return body != "" && (isKagane || !isInterstitial(body)) }
if err := f.run(ctx, seriesURL, read, done); err != nil {
// Challenge never cleared, or the API refused. Indistinguishable from
// here and handled identically by the caller.
if errors.Is(err, errChallengeHeld) {
return "", 403, nil
}
err := chromedp.Run(tabCtx,
chromedp.Navigate(seriesURL),
// The challenge reloads the page itself when it passes; waiting for the
// site's own root element is what tells us we are through it.
chromedp.WaitReady("body", chromedp.ByQuery),
read,
)
if err != nil {
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
}
if body == "" {
// Challenge still up, or the API refused. Indistinguishable from here
// and handled identically by the caller.
return "", 403, nil
}
return body, 200, nil
}
// Image retrieves one cover's bytes through the browser sidecar, and its
// content type.
//
// It exists because kagane serves covers behind the same challenge as its
// pages *and* with `cross-origin-resource-policy: same-origin`, so the bytes
// are only reachable from inside a browser that already holds the clearance
// cookie (verified 2026-08-08). Acquisition through the sidecar is the only
// route.
//
// The image URL is navigated to rather than fetched from some other kagane
// page: the challenge only runs on a top-level navigation, and once it clears
// the document *is* the image, so a same-origin fetch of location.href reads
// it straight back out of the cache.
//
// The challenge is not solved by the first read: WaitReady("body") is satisfied
// by the interstitial too. run holds the tab open until the in-page fetch
// succeeds, which is what gives the challenge script the seconds it needs.
func (f *BrowserFetcher) Image(ctx context.Context, imageURL string) ([]byte, string, error) {
m := kaganeImageURLRe.FindStringSubmatch(imageURL)
if m == nil {
return nil, "", fmt.Errorf("not a browser-fetchable cover url: %q", imageURL)
}
imageID := m[1]
var dataURL string
err := f.run(ctx, imageURL,
chromedp.Evaluate(`fetch(location.href).then(r => r.ok
? r.blob().then(b => new Promise(res => {
const fr = new FileReader();
fr.onload = () => res(fr.result);
fr.readAsDataURL(b);
}))
: "")`, &dataURL, awaitPromise),
func() bool { return dataURL != "" })
if err != nil {
return nil, "", fmt.Errorf("browser image %s: %w", imageID, err)
}
// "data:image/webp;base64,<payload>".
head, payload, ok := strings.Cut(dataURL, ";base64,")
if !ok {
return nil, "", fmt.Errorf("browser image %s: not a data url", imageID)
}
raw, err := base64.StdEncoding.DecodeString(payload)
if err != nil {
return nil, "", fmt.Errorf("browser image %s: %w", imageID, err)
}
return raw, strings.TrimPrefix(head, "data:"), nil
}
// errChallengeHeld reports that the budget ran out with the interstitial still
// up. Distinct from a transport failure: it means "this site said no", which
// the poller answers with a 403 and its ordinary cooldown.
var errChallengeHeld = errors.New("challenge held")
// errBrowserInterrupted distinguishes a remote Chrome restart from the
// caller's own deadline. chromedp reports both as context.Canceled.
var errBrowserInterrupted = errors.New("browser interrupted")
func classifyBrowserError(ctx context.Context, browserLost bool, err error) error {
if err == nil || ctx.Err() != nil {
return err
}
if !browserLost {
return err
}
if !errors.Is(err, context.Canceled) {
return err
}
return fmt.Errorf("%w: %w", errBrowserInterrupted, err)
}
func browserConnectionLost(ctx context.Context) bool {
c := chromedp.FromContext(ctx)
if c == nil || c.Browser == nil {
return true
}
select {
case <-c.Browser.LostConnection:
return true
default:
return false
}
}
// challengePollInterval paces re-reads while a challenge solves itself.
const challengePollInterval = 2 * time.Second
// isInterstitial reports whether html is Cloudflare's challenge page rather
// than the site's own. Matched on the challenge runtime's script path, which is
// stable across the interstitial's wording and locale — the visible "Just a
// moment..." title is neither.
func isInterstitial(html string) bool {
return strings.Contains(html, "/cdn-cgi/challenge-platform/")
}
// run navigates to target and re-reads until done reports an answer, bounded by
// challengeTimeout and by the caller's own deadline, in a tab that is closed on
// return so one wedged page cannot poison later calls.
//
// Holding the tab open across re-reads is the whole point. A Cloudflare
// interstitial needs several seconds of a live page to solve itself and write
// clearance into the browser's shared cookie jar; reading once and closing the
// tab — which is what this did before 2026-08-08 — never gives it that window,
// so every fetch lands on the interstitial and the clearance that would have
// unblocked all the later ones is never obtained.
func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.Action, done func() bool) error {
f.mu.Lock()
defer f.mu.Unlock()
callerCtx := ctx
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
defer cancel()
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
defer cancelTab()
// Bind the caller's deadline to the tab.
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
defer cancelDeadline()
go func() {
<-ctx.Done()
cancelDeadline()
}()
if err := chromedp.Run(tabCtx,
chromedp.Navigate(target),
chromedp.WaitReady("body", chromedp.ByQuery),
); err != nil {
return classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
}
var lastErr error
for {
// The challenge reloads the page when it passes, which tears down the
// execution context mid-read. That is a retry, not a failure.
if err := chromedp.Run(tabCtx, read); err != nil {
err = classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
if errors.Is(err, errBrowserInterrupted) {
return err
}
lastErr = err
} else if done() {
return nil
}
select {
case <-ctx.Done():
if err := callerCtx.Err(); err != nil {
return err
}
if lastErr != nil {
return fmt.Errorf("%w (last read: %v)", errChallengeHeld, lastErr)
}
return errChallengeHeld
case <-time.After(challengePollInterval):
}
}
}
// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
// chapter list. Returning false for anything else is a second line of defence
// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
+1 -19
View File
@@ -1,10 +1,6 @@
package latest
import (
"context"
"errors"
"testing"
)
import "testing"
func TestKaganeAPIURL(t *testing.T) {
const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
@@ -61,17 +57,3 @@ func TestNovelfullSeriesURL(t *testing.T) {
})
}
}
func TestClassifyBrowserInterruption(t *testing.T) {
if err := classifyBrowserError(context.Background(), true, context.Canceled); !errors.Is(err, errBrowserInterrupted) {
t.Fatalf("classifyBrowserError(context.Canceled) = %v, want browser interruption", err)
}
if err := classifyBrowserError(context.Background(), false, context.Canceled); errors.Is(err, errBrowserInterrupted) {
t.Fatalf("ordinary cancellation misclassified as browser interruption: %v", err)
}
caller, cancel := context.WithCancel(context.Background())
cancel()
if err := classifyBrowserError(caller, true, context.Canceled); errors.Is(err, errBrowserInterrupted) {
t.Fatalf("caller cancellation misclassified as browser interruption: %v", err)
}
}
-204
View File
@@ -1,204 +0,0 @@
package latest
import (
"context"
"errors"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/netip"
"net/url"
"strings"
"time"
"bookmarkmanager/backend/internal/store"
)
// CoverBytesFetcher retrieves one cover from its source URL. The caller owns
// persistence; this seam keeps network policy independent from the store.
type CoverBytesFetcher interface {
Fetch(ctx context.Context, sourceURL string) (body []byte, contentType string, err error)
}
// fetchCoverBytes routes a cover's byte retrieval by URL shape, not by Site
// name: the browser fetcher's module claims the addresses only it can fetch
// (kagane's image route answers a plain fetch with a challenge and
// `cross-origin-resource-policy: same-origin`), and everything else goes over
// plain TLS. Missing fetchers degrade to an error the caller logs, never a
// fallback onto a path that cannot succeed. One routing rule for the poll and
// the acquirer, so the two cannot drift apart.
func fetchCoverBytes(ctx context.Context, cover string, browser BrowserCoverFetcher, tls CoverBytesFetcher) ([]byte, string, error) {
if browserOnlyCoverURL(cover) {
if browser == nil {
return nil, "", errors.New("no cover fetcher")
}
return browser.Image(ctx, cover)
}
if tls == nil {
return nil, "", errors.New("no cover fetcher")
}
return tls.Fetch(ctx, cover)
}
// CoverResolver resolves a host before any connection is attempted. Tests
// inject it to exercise hostile DNS results without touching the live network.
type CoverResolver func(context.Context, string) ([]netip.Addr, error)
// TLSCoverFetcher retrieves image bytes with the standard HTTPS client. Unlike
// TLSFetcher, it does not need a browser fingerprint: cover hosts are public
// CDNs and the response is accepted only after the destination gate passes.
type TLSCoverFetcher struct {
client *http.Client
resolve CoverResolver
}
var _ CoverBytesFetcher = (*TLSCoverFetcher)(nil)
const coverRequestTimeout = 30 * time.Second
var carrierGradeNAT = netip.MustParsePrefix("100.64.0.0/10")
// NewCoverFetcher builds the production cover client with the real resolver.
func NewCoverFetcher() *TLSCoverFetcher {
return NewCoverFetcherWithResolver(nil)
}
// NewCoverFetcherWithResolver builds a cover client using resolve, or the real
// system resolver when resolve is nil.
func NewCoverFetcherWithResolver(resolve CoverResolver) *TLSCoverFetcher {
if resolve == nil {
resolve = defaultCoverResolver
}
return newCoverFetcher(newCoverHTTPClient(resolve), resolve)
}
func newCoverFetcher(client *http.Client, resolve CoverResolver) *TLSCoverFetcher {
f := &TLSCoverFetcher{client: client, resolve: resolve}
client.CheckRedirect = func(req *http.Request, _ []*http.Request) error {
if err := f.validateURL(req.Context(), req.URL); err != nil {
return fmt.Errorf("redirect destination: %w", err)
}
return nil
}
return f
}
func defaultCoverResolver(ctx context.Context, host string) ([]netip.Addr, error) {
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
}
func newCoverHTTPClient(resolve CoverResolver) *http.Client {
base, ok := http.DefaultTransport.(*http.Transport)
if !ok {
base = &http.Transport{}
}
transport := base.Clone()
// A proxy would make the dial target the proxy rather than the cover host,
// defeating destination classification. Cover fetching is direct by design.
transport.Proxy = nil
dialer := &net.Dialer{}
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return nil, fmt.Errorf("split cover address %q: %w", address, err)
}
addrs, err := resolveCoverHost(ctx, host, resolve)
if err != nil {
return nil, err
}
for _, addr := range addrs {
if !publicCoverAddress(addr) {
return nil, fmt.Errorf("cover host resolves to refused address %s", addr)
}
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(addr.String(), port))
if err == nil {
return conn, nil
}
}
return nil, fmt.Errorf("cover host %q has no reachable address", host)
}
return &http.Client{Transport: transport, Timeout: coverRequestTimeout}
}
func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
u, err := url.Parse(sourceURL)
if err != nil {
return nil, "", fmt.Errorf("parse cover URL: %w", err)
}
if err := f.validateURL(ctx, u); err != nil {
return nil, "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, "", fmt.Errorf("build cover request: %w", err)
}
resp, err := f.client.Do(req)
if err != nil {
return nil, "", fmt.Errorf("fetch cover: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("fetch cover: status %d", resp.StatusCode)
}
raw, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err != nil {
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", resp.Header.Get("Content-Type"))
}
contentType, ok := store.CoverContentType(raw)
if !ok {
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", raw)
}
if resp.ContentLength > maxBodyBytes {
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes+1))
if err != nil {
return nil, "", fmt.Errorf("read cover: %w", err)
}
if len(body) > maxBodyBytes {
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
}
return body, contentType, nil
}
// This gate deliberately differs from fetchableSeriesURL: cover hosts are
// site-independent CDNs, so a Site host allowlist would reject valid covers.
func (f *TLSCoverFetcher) validateURL(ctx context.Context, u *url.URL) error {
if u == nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
return errors.New("cover URL must use HTTPS without credentials")
}
host := u.Hostname()
if host == "" {
return errors.New("cover URL has no host")
}
addrs, err := resolveCoverHost(ctx, host, f.resolve)
if err != nil {
return fmt.Errorf("resolve cover host %q: %w", host, err)
}
if len(addrs) == 0 {
return fmt.Errorf("resolve cover host %q: no addresses", host)
}
for _, addr := range addrs {
if !publicCoverAddress(addr) {
return fmt.Errorf("cover host %q resolves to refused address %s", host, addr)
}
}
return nil
}
func resolveCoverHost(ctx context.Context, host string, resolve CoverResolver) ([]netip.Addr, error) {
if literal, err := netip.ParseAddr(host); err == nil {
return []netip.Addr{literal.Unmap()}, nil
}
return resolve(ctx, strings.TrimSuffix(host, "."))
}
func publicCoverAddress(addr netip.Addr) bool {
addr = addr.Unmap()
return addr.IsValid() && addr.IsGlobalUnicast() &&
!addr.IsLoopback() && !addr.IsPrivate() && !addr.IsLinkLocalUnicast() &&
!carrierGradeNAT.Contains(addr)
}
-226
View File
@@ -1,226 +0,0 @@
package latest
import (
"bytes"
"context"
"crypto/tls"
"io"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"testing"
)
func TestCoverFetcherFetchesPublicHTTPSImage(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.TLS == nil {
t.Fatal("cover request was not made over TLS")
}
w.Header().Set("Content-Type", "image/jpeg")
io.WriteString(w, "cover-bytes")
}))
defer server.Close()
transport := server.Client().Transport.(*http.Transport).Clone()
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // test server certificate
transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String())
}
client := &http.Client{Transport: transport}
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
})
body, contentType, err := fetcher.Fetch(context.Background(), "https://cdn.example/cover.jpg")
if err != nil {
t.Fatalf("Fetch: %v", err)
}
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
t.Fatalf("Fetch = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
}
}
func TestNewCoverFetcherRechecksResolverBeforeConnection(t *testing.T) {
var requests int
server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
requests++
}))
defer server.Close()
_, port, err := net.SplitHostPort(server.Listener.Addr().String())
if err != nil {
t.Fatalf("server address: %v", err)
}
resolves := 0
fetcher := NewCoverFetcherWithResolver(func(context.Context, string) ([]netip.Addr, error) {
resolves++
if resolves == 1 {
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
}
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
})
_, _, err = fetcher.Fetch(context.Background(), "https://cdn.example:"+port+"/cover.jpg")
if err == nil {
t.Fatal("Fetch accepted a destination that became private")
}
if resolves != 2 {
t.Fatalf("resolver calls = %d, want preflight and dial checks", resolves)
}
if requests != 0 {
t.Fatalf("requests = %d, want 0", requests)
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func coverResponse(status int, contentType, location string, body []byte) *http.Response {
header := make(http.Header)
if contentType != "" {
header.Set("Content-Type", contentType)
}
if location != "" {
header.Set("Location", location)
}
return &http.Response{
StatusCode: status,
Status: http.StatusText(status),
Header: header,
Body: io.NopCloser(bytes.NewReader(body)),
ContentLength: int64(len(body)),
}
}
func TestCoverFetcherRefusesUnsafeDestinationsBeforeRequest(t *testing.T) {
var calls int
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
return coverResponse(http.StatusOK, "image/jpeg", "", []byte("must not reach network")), nil
})}
resolve := func(_ context.Context, host string) ([]netip.Addr, error) {
switch host {
case "loopback.example":
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
case "private.example":
return []netip.Addr{netip.MustParseAddr("10.0.0.1")}, nil
case "linklocal.example":
return []netip.Addr{netip.MustParseAddr("169.254.1.1")}, nil
case "unique-local.example":
return []netip.Addr{netip.MustParseAddr("fc00::1")}, nil
case "cgnat.example":
return []netip.Addr{netip.MustParseAddr("100.64.0.1")}, nil
default:
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
}
}
fetcher := newCoverFetcher(client, resolve)
tests := []string{
"http://public.example/cover.jpg",
"https://127.0.0.1/cover.jpg",
"https://10.0.0.1/cover.jpg",
"https://169.254.1.1/cover.jpg",
"https://[fc00::1]/cover.jpg",
"https://100.64.0.1/cover.jpg",
"https://loopback.example/cover.jpg",
"https://private.example/cover.jpg",
"https://linklocal.example/cover.jpg",
"https://unique-local.example/cover.jpg",
"https://cgnat.example/cover.jpg",
}
for _, sourceURL := range tests {
t.Run(sourceURL, func(t *testing.T) {
calls = 0
if _, _, err := fetcher.Fetch(context.Background(), sourceURL); err == nil {
t.Fatal("Fetch accepted refused destination")
}
if calls != 0 {
t.Fatalf("network calls = %d, want 0", calls)
}
})
}
}
func TestCoverFetcherStopsRedirectIntoPrivateAddress(t *testing.T) {
var calls int
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
calls++
if req.URL.Hostname() != "cdn.example" {
t.Fatalf("redirect reached %s", req.URL)
}
return coverResponse(http.StatusFound, "", "https://internal.example/cover.jpg", nil), nil
})}
fetcher := newCoverFetcher(client, func(_ context.Context, host string) ([]netip.Addr, error) {
if host == "internal.example" {
return []netip.Addr{netip.MustParseAddr("192.168.1.1")}, nil
}
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
})
if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/cover.jpg"); err == nil {
t.Fatal("Fetch followed redirect into private address")
}
if calls != 1 {
t.Fatalf("network calls = %d, want only public first hop", calls)
}
}
func TestCoverFetcherRejectsOversizedBody(t *testing.T) {
var calls int
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxBodyBytes+1))
response.ContentLength = -1
return response, nil
})}
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
})
if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/large.webp"); err == nil {
t.Fatal("Fetch accepted oversized body")
}
if calls != 1 {
t.Fatalf("network calls = %d, want 1", calls)
}
}
func TestCoverFetcherRejectsNonImage(t *testing.T) {
var calls int
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
return coverResponse(http.StatusOK, "text/html", "", []byte("challenge")), nil
})}
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
})
if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/challenge"); err == nil {
t.Fatal("Fetch accepted non-image response")
}
if calls != 1 {
t.Fatalf("network calls = %d, want 1", calls)
}
}
// comix labels its covers "image/jpg", which is not a registered type but is
// what the Site actually answers with; the bytes are stored under the real
// name so one image cannot land under two spellings.
func TestCoverFetcherCanonicalisesJpgAlias(t *testing.T) {
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return coverResponse(http.StatusOK, "image/jpg", "", []byte("cover-bytes")), nil
})}
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
})
body, contentType, err := fetcher.Fetch(context.Background(), "https://static.comix.to/cover.jpg")
if err != nil {
t.Fatalf("Fetch: %v", err)
}
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
t.Fatalf("Fetch = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
}
}
+70 -146
View File
@@ -4,7 +4,6 @@ import (
"context"
"log"
"net/url"
"slices"
"time"
"bookmarkmanager/backend/internal/store"
@@ -16,13 +15,6 @@ type Fetcher interface {
Get(ctx context.Context, url string) (body string, status int, err error)
}
// BrowserCoverFetcher retrieves one cover's bytes through the browser-backed
// path — the only route that clears the challenge kagane's image URLs answer
// a plain fetch with. Satisfied by BrowserFetcher.
type BrowserCoverFetcher interface {
Image(ctx context.Context, imageURL string) (body []byte, contentType string, err error)
}
// Poller re-checks each bookmarked series' newest published chapter on a
// schedule, independent of the userscript's own in-browser checks. The two run
// in parallel and report the same observable fact, so whichever writes last wins
@@ -31,116 +23,37 @@ type BrowserCoverFetcher interface {
// Two clocks, deliberately independent:
//
// - Interval is how often this goroutine wakes up and looks.
// - Cooldowns are how long a series rests since its own last check. Browser-
// backed sites use the longer BrowserCooldown.
// - Cooldown is how long one bookmark rests since its own last check.
//
// Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
// by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
// it only makes the poller wake up and find nothing due more often.
// Only the cooldown is per bookmark, and it is enforced by the WHERE clause in
// DueForLatestCheck rather than by any timer. Shortening Interval therefore
// cannot shorten anyone's cooldown; it only makes the poller wake up and find
// nothing due more often.
type Poller struct {
Store *store.Store
Fetch Fetcher
Store *store.Store
Fetch Fetcher
// BrowserFetch handles sites behind a JavaScript challenge that Fetch
// cannot clear. Nil disables those sites entirely rather than falling back
// to Fetch, which would only ever retrieve a challenge page.
BrowserFetch Fetcher
// CoverFetch is optional; failures are logged and never affect the chapter poll.
CoverFetch BrowserCoverFetcher
// CoverBytesFetch is optional; it handles plain-TLS sources through the
// same failure-isolated prefetch path.
CoverBytesFetch CoverBytesFetcher
Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
var browserBackedSites = []string{"kagane", "novelfull"}
// fillBlankCover gives a Series its Cover when it has none. The blank state is
// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows
// created before acquisition existed, and rows whose creation-time fetch
// failed, both heal here. A non-blank CoverAddress is left alone — refetching
// would add a request per Series per cycle and change artwork under the Reader
// for no visible reason. A row that already carries a source URL is owned by
// prefetchCover instead; this path only extracts from the series page.
//
// Failures are logged against the Series and never returned: the chapter poll
// must not notice. A failed fill is retried the next time this Series is due;
// there is no separate retry queue.
func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, body string) {
if sr.CoverAddress != "" || sr.Cover != "" {
return
}
cover, ok := coverFrom(sr.Site, sr.SeriesURL, body)
if !ok {
return
}
p.storeCover(ctx, sr, cover)
}
// prefetchCover heals Series that already carry a third-party source URL but
// no stored address — the state left by client-supplied covers before
// acquisition moved server-side. Every Site takes the same path; fetchCoverBytes
// routes by URL shape, so browser-claimed URLs still need the sidecar. New
// blanks have no source URL and go through fillBlankCover from the series page
// instead.
func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) {
if sr.Cover == "" || sr.CoverAddress != "" {
return
}
body, contentType, found, err := p.Store.GetCover(sr.Cover)
if err != nil {
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
return
}
if found {
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil {
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
}
return
}
p.storeCover(ctx, sr, sr.Cover)
}
// storeCover fetches bytes for sourceURL and points the Series at them. Every
// failure is logged against the Series and swallowed so the chapter poll
// cannot see it.
func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL string) {
bytes, contentType, err := fetchCoverBytes(ctx, sourceURL, p.CoverFetch, p.CoverBytesFetch)
if err != nil {
log.Printf("latest poll %q: fetch cover %s: %v", sr.Key(), sourceURL, err)
return
}
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sourceURL, bytes, contentType); err != nil {
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
}
}
// fetcherFor returns the fetcher a site's page needs, or nil when the site
// cannot be fetched at all right now. kagane and novelfull pages sit behind a
// Cloudflare JavaScript challenge that no TLS fingerprint clears (kagane
// verified 2026-08-03, novelfull verified 2026-08-05, both against the same
// Chrome_133 profile TLSFetcher uses), so both prefer the browser; novelfull
// alone falls back to the plain-TLS fetcher when no browser is configured,
// because its challenge is a live time-varying fact (AGENTS.md) and its cover
// bytes never need the browser. kagane never falls back: a plain fetch of a
// kagane page or cover would only ever retrieve a challenge page. One routing
// rule for the poll and the acquirer, so the two cannot drift apart.
func fetcherFor(site string, browser, tls Fetcher) Fetcher {
switch {
case site == "kagane":
return browser
case slices.Contains(browserBackedSites, site): // novelfull
if browser != nil {
return browser
}
return tls
default:
return tls
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
// JavaScript challenge that no TLS fingerprint clears — kagane verified
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
// profile TLSFetcher uses — so they are browser-only or nothing.
func (p *Poller) fetcherFor(site string) Fetcher {
switch site {
case "kagane", "novelfull":
return p.BrowserFetch
}
return p.Fetch
}
// Run polls until ctx is cancelled.
@@ -150,8 +63,8 @@ func fetcherFor(site string, browser, tls Fetcher) Fetcher {
// failure mode for a misconfigured batch x stagger: a slower cadence, never
// concurrent fetch storms.
func (p *Poller) Run(ctx context.Context) {
log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s",
p.Interval, p.Cooldown, p.Batch, p.Stagger)
t := time.NewTicker(p.Interval)
defer t.Stop()
for {
@@ -165,19 +78,17 @@ func (p *Poller) Run(ctx context.Context) {
}
}
// runOnce processes one batch of due series.
// runOnce processes one batch of due bookmarks.
func (p *Poller) runOnce(ctx context.Context) {
now := p.Now()
cutoff := now.Add(-p.Cooldown).UnixMilli()
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
cutoff := p.Now().Add(-p.Cooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, p.Batch)
if err != nil {
log.Printf("latest poll: due query: %v", err)
return
}
checked := 0
for i, sr := range due {
for i, b := range due {
if ctx.Err() != nil {
break
}
@@ -196,7 +107,7 @@ func (p *Poller) runOnce(ctx context.Context) {
if stopped {
break
}
p.checkOne(ctx, sr)
p.checkOne(ctx, b)
checked++
}
// due vs checked is how you tell which constraint is binding: ticks that
@@ -208,10 +119,10 @@ func (p *Poller) runOnce(ctx context.Context) {
// checkOne re-checks one series. Every failure path here is "log and move on":
// the poller is a best-effort enhancement, and no single bad series may stall a
// batch or take down the process.
func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
defer func() {
if r := recover(); r != nil {
log.Printf("latest poll %q: recovered from panic: %v", sr.Key(), r)
log.Printf("latest poll %q: recovered from panic: %v", b.Key, r)
}
}()
@@ -219,8 +130,8 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
// mid-request still consumes the cooldown. Otherwise a renamed or deleted
// series would be retried on every single tick forever. The userscript
// stamps in the same order and for the same reason (L471-473).
if err := p.Store.MarkLatestChecked(sr.Site, sr.SeriesID, p.Now().UnixMilli()); err != nil {
log.Printf("latest poll %q: mark checked: %v", sr.Key(), err)
if err := p.Store.MarkLatestChecked(b.Key, p.Now().UnixMilli()); err != nil {
log.Printf("latest poll %q: mark checked: %v", b.Key, err)
return
}
@@ -231,56 +142,69 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
// link-local/internal addresses or non-https schemes. The cooldown above
// is already consumed, so a row that never passes this check is retried at
// cooldown pace rather than hot-looping.
if !fetchableSeriesURL(sr.Site, sr.SeriesURL) {
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
if !fetchableSeriesURL(b.Site, b.SeriesURL) {
log.Printf("latest poll %q: not fetchable: site=%q url=%q", b.Key, b.Site, b.SeriesURL)
return
}
f := fetcherFor(sr.Site, p.BrowserFetch, p.Fetch)
f := p.fetcherFor(b.Site)
if f == nil {
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
log.Printf("latest poll %q: no fetcher for site %q", b.Key, b.Site)
return
}
p.prefetchCover(ctx, sr)
body, status, err := f.Get(ctx, sr.SeriesURL)
body, status, err := f.Get(ctx, b.SeriesURL)
if err != nil {
log.Printf("latest poll %q: fetch %s: %v", sr.Key(), sr.SeriesURL, err)
log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
return
}
if status != 200 {
log.Printf("latest poll %q: fetch %s: status %d", sr.Key(), sr.SeriesURL, status)
log.Printf("latest poll %q: fetch %s: status %d", b.Key, b.SeriesURL, status)
return
}
latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body)
// Cover fill is independent of the chapter signal: a page that lost its
// chapter list may keep its og:image, and a blank Series heals either way.
p.fillBlankCover(ctx, sr, body)
latest, ok := latestChapterFrom(b.Site, b.SeriesURL, body)
if !ok {
// Most likely a challenge page or a layout change. Either way the row is
// already stamped, so this waits out a cooldown instead of hot-looping.
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), len(body))
log.Printf("latest poll %q: no chapter links in %d bytes", b.Key, len(body))
return
}
// Re-read: the row may have been updated or deleted while the fetch was in
// flight, and writing b back wholesale would undo that.
//
// ponytail: non-transactional read-modify-write, wrap Get+Upsert in a tx if
// this ever runs for more than one user. A client PUT that commits between
// these two statements is lost to the stale re-read — reverting read
// progress or a status change, and moving updated_at because the stored
// value now differs. Accepted for a single-user deployment: the window is
// milliseconds and the loser is one poll cycle.
cur, found, err := p.Store.Get(b.Key)
if err != nil {
log.Printf("latest poll %q: reread: %v", b.Key, err)
return
}
if !found {
return
}
// Equality, not >, mirroring the userscript (L427): a site that retracts a
// chapter should correct the stored number downward. The comparison is
// against the due-query snapshot; a concurrent write in between only costs
// one redundant UPDATE of the same absolute value, never a wrong one.
if sr.LatestChapterNum != nil && *sr.LatestChapterNum == latest.Num {
// chapter should correct the stored number downward.
if cur.LatestChapterNum != nil && *cur.LatestChapterNum == latest.Num {
return
}
// Series-level write: the row is shared, so one update refreshes every
// bookmark joining to it, and the bookmark's updated_at is never touched —
// a newly published chapter is not reading progress and must not reorder
// the list.
if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, latest.Label, latest.Num); err != nil {
log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err)
num := latest.Num
cur.LatestChapter = latest.Label
cur.LatestChapterNum = &num
// A candidate only. last_chapter_num is untouched, so the CASE in Upsert
// keeps the stored updated_at and the bookmark list does not reorder.
cur.UpdatedAt = p.Now().UnixMilli()
if _, err := p.Store.Upsert(cur); err != nil {
log.Printf("latest poll %q: upsert: %v", b.Key, err)
return
}
log.Printf("latest poll %q: latest is now %s", sr.Key(), latest.Label)
log.Printf("latest poll %q: latest is now %s", b.Key, latest.Label)
}
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
File diff suppressed because it is too large Load Diff
+11 -145
View File
@@ -1,12 +1,12 @@
package latest
import (
"encoding/json"
"html"
"net/url"
"regexp"
"strconv"
"strings"
"bookmarkmanager/backend/internal/store"
)
// latestChapter is the newest chapter a series page advertises.
@@ -22,12 +22,6 @@ type latestChapter struct {
// before using the slug to scope anything.
var asuraSlugRe = regexp.MustCompile(`/comics/([^/?#]+)`)
// asuraBuildHash matches the trailing "-xxxxxxxx" site-wide build ID Asura
// appends to every series slug. It rotates on each site redeploy, so it is
// never part of a stable series_id. Must stay in sync with stripBuildHash in
// userscript/manga-bookmark.user.js.
var asuraBuildHash = regexp.MustCompile(`-[0-9a-f]{8}$`)
// demonicChapterRe matches the pre-redirect anchors demonic series pages link
// through. Both the raw "&" and the HTML-escaped "&amp;" forms occur.
var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?chapter=([0-9.]+)`)
@@ -36,18 +30,6 @@ var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?ch
// Only the id prefix is stable; the slug tail follows the title.
var comixSlugRe = regexp.MustCompile(`/title/([^/?#]+)`)
func comixSeriesID(seriesURL string) (string, bool) {
m := comixSlugRe.FindStringSubmatch(seriesURL)
if m == nil {
return "", false
}
id := m[1]
if i := strings.Index(id, "-"); i != -1 {
id = id[:i]
}
return id, true
}
// kaganeChapterRe matches the chapter numbers in a kagane API response. This
// branch is fed by the browser fetcher, so the body is JSON rather than HTML —
// there are no anchors to scan.
@@ -92,23 +74,27 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
}
// Stored URLs predating a redeploy may carry a stale build hash;
// chapter hrefs in the fetched body carry the current one. Strip to
// the stable ID and make the hash optional in the pattern, so scoping
// survives rotations.
slug := asuraBuildHash.ReplaceAllString(m[1], "")
// the stable ID (same rule as migrateAsuraKeys) and make the hash
// optional in the pattern, so scoping survives rotations.
slug := store.AsuraBuildHash.ReplaceAllString(m[1], "")
// Compiled per call rather than cached: this runs once per fetch, which
// is at most a few times a minute, and the slug varies per series.
re = regexp.MustCompile(`/comics/` + regexp.QuoteMeta(slug) + `(?:-[0-9a-f]{8})?/chapter/([0-9.]+)`)
case "demonic":
re = demonicChapterRe
case "comix":
id, ok := comixSeriesID(seriesURL)
if !ok {
m := comixSlugRe.FindStringSubmatch(seriesURL)
if m == nil {
return latestChapter{}, false
}
// comix ships an SPA: the served HTML carries a JSON state blob instead
// of chapter anchors, and latestChapterUrl is the only place the newest
// chapter appears. Scoping to this series' id prefix keeps a
// "recommended" strip's entries from winning the maximum.
id := m[1]
if i := strings.Index(id, "-"); i != -1 {
id = id[:i]
}
re = regexp.MustCompile(`"latestChapterUrl":"/title/` + regexp.QuoteMeta(id) + `-[^"]*-chapter-([0-9.]+)"`)
case "kagane":
re = kaganeChapterRe
@@ -156,123 +142,3 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
}
return best, found
}
var metaTagRe = regexp.MustCompile(`(?is)<meta\b[^>]*>`)
var doubleQuotedMetaAttrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*"([^"]*)"`)
var singleQuotedMetaAttrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*'([^']*)'`)
// comix's server-rendered page embeds query data in this JSON script; parsing
// the target detail entry avoids matching posters from recommended results.
var comixInitialDataRe = regexp.MustCompile(`(?is)<script\b[^>]*\bid\s*=\s*["']initial-data["'][^>]*>(.*?)</script>`)
// kaganeImageURLRe matches the canonical compressed image route kagane's API
// publishes — the only cover URL form the extractor emits and the browser
// fetcher accepts. The URL is matched in full (scheme, host, id shape) rather
// than trusted: the value a fetcher is pointed at may have been client-
// supplied, and a headless browser is a strong SSRF primitive.
var kaganeImageURLRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`)
// browserOnlyCoverURL reports whether the browser sidecar is the only fetcher
// for cover bytes at imageURL. kagane's image route answers a plain fetch with
// a challenge and `cross-origin-resource-policy: same-origin`, so a TLS fetch
// would only ever retrieve a challenge page and must not be attempted
// (ADR-0007). This is the byte-fetch router's per-Site knowledge; it lives in
// the extraction module, which owns kagane's URL shapes.
func browserOnlyCoverURL(imageURL string) bool {
return kaganeImageURLRe.MatchString(imageURL)
}
// kagane's browser-fetched series response publishes cover image IDs under
// series_covers. The API's canonical compressed image route is the only URL
// form accepted by the store and browser fetcher; no rendition is guessed.
func kaganeCoverURL(body string) string {
var response struct {
SeriesCovers []struct {
ImageID string `json:"image_id"`
} `json:"series_covers"`
}
if err := json.Unmarshal([]byte(body), &response); err != nil {
return ""
}
for _, cover := range response.SeriesCovers {
// Validate the assembled URL against the same regex the browser
// fetcher enforces, so the extractor can never emit an address the
// fetch would refuse.
imageURL := "https://kagane.to/api/v2/image/" + cover.ImageID + "/compressed"
if kaganeImageURLRe.MatchString(imageURL) {
return imageURL
}
}
return ""
}
func comixCoverURL(seriesURL, body string) string {
id, ok := comixSeriesID(seriesURL)
if !ok {
return ""
}
data := comixInitialDataRe.FindStringSubmatch(body)
if data == nil {
return ""
}
var state struct {
Queries map[string]json.RawMessage `json:"queries"`
}
if err := json.Unmarshal([]byte(data[1]), &state); err != nil {
return ""
}
raw := state.Queries[`["manga","detail","`+id+`"]`]
if len(raw) == 0 {
return ""
}
var detail struct {
Poster struct {
Medium string `json:"medium"`
} `json:"poster"`
}
if err := json.Unmarshal(raw, &detail); err != nil {
return ""
}
return publishedCoverURL(detail.Poster.Medium)
}
// coverFrom reports false for unknown sites, challenge bodies, and pages with
// no usable cover. Metadata extraction keeps scanning after an empty match so
// a later published cover is not hidden by an empty tag.
func coverFrom(site, seriesURL, body string) (string, bool) {
var cover string
switch site {
case "asura", "demonic", "lightnovelworld":
cover = metaContent(body, "property", "og:image")
case "novelfull":
cover = metaContent(body, "name", "image")
case "comix":
cover = comixCoverURL(seriesURL, body)
case "kagane":
cover = kaganeCoverURL(body)
}
return cover, cover != ""
}
func metaContent(body, attrName, attrValue string) string {
for _, tag := range metaTagRe.FindAllString(body, -1) {
attrs := make(map[string]string)
for _, m := range doubleQuotedMetaAttrRe.FindAllStringSubmatch(tag, -1) {
attrs[strings.ToLower(m[1])] = m[2]
}
for _, m := range singleQuotedMetaAttrRe.FindAllStringSubmatch(tag, -1) {
attrs[strings.ToLower(m[1])] = m[2]
}
if strings.EqualFold(attrs[strings.ToLower(attrName)], attrValue) {
if cover := publishedCoverURL(attrs["content"]); cover != "" {
return cover
}
}
}
return ""
}
func publishedCoverURL(value string) string {
value = strings.TrimSpace(html.UnescapeString(value))
return strings.ReplaceAll(value, " ", "%20")
}
-113
View File
@@ -79,119 +79,6 @@ const lnwSeriesFixture = `
<a href="https://lightnovelworld.net/overgeared-chapter-9999/">Chapter 9999</a>
`
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
// (redirected to ...-00dcbf97) on 2026-08-10.
const asuraCoverFixture = `<meta property="og:image" content="https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp">`
// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer on 2026-08-10.
// The source publishes the raw space in this URL.
const demonicCoverFixture = `<meta property="og:image" content="https://readermc.org/images/thumbnails/Catastrophic Necromancer.webp">`
// Trimmed from https://comix.to/title/n8we-dungeons-and-crayons on 2026-08-10.
// The state includes a recommended poster before the target detail object and
// nested IDs inside that object; no og:image is present.
const comixCoverFixture = `<script type="application/json" id="initial-data">{"queries":{"[\"manga\",\"recommended\",\"n8we\",1]":{"poster":{"medium":"https://static.comix.to/recommended@280.jpg","large":"https://static.comix.to/recommended.jpg"}},"[\"manga\",\"detail\",\"n8we\"]":{"chapters":[{"hid":"nested"}],"poster":{"medium":"https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg","large":"https://static.comix.to/039d/i/1/34/6a6742bf15736.jpg"}}}}</script>`
// Trimmed from GET https://kagane.to/api/v2/series/019fe11a-8670-7cf3-8343-0b02057d3787 on 2026-08-10.
const kaganeCoverFixture = `{"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}]}`
// Trimmed from https://novelfull.com/reverend-insanity.html on 2026-08-10.
const novelfullCoverFixture = `<meta name="image" content="https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp">`
// Trimmed from https://lightnovelworld.net/novel/a-will-eternal/ on 2026-08-10.
const lnwCoverFixture = `<meta content='https://lightnovelworld.net/wp-content/uploads/2026/03/a-will-eternal-1.webp' property='og:image'>`
func TestCoverFrom(t *testing.T) {
const comixURL = "https://comix.to/title/n8we-dungeons-and-crayons"
tests := []struct {
name string
site string
seriesURL string
body string
wantOK bool
wantCover string
}{
{
name: "asura uses published metadata URL",
site: "asura", body: asuraCoverFixture, wantOK: true,
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
},
{
name: "demonic escapes raw spaces",
site: "demonic", body: demonicCoverFixture, wantOK: true,
wantCover: "https://readermc.org/images/thumbnails/Catastrophic%20Necromancer.webp",
},
{
name: "comix takes target medium poster",
site: "comix", seriesURL: comixURL, body: comixCoverFixture, wantOK: true,
wantCover: "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg",
},
{
name: "kagane reads API cover image ID",
site: "kagane", body: kaganeCoverFixture, wantOK: true,
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
},
{
name: "novelfull reads image metadata",
site: "novelfull", body: novelfullCoverFixture, wantOK: true,
wantCover: "https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp",
},
{
name: "lightnovelworld reads og image",
site: "lightnovelworld", body: lnwCoverFixture, wantOK: true,
wantCover: "https://lightnovelworld.net/wp-content/uploads/2026/03/a-will-eternal-1.webp",
},
{
name: "later metadata cover survives empty match",
site: "asura",
body: `<meta property="og:image" content="">` + asuraCoverFixture,
wantOK: true,
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
},
{
name: "page without cover is empty",
site: "asura", body: `<meta property="og:title" content="No Cover">`,
},
{
name: "unknown site is empty",
site: "unknown", body: asuraCoverFixture,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := coverFrom(tt.site, tt.seriesURL, tt.body)
if ok != tt.wantOK {
t.Fatalf("ok = %v, want %v (got %q)", ok, tt.wantOK, got)
}
if got != tt.wantCover {
t.Errorf("cover = %q, want %q", got, tt.wantCover)
}
})
}
}
func TestCoverFromChallenge(t *testing.T) {
tests := []struct {
site string
seriesURL string
}{
{"asura", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"},
{"demonic", "https://demonicscans.org/manga/Catastrophic-Necromancer"},
{"comix", "https://comix.to/title/n8we-dungeons-and-crayons"},
{"kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"},
{"novelfull", "https://novelfull.com/reverend-insanity.html"},
{"lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/"},
}
for _, tt := range tests {
t.Run(tt.site, func(t *testing.T) {
if got, ok := coverFrom(tt.site, tt.seriesURL, challengeFixture); ok || got != "" {
t.Fatalf("cover = %q, ok = %v, want empty", got, ok)
}
})
}
}
func TestLatestChapterFrom(t *testing.T) {
const asuraURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
const demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
-157
View File
@@ -1,157 +0,0 @@
package latest
import (
"context"
"net/http"
"os"
"testing"
"time"
"bookmarkmanager/backend/internal/store"
)
// TestSmokeKaganeImage is the live proof that the acquisition path's browser
// fetch actually clears Cloudflare and returns image bytes. It needs the real
// browser unit with outbound network, so it runs only when SMOKE_BROWSER_WS_URL
// is set:
//
// cd chrome && BROWSER_BIND_ADDR=127.0.0.1 docker compose up -d --build
// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeKaganeImage ./internal/latest
//
// Not chromedp/headless-shell: its challenge never clears (see chrome/Dockerfile),
// so a red run there proves nothing about kagane.
func TestSmokeKaganeImage(t *testing.T) {
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
if ws == "" {
t.Skip("SMOKE_BROWSER_WS_URL unset")
}
const imageURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed" // SP Baby's cover
// The same URL through a plain client is what any other fetcher would get.
// Asserting on it keeps the test honest about why the browser is needed.
req, err := http.NewRequest(http.MethodGet, imageURL, nil)
if err != nil {
t.Fatal(err)
}
if res, err := (&http.Client{Timeout: 15 * time.Second}).Do(req); err == nil {
res.Body.Close()
if res.StatusCode == http.StatusOK {
t.Log("note: kagane answered a plain request 200 — the challenge is not up right now")
}
}
f, err := NewBrowserFetcher(ws)
if err != nil {
t.Fatalf("NewBrowserFetcher: %v", err)
}
defer f.Close()
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
body, contentType, err := f.Image(ctx, imageURL)
if err != nil {
t.Fatalf("Image: %v", err)
}
if len(body) < 1000 {
t.Fatalf("body is %d bytes, want a real image", len(body))
}
if contentType != "image/webp" {
t.Fatalf("content type = %q, want image/webp", contentType)
}
// WebP files start with "RIFF....WEBP".
if string(body[:4]) != "RIFF" || string(body[8:12]) != "WEBP" {
t.Fatalf("body is not a WebP: % x", body[:12])
}
t.Logf("fetched %d bytes of %s", len(body), contentType)
// The browser module claims only the cover URL shape it can clear a
// challenge for; anything else must be refused before any navigation.
if _, _, err := f.Image(ctx, "https://cdn.example/cover.jpg"); err == nil {
t.Fatal("Image accepted a cover URL the browser module does not claim")
}
}
// Control for the test above: the poller's own kagane path, same sidecar. If
// this fails too, the sidecar is not clearing the challenge at all and the
// image result says nothing about Image itself.
func TestSmokeKaganeGet(t *testing.T) {
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
if ws == "" {
t.Skip("SMOKE_BROWSER_WS_URL unset")
}
f, err := NewBrowserFetcher(ws)
if err != nil {
t.Fatalf("NewBrowserFetcher: %v", err)
}
defer f.Close()
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
body, status, err := f.Get(ctx, "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787")
if err != nil {
t.Fatalf("Get: %v", err)
}
t.Logf("status=%d bytes=%d head=%.80q", status, len(body), body)
if status != 200 {
t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status)
}
}
// TestSmokeAcquireKaganeCover proves the #62 acquisition path end to end
// against the real browser: a kagane Series bookmarked at creation gets its
// Cover, bytes fetched through the sidecar into the content-addressed store.
// Same SMOKE_BROWSER_WS_URL gate as the tests above; a red run means the
// challenge is not clearing from this IP (a live fact to re-check), not
// necessarily a defect in the pipeline.
func TestSmokeAcquireKaganeCover(t *testing.T) {
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
if ws == "" {
t.Skip("SMOKE_BROWSER_WS_URL unset")
}
const (
seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
coverURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed"
)
s, _ := newTestStore(t)
bf, err := NewBrowserFetcher(ws)
if err != nil {
t.Fatalf("NewBrowserFetcher: %v", err)
}
defer bf.Close()
tlsF, err := NewTLSFetcher()
if err != nil {
t.Fatalf("NewTLSFetcher: %v", err)
}
acq := &Acquirer{
Store: s, Fetch: tlsF, BrowserFetch: bf,
BrowserCoverFetch: bf, Covers: NewCoverFetcher(),
}
s.OnSeriesCreated = acq.Acquire
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: "kagane:" + seriesID, Site: "kagane", SeriesID: seriesID,
Title: "smoke", SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
acq.Wait()
got, found, err := s.Get(s.OwnerID(), "kagane:"+seriesID)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
t.Fatalf("Cover = %q, want %q — the acquire path did not store the browser-fetched bytes", got.Cover, want)
}
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
if err != nil || !ok {
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
}
if len(body) < 1000 {
t.Fatalf("stored cover is %d bytes, want a real image", len(body))
}
if contentType != "image/webp" {
t.Fatalf("content type = %q, want image/webp", contentType)
}
t.Logf("stored %d bytes of %s", len(body), contentType)
}
-120
View File
@@ -1,120 +0,0 @@
// Package pgtest runs the Postgres the test suite needs: one throwaway
// container per test binary, one fresh database per test. Docker is therefore
// a hard prerequisite for `go test ./...`.
//
// Rolled by hand rather than pulled in as a dependency — it is one `docker
// run`, one `docker port` and a ping loop, against a module list that is
// otherwise stdlib plus what the poller genuinely needs.
package pgtest
import (
"database/sql"
"fmt"
"os/exec"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
_ "github.com/jackc/pgx/v5/stdlib"
)
const (
image = "postgres:17-alpine"
readyLimit = 60 * time.Second
)
var (
adminURL string
dbSeq atomic.Int64
)
// Main starts the container, runs the package's tests and tears the container
// down. Every test package that touches the store calls it from TestMain:
//
// func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
func Main(m *testing.M) int {
id, url, err := start()
if err != nil {
fmt.Println("pgtest:", err)
return 1
}
defer exec.Command("docker", "rm", "-f", id).Run()
adminURL = url
return m.Run()
}
// URL creates a database of its own for t and returns a connection URL for it.
// Nothing drops it again: the container goes away wholesale when Main returns.
func URL(t testing.TB) string {
t.Helper()
if adminURL == "" {
t.Fatal("pgtest: no container; this package needs TestMain to call pgtest.Main")
}
// Generated, never derived from the test name, so it needs no quoting and
// cannot collide when tests run in parallel.
name := "test_" + strconv.FormatInt(dbSeq.Add(1), 10)
admin, err := sql.Open("pgx", adminURL)
if err != nil {
t.Fatalf("pgtest: open admin connection: %v", err)
}
defer admin.Close()
if _, err := admin.Exec(`CREATE DATABASE ` + name); err != nil {
t.Fatalf("pgtest: create database %s: %v", name, err)
}
return strings.Replace(adminURL, "/postgres?", "/"+name+"?", 1)
}
// start launches the container and waits for it to accept queries, returning
// its id and a connection URL for the default database.
func start() (id, url string, err error) {
out, err := exec.Command("docker", "run", "-d", "--rm",
"-e", "POSTGRES_PASSWORD=pgtest",
"-P", image,
// Durability buys nothing for a database that dies with the test
// binary, and turning it off is most of the container's start-up cost.
"-c", "fsync=off", "-c", "full_page_writes=off",
).Output()
if err != nil {
return "", "", fmt.Errorf("docker run %s: %w", image, err)
}
id = strings.TrimSpace(string(out))
port, err := exec.Command("docker", "port", id, "5432/tcp").Output()
if err != nil {
exec.Command("docker", "rm", "-f", id).Run()
return "", "", fmt.Errorf("docker port: %w", err)
}
// "0.0.0.0:32768" (and possibly a second, IPv6 line); the port is all we want.
first, _, _ := strings.Cut(strings.TrimSpace(string(port)), "\n")
url = fmt.Sprintf("postgres://postgres:pgtest@127.0.0.1:%s/postgres?sslmode=disable",
first[strings.LastIndex(first, ":")+1:])
if err := waitReady(url); err != nil {
exec.Command("docker", "rm", "-f", id).Run()
return "", "", err
}
return id, url, nil
}
func waitReady(url string) error {
db, err := sql.Open("pgx", url)
if err != nil {
return err
}
defer db.Close()
deadline := time.Now().Add(readyLimit)
for {
if err = db.Ping(); err == nil {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("postgres not ready after %s: %w", readyLimit, err)
}
time.Sleep(200 * time.Millisecond)
}
}
+51 -19
View File
@@ -1,10 +1,13 @@
package session
import (
"crypto/rand"
"encoding/hex"
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"net"
"net/http"
"strconv"
"strings"
"sync"
"time"
@@ -13,18 +16,49 @@ import (
const (
CookieName = "bmgr_session"
// 60 days: long enough that a phone stays logged in between reading spells.
SessionTTL = 60 * 24 * time.Hour
sessionTTL = 60 * 24 * time.Hour
// Domain separation, so the session key can never collide with any other
// use of the secrets it is derived from. Changing this string logs
// everyone out.
sessionKeyPurpose = "bmgr-web-session-v1"
)
// NewID returns an opaque session id: 32 random bytes, hex-encoded. The id is
// all the cookie carries and all the sessions table keys on, so its entropy is
// what stops a guessed id from being someone else's session.
func NewID() string {
var b [32]byte
if _, err := rand.Read(b[:]); err != nil {
panic("session id: " + err.Error())
// Key derives the cookie-signing key from both secrets. Sessions are
// stateless — there is no session table — so rotating either API_TOKEN or
// WEB_PASSWORD invalidates every outstanding cookie at once. The \x00
// separator prevents the concatenation ambiguity a bare apiToken+webPassword
// would have (e.g. "ab"+"c" colliding with "a"+"bc").
func Key(apiToken, webPassword string) []byte {
sum := sha256.Sum256([]byte(apiToken + "\x00" + webPassword + sessionKeyPurpose))
return sum[:]
}
// Sign encodes "<expiryMs>.<base64url HMAC(expiryMs)>".
func Sign(key []byte, expiryMs int64) string {
payload := strconv.FormatInt(expiryMs, 10)
return payload + "." + sessionMAC(key, payload)
}
func sessionMAC(key []byte, payload string) string {
mac := hmac.New(sha256.New, key)
mac.Write([]byte(payload))
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
}
// Verify checks shape, then expiry, then the signature — in that order.
// The signature comparison is constant-time; the checks before it only look at
// data the holder already supplied, so their timing leaks nothing.
func Verify(key []byte, value string, nowMs int64) bool {
payload, sig, ok := strings.Cut(value, ".")
if !ok {
return false
}
return hex.EncodeToString(b[:])
expiry, err := strconv.ParseInt(payload, 10, 64)
if err != nil || expiry <= nowMs {
return false
}
want := sessionMAC(key, payload)
return subtle.ConstantTimeCompare([]byte(sig), []byte(want)) == 1
}
// isHTTPS reports whether the browser's connection is encrypted. Behind Traefik
@@ -35,14 +69,12 @@ func isHTTPS(r *http.Request) bool {
return r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"
}
// SetCookie writes the session cookie. The value is the session id and nothing
// else; the row behind it is looked up on every request.
func SetCookie(w http.ResponseWriter, r *http.Request, id string) {
func SetCookie(w http.ResponseWriter, r *http.Request, key []byte) {
http.SetCookie(w, &http.Cookie{
Name: CookieName,
Value: id,
Value: Sign(key, time.Now().Add(sessionTTL).UnixMilli()),
Path: "/",
MaxAge: int(SessionTTL / time.Second),
MaxAge: int(sessionTTL / time.Second),
HttpOnly: true,
Secure: isHTTPS(r),
SameSite: http.SameSiteLaxMode,
@@ -88,14 +120,14 @@ func ClientIP(r *http.Request) string {
return host
}
// LoginLimiter throttles failed sign-in attempts: MaxFailures failures inside
// a rolling Window blocks further attempts from that IP until the oldest one
// LoginLimiter throttles password guessing: MaxFailures failures inside a
// rolling Window blocks further attempts from that IP until the oldest one
// ages out. There is no permanent ban and no unlock step.
//
// Behind carrier-grade NAT this budget is shared with every other subscriber on
// the same public address, so a stranger can lock the owner out for up to one
// window. That is accepted: the block self-heals, and ten attempts is generous
// for the occasional fumbled sign-in.
// for a mistyped password.
//
// State is in memory and per-process, so a restart clears it. Entries are
// pruned lazily on access; for a single-user deployment the map cannot grow
+63 -19
View File
@@ -9,19 +9,66 @@ import (
"time"
)
func TestNewID(t *testing.T) {
a := NewID()
b := NewID()
if a == b {
t.Fatal("NewID returned the same value twice")
func TestSessionRoundTrip(t *testing.T) {
key := Key("token-abc", "pw-abc")
now := time.Now().UnixMilli()
value := Sign(key, now+60_000)
if !Verify(key, value, now) {
t.Fatal("Verify = false for a freshly signed cookie, want true")
}
if len(a) != 64 { // 32 random bytes, hex
t.Fatalf("NewID() length = %d, want 64", len(a))
}
func TestSessionRejects(t *testing.T) {
key := Key("token-abc", "pw-abc")
now := time.Now().UnixMilli()
valid := Sign(key, now+60_000)
payload, sig, _ := strings.Cut(valid, ".")
cases := []struct {
name string
value string
}{
{"empty", ""},
{"no separator", payload + sig},
{"unparseable expiry", "notanumber." + sig},
{"expired", Sign(key, now-1)},
{"tampered signature", payload + "." + flipLastChar(sig)},
{"tampered expiry", "99999999999999." + sig},
{"signed with another key", Sign(Key("other-token", "pw-abc"), now+60_000)},
}
for _, r := range a {
if !strings.ContainsRune("0123456789abcdef", r) {
t.Fatalf("NewID() = %q, want hex", a)
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if Verify(key, tc.value, now) {
t.Fatalf("Verify(%q) = true, want false", tc.value)
}
})
}
}
func flipLastChar(s string) string {
if s == "" {
return "x"
}
last := s[len(s)-1]
if last == 'A' {
return s[:len(s)-1] + "B"
}
return s[:len(s)-1] + "A"
}
func TestSessionKeyDependsOnToken(t *testing.T) {
a := Key("token-a", "pw-abc")
b := Key("token-b", "pw-abc")
if string(a) == string(b) {
t.Fatal("Key collided for different API tokens")
}
}
func TestSessionKeyDependsOnWebPassword(t *testing.T) {
a := Key("token-abc", "pw-a")
b := Key("token-abc", "pw-b")
if string(a) == string(b) {
t.Fatal("Key collided for different web passwords with the same API token")
}
}
@@ -39,7 +86,7 @@ func TestSetSessionCookieAttributes(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRequest(http.MethodPost, "/", nil)
r := httptest.NewRequest(http.MethodPost, "/login", nil)
if tc.tls {
r.TLS = &tls.ConnectionState{}
}
@@ -47,7 +94,7 @@ func TestSetSessionCookieAttributes(t *testing.T) {
r.Header.Set("X-Forwarded-Proto", tc.forwarded)
}
rr := httptest.NewRecorder()
SetCookie(rr, r, "abc123")
SetCookie(rr, r, Key("token-abc", "pw-abc"))
cookies := rr.Result().Cookies()
if len(cookies) != 1 {
@@ -57,9 +104,6 @@ func TestSetSessionCookieAttributes(t *testing.T) {
if c.Name != CookieName {
t.Fatalf("cookie name = %q, want %q", c.Name, CookieName)
}
if c.Value != "abc123" {
t.Fatalf("cookie value = %q, want the session id verbatim", c.Value)
}
if !c.HttpOnly {
t.Fatal("cookie HttpOnly = false, want true")
}
@@ -72,8 +116,8 @@ func TestSetSessionCookieAttributes(t *testing.T) {
if c.Secure != tc.wantSecure {
t.Fatalf("cookie Secure = %v, want %v", c.Secure, tc.wantSecure)
}
if c.MaxAge != int(SessionTTL/time.Second) {
t.Fatalf("cookie MaxAge = %d, want %d", c.MaxAge, int(SessionTTL/time.Second))
if c.MaxAge != int(sessionTTL/time.Second) {
t.Fatalf("cookie MaxAge = %d, want %d", c.MaxAge, int(sessionTTL/time.Second))
}
})
}
@@ -119,7 +163,7 @@ func TestClientIP(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRequest(http.MethodPost, "/", nil)
r := httptest.NewRequest(http.MethodPost, "/login", nil)
r.RemoteAddr = tc.remoteAddr
for _, v := range tc.xff {
r.Header.Add("X-Forwarded-For", v)
@@ -1,28 +0,0 @@
-- One row per tracked series, keyed "<site>:<series_id>".
--
-- Everything is NOT NULL with a default except latest_chapter_num, where NULL
-- is a distinct state: nothing has been captured yet, which is not the same as
-- chapter zero.
--
-- Timestamps are unix milliseconds as bigint, not timestamptz: the userscripts
-- send Date.now() over the wire and the ordering rule compares them directly.
CREATE TABLE bookmarks (
key text PRIMARY KEY,
site text NOT NULL,
series_id text NOT NULL,
title text NOT NULL DEFAULT '',
series_url text NOT NULL DEFAULT '',
cover text NOT NULL DEFAULT '',
last_chapter text NOT NULL DEFAULT '',
last_chapter_num double precision NOT NULL DEFAULT 0,
last_chapter_url text NOT NULL DEFAULT '',
favorite boolean NOT NULL DEFAULT false,
latest_chapter text NOT NULL DEFAULT '',
latest_chapter_num double precision,
-- When the server last polled this series, unix ms; 0 means never, and sorts
-- first so a new bookmark is picked up on the next tick with no special case.
latest_checked_at bigint NOT NULL DEFAULT 0,
status text NOT NULL DEFAULT 'reading',
kind text NOT NULL DEFAULT 'manga',
updated_at bigint NOT NULL
);
@@ -1,45 +0,0 @@
-- One row per distinct work, shared by every bookmark that tracks it
-- (ADR-0003). Keyed (site, series_id), the pair a bookmark key decomposes
-- into. title/series_url/cover are written once, at creation, and never
-- again: client-supplied values are ignored once the row exists and the
-- poller is the only party that may change them. kind and the latest-chapter
-- fields are last-write-wins like the bookmark's own fields.
CREATE TABLE series (
site text NOT NULL,
series_id text NOT NULL,
title text NOT NULL DEFAULT '',
series_url text NOT NULL DEFAULT '',
cover text NOT NULL DEFAULT '',
kind text NOT NULL DEFAULT 'manga',
latest_chapter text NOT NULL DEFAULT '',
latest_chapter_num double precision,
-- When the server last polled this series, unix ms; 0 means never, and sorts
-- first so a new bookmark is picked up on the next tick with no special case.
latest_checked_at bigint NOT NULL DEFAULT 0,
PRIMARY KEY (site, series_id)
);
-- Backfill from today's rows. The bookmark key's uniqueness makes
-- (site, series_id) unique in practice; DISTINCT is belt and braces.
INSERT INTO series (site, series_id, title, series_url, cover, kind,
latest_chapter, latest_chapter_num, latest_checked_at)
SELECT DISTINCT site, series_id, title, series_url, cover, kind,
latest_chapter, latest_chapter_num, latest_checked_at
FROM bookmarks;
-- The bookmark keeps only what differs between readers (ADR-0003): progress,
-- favourite, lifecycle bucket. The dropped columns now live on series.
ALTER TABLE bookmarks
DROP COLUMN title,
DROP COLUMN series_url,
DROP COLUMN cover,
DROP COLUMN kind,
DROP COLUMN latest_chapter,
DROP COLUMN latest_chapter_num,
DROP COLUMN latest_checked_at;
-- A bookmark may not point at a series that does not exist. No cascade: a
-- series outlives its last bookmark, and deleting one is not a store operation.
ALTER TABLE bookmarks
ADD CONSTRAINT bookmarks_series_fk
FOREIGN KEY (site, series_id) REFERENCES series (site, series_id);
@@ -1,17 +0,0 @@
-- One row per person. Keyed by their Discord user ID; carries the SHA-256 of
-- their userscript token and when they were created. Hashed because a token
-- in the database is a token anyone with the database can replay; SHA-256 is
-- enough because the tokens are high-entropy random values with nothing to
-- brute-force. No one can register yet, so this table holds exactly the one
-- owner row the seed creates at startup (see Store.Open).
CREATE TABLE readers (
id bigserial PRIMARY KEY,
discord_id text NOT NULL UNIQUE,
token_sha256 bytea NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT now()
);
-- Every bookmark now belongs to a reader. Added nullable: rows created before
-- this migration have no owner yet — 0004 attaches them to the seeded owner
-- before NOT NULL and the composite key land.
ALTER TABLE bookmarks ADD COLUMN reader_id bigint;
@@ -1,19 +0,0 @@
-- Attach every pre-existing bookmark to the owner reader, seeded between the
-- two migrate passes (Store.Open). The oldest reader is the owner by
-- construction: only the seed creates readers, and it runs once per database.
-- Run-once via the version table, like every migration.
UPDATE bookmarks SET reader_id = (SELECT id FROM readers ORDER BY id LIMIT 1);
-- Ownership lands structurally: reader_id becomes part of the key, so a
-- bookmark is one Reader's progress on one Series and a duplicate for the
-- same pair is impossible at the database level. Deleting a Reader takes
-- their bookmarks with them. The old text key is gone — the wire "key" is
-- derived as site:series_id on read, and nothing references the column.
-- Dropping it drops the primary key it carried; the composite key replaces
-- it, and the FK index the series constraint needs is created automatically.
ALTER TABLE bookmarks
ALTER COLUMN reader_id SET NOT NULL,
DROP COLUMN key,
ADD PRIMARY KEY (reader_id, site, series_id),
ADD CONSTRAINT bookmarks_reader_fk
FOREIGN KEY (reader_id) REFERENCES readers (id) ON DELETE CASCADE;
@@ -1,11 +0,0 @@
-- One row per browser session. The id is an opaque random value the cookie
-- carries verbatim; a request is authenticated by looking the row up, and
-- deleting the row is how a session is revoked. Expired rows are removed
-- lazily on lookup and swept by the next login, so nothing runs a background
-- cleanup.
CREATE TABLE sessions (
id text PRIMARY KEY,
reader_id bigint NOT NULL REFERENCES readers (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT now(),
expires_at timestamptz NOT NULL
);
@@ -1,7 +0,0 @@
-- Rotation is an epoch bump: a Reader's credential is derived from the
-- deployment secret, their Discord id and this epoch, so bumping it issues a
-- new credential and the rewritten token_sha256 invalidates the old one the
-- moment the transaction commits. The seed's ON CONFLICT refresh (Store.Open)
-- is gated on this being 0, so a restart can never undo a rotation by
-- restoring the epoch-0 hash.
ALTER TABLE readers ADD COLUMN token_epoch bigint NOT NULL DEFAULT 0;
@@ -1,8 +0,0 @@
-- Kagane cover bytes belong in their own table so image blobs never enter the
-- series queries that drive the latest-chapter poller.
CREATE TABLE covers (
image_id text PRIMARY KEY,
body bytea NOT NULL,
content_type text NOT NULL,
fetched_at timestamptz NOT NULL DEFAULT now()
);
@@ -1,9 +0,0 @@
-- Cover bytes move out of Postgres. Existing rows are intentionally dropped:
-- the old kagane path already refetches missing Covers on demand.
DROP TABLE covers;
CREATE TABLE covers (
address text PRIMARY KEY,
path text NOT NULL,
content_type text NOT NULL
);
@@ -1,8 +0,0 @@
-- The Cover splits into two facts. `cover` keeps the third-party address the
-- bytes come from, which is what the acquisition path refetches and dedupes
-- on; `cover_address` is the content address of the bytes once they are
-- actually stored, and is what the wire's absolute URL is built from.
--
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
-- that 404s", which is the distinction the API and the UI both depend on.
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
-80
View File
@@ -1,80 +0,0 @@
package store
import (
"database/sql"
"fmt"
"time"
)
// Session is one browser login: an opaque id the cookie carries verbatim,
// the Reader it belongs to, and when it stops being valid.
type Session struct {
ID string
ReaderID int64
ExpiresAt time.Time
}
// CreateSession stores a new session row for reader. The id is generated by
// the caller (session.NewID) — the store only persists it. Expired rows that
// were never looked up are swept in the same transaction: this is the one
// write every login makes, so the table stays bounded without a background
// job.
func (s *Store) CreateSession(id string, readerID int64, ttl time.Duration) (Session, error) {
tx, err := s.db.Begin()
if err != nil {
return Session{}, err
}
defer tx.Rollback()
expires := time.Now().Add(ttl)
if _, err := tx.Exec(`INSERT INTO sessions (id, reader_id, expires_at) VALUES ($1, $2, $3)`,
id, readerID, expires); err != nil {
return Session{}, err
}
if _, err := tx.Exec(`DELETE FROM sessions WHERE expires_at < now()`); err != nil {
return Session{}, err
}
if err := tx.Commit(); err != nil {
return Session{}, err
}
return Session{ID: id, ReaderID: readerID, ExpiresAt: expires}, nil
}
// GetSession returns the live session row for id, or ok=false when the id is
// unknown or expired. An expired row is deleted on the way out, so the table
// never grows past sessions that are still valid.
func (s *Store) GetSession(id string, now time.Time) (Session, bool, error) {
var sess Session
err := s.db.QueryRow(
`SELECT id, reader_id, expires_at FROM sessions WHERE id = $1`, id,
).Scan(&sess.ID, &sess.ReaderID, &sess.ExpiresAt)
if err == sql.ErrNoRows {
return Session{}, false, nil
}
if err != nil {
return Session{}, false, err
}
if !sess.ExpiresAt.After(now) {
// Best-effort: the row is dead either way; failing the request over a
// cleanup delete would only hide the real error. CreateSession's
// sweep catches anything this misses.
_, _ = s.db.Exec(`DELETE FROM sessions WHERE id = $1`, id)
return Session{}, false, nil
}
return sess, true, nil
}
// DeleteSession revokes one session. Deleting an unknown id is not an error.
func (s *Store) DeleteSession(id string) error {
_, err := s.db.Exec(`DELETE FROM sessions WHERE id = $1`, id)
return err
}
// DeleteReaderSessions revokes every session one Reader holds — the owner's
// remedy when a Reader's browser must be logged out everywhere at once. The
// next request carrying any of those cookies finds no row and is rejected.
func (s *Store) DeleteReaderSessions(readerID int64) error {
if _, err := s.db.Exec(`DELETE FROM sessions WHERE reader_id = $1`, readerID); err != nil {
return fmt.Errorf("delete sessions for reader %d: %w", readerID, err)
}
return nil
}
-84
View File
@@ -1,84 +0,0 @@
package store
import (
"testing"
"time"
)
func TestCreateAndGetSession(t *testing.T) {
s := newTestStore(t)
owner := s.OwnerID()
sess, err := s.CreateSession("sess-1", owner, time.Hour)
if err != nil {
t.Fatalf("CreateSession: %v", err)
}
if sess.ID != "sess-1" || sess.ReaderID != owner {
t.Fatalf("CreateSession returned %+v, want id sess-1 reader %d", sess, owner)
}
got, ok, err := s.GetSession("sess-1", time.Now())
if err != nil || !ok {
t.Fatalf("GetSession: ok=%v err=%v, want ok", ok, err)
}
if got.ReaderID != owner {
t.Fatalf("session reader = %d, want %d", got.ReaderID, owner)
}
}
func TestGetSessionUnknownID(t *testing.T) {
s := newTestStore(t)
if _, ok, err := s.GetSession("nope", time.Now()); err != nil || ok {
t.Fatalf("GetSession(unknown) = ok=%v err=%v, want ok=false", ok, err)
}
}
func TestExpiredSessionIsGone(t *testing.T) {
s := newTestStore(t)
owner := s.OwnerID()
if _, err := s.CreateSession("sess-exp", owner, -time.Minute); err != nil {
t.Fatalf("CreateSession: %v", err)
}
now := time.Now()
if _, ok, err := s.GetSession("sess-exp", now); err != nil || ok {
t.Fatalf("GetSession(expired) = ok=%v err=%v, want ok=false", ok, err)
}
// The expired row is deleted on lookup, so the next call cannot revive it.
if _, ok, err := s.GetSession("sess-exp", now.Add(-time.Hour)); err != nil || ok {
t.Fatalf("GetSession(expired again) = ok=%v err=%v, want ok=false", ok, err)
}
}
func TestDeleteSessionRevokes(t *testing.T) {
s := newTestStore(t)
owner := s.OwnerID()
if _, err := s.CreateSession("sess-del", owner, time.Hour); err != nil {
t.Fatalf("CreateSession: %v", err)
}
if err := s.DeleteSession("sess-del"); err != nil {
t.Fatalf("DeleteSession: %v", err)
}
if _, ok, err := s.GetSession("sess-del", time.Now()); err != nil || ok {
t.Fatalf("GetSession after delete = ok=%v err=%v, want ok=false", ok, err)
}
// Deleting twice is not an error.
if err := s.DeleteSession("sess-del"); err != nil {
t.Fatalf("DeleteSession twice: %v", err)
}
}
func TestDeleteSessionIsPerReader(t *testing.T) {
s := newTestStore(t)
other := secondReader(t, s)
if _, err := s.CreateSession("sess-other", other, time.Hour); err != nil {
t.Fatalf("CreateSession: %v", err)
}
got, ok, err := s.GetSession("sess-other", time.Now())
if err != nil || !ok {
t.Fatalf("GetSession: ok=%v err=%v, want ok", ok, err)
}
if got.ReaderID != other {
t.Fatalf("session reader = %d, want %d", got.ReaderID, other)
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-37
View File
@@ -1,37 +0,0 @@
package token
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"strconv"
)
// Token derives one Reader's userscript credential from the deployment
// secret, the Reader's Discord id and their token epoch.
//
// The credential is deterministic rather than stored random because the
// server must be able to rebuild the install URL after a restart while the
// database holds only hashes: a random token with no plaintext copy anywhere
// would be unreconstructible, and keeping plaintext in memory would break
// every install link on restart. HMAC output is high-entropy, indistinguishable
// from random to anyone without the secret, and changes whenever the epoch
// does — which is what rotation is. The stored form is Hash of this value,
// so a database leak yields nothing but hashes of unguessable strings.
func Token(key []byte, discordID string, epoch int64) string {
mac := hmac.New(sha256.New, key)
// The separator is unambiguous: discord ids are decimal snowflakes and
// epochs are plain integers, so no two (id, epoch) pairs can collide.
mac.Write([]byte(discordID))
mac.Write([]byte{0})
mac.Write([]byte(strconv.FormatInt(epoch, 10)))
return hex.EncodeToString(mac.Sum(nil))
}
// Hash is the SHA-256 of a credential — the only form that ever touches the
// database (readers.token_sha256). SHA-256 rather than a password hash is
// deliberate: these are unguessable values with nothing to brute-force, so a
// slow hash would only add per-request cost.
func Hash(cred string) [32]byte {
return sha256.Sum256([]byte(cred))
}
-53
View File
@@ -1,53 +0,0 @@
package token
import (
"bytes"
"crypto/sha256"
"testing"
)
func TestTokenDeterministicPerReaderAndEpoch(t *testing.T) {
key := []byte("deployment-secret")
a := Token(key, "reader-1", 0)
b := Token(key, "reader-1", 0)
if a != b {
t.Fatal("same (reader, epoch) derived different credentials")
}
if a == Token(key, "reader-2", 0) {
t.Fatal("different readers derived the same credential")
}
if a == Token(key, "reader-1", 1) {
t.Fatal("rotation epoch derived the same credential")
}
}
func TestTokenChangesWithSecret(t *testing.T) {
a := Token([]byte("key-1"), "reader-1", 0)
b := Token([]byte("key-2"), "reader-1", 0)
if a == b {
t.Fatal("different secrets derived the same credential")
}
}
func TestTokenFormat(t *testing.T) {
cred := Token([]byte("key"), "reader-1", 0)
// 32 bytes of HMAC-SHA256, hex-encoded: the length the install URL and
// the committed placeholder both assume.
if len(cred) != 64 {
t.Fatalf("credential length = %d, want 64", len(cred))
}
for _, c := range cred {
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
t.Fatalf("credential contains non-hex byte %q", c)
}
}
}
func TestHashIsSha256OfCredential(t *testing.T) {
cred := Token([]byte("key"), "reader-1", 0)
got := Hash(cred)
want := sha256.Sum256([]byte(cred))
if !bytes.Equal(got[:], want[:]) {
t.Fatal("Hash is not the SHA-256 of the credential")
}
}
+25 -65
View File
@@ -1,24 +1,14 @@
package userscript
import (
"bytes"
"crypto/subtle"
"log"
"net/http"
"os"
"regexp"
"time"
"bookmarkmanager/backend/internal/httpmw"
"bookmarkmanager/backend/internal/store"
)
// tokenPlaceholder is what the bindmounted userscript carries where the
// Reader's credential goes: in the API_TOKEN constant and in the @downloadURL
// and @updateURL metadata lines. The handler substitutes the requesting
// Reader's credential for it at serve time, so no credential literal is ever
// committed or deployed, and each Reader's copy carries exactly their own.
var tokenPlaceholder = []byte("__API_TOKEN__")
// versionLine matches the userscript metadata block's @version directive.
var versionLine = regexp.MustCompile(`(?m)^// @version[ \t]+.*$`)
@@ -36,65 +26,35 @@ func stampVersion(src []byte, mod time.Time) []byte {
return versionLine.ReplaceAll(src, []byte("// @version "+mod.UTC().Format("2006.01.02.1504")))
}
// substituteToken replaces every tokenPlaceholder with the Reader's
// credential. A file without the placeholder is returned unchanged so Render
// can warn about it rather than silently serving a credential-less script.
func substituteToken(src []byte, credential string) []byte {
return bytes.ReplaceAll(src, tokenPlaceholder, []byte(credential))
}
// Render writes one userscript file with the credential substituted and the
// mtime-derived version stamped. Shared by the download path (Handler) and
// the web UI's install endpoints, so both serve byte-identical scripts.
// userscriptHandler serves the userscript to Violentmonkey's updater.
//
// The file is read per request — that is what lets a bindmounted copy be
// edited on the host without a restart. It is ~50 KB and polled about once a
// day.
func Render(w http.ResponseWriter, r *http.Request, path, credential string) {
info, err := os.Stat(path)
if err != nil {
log.Printf("userscript: stat %s: %v", path, err)
http.NotFound(w, r)
return
}
src, err := os.ReadFile(path)
if err != nil {
log.Printf("userscript: read %s: %v", path, err)
http.NotFound(w, r)
return
}
rendered := substituteToken(src, credential)
if bytes.Equal(rendered, src) {
// The bindmounted file was not built for per-Reader rendering. Serving
// it as written is the operator's freedom, but a credential-less copy
// is a deployment bug worth one log line — the symptom (silent 401s on
// every device) is otherwise indistinguishable from a network fault.
log.Printf("userscript: %s has no %s placeholder; serving as written", path, tokenPlaceholder)
}
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
w.Header().Set("Cache-Control", "no-cache")
w.Write(stampVersion(rendered, info.ModTime()))
}
// Handler serves the userscript to Violentmonkey's updater, rendered for the
// Reader whose credential is in the path.
// The token lives in the path because the update poll sends no Authorization
// header, and the file embeds API_TOKEN in plain text, so an open path would
// hand that token to anyone who guessed the URL. A mismatch answers 404 rather
// than 401: a prober learns nothing about whether the route exists.
//
// The credential lives in the path because the update poll sends no
// Authorization header, and the rendered file embeds the credential in
// plaintext, so an open path would hand it to anyone who guessed the URL. A
// mismatch answers 404 rather than 401: a prober learns nothing about whether
// the route exists. The same credential authenticates the API bearer header,
// so the two are one secret with one blast radius.
//
// The path segment is the credential itself, so once it resolves it is also
// exactly what the served copy must carry — no re-derivation needed.
func Handler(s *store.Store, path string) http.HandlerFunc {
// The file is read per request — that is what lets a bindmounted copy be edited
// on the host without a restart. It is ~50 KB and polled about once a day.
func Handler(token, path string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
cred := r.PathValue("token")
if _, ok := httpmw.ResolveReader(s, cred); !ok {
if subtle.ConstantTimeCompare([]byte(r.PathValue("token")), []byte(token)) != 1 {
http.NotFound(w, r)
return
}
Render(w, r, path, cred)
info, err := os.Stat(path)
if err != nil {
log.Printf("userscript: stat %s: %v", path, err)
http.NotFound(w, r)
return
}
src, err := os.ReadFile(path)
if err != nil {
log.Printf("userscript: read %s: %v", path, err)
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
w.Header().Set("Cache-Control", "no-cache")
w.Write(stampVersion(src, info.ModTime()))
}
}
+89 -41
View File
@@ -1,67 +1,115 @@
package userscript
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
const testToken = "s3cret-token"
// sampleScript is a stand-in for the real userscript: a metadata block with a
// @version line, the credential placeholder in its metadata and body, plus
// content that must survive the rewrites untouched.
// @version line, plus a body that must survive the rewrite untouched.
const sampleScript = `// ==UserScript==
// @name Manga Bookmark Sync
// @version 1.5.0
// @downloadURL https://api.example/u/__API_TOKEN__/manga-bookmark.user.js
// @match https://asurascans.com/*
// ==/UserScript==
(function () { "use strict";
const API_TOKEN = "__API_TOKEN__";
})();
(function () { "use strict"; })();
`
func TestStampVersionReplacesVersionLineOnly(t *testing.T) {
// writeScript drops a userscript in a temp dir with a known mtime and returns
// its path plus the version string the handler is expected to stamp.
func writeScript(t *testing.T, body string) (path, wantVersion string) {
t.Helper()
path = filepath.Join(t.TempDir(), "manga-bookmark.user.js")
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatalf("write script: %v", err)
}
mod := time.Date(2026, 7, 28, 16, 42, 0, 0, time.UTC)
got := string(stampVersion([]byte(sampleScript), mod))
if err := os.Chtimes(path, mod, mod); err != nil {
t.Fatalf("chtimes: %v", err)
}
return path, "2026.07.28.1642"
}
if !strings.Contains(got, "// @version "+mod.UTC().Format("2006.01.02.1504")) {
t.Errorf("body has no stamped version:\n%s", got)
// newTestMux registers Handler the same way main.go's router does, without
// pulling in the store or the rest of the app.
func newTestMux(token, path string) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js", Handler(token, path))
return mux
}
func getScript(t *testing.T, srv http.Handler, token string) *httptest.ResponseRecorder {
t.Helper()
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+token+"/manga-bookmark.user.js", nil))
return rr
}
func TestUserscriptServedWithStampedVersion(t *testing.T) {
path, wantVersion := writeScript(t, sampleScript)
rr := getScript(t, newTestMux(testToken, path), testToken)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code)
}
if strings.Contains(got, "1.5.0") {
t.Errorf("body still carries the file's own version:\n%s", got)
if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/javascript") {
t.Errorf("Content-Type = %q, want text/javascript", ct)
}
// Everything outside the @version line is served verbatim, including the
// placeholder — stamping must not do the substitution's job.
if !strings.Contains(got, `const API_TOKEN = "__API_TOKEN__";`) {
t.Errorf("body was altered beyond the version line:\n%s", got)
if cc := rr.Header().Get("Cache-Control"); cc != "no-cache" {
t.Errorf("Cache-Control = %q, want no-cache", cc)
}
body := rr.Body.String()
if !strings.Contains(body, "// @version "+wantVersion) {
t.Errorf("body has no stamped version %q:\n%s", wantVersion, body)
}
if strings.Contains(body, "1.5.0") {
t.Errorf("body still carries the file's own version:\n%s", body)
}
// Everything outside the @version line is served verbatim.
if !strings.Contains(body, `(function () { "use strict"; })();`) {
t.Errorf("body was altered beyond the version line:\n%s", body)
}
if !strings.Contains(body, "// @name Manga Bookmark Sync") {
t.Errorf("metadata block was altered:\n%s", body)
}
}
func TestStampVersionWithoutVersionLineServedUnmodified(t *testing.T) {
// The empty-token case ("/u//manga-bookmark.user.js") is covered at the
// router level (see backend's guardEmptyUserscriptToken): ServeMux 307s it to
// "/u/manga-bookmark.user.js" before this handler's own token check ever runs.
func TestUserscriptWrongTokenIs404(t *testing.T) {
path, _ := writeScript(t, sampleScript)
srv := newTestMux(testToken, path)
for _, tok := range []string{"wrong", testToken + "x", testToken[:3]} {
if got := getScript(t, srv, tok).Code; got != http.StatusNotFound {
t.Errorf("token %q: status = %d, want 404", tok, got)
}
}
}
func TestUserscriptMissingFileIs404(t *testing.T) {
srv := newTestMux(testToken, filepath.Join(t.TempDir(), "absent.user.js"))
if got := getScript(t, srv, testToken).Code; got != http.StatusNotFound {
t.Fatalf("status = %d, want 404", got)
}
}
func TestUserscriptWithoutVersionLineServedUnmodified(t *testing.T) {
const noVersion = "// ==UserScript==\n// @name x\n// ==/UserScript==\nconsole.log(1);\n"
if got := string(stampVersion([]byte(noVersion), time.Now())); got != noVersion {
t.Errorf("stampVersion altered a file with no @version line:\n%s", got)
}
}
func TestSubstituteTokenReplacesEveryPlaceholder(t *testing.T) {
got := string(substituteToken([]byte(sampleScript), "abc123"))
if strings.Contains(got, "__API_TOKEN__") {
t.Errorf("placeholder survived substitution:\n%s", got)
}
// The credential lands in the constant and in both metadata lines.
if want := `const API_TOKEN = "abc123";`; !strings.Contains(got, want) {
t.Errorf("no substituted constant %q:\n%s", want, got)
}
if want := "https://api.example/u/abc123/manga-bookmark.user.js"; !strings.Contains(got, want) {
t.Errorf("no substituted download URL %q:\n%s", want, got)
}
}
func TestSubstituteTokenWithoutPlaceholderServedUnmodified(t *testing.T) {
const noPlaceholder = "// ==UserScript==\n// @name x\n// ==/UserScript==\n"
if got := string(substituteToken([]byte(noPlaceholder), "abc123")); got != noPlaceholder {
t.Errorf("substituteToken altered a file without the placeholder:\n%s", got)
path, _ := writeScript(t, noVersion)
rr := getScript(t, newTestMux(testToken, path), testToken)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code)
}
if rr.Body.String() != noVersion {
t.Fatalf("body = %q, want it unmodified", rr.Body.String())
}
}
-320
View File
@@ -1,320 +0,0 @@
package web
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"sync"
"time"
"bookmarkmanager/backend/internal/session"
"bookmarkmanager/backend/internal/token"
)
const (
// oauthStateTTL bounds how long a started sign-in stays valid. Ten
// minutes is generous for Discord's round trip and short enough that a
// captured state is stale before it is worth replaying.
oauthStateTTL = 10 * time.Minute
// maxStates caps the state map so a flood of /auth/discord hits cannot
// grow memory; past the cap the oldest state is evicted, which at worst
// invalidates an in-flight sign-in.
maxStates = 256
// maxResponseBytes caps Discord API bodies; they are small, and an
// unbounded read is an OOM handed to Discord's CDN.
maxResponseBytes = 1 << 20
// discordTimeout keeps a hung Discord request from hanging the login
// callback forever.
discordTimeout = 15 * time.Second
)
// DiscordConfig is the OAuth application this service registers as, plus the
// guild that gates access.
type DiscordConfig struct {
ClientID string
ClientSecret string
GuildID string
// RequiredRole, when non-empty, is a role ID a member must hold on top of
// guild membership. Empty by default: membership alone suffices.
RequiredRole string
// APIBase is the Discord API root; configurable so tests run the whole
// flow against a local stub.
APIBase string
// RedirectURI is the full public URL of the callback — Discord requires
// the exact string, so it is configured, never derived from headers.
RedirectURI string
}
// oauthStates stores one-time sign-in states. A state is generated at
// /auth/discord, echoed back by Discord at the callback, and consumed there.
type oauthStates struct {
mu sync.Mutex
expiry map[string]time.Time
}
func newOAuthStates() *oauthStates {
return &oauthStates{expiry: make(map[string]time.Time)}
}
func (s *oauthStates) put(state string, expires time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
now := time.Now()
for k, at := range s.expiry {
if !at.After(now) {
delete(s.expiry, k)
}
}
// Evict the state closest to expiring when full, so a flood of starts
// cannot grow memory; at worst it invalidates an in-flight sign-in.
if len(s.expiry) >= maxStates {
var oldest string
var oldestAt time.Time
for k, at := range s.expiry {
if oldest == "" || at.Before(oldestAt) {
oldest, oldestAt = k, at
}
}
delete(s.expiry, oldest)
}
s.expiry[state] = expires
}
// take validates and consumes a state in one step: a state works exactly
// once, which is what makes a replayed callback useless.
func (s *oauthStates) take(state string) bool {
s.mu.Lock()
defer s.mu.Unlock()
expires, ok := s.expiry[state]
if !ok || !expires.After(time.Now()) {
return false
}
delete(s.expiry, state)
return true
}
// discordStart begins the authorization code grant: a fresh state, then a
// redirect to Discord's authorize page.
func (h *Handler) discordStart(w http.ResponseWriter, r *http.Request) {
state := session.NewID()
h.states.put(state, time.Now().Add(oauthStateTTL))
u := h.discord.APIBase + "/oauth2/authorize?" + url.Values{
"client_id": {h.discord.ClientID},
"redirect_uri": {h.discord.RedirectURI},
"response_type": {"code"},
"scope": {"identify guilds.members.read"},
"state": {state},
}.Encode()
http.Redirect(w, r, u, http.StatusSeeOther)
}
// discordCallback completes the grant: exchange the code, verify identity,
// membership and role, then mint a session. Every failure path renders the
// login page with an author-written message — nothing Discord supplied is
// ever interpolated into a page, and no secret reaches a log line.
func (h *Handler) discordCallback(w http.ResponseWriter, r *http.Request) {
ip := session.ClientIP(r)
if wait := h.limiter.RetryAfter(ip, time.Now()); wait > 0 {
secs := int(wait.Seconds()) + 1
w.Header().Set("Retry-After", strconv.Itoa(secs))
h.renderLogin(w, http.StatusTooManyRequests,
"Too many attempts. Try again in "+strconv.Itoa((secs+59)/60)+" min.")
return
}
// Discord refuses the grant (the reader hit cancel, or the application
// was misconfigured). The state is consumed so the flow is cleanly over;
// this makes no Discord calls, so it is not a failure the limiter counts.
if oerr := r.URL.Query().Get("error"); oerr != "" {
h.states.take(r.URL.Query().Get("state"))
h.renderLogin(w, http.StatusBadRequest, "Sign-in was cancelled.")
return
}
code := r.URL.Query().Get("code")
if code == "" || !h.states.take(r.URL.Query().Get("state")) {
h.limiter.Fail(ip, time.Now())
h.renderLogin(w, http.StatusBadRequest,
"This sign-in link was invalid or already used. Start again.")
return
}
tok, err := h.exchangeToken(r.Context(), code)
if err != nil {
h.limiter.Fail(ip, time.Now())
log.Printf("discord token exchange: %v", err)
h.renderLogin(w, http.StatusBadGateway,
"Discord sign-in is unavailable right now. Try again in a moment.")
return
}
userID, err := h.discordUserID(r.Context(), tok.AccessToken)
if err != nil {
h.limiter.Fail(ip, time.Now())
log.Printf("discord users/@me: %v", err)
h.renderLogin(w, http.StatusBadGateway,
"Discord sign-in is unavailable right now. Try again in a moment.")
return
}
member, isMember, err := h.discordMember(r.Context(), tok.AccessToken)
if err != nil {
h.limiter.Fail(ip, time.Now())
log.Printf("discord member check: %v", err)
h.renderLogin(w, http.StatusBadGateway,
"Discord sign-in is unavailable right now. Try again in a moment.")
return
}
// The refusal is the same for a non-member and a member without the
// required role, and it names neither the guild nor its id: an outsider
// cannot tell whether the guild exists, let alone which one gates.
//
// It also returns before EnsureReader, so a refused sign-in leaves no
// Reader row behind — the gate is the only thing standing between guild
// membership and a library.
if !isMember || (h.discord.RequiredRole != "" && !slices.Contains(member.Roles, h.discord.RequiredRole)) {
h.limiter.Fail(ip, time.Now())
h.renderLogin(w, http.StatusForbidden,
"This Discord account is not a member of this community.")
return
}
// Registration is the login (issue #27): first sight of a guild member
// creates their Reader, every later sight returns the same one. Their
// userscript credential is derived at epoch 0 the way the owner's is, so
// the install links work before they have read anything.
readerID, err := h.store.EnsureReader(userID, token.Hash(token.Token(h.tokenKey, userID, 0)))
if err != nil {
log.Printf("register reader: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.limiter.Reset(ip)
sess, err := h.store.CreateSession(session.NewID(), readerID, session.SessionTTL)
if err != nil {
log.Printf("create session: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
session.SetCookie(w, r, sess.ID)
http.Redirect(w, r, "/", http.StatusSeeOther)
}
// exchangeToken trades an authorization code for an access token. The body is
// form-encoded because that is what Discord accepts — it rejects a JSON
// payload — so the wire format is fixed here, not in a client library.
func (h *Handler) exchangeToken(ctx context.Context, code string) (discordToken, error) {
form := url.Values{
"client_id": {h.discord.ClientID},
"client_secret": {h.discord.ClientSecret},
"grant_type": {"authorization_code"},
"code": {code},
"redirect_uri": {h.discord.RedirectURI},
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
h.discord.APIBase+"/oauth2/token", strings.NewReader(form.Encode()))
if err != nil {
return discordToken{}, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
resp, err := h.httpClient.Do(req)
if err != nil {
return discordToken{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return discordToken{}, fmt.Errorf("status %d", resp.StatusCode)
}
var tok discordToken
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&tok); err != nil {
return discordToken{}, err
}
if tok.AccessToken == "" {
return discordToken{}, errors.New("empty access token")
}
return tok, nil
}
// discordUserID fetches the signed-in user's id via the identify scope.
func (h *Handler) discordUserID(ctx context.Context, accessToken string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
h.discord.APIBase+"/users/@me", nil)
if err != nil {
return "", err
}
req.Header.Set("Authorization", "Bearer "+accessToken)
req.Header.Set("Accept", "application/json")
resp, err := h.httpClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("status %d", resp.StatusCode)
}
var u struct {
ID string `json:"id"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&u); err != nil {
return "", err
}
if u.ID == "" {
return "", errors.New("empty user id")
}
return u.ID, nil
}
type discordMember struct {
Roles []string `json:"roles"`
}
// discordMember fetches the current user's membership in the configured guild.
//
// This is the OAuth endpoint (Get Current User Guild Member), the one the
// guilds.members.read scope grants. Its bot-side twin, GET /guilds/{id}/
// members/{user}, reads almost identically and is the wrong one: it wants a
// Bot token and the application present in the guild, and answers a user
// Bearer token with 401 — which fails as an outage rather than a refusal, so
// nobody could sign in at all.
//
// A 404 or 403 (not in the guild, or the token lacks the scope) is a
// non-member, not an error.
func (h *Handler) discordMember(ctx context.Context, accessToken string) (discordMember, bool, error) {
u := h.discord.APIBase + "/users/@me/guilds/" +
url.PathEscape(h.discord.GuildID) + "/member"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return discordMember{}, false, err
}
req.Header.Set("Authorization", "Bearer "+accessToken)
req.Header.Set("Accept", "application/json")
resp, err := h.httpClient.Do(req)
if err != nil {
return discordMember{}, false, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden {
return discordMember{}, false, nil
}
if resp.StatusCode != http.StatusOK {
return discordMember{}, false, fmt.Errorf("status %d", resp.StatusCode)
}
var m discordMember
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&m); err != nil {
return discordMember{}, false, err
}
return m, true, nil
}
type discordToken struct {
AccessToken string `json:"access_token"`
}
-55
View File
@@ -1,55 +0,0 @@
package web
import (
"testing"
"time"
)
func TestOAuthStateSingleUse(t *testing.T) {
s := newOAuthStates()
s.put("st", time.Now().Add(time.Minute))
if !s.take("st") {
t.Fatal("take of a fresh state = false, want true")
}
if s.take("st") {
t.Fatal("take of a consumed state = true, want false")
}
}
func TestOAuthStateUnknownOrExpired(t *testing.T) {
s := newOAuthStates()
if s.take("never-seen") {
t.Fatal("take of an unknown state = true, want false")
}
s.put("stale", time.Now().Add(-time.Minute))
if s.take("stale") {
t.Fatal("take of an expired state = true, want false")
}
}
// The map is capped: a flood of starts evicts older states instead of growing,
// and consumed states must not change that.
func TestOAuthStateEviction(t *testing.T) {
s := newOAuthStates()
key := func(i, salt int) string {
return string(rune('a'+i%26)) + string(rune('0'+i/26+salt*16))
}
now := time.Now().Add(time.Hour)
for i := 0; i < maxStates*2; i++ {
s.put(key(i, 0), now)
}
if got := len(s.expiry); got != maxStates {
t.Fatalf("states after a flood = %d, want %d", got, maxStates)
}
// Consume everything, then flood again: the map stays bounded.
for state := range s.expiry {
s.take(state)
}
for i := 0; i < maxStates; i++ {
s.put(key(i, 1), now)
}
if got := len(s.expiry); got != maxStates {
t.Fatalf("states after consume+flood = %d, want %d", got, maxStates)
}
}
+21 -81
View File
@@ -243,80 +243,6 @@ button { cursor: pointer; }
/* The label is 15px tall by design; the thumb gets 44 without moving it. */
.ghost::after { content: ""; position: absolute; inset: -15px -12px; }
/* ---- userscript setup: collapsed by default, one hairline, no card ---- */
.setup {
margin: 0 20px;
padding: 12px 0 0;
border-bottom: 1px solid var(--rule);
color: var(--mute);
}
.setup summary {
display: flex;
align-items: center;
min-height: 44px;
padding: 0;
font: 500 10px/1 var(--font-mono);
letter-spacing: .2em;
text-transform: uppercase;
color: var(--mute-2);
cursor: pointer;
list-style: none;
}
.setup summary::-webkit-details-marker { display: none; }
.setup summary:hover { color: var(--paper); }
.setup[open] { padding-bottom: 16px; }
.setup-copy {
margin: 0;
padding: 4px 0 12px;
font: 14px/1.55 var(--font-body);
color: var(--mute);
}
.setup-links {
display: flex;
flex-wrap: wrap;
gap: 8px 20px;
margin: 0 0 14px;
}
.setup-links .ghost { font-size: 11px; }
.setup-rotate { margin: 0; }
/* Rotation confirmation: the one hot state the panel wears, and it is
destruction, not new-chapter signal — danger, never ember. */
.setup-warn {
margin: 0;
padding: 10px 12px;
border: 1px solid var(--danger);
color: var(--danger);
font: 500 12px/1.5 var(--font-mono);
letter-spacing: .04em;
}
/* ---- reader roster (owner only): same hairline panel, one row per Reader ---- */
.readerlist { margin: 0; padding: 0; list-style: none; }
.readerlist li {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 4px 16px;
min-height: 44px;
border-top: 1px solid var(--rule);
}
.readerlist form { margin: 0 0 0 auto; }
.reader-id {
font: 500 13px/1.4 var(--font-mono);
letter-spacing: .04em;
color: var(--paper);
}
.reader-sessions {
font: 500 10px/1 var(--font-mono);
letter-spacing: .14em;
text-transform: uppercase;
color: var(--mute);
}
/* Revocation cuts someone off, so it wears --danger. Ember stays reserved for
the new-chapter signal. */
.ghost.danger { color: var(--danger); }
.ghost.danger:hover { color: var(--danger); border-bottom-color: var(--danger); }
.chrome { display: flex; flex-direction: column; }
.searchbar {
@@ -848,6 +774,26 @@ button { cursor: pointer; }
filter: drop-shadow(0 0 34px var(--ember-wash)) drop-shadow(0 18px 24px rgba(0,0,0,.5));
}
.login-card form { display: flex; flex-direction: column; gap: 18px; }
.login-card label {
font: 500 10px/1 var(--font-mono);
letter-spacing: .16em;
text-transform: uppercase;
color: var(--mute);
}
.login-card input {
width: 100%;
height: 54px;
margin-top: 9px;
padding: 0 2px;
border: none;
border-bottom: 1px solid var(--field-line);
background: transparent;
color: var(--paper);
font: 500 20px var(--font-mono);
letter-spacing: .16em;
outline: none;
}
.login-card input:focus { border-bottom-color: var(--paper); }
.login-card .error {
margin: 0;
min-height: 20px;
@@ -864,13 +810,7 @@ button { cursor: pointer; }
.login-card button:hover {
background: var(--ember);
border-color: var(--ember);
color: var(--ember-ink);
}
.login-card .login-note {
margin: 14px 0 0;
text-align: center;
font: 400 12px/1.4 var(--font-body);
color: var(--mute);
color: #fff;
}
/* ---- laptop and up: the whole sheet is drawn 20% larger, which is what
-4
View File
@@ -74,10 +74,6 @@
</nav>
</div>
{{template "setup" .}}
{{if .Owner}}{{template "readers" .}}{{end}}
{{template "keyrow" .}}
{{template "recent" .}}
-11
View File
@@ -16,17 +16,6 @@
<div class="empty"><strong>Nothing archived.</strong><p>Shelve a series to park it here — it keeps getting checked for new chapters.</p></div>
{{else if eq .Tab "finished"}}
<div class="empty"><strong>Nothing finished yet.</strong><p>Mark a series finished and it moves out of your reading list.</p></div>
{{else if .EmptyLibrary}}
{{/* Nothing in either library, so the links are the only thing this page can
usefully say. Both scripts: the two libraries are separate installs. */}}
<div class="empty">
<strong>Nothing here yet.</strong>
<p>Install the userscripts, then open a series and read a chapter — bookmarks arrive on their own.</p>
<p class="setup-links">
<a class="ghost" href="/install/manga-bookmark.user.js">Install Manga script</a>
<a class="ghost" href="/install/novel-bookmark.user.js">Install Novels script</a>
</p>
</div>
{{else}}
<div class="empty"><strong>Nothing here yet.</strong><p>Bookmarks appear once the userscript records a chapter.</p></div>
{{end}}
+7 -3
View File
@@ -19,13 +19,17 @@
<figure class="login-art" aria-hidden="true">
<img src="/static/login-art.png" alt="">
</figure>
<form method="get" action="/auth/discord">
<form method="post" action="/login">
<div>
<label for="password">Password</label>
<input id="password" name="password" type="password"
autocomplete="current-password" autofocus required>
</div>
{{/* The page reloads on a failed sign-in, so the message is present from
the start; role=alert is what gets it announced anyway. */}}
<p class="error" role="alert">{{.Error}}</p>
<button type="submit">Continue with Discord</button>
<button type="submit">Sign in</button>
</form>
<p class="login-note">Guild membership is required to sign in.</p>
</main>
</body>
</html>
@@ -1,29 +0,0 @@
{{/* The owner's Reader roster. Rendered only for the owner (listView.Owner),
and re-rendered whole as the response to a revocation so the session
counts cannot describe the state before the tap. Revocation is
confirm-gated: it signs someone out of every device at once. */}}
{{define "readers"}}
<details class="setup" id="readers">
<summary>Readers</summary>
<p class="setup-copy">Everyone who has signed in through Discord. Revoking
signs a Reader out of every device; their library and bookmarks are
untouched, and they can sign in again.</p>
<ul class="readerlist">
{{range .Readers}}
<li>
<span class="reader-id">{{.DiscordID}}</span>
<span class="reader-sessions">{{.Sessions}} session{{if ne .Sessions 1}}s{{end}}</span>
{{/* The owner's own row never offers Revoke: it is the one row where the
button would sign the tapping browser out, and the endpoint refuses
it anyway. Logout is the deliberate way to do that. */}}
{{if and .Sessions (ne .ID $.OwnerID)}}
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
<button type="submit" class="ghost danger">Revoke sessions</button>
</form>
{{end}}
</li>
{{end}}
</ul>
</details>
{{end}}
-34
View File
@@ -1,34 +0,0 @@
{{/* The userscript install panel. Each link serves the script rendered
with the acting Reader's credential inside it, so the credential never
appears in this page's markup, the address bar, or a redirect. Rotation
is confirm-gated because it invalidates every installed copy at once;
the response swaps this same panel open with the reinstall warning. */}}
{{define "setup"}}
<details class="setup" id="setup"{{if .Rotated}} open{{end}}>
<summary>Userscripts</summary>
<p class="setup-copy">Install each script once per device. They keep your
bookmarks in sync across every site and update themselves from here.</p>
<p class="setup-links">
<a class="ghost" href="/install/manga-bookmark.user.js">Install Manga script</a>
<a class="ghost" href="/install/novel-bookmark.user.js">Install Novels script</a>
</p>
<p class="setup-copy">On mobile, Violentmonkey does not pick up the install
links — the script opens as text. Download the file instead, then add it
from Violentmonkey's own menu.</p>
<p class="setup-links">
<a class="ghost" href="/install/manga-bookmark.user.js?download=1">Download Manga script</a>
<a class="ghost" href="/install/novel-bookmark.user.js?download=1">Download Novels script</a>
</p>
{{if .Rotated}}
<p class="setup-warn" role="status">Credential rotated — the old one no
longer works. Reinstall both scripts on every device now, or they will
silently stop syncing.</p>
{{else}}
<form class="setup-rotate" hx-post="/rotate-token" hx-target="#setup"
hx-swap="outerHTML"
hx-confirm="Rotation invalidates the current credential on every device immediately. You will have to reinstall both scripts everywhere. Rotate?">
<button type="submit" class="ghost">Rotate credential</button>
</form>
{{end}}
</details>
{{end}}
+57 -212
View File
@@ -1,7 +1,7 @@
package web
import (
"context"
"crypto/subtle"
"embed"
"html/template"
"io/fs"
@@ -16,8 +16,6 @@ import (
"bookmarkmanager/backend/internal/session"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/token"
"bookmarkmanager/backend/internal/userscript"
)
//go:embed templates
@@ -33,23 +31,11 @@ const RecentCount = 5
// It is a separate handler from api.Handler because the two speak different
// representations (HTML versus JSON) to different clients under different auth.
type Handler struct {
store *store.Store
// tokenKey derives Readers' userscript credentials (internal/token): the
// install endpoints render the scripts with the credential inside, which
// is the one place the UI needs the secret.
tokenKey []byte
// mangaUserscriptPath / novelUserscriptPath are the bindmounted script
// files the install endpoints render — the same files the /u/ download
// paths serve.
mangaUserscriptPath string
novelUserscriptPath string
tmpl *template.Template
discord DiscordConfig
states *oauthStates
limiter *session.LoginLimiter
// httpClient is the plain stdlib client that talks to Discord. It is not
// an injected interface: tests point APIBase at a stub server instead.
httpClient *http.Client
store *store.Store
tmpl *template.Template
key []byte
password string
limiter *session.LoginLimiter
}
// listView is what every list-rendering template receives.
@@ -57,7 +43,7 @@ type listView struct {
// Lib is the library this view renders: store.KindManga or store.KindNovel.
// Manga is the default and carries no query parameter, so every pre-novel
// URL keeps meaning exactly what it did.
Lib string
Lib string
Tab string // "all", "fav", or "new"
Recent []store.Bookmark
Items []store.Bookmark
@@ -69,22 +55,6 @@ type listView struct {
// OOB marks a render of the chrome partials as an out-of-band swap rather
// than the inline copy app.html lays out.
OOB bool
// Rotated marks the setup panel as having just rotated the credential:
// it swaps the reinstall warning in over the button row.
Rotated bool
// EmptyLibrary means this Reader holds no bookmarks in either library, so
// the empty state can offer the installs instead of reporting on a filter.
// It is not "newly registered": a Reader who deletes their last bookmark is
// in the same position and needs the same links.
EmptyLibrary bool
// Owner marks the acting Reader as the deployment's owner, which unlocks
// the Readers panel. Nothing else in the UI differs.
Owner bool
// Readers is the owner's roster, populated only for the owner's own page
// render and the revocation fragment. OwnerID travels with it so the roster
// can tell the owner's own row apart from the Readers they may revoke.
Readers []store.ReaderSummary
OwnerID int64
}
// PageURL and ListURL are the two link shapes every tab needs. Building them
@@ -111,28 +81,23 @@ type loginView struct {
// New parses every template up front so a broken one kills the process at
// startup rather than the first request that touches it.
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string) (*Handler, error) {
func New(s *store.Store, apiToken, webPassword string) (*Handler, error) {
tmpl, err := template.ParseFS(templateFS, "templates/*.html")
if err != nil {
return nil, err
}
return &Handler{
store: s,
tokenKey: tokenKey,
mangaUserscriptPath: mangaPath,
novelUserscriptPath: novelPath,
tmpl: tmpl,
discord: discord,
states: newOAuthStates(),
limiter: session.NewLoginLimiter(),
httpClient: &http.Client{Timeout: discordTimeout},
store: s,
tmpl: tmpl,
key: session.Key(apiToken, webPassword),
password: webPassword,
limiter: session.NewLoginLimiter(),
}, nil
}
func (h *Handler) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /{$}", h.index)
mux.HandleFunc("GET /auth/discord", h.discordStart)
mux.HandleFunc("GET /auth/discord/callback", h.discordCallback)
mux.HandleFunc("POST /login", h.login)
mux.HandleFunc("POST /logout", h.logout)
mux.Handle("GET /static/", staticHandler())
@@ -141,17 +106,6 @@ func (h *Handler) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /ui/bookmarks/{key}/status", h.requireSession(h.uiStatus))
mux.HandleFunc("POST /ui/bookmarks/{key}/chapter", h.requireSession(h.uiChapter))
mux.HandleFunc("DELETE /ui/bookmarks/{key}", h.requireSession(h.uiDelete))
// Install endpoints render the script directly under the session: the
// credential travels inside the served bytes, never in the address bar or
// the page markup. Updates after install use the credential-bearing /u/
// path the script embeds, which needs no session.
mux.HandleFunc("GET /install/manga-bookmark.user.js", h.requireSession(h.installUserscript("manga-bookmark.user.js")))
mux.HandleFunc("GET /install/novel-bookmark.user.js", h.requireSession(h.installUserscript("novel-bookmark.user.js")))
mux.HandleFunc("POST /rotate-token", h.requireSession(h.rotateToken))
// Owner-only: the one place the UI crosses the Reader boundary.
mux.HandleFunc("POST /readers/{id}/revoke", h.requireSession(h.revokeReaderSessions))
}
// staticHandler serves the embedded assets. An hour, not longer: assets are
@@ -177,26 +131,10 @@ func staticHandler() http.Handler {
}))
}
type ctxKey int
// readerCtxKey is where requireSession stashes the authenticated Reader id.
const readerCtxKey ctxKey = iota
// sessionReader reports whether the request carries a live session, and for
// whom. The cookie holds only the session id; the row behind it is looked up
// on every request, so deleting a session takes effect immediately. Expiry is
// enforced here, in the store, which also removes rows that have lapsed.
func (h *Handler) sessionReader(r *http.Request) (int64, bool) {
// authed reports whether the request carries a valid session cookie.
func (h *Handler) authed(r *http.Request) bool {
c, err := r.Cookie(session.CookieName)
if err != nil {
return 0, false
}
sess, ok, err := h.store.GetSession(c.Value, time.Now())
if err != nil {
log.Printf("session lookup: %v", err)
return 0, false
}
return sess.ReaderID, ok
return err == nil && session.Verify(h.key, c.Value, time.Now().UnixMilli())
}
// requireSession guards the fragment endpoints. It answers 401 rather than
@@ -204,18 +142,14 @@ func (h *Handler) sessionReader(r *http.Request) (int64, bool) {
// redirected login page would be spliced into the card list.
func (h *Handler) requireSession(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
readerID, ok := h.sessionReader(r)
if !ok {
if !h.authed(r) {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
next(w, r.WithContext(context.WithValue(r.Context(), readerCtxKey, readerID)))
next(w, r)
}
}
// readerOf returns the authenticated Reader id requireSession stashed.
func readerOf(r *http.Request) int64 { return r.Context().Value(readerCtxKey).(int64) }
func (h *Handler) render(w http.ResponseWriter, status int, name string, data any) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(status)
@@ -229,25 +163,16 @@ func (h *Handler) render(w http.ResponseWriter, status int, name string, data an
// page is served at / with status 200 rather than as a redirect to a separate
// URL: one page, no redirect loop to reason about.
func (h *Handler) index(w http.ResponseWriter, r *http.Request) {
readerID, ok := h.sessionReader(r)
if !ok {
if !h.authed(r) {
h.render(w, http.StatusOK, "login", loginView{})
return
}
view, err := h.buildListView(readerID, libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
view, err := h.buildListView(libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
if err != nil {
log.Printf("index: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
if readerID == h.store.OwnerID() {
view.Owner, view.OwnerID = true, readerID
if view.Readers, err = h.store.Readers(); err != nil {
log.Printf("index readers: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
}
h.render(w, http.StatusOK, "app", view)
}
@@ -283,22 +208,18 @@ func libOf(q string) string {
return store.KindManga
}
// buildListView loads one reader's list once and derives both the tab-filtered
// items and the recent strip from it.
// buildListView loads the list once and derives both the tab-filtered items and
// the recent strip from it.
//
// Archived and finished series appear in their own tab and nowhere else — not
// in All, not in Updated, not in Favourites, and not in the recent strip. An
// archived favourite therefore shows only under Archived: Favourites means
// "favourites I am currently reading".
func (h *Handler) buildListView(readerID int64, lib, tab string) (listView, error) {
all, err := h.store.List(readerID) // already ordered updated_at DESC
func (h *Handler) buildListView(lib, tab string) (listView, error) {
all, err := h.store.List() // already ordered updated_at DESC
if err != nil {
return listView{}, err
}
// Taken before the filter narrows the slice: a Reader with novels but no
// manga has a working install already, and does not need to be told to go
// and get one.
emptyLibrary := len(all) == 0
// Narrow to one library first: reading, withNew and recent all derive from
// this slice, so doing it later would let the other library's rows into the
// strip and the Updated badge.
@@ -342,12 +263,11 @@ func (h *Handler) buildListView(readerID int64, lib, tab string) (listView, erro
recent = recent[:RecentCount]
}
}
return listView{Lib: lib, Tab: tab, Recent: recent, Items: items,
NewCount: len(withNew), EmptyLibrary: emptyLibrary}, nil
return listView{Lib: lib, Tab: tab, Recent: recent, Items: items, NewCount: len(withNew)}, nil
}
func (h *Handler) uiList(w http.ResponseWriter, r *http.Request) {
view, err := h.buildListView(readerOf(r), libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
view, err := h.buildListView(libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
if err != nil {
log.Printf("ui list: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
@@ -405,7 +325,7 @@ func (h *Handler) writeChromeOOB(w http.ResponseWriter, view listView) {
// refreshChrome rebuilds the chrome for the reader's current tab after a
// mutation and appends it to the response.
func (h *Handler) refreshChrome(w http.ResponseWriter, r *http.Request) {
view, err := h.buildListView(readerOf(r), currentLib(r), currentTab(r))
view, err := h.buildListView(currentLib(r), currentTab(r))
if err != nil {
log.Printf("ui chrome: %v", err)
return
@@ -413,21 +333,35 @@ func (h *Handler) refreshChrome(w http.ResponseWriter, r *http.Request) {
h.writeChromeOOB(w, view)
}
// renderLogin renders the login page with an error message, for refused or
// failed sign-ins. Every message is author-written text — nothing Discord
// supplied is ever interpolated into a page.
func (h *Handler) renderLogin(w http.ResponseWriter, status int, msg string) {
h.render(w, status, "login", loginView{Error: msg})
func (h *Handler) login(w http.ResponseWriter, r *http.Request) {
ip := session.ClientIP(r)
if wait := h.limiter.RetryAfter(ip, time.Now()); wait > 0 {
secs := int(wait.Seconds()) + 1
w.Header().Set("Retry-After", strconv.Itoa(secs))
h.render(w, http.StatusTooManyRequests, "login", loginView{
Error: "Too many attempts. Try again in " +
strconv.Itoa((secs+59)/60) + " min.",
})
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, "invalid form", http.StatusBadRequest)
return
}
got := r.PostFormValue("password")
if subtle.ConstantTimeCompare([]byte(got), []byte(h.password)) != 1 {
h.limiter.Fail(ip, time.Now())
h.render(w, http.StatusUnauthorized, "login", loginView{Error: "Wrong password."})
return
}
h.limiter.Reset(ip)
session.SetCookie(w, r, h.key)
http.Redirect(w, r, "/", http.StatusSeeOther)
}
// logout revokes the session row and clears the cookie in one step: the next
// request finds no row and is rejected.
func (h *Handler) logout(w http.ResponseWriter, r *http.Request) {
if c, err := r.Cookie(session.CookieName); err == nil {
if err := h.store.DeleteSession(c.Value); err != nil {
log.Printf("delete session: %v", err)
}
}
session.ClearCookie(w, r)
http.Redirect(w, r, "/", http.StatusSeeOther)
}
@@ -440,7 +374,7 @@ func (h *Handler) loadForMutation(w http.ResponseWriter, r *http.Request) (store
http.Error(w, "missing key", http.StatusBadRequest)
return store.Bookmark{}, false
}
b, ok, err := h.store.Get(readerOf(r), key)
b, ok, err := h.store.Get(key)
if err != nil {
log.Printf("ui get %q: %v", key, err)
http.Error(w, "internal error", http.StatusInternalServerError)
@@ -463,7 +397,7 @@ func (h *Handler) loadForMutation(w http.ResponseWriter, r *http.Request) (store
// describe the whole library, so they are rebuilt out of band on every
// mutation, at the cost of one extra list read per toggle.
func (h *Handler) saveAndRenderCard(w http.ResponseWriter, r *http.Request, b store.Bookmark) {
stored, err := h.store.Upsert(readerOf(r), b)
stored, err := h.store.Upsert(b)
if err != nil {
log.Printf("ui upsert %q: %v", b.Key, err)
http.Error(w, "internal error", http.StatusInternalServerError)
@@ -555,7 +489,7 @@ func (h *Handler) uiDelete(w http.ResponseWriter, r *http.Request) {
http.Error(w, "missing key", http.StatusBadRequest)
return
}
if err := h.store.Delete(readerOf(r), key); err != nil {
if err := h.store.Delete(key); err != nil {
log.Printf("ui delete %q: %v", key, err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
@@ -566,92 +500,3 @@ func (h *Handler) uiDelete(w http.ResponseWriter, r *http.Request) {
// the library got smaller.
h.refreshChrome(w, r)
}
// installUserscript renders the bindmounted script with the acting Reader's
// derived credential substituted in. The credential is derived, not stored,
// so installs work after any restart; the Reader never types or copies it —
// clicking Install is the whole setup.
//
// ?download=1 forces a save instead. Mobile Violentmonkey (Chromium) does not
// intercept navigation to a .user.js URL, so the Install link only renders the
// source as text there; the Reader needs the file on disk to add it by hand.
func (h *Handler) installUserscript(name string) http.HandlerFunc {
path := h.mangaUserscriptPath
if name == "novel-bookmark.user.js" {
path = h.novelUserscriptPath
}
return func(w http.ResponseWriter, r *http.Request) {
discordID, epoch, err := h.store.ReaderTokenInfo(readerOf(r))
if err != nil {
log.Printf("install %s: %v", name, err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
if r.URL.Query().Has("download") {
w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`)
}
userscript.Render(w, r, path, token.Token(h.tokenKey, discordID, epoch))
}
}
// rotateToken issues the acting Reader a new credential: the epoch bumps and
// the stored hash is rewritten, so the old credential stops authenticating
// the moment the statement commits. Every device must reinstall, or its
// script keeps failing silently — the setup panel states that warning next
// to the button, and the response repeats it as confirmation.
func (h *Handler) rotateToken(w http.ResponseWriter, r *http.Request) {
readerID := readerOf(r)
discordID, epoch, err := h.store.ReaderTokenInfo(readerID)
if err != nil {
log.Printf("rotate token: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
// The hash is computed for epoch+1 and guarded by it in the store, so a
// concurrent rotation cannot leave the stored hash describing another
// epoch.
if err := h.store.RotateToken(readerID, epoch, token.Hash(token.Token(h.tokenKey, discordID, epoch+1))); err != nil {
log.Printf("rotate token: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
view := listView{Lib: store.KindManga, Rotated: true}
h.render(w, http.StatusOK, "setup", view)
}
// revokeReaderSessions logs one Reader out of every browser they are signed
// in on. Owner-only: it reaches across the Reader boundary every other handler
// respects, so the guard is a comparison against the seeded owner rather than
// a role a Reader could acquire. A non-owner gets 404 — the panel does not
// exist for them, so neither should the endpoint.
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
if readerOf(r) != h.store.OwnerID() {
http.NotFound(w, r)
return
}
target, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil {
http.Error(w, "bad reader id", http.StatusBadRequest)
return
}
// The owner is not one of the Readers this endpoint reaches: revoking
// themselves would sign out the browser making the request, which is what
// logout is for. The roster hides the button; this refuses the hand-rolled
// POST behind it.
if target == h.store.OwnerID() {
http.NotFound(w, r)
return
}
if err := h.store.DeleteReaderSessions(target); err != nil {
log.Printf("revoke sessions: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
readers, err := h.store.Readers()
if err != nil {
log.Printf("revoke sessions: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.render(w, http.StatusOK, "readers", listView{Owner: true, Readers: readers, OwnerID: h.store.OwnerID()})
}
+70 -192
View File
@@ -16,38 +16,18 @@ import (
"bookmarkmanager/backend/internal/httpmw"
"bookmarkmanager/backend/internal/latest"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/token"
"bookmarkmanager/backend/internal/userscript"
"bookmarkmanager/backend/internal/web"
)
// Config holds all runtime settings, sourced from environment variables.
type Config struct {
// TokenKey derives every Reader's userscript credential (internal/token).
// Required: without it no install URL can ever be built.
TokenKey string
Token string
AllowedOrigins []string
// DatabaseURL is the Postgres connection URL; required, no default,
// because a wrong guess would silently start on an empty database.
DatabaseURL string
// CoverDir is the filesystem volume for immutable cover bytes. Required:
// serving a stored address without durable bytes would be worse than a
// startup failure.
CoverDir string
// PublicBaseURL is the origin this deployment answers on, e.g.
// "https://bookmarks.example.com". Required: cover URLs go out absolute
// because the userscript renders them on third-party origins, where a
// relative path would resolve against the Site (ADR-0007), and there is
// no way to guess it from a request the poller never sees.
PublicBaseURL string
Port string
// OwnerDiscordID identifies the seeded owner Reader (issue #22). Required:
// bookmarks are scoped to a Reader, and a fresh deployment needs one
// before anybody logs in. The owner is also the only Reader who can revoke
// another Reader's sessions.
OwnerDiscordID string
// Discord is the OAuth application the browser UI signs in with.
Discord web.DiscordConfig
DBPath string
Port string
// WebPassword gates the browser UI. Empty disables the web routes entirely.
WebPassword string
// UserscriptPath is the file served at /u/{token}/manga-bookmark.user.js.
// Supplied by a bindmount so the script can be edited without a rebuild.
UserscriptPath string
@@ -66,20 +46,18 @@ type Config struct {
// deployment. Past that nothing breaks; the effective cadence stretches to
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
type LatestPoll struct {
Enabled bool
Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
Enabled bool
Cooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
const (
defaultPollCooldown = time.Hour
defaultBrowserPollCooldown = 6 * time.Hour
defaultPollInterval = 10 * time.Minute
defaultPollStagger = 20 * time.Second
defaultPollBatch = 14
defaultPollCooldown = time.Hour
defaultPollInterval = 10 * time.Minute
defaultPollStagger = 20 * time.Second
defaultPollBatch = 14
// minPollCooldown keeps a typo from turning a polite background check into
// a hammer against sites that are already bot-scoring us.
minPollCooldown = 15 * time.Minute
@@ -137,27 +115,20 @@ func envInt(key string, def int) int {
return n
}
func clampPollCooldown(name string, d time.Duration) time.Duration {
if d < minPollCooldown {
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
return minPollCooldown
}
return d
}
// loadLatestPoll reads the poller's settings, clamping anything that would make
// it antisocial.
func loadLatestPoll() LatestPoll {
p := LatestPoll{
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
}
if p.Cooldown < minPollCooldown {
log.Printf("config: cooldown %s is below the %s floor, clamping", p.Cooldown, minPollCooldown)
p.Cooldown = minPollCooldown
}
p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
// batch x stagger has to fit inside one tick or a batch is still running
// when the next one is due. Run() serialises them, so this degrades to a
// slower cadence rather than to overlapping fetches — worth a warning, not
@@ -171,24 +142,14 @@ func loadLatestPoll() LatestPoll {
func loadConfig() Config {
c := Config{
TokenKey: os.Getenv("TOKEN_KEY"),
DatabaseURL: os.Getenv("DATABASE_URL"),
CoverDir: os.Getenv("COVER_DIR"),
PublicBaseURL: os.Getenv("PUBLIC_BASE_URL"),
Token: os.Getenv("API_TOKEN"),
DBPath: envOr("DB_PATH", "/data/bookmarks.db"),
Port: envOr("PORT", "8080"),
OwnerDiscordID: os.Getenv("OWNER_DISCORD_ID"),
WebPassword: os.Getenv("WEB_PASSWORD"),
UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"),
NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"),
LatestPoll: loadLatestPoll(),
}
c.Discord = web.DiscordConfig{
ClientID: os.Getenv("DISCORD_CLIENT_ID"),
ClientSecret: os.Getenv("DISCORD_CLIENT_SECRET"),
GuildID: os.Getenv("DISCORD_GUILD_ID"),
RequiredRole: os.Getenv("DISCORD_REQUIRED_ROLE"),
APIBase: envOr("DISCORD_API_BASE", "https://discord.com/api/v10"),
RedirectURI: os.Getenv("DISCORD_REDIRECT_URI"),
}
for _, o := range strings.Split(os.Getenv("ALLOWED_ORIGINS"), ",") {
if o = strings.TrimSpace(o); o != "" {
c.AllowedOrigins = append(c.AllowedOrigins, o)
@@ -202,40 +163,34 @@ func loadConfig() Config {
// /healthz is public.
func newRouter(s *store.Store, cfg Config) http.Handler {
mux := http.NewServeMux()
h := &api.Handler{Store: s}
mux.HandleFunc("GET /healthz", api.Healthz)
// Public: cover bytes are rendered by the userscript on origins that may
// not send our credentials, and the address is the hash of a URL the Site
// already publishes (ADR-0007).
mux.HandleFunc("GET /covers/{address}", h.Cover)
// Outside httpmw.Auth (the updater sends no Authorization header) and
// outside the web UI's Discord auth (the script must be installable
// without a browser session). The path segment carries the credential
// instead, and the script is rendered with the resolved Reader's
// credential substituted in.
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js",
userscript.Handler(s, cfg.UserscriptPath))
mux.HandleFunc("GET /u/{token}/novel-bookmark.user.js",
userscript.Handler(s, cfg.NovelUserscriptPath))
// outside the WEB_PASSWORD gate (the script must be installable either
// way). The path segment carries the token instead.
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js", userscript.Handler(cfg.Token, cfg.UserscriptPath))
mux.HandleFunc("GET /u/{token}/novel-bookmark.user.js", userscript.Handler(cfg.Token, cfg.NovelUserscriptPath))
h := &api.Handler{Store: s}
protected := http.NewServeMux()
protected.HandleFunc("GET /bookmarks", h.List)
protected.HandleFunc("PUT /bookmarks/{key}", h.Put)
protected.HandleFunc("DELETE /bookmarks/{key}", h.Delete)
auth := httpmw.Auth(s, protected)
auth := httpmw.Auth(cfg.Token, protected)
mux.Handle("/bookmarks", auth)
mux.Handle("/bookmarks/", auth)
// The browser UI is always registered; signing in is Discord OAuth, so
// there is no password to forget and no gate to leave unset.
wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey),
cfg.UserscriptPath, cfg.NovelUserscriptPath)
if err != nil {
log.Fatalf("web handler: %v", err)
// The browser UI is registered only when a password is configured, so a
// deployment that forgets WEB_PASSWORD exposes nothing rather than
// exposing an unprotected list.
if cfg.WebPassword != "" {
wh, err := web.New(s, cfg.Token, cfg.WebPassword)
if err != nil {
log.Fatalf("web handler: %v", err)
}
wh.Register(mux)
}
wh.Register(mux)
return httpmw.CORS(cfg.AllowedOrigins, httpmw.Gzip(guardEmptyUserscriptToken(mux)))
}
@@ -257,42 +212,11 @@ func guardEmptyUserscriptToken(next http.Handler) http.Handler {
func main() {
cfg := loadConfig()
if cfg.TokenKey == "" {
log.Fatal("TOKEN_KEY is required")
}
if cfg.OwnerDiscordID == "" {
log.Fatal("OWNER_DISCORD_ID is required")
}
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required")
}
if cfg.CoverDir == "" {
log.Fatal("COVER_DIR is required")
}
if cfg.PublicBaseURL == "" {
log.Fatal("PUBLIC_BASE_URL is required")
}
// The web UI signs in through Discord, so a deployment without the OAuth
// application is misconfigured rather than passwordless.
for key, v := range map[string]string{
"DISCORD_CLIENT_ID": cfg.Discord.ClientID,
"DISCORD_CLIENT_SECRET": cfg.Discord.ClientSecret,
"DISCORD_GUILD_ID": cfg.Discord.GuildID,
"DISCORD_REDIRECT_URI": cfg.Discord.RedirectURI,
} {
if v == "" {
log.Fatalf("%s is required", key)
}
}
// The owner's userscript credential is derived from TOKEN_KEY at epoch 0
// (internal/token); the readers row carries its SHA-256, not the
// credential itself.
owner := store.Owner{
DiscordID: cfg.OwnerDiscordID,
TokenHash: token.Hash(token.Token([]byte(cfg.TokenKey), cfg.OwnerDiscordID, 0)),
if cfg.Token == "" {
log.Fatal("API_TOKEN is required")
}
s, err := store.Open(cfg.DatabaseURL, owner, cfg.CoverDir, cfg.PublicBaseURL)
s, err := store.Open(cfg.DBPath)
if err != nil {
log.Fatalf("open store: %v", err)
}
@@ -301,52 +225,9 @@ func main() {
// The poller is off the request path entirely: if it cannot start, the
// service still serves bookmarks and the userscript still captures latest
// chapters on its own.
//
// One headless browser serves both consumers that need a Cloudflare
// challenge cleared: the poller's kagane/novelfull page fetches and
// kagane's cover bytes. Optional — unset leaves kagane unpolled and its
// Covers blank until the bytes exist.
var browser latest.Fetcher
pollCtx, stopPoll := context.WithCancel(context.Background())
defer stopPoll()
if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" {
bf, err := latest.NewBrowserFetcher(ws)
if err != nil {
log.Printf("browser fetcher disabled: %v", err)
} else {
browser = bf
context.AfterFunc(pollCtx, bf.Close)
log.Printf("browser fetcher at %s", ws)
}
}
// A Series nobody had bookmarked before gets its Latest Chapter and its
// Cover from one fetch, at creation, instead of waiting out a poll queue
// ordered by Reader count. Off the write path: the hook returns as soon
// as the goroutine is started.
var tlsFetch latest.Fetcher
if f, err := latest.NewTLSFetcher(); err != nil {
log.Printf("creation-time acquisition: plain-TLS Sites disabled, cannot build client: %v", err)
} else {
tlsFetch = f
}
var browserCover latest.BrowserCoverFetcher
if b, ok := browser.(latest.BrowserCoverFetcher); ok {
browserCover = b
}
// The Acquirer must survive a TLS client failure: kagane needs only the
// sidecar, and novelfull degrades to whatever is left.
if tlsFetch != nil || browser != nil {
acq := &latest.Acquirer{
Store: s,
Fetch: tlsFetch,
BrowserFetch: browser,
BrowserCoverFetch: browserCover,
Covers: latest.NewCoverFetcher(),
Ctx: pollCtx,
}
s.OnSeriesCreated = acq.Acquire
}
startLatestPoller(pollCtx, s, cfg.LatestPoll, browser)
startLatestPoller(pollCtx, s, cfg.LatestPoll)
srv := &http.Server{
Addr: ":" + cfg.Port,
@@ -355,8 +236,7 @@ func main() {
}
go func() {
// The connection URL carries a password, so it stays out of the log.
log.Printf("listening on :%s (origins=%v)", cfg.Port, cfg.AllowedOrigins)
log.Printf("listening on :%s (db=%s, origins=%v)", cfg.Port, cfg.DBPath, cfg.AllowedOrigins)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("serve: %v", err)
}
@@ -377,32 +257,11 @@ func main() {
}
}
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
var covers latest.BrowserCoverFetcher
if f, ok := browser.(latest.BrowserCoverFetcher); ok {
covers = f
}
return &latest.Poller{
Store: s,
Fetch: fetch,
BrowserFetch: browser,
CoverFetch: covers,
CoverBytesFetch: latest.NewCoverFetcher(),
Now: time.Now,
Cooldown: cfg.Cooldown,
BrowserCooldown: cfg.BrowserCooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
}
// startLatestPoller launches the background poller unless it is disabled or its
// HTTP client cannot be built. Any problem here is logged and skipped: this
// feature going missing degrades the service to userscript-only latest-chapter
// tracking, which is exactly how it behaved before.
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) {
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll) {
if !cfg.Enabled {
log.Println("latest-chapter poller: disabled by config")
return
@@ -412,10 +271,29 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, brow
log.Printf("latest-chapter poller: disabled, cannot build client: %v", err)
return
}
// Nil browser: sites behind a JavaScript challenge are simply not polled,
// and their latest_chapter comes from the userscript alone — which is how
// the service behaved before the sidecar existed.
p := newLatestPoller(s, cfg, f, browser)
p := &latest.Poller{
Store: s,
Fetch: f,
Now: time.Now,
Cooldown: cfg.Cooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
// Optional: without it, sites behind a JavaScript challenge are simply not
// polled, and their latest_chapter comes from the userscript alone — which
// is how the service behaved before the sidecar existed.
if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" {
bf, err := latest.NewBrowserFetcher(ws)
if err != nil {
log.Printf("latest-chapter poller: browser fetcher disabled: %v", err)
} else {
p.BrowserFetch = bf
context.AfterFunc(ctx, bf.Close)
log.Printf("latest-chapter poller: browser fetcher at %s", ws)
}
}
go p.Run(ctx)
}
+8 -50
View File
@@ -17,33 +17,25 @@ import (
func TestLoadLatestPollDefaults(t *testing.T) {
for _, k := range []string{
"LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN",
"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", "LATEST_CHAPTER_POLL_INTERVAL",
"LATEST_CHAPTER_POLL_STAGGER", "LATEST_CHAPTER_POLL_BATCH",
"LATEST_CHAPTER_POLL_INTERVAL", "LATEST_CHAPTER_POLL_STAGGER",
"LATEST_CHAPTER_POLL_BATCH",
} {
t.Setenv(k, "")
}
got := loadLatestPoll()
want := LatestPoll{
Enabled: true,
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute,
Stagger: 20 * time.Second,
Batch: 14,
Enabled: true,
Cooldown: time.Hour,
Interval: 10 * time.Minute,
Stagger: 20 * time.Second,
Batch: 14,
}
if got != want {
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, want)
}
}
func TestLoadConfigReadsCoverDirectory(t *testing.T) {
t.Setenv("COVER_DIR", "/covers")
if got := loadConfig().CoverDir; got != "/covers" {
t.Fatalf("CoverDir = %q, want /covers", got)
}
}
func TestLoadLatestPollEnabledParsing(t *testing.T) {
tests := []struct {
raw string
@@ -82,30 +74,6 @@ func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
wantFrom: func(p LatestPoll) any { return p.Cooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown below the floor is clamped up",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "1m"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown at the floor is kept",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "15m"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown override is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "8h"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 8 * time.Hour,
},
{
name: "browser cooldown unparseable value falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "six hours"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 6 * time.Hour,
},
{
name: "a valid override is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"},
@@ -155,16 +123,6 @@ func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
}
}
func TestNewLatestPollerWiresCooldowns(t *testing.T) {
p := newLatestPoller(nil, LatestPoll{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
}, nil, nil)
if p.Cooldown != time.Hour || p.BrowserCooldown != 6*time.Hour {
t.Fatalf("poller cooldowns = %s/%s, want 1h/6h", p.Cooldown, p.BrowserCooldown)
}
}
func TestPutStatusValidation(t *testing.T) {
cases := []struct {
name string
@@ -244,7 +202,7 @@ func TestPutOmittedStatusPreservesArchivedAndAppliesProgress(t *testing.T) {
}
func TestGzipCompressesTextNotFonts(t *testing.T) {
srv, _ := newWebTestServer(t, testConfig())
srv, _ := newWebTestServer(t, webConfig())
cases := []struct {
path string
-356
View File
@@ -1,356 +0,0 @@
package main
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/token"
)
// registerReader creates an extra Reader the way a first login does and
// returns its id. The credential is derived the same way the owner's is, so it
// authenticates through the real router.
func registerReader(t *testing.T, s *store.Store, discordID string) int64 {
t.Helper()
id, err := s.EnsureReader(discordID, token.Hash(readerCredential(discordID)))
if err != nil {
t.Fatalf("register reader %q: %v", discordID, err)
}
return id
}
// credRequest builds a request authenticated as the Reader whose credential
// is passed.
func credRequest(method, target, cred string) *http.Request {
req := httptest.NewRequest(method, target, nil)
req.Header.Set("Authorization", "Bearer "+cred)
return req
}
// readerCredential is the epoch-0 derived credential of an arbitrary Reader.
func readerCredential(discordID string) string {
return token.Token([]byte(testTokenKey), discordID, 0)
}
// withBody attaches a request body, for PUTs that carry a JSON payload.
func withBody(req *http.Request, body string) *http.Request {
req.Body = io.NopCloser(strings.NewReader(body))
req.ContentLength = int64(len(body))
return req
}
// A refused credential is refused however plausible it looks: only a hash the
// readers table holds authenticates anything.
func TestUnknownCredentialRejected(t *testing.T) {
srv := newRouter(newTestStore(t), testConfig())
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("never-registered")))
if rr.Code != http.StatusUnauthorized {
t.Fatalf("unregistered Reader's credential: status = %d, want 401", rr.Code)
}
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", ownerCredential()))
if rr.Code != http.StatusOK {
t.Fatalf("owner's derived credential: status = %d, want 200", rr.Code)
}
}
// A Reader's credential authenticates exactly that Reader: rows written under
// one credential are invisible to the other, on the same key.
func TestPerReaderIsolation(t *testing.T) {
s := newTestStore(t)
registerReader(t, s, "other-reader")
srv := newRouter(s, testConfig())
ownerKey := "asura:solo"
putBookmark(t, srv, ownerKey, store.Bookmark{
Key: ownerKey, Site: "asura", SeriesID: "solo",
Title: "Solo Leveling", UpdatedAt: 1,
})
// The other Reader's list is empty even though the owner holds the key.
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
if rr.Code != http.StatusOK {
t.Fatalf("other reader list: status = %d, want 200", rr.Code)
}
var theirs []store.Bookmark
if err := json.Unmarshal(rr.Body.Bytes(), &theirs); err != nil {
t.Fatalf("decode: %v", err)
}
if len(theirs) != 0 {
t.Fatalf("other reader sees %d bookmarks, want 0 (owner's rows leaked)", len(theirs))
}
// The other Reader writes the same key; both rows coexist, each visible
// only to its owner. The series title is shared (ADR-0003) — the
// reader-owned fields are progress and updated_at.
req := credRequest(http.MethodPut, "/bookmarks/"+ownerKey, readerCredential("other-reader"))
req.Header.Set("Content-Type", "application/json")
body := `{"key":"asura:solo","site":"asura","series_id":"solo","title":"Theirs","last_chapter_num":3}`
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, withBody(req, body))
if rr.Code != http.StatusOK {
t.Fatalf("other reader put: status = %d, want 200", rr.Code)
}
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
var theirs2 []store.Bookmark
if err := json.Unmarshal(rr.Body.Bytes(), &theirs2); err != nil {
t.Fatalf("decode: %v", err)
}
if len(theirs2) != 1 || theirs2[0].LastChapterNum != 3 {
t.Fatalf("other reader list = %+v, want their own row with their progress", theirs2)
}
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", ownerCredential()))
var owners []store.Bookmark
if err := json.Unmarshal(rr.Body.Bytes(), &owners); err != nil {
t.Fatalf("decode: %v", err)
}
if len(owners) != 1 || owners[0].Title != "Solo Leveling" || owners[0].LastChapterNum != 0 {
t.Fatalf("owner list = %+v, want their own row at their own progress", owners)
}
// The mirror: the owner's write does not move the other Reader's progress
// either. Without it, isolation is only asserted in one direction.
req = credRequest(http.MethodPut, "/bookmarks/"+ownerKey, ownerCredential())
req.Header.Set("Content-Type", "application/json")
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, withBody(req, `{"key":"asura:solo","site":"asura","series_id":"solo","title":"Solo Leveling","last_chapter_num":9}`))
if rr.Code != http.StatusOK {
t.Fatalf("owner put: status = %d, want 200", rr.Code)
}
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
var theirs3 []store.Bookmark
if err := json.Unmarshal(rr.Body.Bytes(), &theirs3); err != nil {
t.Fatalf("decode: %v", err)
}
if len(theirs3) != 1 || theirs3[0].LastChapterNum != 3 {
t.Fatalf("other reader list = %+v, want progress 3 after the owner's write", theirs3)
}
// DELETE is scoped to its caller too, asserted in both directions: each
// Reader's delete on the shared key takes only their own row.
list := func(cred string) []store.Bookmark {
t.Helper()
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", cred))
var got []store.Bookmark
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
return got
}
del := func(cred string) {
t.Helper()
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodDelete, "/bookmarks/"+ownerKey, cred))
if rr.Code != http.StatusNoContent {
t.Fatalf("delete: status = %d, want 204", rr.Code)
}
}
del(readerCredential("other-reader"))
if got := list(ownerCredential()); len(got) != 1 {
t.Fatalf("owner's row was deletable by the other Reader: %+v", got)
}
if got := list(readerCredential("other-reader")); len(got) != 0 {
t.Fatalf("other Reader's own delete left %+v behind", got)
}
// The mirror: the other Reader takes the key again, the owner deletes
// theirs, and the other's row is untouched.
req = credRequest(http.MethodPut, "/bookmarks/"+ownerKey, readerCredential("other-reader"))
req.Header.Set("Content-Type", "application/json")
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, withBody(req, body))
if rr.Code != http.StatusOK {
t.Fatalf("other reader re-put: status = %d, want 200", rr.Code)
}
del(ownerCredential())
if got := list(readerCredential("other-reader")); len(got) != 1 {
t.Fatalf("other Reader's row was deletable by the owner: %+v", got)
}
if got := list(ownerCredential()); len(got) != 0 {
t.Fatalf("owner's own delete left %+v behind", got)
}
}
// The install endpoints are session-gated and render the script directly
// with the Reader's credential inside: the credential never appears in the
// address bar, the page markup, or any Location header.
func TestInstallServesScriptWithCredential(t *testing.T) {
cfg := testConfig()
dir := t.TempDir()
path := filepath.Join(dir, "manga-bookmark.user.js")
novelPath := filepath.Join(dir, "novel-bookmark.user.js")
for _, p := range []string{path, novelPath} {
if err := os.WriteFile(p, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
t.Fatalf("write script: %v", err)
}
}
cfg.UserscriptPath = path
cfg.NovelUserscriptPath = novelPath
srv, st := newWebTestServer(t, cfg)
for _, script := range []string{"manga-bookmark.user.js", "novel-bookmark.user.js"} {
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/install/"+script, nil))
if rr.Code != http.StatusUnauthorized {
t.Fatalf("%s without session: status = %d, want 401", script, rr.Code)
}
req := httptest.NewRequest(http.MethodGet, "/install/"+script, nil)
req.AddCookie(sessionCookie(t, st))
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("%s with session: status = %d, want 200", script, rr.Code)
}
body := rr.Body.String()
if strings.Contains(body, "__API_TOKEN__") {
t.Fatalf("%s served with an unsubstituted placeholder", script)
}
// The credential rides inside the served script — nowhere visible in
// the UI — and is the session holder's own.
if !strings.Contains(body, `API_TOKEN = "`+ownerCredential()+`"`) {
t.Fatalf("%s does not carry the owner's credential:\n%s", script, body)
}
if loc := rr.Header().Get("Location"); loc != "" {
t.Fatalf("%s answered with a redirect, credential in Location %q", script, loc)
}
// The plain link must stay inline: Violentmonkey's updater polls the
// /u/ path and an attachment disposition there would break updates.
if cd := rr.Header().Get("Content-Disposition"); cd != "" {
t.Fatalf("%s served as %q, want inline", script, cd)
}
// ?download=1 is the mobile path: Violentmonkey on Chromium ignores a
// .user.js navigation, so the Reader saves the file and adds it by hand.
req = httptest.NewRequest(http.MethodGet, "/install/"+script+"?download=1", nil)
req.AddCookie(sessionCookie(t, st))
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("%s?download=1: status = %d, want 200", script, rr.Code)
}
if got, want := rr.Header().Get("Content-Disposition"), `attachment; filename="`+script+`"`; got != want {
t.Fatalf("%s?download=1: Content-Disposition = %q, want %q", script, got, want)
}
if !strings.Contains(rr.Body.String(), `API_TOKEN = "`+ownerCredential()+`"`) {
t.Fatalf("%s?download=1 does not carry the owner's credential", script)
}
}
}
// Rotation through the web UI invalidates the old credential immediately,
// mints one that authenticates the API and the script path, and warns that
// every device must reinstall.
func TestRotateCredentialViaWebUI(t *testing.T) {
s, _ := newTestStoreURL(t)
path := filepath.Join(t.TempDir(), "manga-bookmark.user.js")
if err := os.WriteFile(path, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
t.Fatalf("write script: %v", err)
}
cfg := testConfig()
cfg.UserscriptPath = path
srv := newRouter(s, cfg)
oldCred := ownerCredential()
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", oldCred))
if rr.Code != http.StatusOK {
t.Fatalf("old credential before rotation: status = %d, want 200", rr.Code)
}
req := httptest.NewRequest(http.MethodPost, "/rotate-token", nil)
req.AddCookie(sessionCookie(t, s))
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("rotate: status = %d, want 200", rr.Code)
}
if !strings.Contains(rr.Body.String(), "Credential rotated") {
t.Fatalf("rotation response does not warn about reinstall:\n%s", rr.Body.String())
}
// The old credential is dead on the API and on the script path.
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", oldCred))
if rr.Code != http.StatusUnauthorized {
t.Fatalf("old credential after rotation: status = %d, want 401", rr.Code)
}
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+oldCred+"/manga-bookmark.user.js", nil))
if rr.Code != http.StatusNotFound {
t.Fatalf("old credential script path after rotation: status = %d, want 404", rr.Code)
}
// The new credential authenticates the API and the script path, and is
// substituted into the served script.
newCred := token.Token([]byte(testTokenKey), testDiscordID, 1)
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", newCred))
if rr.Code != http.StatusOK {
t.Fatalf("new credential after rotation: status = %d, want 200", rr.Code)
}
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+newCred+"/manga-bookmark.user.js", nil))
if rr.Code != http.StatusOK {
t.Fatalf("new credential script path: status = %d, want 200", rr.Code)
}
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+newCred+`"`) {
t.Fatalf("served script does not carry the rotated credential:\n%s", got)
}
// The install link now renders the script with the new credential.
req = httptest.NewRequest(http.MethodGet, "/install/manga-bookmark.user.js", nil)
req.AddCookie(sessionCookie(t, s))
rr = httptest.NewRecorder()
srv.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("install after rotation: status = %d, want 200", rr.Code)
}
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+newCred+`"`) {
t.Fatalf("install after rotation does not carry the new credential:\n%s", got)
}
}
// The app page offers the install links; the credential never appears in its
// markup.
func TestIndexShowsSetupPanelWithoutCredential(t *testing.T) {
srv, st := newWebTestServer(t, testConfig())
req := httptest.NewRequest(http.MethodGet, "/", nil)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, req)
body := rr.Body.String()
for _, want := range []string{
`href="/install/manga-bookmark.user.js"`,
`href="/install/novel-bookmark.user.js"`,
`href="/install/manga-bookmark.user.js?download=1"`,
`href="/install/novel-bookmark.user.js?download=1"`,
"Rotate credential",
} {
if !strings.Contains(body, want) {
t.Errorf("app page lacks %q", want)
}
}
if strings.Contains(body, ownerCredential()) {
t.Fatal("app page leaks the credential")
}
}
+139 -722
View File
File diff suppressed because it is too large Load Diff
-28
View File
@@ -1,28 +0,0 @@
# Copy to chrome/.env on the home machine. Never commit the real .env.
#
# This file configures the browser unit only. It is separate from the API
# stack's ../.env on purpose: the two run on different machines.
# The address the CDP port is published on — required, no default.
#
# Use this machine's **tailnet IP**, e.g. 100.x.y.z (`tailscale ip -4`). Not
# 0.0.0.0, not the LAN address: CDP has no authentication of its own, so
# anything that can reach this port has full control of the browser and a
# foothold on this host. Tailscale device identity plus an ACL is the access
# control; the bind address is what enforces it.
#
# For a throwaway local test, 127.0.0.1 is fine — but then only this machine
# can reach it, so the API must run here too.
# Left commented so `cp .env.example .env && docker compose up` fails with the
# variable's own message telling you what to set, rather than Docker rejecting
# "100.x.y.z" as an invalid IP.
# BROWSER_BIND_ADDR=100.x.y.z
# Clock zone the browser reports. Any real zone works and it need not match
# the egress IP's country — but it must not be UTC, which is itself the bot
# signal that stops the challenge clearing. The measurement is in entrypoint.sh.
#
# Unset falls back to the host's /etc/timezone, which is a real zone whenever
# the host clock is set to local time. Set this when the host runs UTC — a UTC
# server is exactly the case that fails.
# BROWSER_TZ=Asia/Jakarta
-45
View File
@@ -1,45 +0,0 @@
# syntax=docker/dockerfile:1
# Real Google Chrome for the latest-chapter poller and the kagane cover proxy.
#
# Not chromedp/headless-shell, which this replaces. headless-shell is a stripped
# Chrome build and Cloudflare's managed challenge on kagane.to never clears for
# it: measured 2026-08-08, 60s of a held-open tab still served the interstitial,
# while stock Chrome from the same IP cleared in ~4s. The tells are structural
# rather than a header — navigator.webdriver true, an empty plugin list, and
# Chromium- rather than Chrome-branded client hints. Overriding webdriver alone
# was tried and did not move it, so the browser build itself is the fix.
#
# zenika/alpine-chrome was also tried: its Chrome is 124 (2024), old enough that
# Cloudflare refuses it outright and old enough to break chromedp's CDP structs.
FROM debian:trixie-slim
# Chrome is deliberately unpinned, against the usual rule. A pinned build goes
# stale, and a stale browser is exactly what Cloudflare turns away — the 124 in
# alpine-chrome is the worked example. Rebuild is the upgrade path.
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates wget gnupg util-linux \
&& wget -qO- https://dl.google.com/linux/linux_signing_key.pub \
| gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg \
&& echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \
> /etc/apt/sources.list.d/google-chrome.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends google-chrome-stable socat \
&& rm -rf /var/lib/apt/lists/*
# Keep the profile path present so Docker initializes the named volume with
# the unprivileged user's ownership.
RUN useradd --create-home --shell /usr/sbin/nologin chrome \
&& mkdir -p /home/chrome/profile /home/chrome/state \
&& chown -R chrome:chrome /home/chrome
# Unprivileged: Chrome refuses to run as root, and the CDP endpoint is a shell
# on whatever user owns it.
USER chrome
WORKDIR /home/chrome
COPY entrypoint.sh /entrypoint.sh
EXPOSE 9222
ENTRYPOINT ["/entrypoint.sh"]
-62
View File
@@ -1,62 +0,0 @@
# The browser, as its own deployable unit.
#
# This does NOT run beside the API. It runs on the home machine, reached from
# the VPS over the tailnet, and is updated without touching the API stack:
#
# cd chrome && docker compose up -d --build
#
# Set BROWSER_BIND_ADDR in chrome/.env to this machine's tailnet IP. See
# ../DEPLOY.md §7 for the full first-time procedure and ../docs/adr/
# 0006-browser-on-the-home-machine.md for why the browser lives here at all.
name: bookmark-browser
services:
browser:
build: .
image: bookmarkmanager-chrome:latest
container_name: bookmark-browser
restart: unless-stopped
environment:
# Any real zone works, but a UTC clock is itself the bot signal and the
# challenge then never clears — measurement in entrypoint.sh. Unset falls
# back to the host's /etc/timezone below, which is a real zone whenever
# the host clock is local; set BROWSER_TZ when the host runs UTC.
TZ: ${BROWSER_TZ:-}
volumes:
# The zone *name*, which is what Chrome's ICU needs — see entrypoint.sh.
# Absent on a non-Debian host, which the entrypoint handles by falling back to UTC.
- /etc/timezone:/etc/timezone:ro
# Cloudflare clearance must survive Chrome reaping and image recreation.
- chrome-profile:/home/chrome/profile
# Bound to the tailnet address only, never 0.0.0.0. CDP authenticates
# nothing: whatever reaches this port drives the browser and, through it,
# this host. On the VPS the safety was Docker network membership; here the
# machine has a real LAN, so the bind address *is* the access control,
# backed by Tailscale device identity. No default — an unset variable must
# fail the deploy rather than silently publish CDP to the LAN.
ports:
- "${BROWSER_BIND_ADDR:?set BROWSER_BIND_ADDR to this machine's tailnet IP}:9222:9222"
# Reaps zombie renderer processes, which otherwise accumulate for the
# container's lifetime.
init: true
# Chrome allocates shared memory per tab and dies on Docker's 64MB default.
# 128MB against a measured 19MB peak: the old 1GB reservation was sized by
# superstition, and this box has 1.8GB total.
shm_size: '128mb'
# The browser is the newcomer on a machine where a Gitea runner already
# holds ~1.2GiB of 1.8GiB. Load-bearing, not decorative: untuned Chrome
# peaked at 645MiB cgroup, which is more than is free here.
#
# memswap_limit is memory+swap combined, so this allows 512MiB of swap —
# Chrome reclaims its own cold pages onto this box's 5.9GiB of SATA swap
# instead of taking resident memory from the runner.
mem_limit: 512m
memswap_limit: 1g
# If the box does run out, the kernel takes the browser and never CI.
oom_score_adj: 800
# A challenge solve yields to a running build. Cold start degrades to ~3s
# at half a CPU, immaterial against a 45-second challenge budget.
cpu_shares: 512
volumes:
chrome-profile:
-225
View File
@@ -1,225 +0,0 @@
#!/bin/sh
set -eu
# A UTC clock is itself the bot signal — Cloudflare treats it as the datacenter
# default — and kagane's challenge then never clears. Measured 2026-08-08 with
# an identical container on one Indonesian egress IP: UTC never cleared in 60s
# (twice), while Asia/Jakarta and America/New_York both cleared in 4s. Any real
# zone will do; the zone does not have to match the IP's country, it just must
# not be UTC. It does have to be right the way Chrome reads it.
#
# TZ must carry the zone *name*. Chrome resolves the zone through ICU, which
# takes the name from /etc/localtime's symlink target and ignores the file's
# contents; bind-mounting the host's /etc/localtime therefore lands on the
# image's own symlink to Etc/UTC and leaves glibc reporting +07 while Chrome
# still reports UTC. /etc/timezone, mounted by docker-compose.yml, is the name.
[ -n "${TZ:-}" ] || TZ=$(cat /etc/timezone 2>/dev/null || echo UTC)
export TZ
state=/home/chrome/state
profile=/home/chrome/profile
lock_file=$state/lock
pid_file=$state/chrome.pid
connections_dir=$state/connections
last_use_file=$state/last-use
idle_seconds=300
mkdir -p "$state" "$profile" "$connections_dir"
exec 9>>"$lock_file"
# Chrome's own UA advertises "HeadlessChrome" under --headless=new, and that
# one token is the difference between kagane.to's challenge clearing in ~4s and
# never clearing at all. Read the installed major version so client hints and
# the UA stay aligned after an image rebuild.
major=$(google-chrome-stable --version | sed -E 's/[^0-9]*([0-9]+)\..*/\1/')
ua="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36"
lock() {
flock 9
}
unlock() {
flock -u 9
}
browser_alive() {
[ -s "$pid_file" ] || return 1
pid=$(cat "$pid_file")
[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null
}
has_connections() {
for marker in "$connections_dir"/*; do
[ -e "$marker" ] || continue
pid=${marker##*/}
if kill -0 "$pid" 2>/dev/null; then
return 0
fi
# A SIGKILLed helper cannot run its cleanup trap. Reconcile its marker
# here so one dead client cannot pin Chrome forever.
rm -f "$marker"
done
return 1
}
start_browser() {
# No --enable-automation: it sets navigator.webdriver, the first thing a
# bot check reads. setsid gives Chrome a process group so the reaper can
# terminate its renderer children with the browser.
# --no-sandbox avoids granting SYS_ADMIN solely for Docker's unavailable
# user namespaces; containment is the unprivileged user and private network.
setsid google-chrome-stable \
--headless=new \
--no-sandbox \
--remote-debugging-port=9223 \
--user-agent="$ua" \
--user-data-dir="$profile" \
--no-first-run \
--no-default-browser-check \
--disable-gpu \
about:blank >/dev/null &
printf '%s\n' "$!" >"$pid_file"
}
stop_browser() {
pid=$(cat "$pid_file")
kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
i=0
while kill -0 "$pid" 2>/dev/null && [ "$i" -lt 100 ]; do
i=$((i + 1))
sleep 0.1
done
if kill -0 "$pid" 2>/dev/null; then
kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
}
wait_for_browser() {
i=0
while [ "$i" -lt 300 ]; do
if wget -qO /dev/null http://127.0.0.1:9223/json/version; then
return 0
fi
browser_alive || return 1
i=$((i + 1))
sleep 0.1
done
return 1
}
finish_connection() {
lock
rm -f "$connection_marker"
date +%s >"$last_use_file"
unlock
}
connection_signal() {
trap - INT TERM HUP
finish_connection
exit 143
}
connection() {
connection_marker=$connections_dir/$$
lock
: >"$connection_marker"
if ! browser_alive; then
rm -f "$pid_file"
start_browser
fi
date +%s >"$last_use_file"
unlock
trap connection_signal INT TERM HUP
if wait_for_browser; then
if socat STDIO TCP:127.0.0.1:9223; then
result=0
else
result=$?
fi
else
# The client only ever sees a bare connection reset here, so this is
# the sole record that the browser, not the network, was the problem.
echo "browser did not come up; dropping connection" >&2
result=1
fi
finish_connection
return "$result"
}
reaper() {
while :; do
sleep 10
lock
if ! has_connections && browser_alive; then
now=$(date +%s)
last=$(cat "$last_use_file" 2>/dev/null || printf '%s' "$now")
if [ $((now - last)) -ge "$idle_seconds" ]; then
stop_browser
fi
fi
unlock
done
}
if [ "${1:-}" = connection ]; then
connection
exit $?
fi
# The files are process state, not the Chrome profile. The profile is a named
# volume in Compose, so clearance survives both a reap and a container rebuild.
for marker in "$connections_dir"/*; do
[ -e "$marker" ] || continue
rm -f "$marker"
done
rm -f "$pid_file" "$last_use_file"
# Chrome's singleton lock names the hostname and pid that took it, and a
# container rebuild changes both — so a Chrome killed uncleanly (OOM, docker
# kill) leaves a lock the next container reads as "another computer holds this
# profile" and refuses to start behind, permanently, with the only symptom a
# bare connection reset at 9222. Clearing it here is safe precisely because
# container_name pins this volume to one container: nothing can be holding the
# profile at the moment this line runs. The lock is process state; the
# clearance cookies it sits beside are not, and are left alone.
rm -f "$profile"/Singleton*
# Chrome binds DevTools to loopback and silently ignores
# --remote-debugging-address. socat remains the network front-end, but each
# accepted connection now starts a browser on demand and is tracked by a
# per-helper marker. A connection held by Go's transport delays reap by its
# idle timeout; the 300-second threshold starts once the last connection closes.
reaper &
reaper_pid=$!
socat TCP-LISTEN:9222,reuseaddr,fork EXEC:'/entrypoint.sh connection',nofork &
front_pid=$!
stop_browser_gracefully() {
lock
if browser_alive; then
# Chrome is a separate session, so stop its process group explicitly;
# this gives its cookie batch time to flush before the container exits.
stop_browser
fi
unlock
}
shutdown() {
trap - INT TERM HUP
stop_browser_gracefully
kill "$front_pid" "$reaper_pid" 2>/dev/null || true
exit 143
}
trap shutdown INT TERM HUP
if wait "$front_pid"; then
status=0
else
status=$?
fi
stop_browser_gracefully
kill "$reaper_pid" 2>/dev/null || true
exit "$status"
+15 -10
View File
@@ -17,12 +17,19 @@ services:
bookmark-api:
# Traffic arrives over the Traefik network, not a published port.
ports: !reset []
# Compose *merges* this list with the base file's, so the service ends up on
# `default`, `db` and `proxy` — only the addition is named here. Do not
# "tidy" the base file down to `db` on the strength of `proxy` being present:
# `db` is `internal: true`, and egress comes from `default`.
environment:
# Must be an IP, not the DNS name — see the base file's comment on this
# same key: Chrome's DevTools HTTP handler 500s any Host header that
# isn't an IP or "localhost".
BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://172.28.0.10:9222}
depends_on:
- headless-shell
# `networks:` here replaces the base file's list entirely, so both must be
# named: `proxy` for Traefik routing, `browser` (defined in the base file)
# to keep reaching headless-shell without putting it on `proxy` too.
networks:
- proxy
- browser
labels:
- "traefik.enable=true"
- "traefik.docker.network=${PROXY_NETWORK:-proxy}"
@@ -40,12 +47,10 @@ services:
- "traefik.http.routers.bmweb.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
- "traefik.http.routers.bmweb.service=bmapi"
# No browser service here. It runs on the home machine as its own unit
# (chrome/docker-compose.yml) and is reached over the tailnet — see
# docs/adr/0006-browser-on-the-home-machine.md. It must never be given a
# service on this host: `proxy` is shared with whatever else sits behind
# Traefik, and an unauthenticated CDP endpoint on it is remote code
# execution for any of them.
# headless-shell is untouched here: it keeps its `browser` network membership
# from the base file and must never join `proxy` — that network is shared
# with whatever else sits behind Traefik on this host, and an exposed
# CDP endpoint on it would be remote code execution for any of them.
networks:
proxy:
+50 -90
View File
@@ -5,54 +5,21 @@
# If your proxy runs in Docker on its own network, use the prod override which
# attaches to that network instead of publishing a port:
# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
#
# The browser is not here. It is its own unit on the home machine —
# chrome/docker-compose.yml — reached over the tailnet via BROWSER_WS_URL.
services:
bookmark-api:
build:
context: ./backend
args:
COVER_DIR: ${COVER_DIR:?set COVER_DIR in .env}
build: ./backend
image: bookmarkmanager-backend:latest
container_name: bookmark-api
restart: unless-stopped
environment:
# TOKEN_KEY derives every Reader's userscript credential (issue #24) —
# compose refuses to start without it.
TOKEN_KEY: ${TOKEN_KEY:?set TOKEN_KEY in .env}
# Owner's Discord user ID — required. Seeds the owner Reader (the
# administrator); every other Reader registers on their first login.
OWNER_DISCORD_ID: ${OWNER_DISCORD_ID:?set OWNER_DISCORD_ID in .env}
# API_TOKEN is required — compose refuses to start without it.
API_TOKEN: ${API_TOKEN:?set API_TOKEN in .env}
ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-https://asuracomic.net,https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to,https://novelfull.com,https://lightnovelworld.net}
# The bookmarks database. Host is the compose service name; the password
# comes from .env so it is never committed.
DATABASE_URL: ${DATABASE_URL:-postgres://bookmarks:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/bookmarks?sslmode=disable}
# Required path inside the API. The build seeds ownership at this path
# and the named volume below mounts there.
COVER_DIR: ${COVER_DIR:?set COVER_DIR in .env}
# Public origin of this deployment, no trailing slash. Required: the
# Cover URLs on the wire are absolute, since the userscript renders them
# on a Site's origin rather than ours (ADR-0007).
PUBLIC_BASE_URL: ${PUBLIC_BASE_URL:?set PUBLIC_BASE_URL in .env}
DB_PATH: /data/bookmarks.db
PORT: "8080"
# Log timestamps only. Go's `log` stamps lines in local time, and this
# service has no other use for a zone: bookmark timestamps are unix ms
# and the two real time columns are timestamptz, both absolute instants.
# Purely so these lines read on the same clock as the browser's. Named
# API_TZ rather than TZ so an operator's exported shell TZ cannot leak
# in; distroless already carries tzdata, so the name just resolves.
TZ: ${API_TZ:-Asia/Jakarta}
# Discord OAuth for the browser UI (ADR-0002). The first four are
# required; DISCORD_REQUIRED_ROLE is optional and empty by default.
# Guild membership is the whole gate: any member becomes a Reader.
DISCORD_CLIENT_ID: ${DISCORD_CLIENT_ID:?set DISCORD_CLIENT_ID in .env}
DISCORD_CLIENT_SECRET: ${DISCORD_CLIENT_SECRET:?set DISCORD_CLIENT_SECRET in .env}
DISCORD_GUILD_ID: ${DISCORD_GUILD_ID:?set DISCORD_GUILD_ID in .env}
DISCORD_REQUIRED_ROLE: ${DISCORD_REQUIRED_ROLE:-}
DISCORD_API_BASE: ${DISCORD_API_BASE:-https://discord.com/api/v10}
DISCORD_REDIRECT_URI: ${DISCORD_REDIRECT_URI:?set DISCORD_REDIRECT_URI in .env}
# Gates the browser UI. Unset means the web routes are not served at all.
WEB_PASSWORD: ${WEB_PASSWORD:-}
# Path inside the container; matches the bindmount above.
USERSCRIPT_PATH: ${USERSCRIPT_PATH:-/userscript/manga-bookmark.user.js}
# Second script from the same bindmount; the novel library is a separate
@@ -62,76 +29,69 @@ services:
# switch; it only takes effect because these are listed here.
LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1}
LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h}
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN: ${LATEST_CHAPTER_POLL_BROWSER_COOLDOWN:-6h}
LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m}
LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14}
LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s}
# CDP endpoint for sites behind a JavaScript challenge (kagane,
# novelfull). The browser is not part of this stack — it runs on the home
# machine as its own unit (chrome/docker-compose.yml) and is reached over
# the tailnet. Unset disables browser polling for those sites and serves
# 404 from the cover proxy for covers not already stored; the userscript
# still covers them. Set it in .env to ws://<home machine tailnet IP>:9222.
#
# Must be an IP, not a MagicDNS hostname: Chrome's DevTools HTTP handler
# rejects the discovery request (GET /json/version) with a 500 unless the
# Host header is an IP address or "localhost" — confirmed 2026-08-03,
# independent of chromedp's own dial logic. The same trap that used to
# force a pinned Docker IP now forbids the tailnet name.
BROWSER_WS_URL: ${BROWSER_WS_URL:-}
# CDP endpoint for sites behind a JavaScript challenge (kagane). Unset
# disables browser polling for those sites; the userscript still covers them.
# Must be an IP, not the "headless-shell" DNS name: Chrome's DevTools HTTP
# handler rejects the discovery request (GET /json/version) with a 500
# unless the Host header is an IP address or "localhost" — confirmed
# 2026-08-03 against chromedp/headless-shell:stable, independent of
# chromedp's own dial logic. The sidecar's static address below exists so
# this URL survives container recreation.
BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://172.28.0.10:9222}
depends_on:
# The migration runner is the first thing the binary does, so a Postgres
# that is still initialising means a crash-loop until it is not.
postgres:
condition: service_healthy
- headless-shell
volumes:
- bookmarks-data:/data
# The userscript is served from here, read fresh on every request. Editing
# the file in this checkout takes effect on the next Violentmonkey poll —
# no rebuild, no restart. `git pull` restores the committed version, which
# is why a redeploy always ships the repo's script.
- ./userscript:/userscript:ro
# Content-addressed cover bytes survive API restarts and redeploys.
- cover-data:${COVER_DIR:?set COVER_DIR in .env}
# Bound to loopback only: the proxy (or curl during smoke test) reaches it,
# the public internet does not.
ports:
- "127.0.0.1:8080:8080"
# `default` is not decoration: `db` is `internal: true`, and a container on
# nothing but an internal network gets neither a published port nor egress
# — which would silently kill every poller fetch.
networks:
- default
- db
- browser
postgres:
image: postgres:17-alpine
headless-shell:
image: chromedp/headless-shell:stable
restart: unless-stopped
environment:
POSTGRES_DB: bookmarks
POSTGRES_USER: bookmarks
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
healthcheck:
test: ["CMD-SHELL", "pg_isready -U bookmarks -d bookmarks"]
interval: 5s
timeout: 3s
retries: 10
volumes:
- postgres-data:/var/lib/postgresql/data
# Deliberately no `ports:` — only bookmark-api, over the `db` network,
# reaches it. Use `docker compose exec postgres psql` for a shell.
# Chrome allocates shared memory per tab and dies on Docker's 64MB default.
shm_size: '1gb'
# Reaps zombie renderer processes, which otherwise accumulate for the
# container's lifetime.
init: true
# Deliberately no `ports:` — an exposed CDP endpoint is remote code
# execution. Only bookmark-api, via the `browser` network below, may reach it.
# Don't pass --remote-debugging-address/--remote-debugging-port here: the
# image's own entrypoint (/headless-shell/run.sh) already starts Chrome on
# 127.0.0.1:9223 and fronts it with a socat proxy listening on 0.0.0.0:9222.
# Redeclaring the port flag here overrides Chrome's, so it binds 9222
# directly (IPv6 loopback only) instead of 9223 — collides with socat's own
# bind on 9222 and leaves nothing listening on 9223, so every external
# connection to headless-shell:9222 fails with EOF. Only pass flags the
# entrypoint doesn't already set.
command:
- --disable-gpu
- --no-sandbox
networks:
- db
browser:
# Pinned so BROWSER_WS_URL can name an IP (required, see above) that
# survives `docker compose up` recreating this container.
ipv4_address: 172.28.0.10
volumes:
postgres-data:
cover-data:
# The pre-Postgres SQLite volume (bookmarks-data) is deliberately no longer
# declared here: undeclared means `docker compose down -v` cannot take it
# with the rest, so the old database survives the cutover until someone
# removes it by hand.
bookmarks-data:
networks:
# Postgres needs no egress and nothing outside bookmark-api needs to reach
# it, so this one really can be cut off from the outside world.
db:
internal: true
# Not `internal: true`: headless Chrome still needs outbound access to reach
# kagane.to. Isolation here comes from membership (only bookmark-api and
# headless-shell join it), not from cutting egress.
browser:
ipam:
config:
- subnet: 172.28.0.0/24
-40
View File
@@ -1,40 +0,0 @@
# Postgres replaces SQLite as the primary datastore
Status: accepted
The project is moving from a single-reader tracker to a service published to a community,
so we replaced `modernc.org/sqlite` with Postgres (`jackc/pgx/v5`, still pure Go, so
`CGO_ENABLED=0` and the distroless image are unaffected). The deciding reason is future
supportability — managed hosting, a datastore that survives the app outgrowing one box —
**not** concurrency, which was measured and found to be a non-issue.
## Considered options
**Stay on SQLite.** Benchmarked against the real store at 1,500 rows (≈50 readers × 30
series): ~9,700 upserts/sec single-writer, plateauing at ~780/sec under 8–50 concurrent
writers, with `List()` holding 90–103 calls/sec under continuous write load. Projected
real load at 50 readers is ~0.02 writes/sec — roughly four and a half orders of magnitude
of headroom. `SetMaxOpenConns(1)` serialises writes but was shown not to starve reads;
an apparent read collapse traced to row count and per-row scanning, not lock contention.
SQLite would have worked. It was rejected for where the project is going, not for what
it does today.
**Postgres.** Chosen. Migrating is cheapest now — 29 rows in one table — and gets
materially harder once there are live readers and a multi-tenant schema.
## Consequences
- Every statement in `internal/store` is rewritten: `?` → `$N`, `IS NOT` →
`IS DISTINCT FROM` (this one is load-bearing; it implements the `updated_at`
ordering rule), `pragma_table_info` → `information_schema.columns`,
`INTEGER`/`REAL` → `bigint`/`double precision`, `favorite` int-as-bool → `boolean`.
- ~75 tests currently get a free isolated database from `t.TempDir()`. They now need a
live server, which makes Docker a hard prerequisite for `go test ./...`. This is the
permanent cost of the decision and the main reason it was close.
- Backups get worse, not better: `VACUUM INTO` produced one self-contained file;
restoring now means `pg_dump`/`pg_restore`, a role, and a password.
- A second stateful container joins the VPS alongside the existing headless-shell.
- **Postgres does not address the real scaling limit.** At batch 14 per 10-minute tick
the poller checks at most 84 series/hour; 50 readers × 30 series is 1,500 bookmarks,
an 18-hour sweep against a configured 1-hour cooldown. That ceiling is an outbound
fetch budget and is fixed by deduplicating polls per Series, not by the datastore.
@@ -1,44 +0,0 @@
# Identity comes from Discord OAuth; we store no passwords and send no email
Status: accepted
The service is being published to a community that already lives on Discord, and we have
no transactional email infrastructure. Rather than build email verification and password
reset to get accounts, Readers sign in with Discord OAuth2 (authorization code grant,
`identify` + `guilds.members.read`), and guild membership replaces both the invite gate
and the email-verification step. No password is ever stored and no mail is ever sent.
## Considered options
**Email + password with invite codes, no verification.** Viable and dependency-free:
an invite code proves community membership, which is what email verification was
standing in for anyway. Rejected because it still requires password hashing, a manual
admin-driven reset path, and a credential store — all of which Discord removes.
**Email + password with a transactional provider** (Resend, Brevo). Rejected as
premature: it builds verification and self-serve reset before anyone has asked for them,
and adds deliverability as an operational concern.
**Discord OAuth.** Chosen. It is less code than either alternative — no hashing, no
reset flow, no invite table — and the authorization question ("is this person in my
community?") is answered by the same call that answers the authentication question.
## Consequences
- **Availability is now coupled to Discord.** If Discord's OAuth endpoint is down,
nobody can start a new session. Existing sessions are unaffected, which bounds the
blast radius.
- **Identity is a Discord snowflake.** Migrating off Discord later means re-identifying
every Reader, because we hold no other credential for them. This is the lock-in the
decision buys, and it is the reason this ADR exists.
- **`guilds.members.read` is checked at login, not continuously.** Someone who leaves
the guild keeps their session until it expires. Acceptable; revocation is a session
delete, not an architectural change.
- **The userscripts cannot use OAuth.** They run in an isolated world on third-party
pages with no redirect surface, so they keep a bearer token — now issued per Reader by
the backend rather than a single shared `API_TOKEN` literal. OAuth gates the web UI;
the web UI is where a Reader obtains their personal userscript.
- `WEB_PASSWORD` disappears, and with it the session HMAC key derivation
(`sha256(API_TOKEN | WEB_PASSWORD | …)`), which needs a replacement secret.
- Seeding the first Reader during migration requires knowing the owner's Discord user
ID up front — a stable snowflake, copied from the Discord client.
@@ -1,44 +0,0 @@
# Series is a shared entity, and only the Poll may update it
Status: accepted
Facts about a Series that are true regardless of who is reading — title, cover, canonical
URL, Latest Chapter — moved off the Bookmark onto a shared `series` row keyed
`(site, series_id)`. A Bookmark now holds only what differs between Readers: Progress,
Favourite, Lifecycle bucket. Fifty Readers tracking one Series produce fifty Bookmarks
and one Series, so the Series is polled once rather than fifty times.
## Why
The poller checks at most 84 series/hour (batch 14 per 10-minute tick). With ~50 Readers
holding ~30 Series each, polling per Bookmark means a 1,500-item sweep — roughly 18 hours
against a configured 1-hour cooldown, quietly breaking the New Chapter signal that is the
product's reason to exist. Deduplicating to distinct Series cuts the sweep several-fold,
and because the Series row now knows how many Readers hold it, the poll queue is ordered
`reader_count DESC, latest_checked_at ASC` — popular Series stay fresh and the long tail
absorbs the shortfall. That ordering is only expressible because the split happened.
Raising throughput instead was rejected: sweeping 400 Series hourly needs the stagger
cut from 20s to ~9s, doubling request rate against sites that already bot-score the
single VPS IP.
## Only the Poll writes Series fields
A client may supply `title`, `cover` and `series_url` only when creating a Series nobody
has bookmarked yet. After that, client-supplied values are ignored; only the backend's
own fetch updates them.
This is a security boundary, not tidiness. Those values are scraped from third-party
pages, which `AGENTS.md` requires be treated as attacker-controlled. Before the split, a
hostile or compromised site could corrupt exactly one Reader's row. After it, the same
write lands on a row every Reader sees — one Reader's browser becomes a write path into
everyone else's UI, and a cover URL can point anywhere. The backend's own fetch is the
higher-trust source: its network, its parser, no third-party JavaScript in the path.
## Consequences
- `Store.Upsert` decomposes one incoming flat body across two tables and enforces the
ownership rule at that seam.
- The `updated_at` ordering rule stays on the Bookmark, where Progress lives. Unchanged.
- Per-Reader title overrides are deliberately not supported; they would reintroduce the
duplication this removes.
@@ -1,32 +0,0 @@
# The wire format stays flat and deliberately does not mirror the schema
Status: accepted
Storage splits a tracked series across two tables (ADR-0003), but `GET /bookmarks` and
`PUT /bookmarks/{key}` keep emitting and accepting one **flat** JSON object with `title`,
`cover`, `last_chapter` and `latest_chapter` as siblings — exactly the shape they had
when there was one table. The server joins on the way out and decomposes on the way in.
## Why a future reader will find this surprising
The obvious move after splitting a table is to nest the JSON to match. Don't "fix" this.
**A nested payload would have broken every installed userscript instantly.** Scripts read
`b.title` directly; moving it to `b.series.title` yields `undefined` — no error, just
blank rows and a New Chapter signal that silently reports nothing forever. Because
Violentmonkey updates roughly once a day per device, the migration relies on old scripts
continuing to work during a 14-day grace window. A nested format and that grace window
are mutually exclusive.
**It is also the better contract independently of compatibility.** A client rendering one
row needs the title and the reading position together; nesting exports the re-stitching
to every browser to mirror a decision about disk layout it should not know about. Keeping
them separate lets storage change again later without a client release — which is the
whole reason this ADR is worth the paragraph.
## Consequence
The flat shape is a contract, not an implementation detail. Changing the storage schema
must not change it. It follows the rule already in force for `updated_at`: the server
owns the truth and returns the row **as stored**, and clients adopt the response rather
than their own payload.
-51
View File
@@ -1,51 +0,0 @@
# ADR-0005: On-demand browser sidecar
Date: 2026-08-09
Status: accepted
Superseded in part by ADR-0006: the lifecycle below is unchanged, but the
service no longer lives in the API stack and the name `headless-shell` is gone.
## Decision
Keep the `headless-shell` service and its CDP port alive, but start Google Chrome
only when the first CDP connection arrives. The entrypoint supervises a `socat`
front-end, serializes browser start/reap state with `flock`, and tracks each
connection with a marker named for its helper PID. A reaper stops Chrome after
300 seconds with no live markers. Marker reconciliation covers a helper killed
before its cleanup trap runs.
Chrome runs in its own process group so reap sends the termination signal to
Chrome and its renderer children. The explicit `/home/chrome/profile` user-data
directory remains: Chrome remaps remote debugging to loopback on modern builds,
and Chrome ignores remote-debugging flags on a default profile. `socat` therefore
continues to front Chrome's loopback CDP port.
The profile is a named Compose volume. Clearance cookies survive both a reap and
`docker compose up --build`; the browser still starts with a fresh debugger UUID,
so chromedp must keep endpoint discovery enabled and must not use
`chromedp.NoModifyURL`.
The socat front-end and explicit profile are retained because Chromium remaps a
non-loopback debugging address to loopback since M113, while Chrome ignores the
remote-debugging flags on a default profile since Chrome 136. Flag tuning is
deliberately not adopted: its roughly 30% idle-footprint saving is irrelevant
to a browser that exists for seconds per wake and risks an untested fingerprint.
## Constraints
The 300-second floor is deliberate. Chromium batches cookie persistence on a
roughly 31-second timer, and Go's default HTTP transport can keep the discovery
connection parked for about 90 seconds after use. Reaping only with zero live
connections holds Chrome through both windows and through the poller's staggered
batch plus cover prefetch.
The anti-bot properties remain unchanged: a plausible non-UTC timezone, a
Chrome-version-derived User-Agent without `HeadlessChrome`, and no automation
flag. A remote browser restart can surface as `context.Canceled`, the same error
as a caller deadline, so the backend wraps cancellation observed with a closed
CDP connection as `browser interrupted`; the focused test asserts that
classification without killing a real browser.
The same process-group stop runs during supervisor shutdown, not only during
idle reap, so Chrome can flush its cookie batch before a container rebuild or
graceful stop.
@@ -1,74 +0,0 @@
# ADR-0006: The browser runs on the home machine, over the tailnet
Date: 2026-08-09
Status: accepted
## Decision
The headless browser is no longer part of the API stack. It is its own compose
unit (`chrome/docker-compose.yml`), deployed on the home machine, and the API on
the VPS reaches it over the existing tailnet through `BROWSER_WS_URL`. No
fallback sidecar remains on the VPS.
The backend needs no code change for this. The CDP endpoint was already a
configuration seam and the fetcher only ever holds the endpoint URL, so
relocation — and reversal — is one environment variable.
## Why
The sidecar held 471 MiB working set (645 MiB peak) on a 1974 MiB VPS with no
swap, which also hosts Traefik, Gitea and its Postgres. That is 24% of the host
and 86% of this project's memory, for a service that at the time answered zero
requests: the poller's due query joins bookmarks, production held four kagane
series and no bookmarks on any of them, and with no kagane bookmark the web UI
never rendered a kagane cover either.
The home machine has 5.9 GiB of swap and a residential egress, which Cloudflare
scores better than a datacenter IP. Both machines were already on the tailnet.
This move is only safe because covers are persisted (ADR-0005's sibling work,
issue #43/#45) and the browser is on-demand (ADR-0005). Without stored covers a
sleeping home machine would blank the library; without on-demand start the CI
runner that already holds ~1.2 GiB of that box's 1.8 GiB would be squeezed
around the clock.
## Constraints
**`BROWSER_WS_URL` must be the tailnet IP, never a MagicDNS hostname.** Chrome's
DevTools HTTP handler answers `/json/version` with a 500 for any `Host` header
that is not an IP or `localhost`. This is the same trap that previously forced a
pinned Docker IP; the pinned subnet is gone, the constraint is not.
**The CDP port binds to the tailnet address only, never `0.0.0.0`.** CDP
authenticates nothing: whatever reaches the port drives the browser and, through
it, the host. On the VPS the safety came from Docker network membership; the
home machine has a real LAN, so a `0.0.0.0` bind is a hole punched into it. The
bind address is the enforcement and Tailscale device identity plus a per-device
ACL is the policy. `BROWSER_BIND_ADDR` deliberately has no default, so an unset
value fails the deploy instead of publishing CDP to the LAN.
No bearer-token proxy is added in front of CDP. It would only defend against a
device already inside the tailnet, and it would be one more thing between the
poller and a browser that is already hard enough to keep clearing challenges.
**Resource limits are load-bearing, not decorative.** The browser is the
newcomer on that box, not the incumbent. A hard 512 MiB cap with 1 GiB
memory+swap makes Chrome reclaim its own cold pages onto the machine's SATA swap
instead of taking resident memory from the runner; untuned Chrome peaked at
645 MiB cgroup, which is more than is free there. `oom_score_adj` biases the
kernel to kill the browser first and never CI. Reduced CPU weight makes a
challenge solve yield to a running build — cold start degrades to about 3 s at
half a CPU, immaterial against a 45-second challenge budget. The shared-memory
reservation drops from 1 GiB to 128 MiB against a measured 19 MiB peak.
## Consequences
An unreachable browser degrades exactly as an unset `BROWSER_WS_URL` already
does: plain-TLS libraries are unaffected, kagane and novelfull log and skip, the
series waits out its cooldown, and stored covers keep serving. A power outage at
home costs chapter freshness on two sites, never the appearance of the library.
The two units are deployed and updated independently. `REDEPLOY.md` §8 covers
the browser; everything before it covers the API stack. A local `docker compose
up` now brings up two services, not three, and polls kagane only if
`BROWSER_WS_URL` is pointed somewhere.
-105
View File
@@ -1,105 +0,0 @@
# ADR-0007: The backend hosts every Site's Cover bytes
Date: 2026-08-09
Status: accepted
## Decision
A Cover is the image a Reader's browser can display for a Series. Where the Site
keeps the picture is the backend's problem, not the client's: the backend fetches
the bytes, stores them, and serves them from its own origin. No client ever
renders a third-party URL, and no client ever supplies one.
Concretely:
- **Acquisition is server-side.** The Cover is extracted from the same series-page
fetch that already yields Latest Chapter. It runs once at Series creation rather
than waiting for the poll queue, so a newly bookmarked Series has both facts in
seconds instead of up to a queue's depth. The poll fills a blank Cover and never
overwrites a non-blank one.
- **Bytes live on a filesystem volume**, content-addressed by the SHA-256 of the
source URL, sharded `${COVER_DIR}/ab/cd/<sha256>`. The database holds the path
and content type, not the bytes.
- **One public route** serves them. No session, no credential.
- **The wire carries an absolute URL** built from a configured public base, and
carries `""` until the bytes exist.
## Why a future reader will find this surprising
Four of the six Sites let anyone hot-link their covers — `static.comix.to` even
answers `access-control-allow-origin: *`. Hosting copies looks like work we were
not obliged to do.
We were obliged. kagane serves covers with `cross-origin-resource-policy:
same-origin` behind a JavaScript challenge (measured 2026-08-08), so no `<img>`
outside kagane.to can load one under any combination of referrer policy and
`crossorigin` attribute. The first fix for that was a kagane-only proxy applied in
the web templates — and it produced issue #47, because the JSON API kept emitting
the raw kagane URL and the userscript rendered it into a broken-image glyph. A
per-Site exception that only one of two clients knows about is not a fix; it is a
bug with a delay on it. Uniformity is the property being bought: every client
renders every Cover the same way, and a Site changing its CORP header or its CDN
cannot break a client again.
## Considered options
**Per-Site exceptions, proxying only what must be proxied.** Cheapest, and what we
had. Rejected: it is what produced #47, and it requires every current and future
client to know which Sites are special.
**A host allowlist for the outbound fetch**, mirroring `fetchableSeriesURL`.
Rejected in favour of destination-class control — see below.
**Cover bytes in Postgres `bytea`**, extending the existing `covers` table.
Rejected: covers are immutable blobs served straight to browsers, which is what a
filesystem is for. The cost is real and accepted — durability is now two things to
back up instead of one, against ADR-0001's grain.
**Per-Reader Cover overrides.** Rejected, consistent with ADR-0003's rejection of
per-Reader title overrides. A Cover is a fact about the Series.
## Two deliberate relaxations
**Destination control is deny-class, not an allowlist.** The outbound fetch
requires `https`, resolves DNS first and refuses loopback, private, link-local and
CGNAT addresses, re-checks on every redirect hop, and caps body size and content
type. It does *not* pin a host set, which is what `fetchableSeriesURL` does for
`series_url`. Cover hosts are CDNs that move: `demonicscans.org` serves its covers
from `readermc.org`, a host with no visible relationship to the Site. An allowlist
would silently stop producing Covers the day a Site switched CDN, and the failure
would look like this bug. The resolved-IP check is the load-bearing part; without
it, an attacker-controlled page need only publish a DNS name pointing at
`127.0.0.1`.
**The cover route is public, where the kagane proxy was session-gated.** An `<img>`
in the userscript panel cannot send a bearer token, and it cannot be given one: the
panel's shadow root is `mode: "open"`, so the host page's own JavaScript can read
any `src` we set. A credential in an image URL is a credential handed to a
third-party site. The route serves public artwork from public Sites and its path
reveals nothing about which Reader holds what. The residual cost is that we can be
hot-linked by others.
## Consequences
- The poll's cover prefetch, today guarded by `sr.Site != "kagane"`, applies to
every Site in both Libraries. Nothing about Covers is conditioned on `kind`.
- Only kagane still needs the CDP browser for its bytes. The other five Sites fetch
over plain TLS — including novelfull, whose HTML answers `cf-mitigated: challenge`
while its image paths answer 200 with `access-control-allow-origin: *`
(measured 2026-08-09).
- Client-side cover scraping is deleted from both userscripts. It could not help: a
scraped URL has no render path left, and it is absent exactly when a Series is
created — neither comix nor lightnovelworld exposes a cover on a chapter page,
which is where a Reader bookmarks mid-read.
- `PUT /bookmarks/{key}` still accepts a `cover` field and ignores it. This extends
ADR-0003's "ignored after creation" to "ignored always", and keeps the flat wire
contract ADR-0004 requires so installed scripts keep working. The field is
therefore permanently inert rather than pending removal, and says so at the
decode site.
- A Cover that fails to load falls back to the placeholder in both clients. The
broken-image glyph reported in #47 is not a state we render.
- The existing kagane `covers` rows are dropped rather than migrated; that path
re-fetches on demand already.
- Two Sites deserve a note for whoever writes the extractor: asura's `.webp` cover
URL answers `Content-Type: image/jpeg`, so trust the header; demonic's `og:image`
carries a raw unencoded space and must be percent-encoded before fetching.
-47
View File
@@ -1,47 +0,0 @@
# Domain Docs
How the engineering skills should consume this repo's domain documentation when exploring the
codebase. Layout: **single-context** — one `CONTEXT.md` plus `docs/adr/` at the repo root.
## Before exploring, read these
- **`CONTEXT.md`** at the repo root — the glossary / ubiquitous language.
- **`docs/adr/`** — read ADRs that touch the area you're about to work in.
If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest
creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and
`/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved.
Neither exists yet in this repo. The existing `AGENTS.md` / `CLAUDE.md` and `docs/design-system.md`
carry the current architecture and design law — read those regardless.
## File structure
```
/
├── CONTEXT.md
├── docs/adr/
│ ├── 0001-....md
│ └── 0002-....md
├── backend/
└── userscript/
```
If this repo ever splits into genuinely separate contexts, add a root `CONTEXT-MAP.md` pointing at
one `CONTEXT.md` per context and update this file.
## Use the glossary's vocabulary
When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a
test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary
explicitly avoids.
If the concept you need isn't in the glossary yet, that's a signal — either you're inventing
language the project doesn't use (reconsider) or there's a real gap (note it for
`/domain-modeling`).
## Flag ADR conflicts
If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:
> _Contradicts ADR-0002 (…) — but worth reopening because…_
-60
View File
@@ -1,60 +0,0 @@
# Issue tracker: Gitea (`tea` CLI)
Issues and specs for this repo live as issues on the self-hosted Gitea instance
`gitea.violetcrown.my.id` (repo `sulthan/mangaBookmark`). **`gh` does not work here** — use
[`tea`](https://gitea.com/gitea/tea) for everything past plain git. Auth lives in `tea login`,
not a `GH_TOKEN` env var. `tea` infers the repo from the local clone's `origin`.
`tea` prints rendered boxes rather than plain text; pass `--output json` (or `-o json`) when a
skill needs to parse the result.
## Conventions
- **Create an issue**: `tea issue create --title "..." --description "..."` (`--labels`,
`--assignees` optional). Multi-line bodies: pass the body through a shell variable or heredoc.
- **Read an issue**: `tea issue <number> --comments` (add `-o json` for machine-readable output).
- **List issues**: `tea issue list --state open -o json --fields index,title,body,labels,state,author`;
filter with `--labels "..."`, `--state open|closed|all`, `--assignee`, `--keyword`.
- **Comment**: `tea comment <number> "..."` (alias of `tea comments add`).
- **Apply / remove labels**: `tea issue edit <number> --add-labels "..."` / `--remove-labels "..."`.
Labels must exist first — see `tea labels list` / `tea labels create --name "..." --color "#rrggbb"`.
- **Close**: `tea issue close <number>` (comment separately with `tea comment`; `close` takes no
`--comment` flag).
## Pull requests as a triage surface
**PRs as a request surface: no.** _(Set to `yes` if this repo treats external PRs as feature
requests; `/triage` reads this flag.)_
When set to `yes`, PRs run through the same labels and states as issues, using the `tea pr`
equivalents: `tea pr <number> --comments`, `tea pr list --state open -o json`,
`tea pr create --head <branch> --base main --title "..." --description "..."`, `tea comment`,
`tea pr close`. Gitea shares one index space across issues and PRs, so a bare `#42` may be either
— resolve with `tea pr 42` and fall back to `tea issue 42`.
## When a skill says "publish to the issue tracker"
Create a Gitea issue with `tea issue create`.
## When a skill says "fetch the relevant ticket"
Run `tea issue <number> --comments`.
## Wayfinding operations
Used by `/wayfinder`. The **map** is a single issue; **tickets** are child issues.
- **Map**: one issue labelled `wayfinder:map` holding the Notes / Decisions-so-far / Fog body.
`tea issue create --labels wayfinder:map --title "..." --description "..."`.
- **Child ticket**: an issue labelled `wayfinder:<type>` (`research`/`prototype`/`grilling`/`task`)
with `Part of #<map>` as the first body line, and a task-list entry in the map body. `tea` has no
sub-issue command, so the task list plus the `Part of` line is the canonical link.
- **Blocking**: a `Blocked by: #<n>, #<n>` line at the top of the child body. Gitea's native issue
dependencies exist in the API but `tea` does not expose them; the body line is the source of
truth. A ticket is unblocked when every listed blocker is closed.
- **Frontier query**: `tea issue list --state open -o json` scoped to the map's task list; drop any
ticket with an open blocker or an assignee; first in map order wins.
- **Claim**: `tea issue edit <n> --add-assignees <your-username>` — the session's first write.
(`tea` has no `@me` shorthand; use the Gitea username from `tea login list`.)
- **Resolve**: `tea comment <n> "<answer>"`, then `tea issue close <n>`, then append a context
pointer to the map's Decisions-so-far via `tea issue edit <map> --description "..."`.
-20
View File
@@ -1,20 +0,0 @@
# Triage Labels
The skills speak in terms of five canonical triage roles. This file maps those roles to the actual
label strings used in this repo's issue tracker (Gitea — see `docs/agents/issue-tracker.md`).
| Label in mattpocock/skills | Label in our tracker | Meaning |
| -------------------------- | -------------------- | ---------------------------------------- |
| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue |
| `needs-info` | `needs-info` | Waiting on reporter for more information |
| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent |
| `ready-for-human` | `ready-for-human` | Requires human implementation |
| `wontfix` | `wontfix` | Will not be actioned |
When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label
string from this table.
Gitea will not auto-create labels on `tea issue edit --add-labels`; create a missing one first with
`tea labels create --name "<label>" --color "#rrggbb"`.
Edit the right-hand column to match whatever vocabulary you actually use.
+5 -25
View File
@@ -2,7 +2,7 @@ Guidance for OpenCode (and Claude Code) working under `userscript/`. See root `A
### Userscript structure (single IIFE, `manga-bookmark.user.js`)
1. **Site adapters** — one per host, `detect(location, document)` return page `type` + IDs. Identify type/IDs from **URL regex** (most stable); pull `title` from **`og:title`** (or the page heading where a site ships no og: tags), not CSS classes. **No adapter reads a cover**: the backend acquires, stores and serves every Cover from its own origin (ADR-0007), the wire's `cover` is already an address on our origin, and `apiPut` strips any `cover` off an outgoing body.
1. **Site adapters** — one per host, `detect(location, document)` return page `type` + IDs. Identify type/IDs from **URL regex** (most stable); pull `title`/`cover` from **`og:title`/`og:image` meta tags**, not CSS classes.
2. **API client** — `apiGet/apiPut/apiDelete` with bearer header; `localStorage` key `bmgr:manga:cache` for instant render + offline fallback.
3. **Progress logic** — auto-upsert `last_chapter` only when `chapterNum >= stored last_chapter_num` (re-reading old chapters must not regress progress; unparseable -> set current). Manual panel override forces any value.
4. **Retry queue** — every write go through `pushBookmark`/`pushDelete`, so
@@ -44,35 +44,15 @@ Guidance for OpenCode (and Claude Code) working under `userscript/`. See root `A
Encodings (incl. triple-encoded punctuation like `%25252D`) identical
on /manga/ and /title/ pages, so decode-once seriesIds match — verified
2026-07-28.
- **comix.to**: series `/title/<id>-<slug>`, chapter
`/title/<id>-<slug>/<uploadId>-chapter-<n>`. Only the leading `<id>` is
identity — the slug re-renders when a series is renamed (`comixSeriesId`).
An SPA that **never rewrites `og:title`**: the server-rendered head keeps
whatever document loaded first, so on a cold load `og:title` is the homepage's
"Comix — Read Comics online for free" and after an in-page hop it is the
*previous* series' name. `document.title` is the one thing client routing does
update, so titles come from there, with the chapter page's `" · Ch.<n>"` tail
stripped. It publishes no `og:image` either, which is one of the reasons cover
acquisition moved to the backend.
- **kagane.to**: series `/series/<uuid>`, reader
`/series/<uuid>/reader/<bookUuid>`. Reader URLs carry no chapter number, so
the number comes out of `og:title`. Two shapes exist: `"<Series> - Chapter
<n>[ - Episode <n>]"` and, for volume-numbered series, `"<Series> - Volume <v>
Chapter <n>"` with no episode name — both must yield a bare series title, or
the volume tail lands in the bookmark's title.
Its covers are challenge- and CORP-protected, so nothing outside kagane.to can
load one directly; the panel renders the backend's own cover address like every
other Site. Behind a Cloudflare JS challenge, so the backend polls it
through the headless browser.
- **novelfull.com** (novel script): series `/<slug>.html`, chapter
`/<slug>/chapter-<n>[-<title-slug>].html`. No `og:*` tags at all — title from
`h3.title` (series) or `a.truyen-title` (chapter); the script reads no cover.
Behind a Cloudflare JS challenge no TLS fingerprint
`h3.title` (series) or `a.truyen-title` (chapter), cover from
`meta[name="image"]`. Behind a Cloudflare JS challenge no TLS fingerprint
clears, so the backend polls it through the headless browser.
- **lightnovelworld.net** (novel script): series `/novel/<slug>/`, chapter
`/<slug>-chapter-<n>/` — flat, at the site root. `h1.entry-title` is the clean
title on a series page and `<Title> Chapter <n>` on a chapter page. Its series
page lists every chapter with an
title on a series page and `<Title> Chapter <n>` on a chapter page. Chapter
pages carry no `og:image`. Its series page lists every chapter with an
absolute href, so the backend polls it with the plain TLS client.
### Second script: `novel-bookmark.user.js`
-1
View File
@@ -1 +0,0 @@
AGENTS.md
+64
View File
@@ -0,0 +1,64 @@
Guidance for Claude Code working under `userscript/`. See root `CLAUDE.md` for the project-wide architecture diagram, hard constraints, and design system.
### Userscript structure (single IIFE, `manga-bookmark.user.js`)
1. **Site adapters** — one per host, `detect(location, document)` return page `type` + IDs. Identify type/IDs from **URL regex** (most stable); pull `title`/`cover` from **`og:title`/`og:image` meta tags**, not CSS classes.
2. **API client** — `apiGet/apiPut/apiDelete` with bearer header; `localStorage` key `bmgr:manga:cache` for instant render + offline fallback.
3. **Progress logic** — auto-upsert `last_chapter` only when `chapterNum >= stored last_chapter_num` (re-reading old chapters must not regress progress; unparseable -> set current). Manual panel override forces any value.
4. **Retry queue** — every write go through `pushBookmark`/`pushDelete`, so
failed mutation park in `localStorage` (`bmgr:manga:queue`) and replayed on
next navigation, reconnect, or `refresh()`. Entries are markers
(`{key, op, sendStatus, attempts}`), never payloads — body read from
cache at send time, so one entry per key give ordering and coalescing for
free. `sendStatus` is **sticky**: while archive pending, later writes to
that key keep carrying bucket, which stop successful
in-between write from silently un-archiving series. `refresh()` drains
before it fetches and overlays anything still pending, so list never
flaps. 400 drops entry, 401 abort pass and keep queue, and
transient failures retry to cap of 10. Latest-chapter writes deliberately
stay out of queue. See
`docs/superpowers/specs/2026-07-27-offline-retry-queue-design.md`.
5. **UI** — rendered inside **Shadow DOM** root to isolate from site CSS
(critical on mobile). Three tabs (All / Favourites / Archived) and row of
link chips to web UI and both manga sites; `WEB_BASE` sits in CONFIG
block next to `API_BASE`. FAB is `7 × 44` edge tab whose *hit* area
widened to `28 × 72` by invisible `#hit` child; `#fab` must keep
`touch-action: none` and must **not** regain `overflow: hidden`. Since
`touch-action` resolved at gesture start, strip can't be both
browser-scrolled and script-dragged, so `makeDraggable` splits by intent: swipe
from `#hit` scrolls via `window.scrollBy`, hold of `ARM_MS` arms
reposition drag, visible sliver drags with no hold. See
`docs/superpowers/specs/2026-07-28-edge-tab-hitbox-design.md`.
6. **SPA navigation** — Asura is Astro, client-routed on comic/chapter pages: patch `history.pushState`/`replaceState` + listen `popstate`, re-run `detect()` on URL change so auto-update fire without reload. Demonic uses classic reloads (initial `document-idle` run suffice).
### Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trust)
- **asurascans.com**: series `/comics/<slug>` (slug carries trailing
site-wide build-hash suffix, e.g. `-059befe1`, that **rotates on every
redeploy**), chapter `/comics/<slug>/chapter/<n>`. `seriesId` must strip
hash (`/-[0-9a-f]{8}$/`, `stripBuildHash` in userscript,
`asuraBuildHash` in backend); URLs keep full slug — stale-hash
URLs 302 to current ones. Astro-rendered; chapter links present in raw
server HTML.
- **demonicscans.org**: series `/manga/<slug>` (slug may URL-encode punctuation, e.g. `%2527` for `'`), chapter `/title/<slug>/chapter/<n>/<page>` (older `chaptered.php?manga=<id>&chapter=<n>` form still exists as redirect, what series-page chapter-list anchors link through).
Encodings (incl. triple-encoded punctuation like `%25252D`) identical
on /manga/ and /title/ pages, so decode-once seriesIds match — verified
2026-07-28.
- **novelfull.com** (novel script): series `/<slug>.html`, chapter
`/<slug>/chapter-<n>[-<title-slug>].html`. No `og:*` tags at all — title from
`h3.title` (series) or `a.truyen-title` (chapter), cover from
`meta[name="image"]`. Behind a Cloudflare JS challenge no TLS fingerprint
clears, so the backend polls it through the headless browser.
- **lightnovelworld.net** (novel script): series `/novel/<slug>/`, chapter
`/<slug>-chapter-<n>/` — flat, at the site root. `h1.entry-title` is the clean
title on a series page and `<Title> Chapter <n>` on a chapter page. Chapter
pages carry no `og:image`. Its series page lists every chapter with an
absolute href, so the backend polls it with the plain TLS client.
### Second script: `novel-bookmark.user.js`
A copy of the manga script with two adapters, `LIBRARY = "novel"` and
`STORE_PREFIX = "bmgr:novel:"`. No migration loop (this script has no previous
installation to carry keys over from). Installed alongside the manga script;
both write to the same backend with the same `LIBRARY` column discriminating
them.
+38 -56
View File
@@ -4,8 +4,8 @@
// @version 1.6.0
// @description Track read progress on Asura, Demonic, Comix & Kagane and sync to a self-hosted backend. Bromite-compatible (no GM_* APIs).
// @author you
// @downloadURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/manga-bookmark.user.js
// @updateURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/manga-bookmark.user.js
// @downloadURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/manga-bookmark.user.js
// @updateURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/manga-bookmark.user.js
// @match https://asuracomic.net/*
// @match https://asurascans.com/*
// @match https://demonicscans.org/*
@@ -22,7 +22,7 @@
// CONFIG — fill these in before installing.
// ============================================================
const API_BASE = "https://bookmark-api.violetcrown.my.id"; // your backend origin, no trailing slash
const API_TOKEN = "__API_TOKEN__"; // substituted by the backend at serve time (issue #24)
const API_TOKEN = "40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df"; // must equal backend API_TOKEN
const WEB_BASE = "https://bookmark.violetcrown.my.id"; // the browser UI, for the panel's nav chips
// This script owns the manga library; the novel script is a separate install
@@ -56,10 +56,9 @@
// ============================================================
// Site adapters
//
// Page type + IDs come from URL regex (most stable); the title comes from
// og: meta tags. Covers are never read here: the backend acquires and serves
// them itself (ADR-0007). Verified live 2026-07-24 against asurascans.com
// and demonicscans.org — see README "Adapter reference".
// Page type + IDs come from URL regex (most stable); title/cover come from
// og: meta tags. Verified live 2026-07-24 against asurascans.com and
// demonicscans.org — see README "Adapter reference".
// ============================================================
function meta(prop) {
@@ -129,6 +128,7 @@
site: this.site,
seriesId: stripBuildHash(m[1]),
title: cleanTitle(meta("og:title")),
cover: meta("og:image") || "",
seriesUrl: loc.origin + "/comics/" + m[1],
chapterLabel: "Chapter " + m[2],
chapterNum: isNaN(num) ? null : num,
@@ -143,6 +143,7 @@
site: this.site,
seriesId: stripBuildHash(m[1]),
title: cleanTitle(meta("og:title")),
cover: meta("og:image") || "",
seriesUrl: loc.origin + "/comics/" + m[1],
chapterLabel: null,
chapterNum: null,
@@ -191,6 +192,7 @@
site: this.site,
seriesId: decodeURIComponent(m[1]),
title: cleanTitle(meta("og:title")),
cover: meta("og:image") || "",
seriesUrl: loc.origin + "/manga/" + m[1],
chapterLabel: "Chapter " + m[2],
chapterNum: isNaN(num) ? null : num,
@@ -205,6 +207,7 @@
site: this.site,
seriesId: decodeURIComponent(m[1]),
title: cleanTitle(meta("og:title")),
cover: meta("og:image") || "",
seriesUrl: loc.origin + "/manga/" + m[1],
chapterLabel: null,
chapterNum: null,
@@ -239,12 +242,6 @@
matches: (loc) => /(^|\.)comix\.to$/.test(loc.hostname),
detect(loc) {
const path = loc.pathname;
// comix client-routes without ever rewriting og:title — the head keeps
// whatever the first server-rendered document carried, so a bookmark
// taken after a client route got the homepage's title, then the
// previous series'. document.title is the one thing its router does
// update. Verified live 2026-08-08; do not "restore" meta("og:title").
const pageTitle = cleanTitle(document.title);
// /title/<id>-<slug>/<uploadId>-chapter-<n>. Several uploads (different
// groups or languages) share one chapter number; the number is the
// progress identity, the upload id is not.
@@ -255,7 +252,8 @@
type: "chapter",
site: this.site,
seriesId: comixSeriesId(m[1]),
title: pageTitle,
title: cleanTitle(meta("og:title")),
cover: coverFromPage(),
seriesUrl: loc.origin + "/title/" + m[1],
chapterLabel: "Chapter " + m[2],
chapterNum: isNaN(num) ? null : num,
@@ -269,7 +267,8 @@
type: "series",
site: this.site,
seriesId: comixSeriesId(m[1]),
title: pageTitle,
title: cleanTitle(meta("og:title")),
cover: coverFromPage(),
seriesUrl: loc.origin + "/title/" + m[1],
chapterLabel: null,
chapterNum: null,
@@ -278,12 +277,24 @@
}
return { type: "other" };
// comix chapter document.title is "<Title> · Ch.<n>"; series is clean.
// comix chapter og:title is "<Title> · Ch.<n>"; series is clean.
function cleanTitle(t) {
if (!t) return "";
return t.replace(/\s*·\s*Ch\.[\d.]+\s*$/i, "").trim();
}
// comix serves no og:image, so this is the one adapter that has to read
// the DOM for a cover. Matching on alt rather than a class keeps it off
// the site's styling: the cover is the image whose alt is the title.
// Do not "simplify" this into meta("og:image") — that returns null.
function coverFromPage() {
const title = cleanTitle(meta("og:title"));
if (!title || !document.querySelectorAll) return "";
for (const img of document.querySelectorAll("img[alt]")) {
if (img.getAttribute("alt") === title) return img.getAttribute("src") || "";
}
return "";
}
},
// Scoped to this series' own id prefix so a recommendation strip's links
// cannot win the maximum. seriesId is passed in because the anchors alone
@@ -302,14 +313,6 @@
},
};
// Kagane builds the reader og:title suffix out of the book's metadata, so
// every combination occurs: the volume part appears only when the book has a
// volume_no, the episode part only when it has a non-empty title. All four
// shapes captured live 2026-08-08 — "SP Baby - Volume 1 Chapter 1" is the one
// the old trailing-space regex missed, which left both the number and the
// series title wrong.
const KAGANE_CHAPTER_SUFFIX = /\s-\s(?:Volume\s[\d.]+\s)?Chapter\s([\d.]+)(?:\s-\s.*)?$/i;
const kagane = {
site: "kagane",
matches: (loc) => /(^|\.)kagane\.to$/.test(loc.hostname),
@@ -326,6 +329,7 @@
site: this.site,
seriesId: m[1],
title: cleanTitle(meta("og:title")),
cover: meta("og:image") || "",
seriesUrl: loc.origin + "/series/" + m[1],
chapterLabel: num === null ? null : "Chapter " + num,
chapterNum: num,
@@ -340,6 +344,7 @@
site: this.site,
seriesId: m[1],
title: cleanTitle(meta("og:title")),
cover: meta("og:image") || "",
seriesUrl: loc.origin + "/series/" + m[1],
chapterLabel: null,
chapterNum: null,
@@ -348,8 +353,9 @@
}
return { type: "other" };
// Reader og:title is "<Title> - Chapter <n> - <episode name>".
function chapterNumFromTitle(t) {
const m = t && t.match(KAGANE_CHAPTER_SUFFIX);
const m = t && t.match(/\s-\sChapter\s([\d.]+)\s/);
if (!m) return null;
const num = parseFloat(m[1]);
return isNaN(num) ? null : num;
@@ -357,7 +363,7 @@
function cleanTitle(t) {
if (!t) return "";
return t.replace(KAGANE_CHAPTER_SUFFIX, "").trim();
return t.replace(/\s-\sChapter\s[\d.]+\s-\s.*$/i, "").trim();
}
},
// Reader hrefs are uuids with no number in them, so no maximum can be taken
@@ -480,10 +486,6 @@
async function apiPut(key, obj, { sendStatus = false } = {}) {
const body = Object.assign({}, obj);
if (!sendStatus) delete body.status;
// Covers belong to the backend, which acquires and serves them itself
// (ADR-0007) and ignores an incoming one; a third-party address must never
// go back on the wire.
delete body.cover;
const res = await fetch(API_BASE + "/bookmarks/" + encodeURIComponent(key), {
method: "PUT",
headers: authHeaders({ "Content-Type": "application/json" }),
@@ -841,6 +843,7 @@
series_id: p.seriesId,
title: p.title || (existing && existing.title) || p.seriesId,
series_url: p.seriesUrl || (existing && existing.series_url) || "",
cover: p.cover || (existing && existing.cover) || "",
last_chapter: p.chapterLabel || (existing && existing.last_chapter) || "",
last_chapter_num:
p.chapterNum != null ? p.chapterNum : existing ? existing.last_chapter_num : null,
@@ -863,6 +866,7 @@
series_id: p.seriesId,
title: existing.title || p.title || p.seriesId,
series_url: existing.series_url || p.seriesUrl || "",
cover: existing.cover || p.cover || "",
last_chapter: p.chapterLabel || existing.last_chapter || "",
last_chapter_num: p.chapterNum != null ? p.chapterNum : existing.last_chapter_num,
last_chapter_url: p.chapterUrl || "",
@@ -1378,15 +1382,7 @@
return el("div", { class: "item" + heat }, [
el("a", { class: "go", href: cont }, [
b.cover
? el("img", {
class: "cover",
src: b.cover,
loading: "lazy",
alt: "",
// A Cover that will not load shows the designed placeholder
// rather than the browser's broken-image glyph (#47).
onerror: (e) => e.target.replaceWith(el("div", { class: "cover ph" })),
})
? el("img", { class: "cover", src: b.cover, loading: "lazy", alt: "" })
: el("div", { class: "cover ph" }),
]),
el("div", { class: "meta" }, [
@@ -1454,15 +1450,8 @@
}
let lastUrl = location.href;
let lastPageSig = "";
function setPage() {
state.page = detect();
lastPageSig = JSON.stringify(state.page);
}
function onNavigate() {
setPage();
state.page = detect();
render();
maybeAutoUpdate();
maybeCaptureLatestOnSeriesPage();
@@ -1495,14 +1484,7 @@
wrap("pushState");
wrap("replaceState");
window.addEventListener("popstate", fire);
// Also catches routes that bypass history — and comix, which fills
// document.title a beat after the route changes, so the 300ms snapshot
// above can still hold the previous page's title. Re-detect whenever what
// we would read has changed, not only when the URL has.
setInterval(() => {
fire();
if (JSON.stringify(detect()) !== lastPageSig) onNavigate();
}, 1500);
setInterval(fire, 1500); // catch routes that bypass history
}
// ============================================================
@@ -1581,7 +1563,7 @@
function init() {
buildUI();
setPage();
state.page = detect();
render();
installNavWatcher();
installLongPress();
+26 -26
View File
@@ -4,8 +4,8 @@
// @version 1.0.0
// @description Track read progress on NovelFull & LightNovelWorld and sync to a self-hosted backend. Bromite-compatible (no GM_* APIs).
// @author you
// @downloadURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/novel-bookmark.user.js
// @updateURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/novel-bookmark.user.js
// @downloadURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/novel-bookmark.user.js
// @updateURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/novel-bookmark.user.js
// @match https://novelfull.com/*
// @match https://lightnovelworld.net/*
// @run-at document-idle
@@ -19,19 +19,12 @@
// CONFIG — fill these in before installing.
// ============================================================
const API_BASE = "https://bookmark-api.violetcrown.my.id"; // your backend origin, no trailing slash
const API_TOKEN = "__API_TOKEN__"; // substituted by the backend at serve time (issue #24)
const API_TOKEN = "40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df"; // must equal backend API_TOKEN
const WEB_BASE = "https://bookmark.violetcrown.my.id"; // the browser UI, for the panel's nav chips
// This script owns the novel library; the manga script is a separate install
// with its own prefix, so the two never share a cache, a queue or a panel.
const STORE_PREFIX = "bmgr:novel:";
const CACHE_KEY = STORE_PREFIX + "cache";
// Per-device record of when each series was last checked for new chapters.
// Deliberately not synced: each device does its own checking.
const LASTCHECKED_KEY = STORE_PREFIX + "lastchecked";
const LATEST_CHECK_THROTTLE_MS = 4 * 60 * 60 * 1000;
const LATEST_CHECK_BATCH = 1; // series fetched per navigation // series fetched per navigation
// Which library this script's rows belong to. The manga script is a separate
// install that declares "manga"; the backend keeps whichever it is told.
@@ -40,11 +33,16 @@
// ============================================================
// Site adapters
//
// Page type + IDs come from URL regex (most stable); the title comes from
// og: meta tags or the page's own heading. Covers are never read here: the
// backend acquires and serves them itself (ADR-0007).
// Page type + IDs come from URL regex (most stable); title/cover come from
// og: meta tags (with the novelfull name= meta as the exception).
// ============================================================
function meta(prop) {
const el = document.querySelector('meta[property="' + prop + '"]');
return el ? el.getAttribute("content") : null;
}
// Chapter lists are read from two places: the page we are standing on, and
// series pages fetched in the background. Both are reduced to {href, text}
// pairs so each adapter needs only one rule for picking the latest chapter.
@@ -68,6 +66,12 @@
return out;
}
// novelfull ships no og: tags at all — its cover lives on a name= meta.
function metaName(name) {
const el = document.querySelector('meta[name="' + name + '"]');
return el ? el.getAttribute("content") : null;
}
function escapeRe(s) {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
@@ -104,6 +108,7 @@
// h3.title on a chapter page is the *chapter's* title; the breadcrumb
// link back to the series page carries the series name.
title: back ? (back.textContent || "").trim() : "",
cover: metaName("image") || "",
seriesUrl: loc.origin + "/" + m[1] + ".html",
chapterLabel: "Chapter " + m[2],
chapterNum: isNaN(num) ? null : num,
@@ -119,6 +124,7 @@
site: this.site,
seriesId: m[1],
title: h3 ? (h3.textContent || "").trim() : "",
cover: metaName("image") || "",
seriesUrl: loc.origin + "/" + m[1] + ".html",
chapterLabel: null,
chapterNum: null,
@@ -151,6 +157,9 @@
seriesId: m[1],
// The heading is "<Series> Chapter <n>"; drop the suffix.
title: heading.replace(/\s*Chapter\s+[0-9.]+\s*$/i, "").trim(),
// Chapter pages carry no og:image. Empty is safe: every write merges
// against the cached row, which keeps the cover the series page gave.
cover: "",
seriesUrl: "https://lightnovelworld.net/novel/" + m[1] + "/",
chapterLabel: "Chapter " + m[2],
chapterNum: isNaN(num) ? null : num,
@@ -166,6 +175,7 @@
site: this.site,
seriesId: m[1],
title: h1 ? (h1.textContent || "").trim() : "",
cover: meta("og:image") || "",
seriesUrl: "https://lightnovelworld.net/novel/" + m[1] + "/",
chapterLabel: null,
chapterNum: null,
@@ -276,10 +286,6 @@
async function apiPut(key, obj, { sendStatus = false } = {}) {
const body = Object.assign({}, obj);
if (!sendStatus) delete body.status;
// Covers belong to the backend, which acquires and serves them itself
// (ADR-0007) and ignores an incoming one; a third-party address must never
// go back on the wire.
delete body.cover;
const res = await fetch(API_BASE + "/bookmarks/" + encodeURIComponent(key), {
method: "PUT",
headers: authHeaders({ "Content-Type": "application/json" }),
@@ -607,6 +613,7 @@
series_id: p.seriesId,
title: p.title || (existing && existing.title) || p.seriesId,
series_url: p.seriesUrl || (existing && existing.series_url) || "",
cover: p.cover || (existing && existing.cover) || "",
last_chapter: p.chapterLabel || (existing && existing.last_chapter) || "",
last_chapter_num:
p.chapterNum != null ? p.chapterNum : existing ? existing.last_chapter_num : null,
@@ -629,6 +636,7 @@
series_id: p.seriesId,
title: existing.title || p.title || p.seriesId,
series_url: existing.series_url || p.seriesUrl || "",
cover: existing.cover || p.cover || "",
last_chapter: p.chapterLabel || existing.last_chapter || "",
last_chapter_num: p.chapterNum != null ? p.chapterNum : existing.last_chapter_num,
last_chapter_url: p.chapterUrl || "",
@@ -1139,15 +1147,7 @@
return el("div", { class: "item" + heat }, [
el("a", { class: "go", href: cont }, [
b.cover
? el("img", {
class: "cover",
src: b.cover,
loading: "lazy",
alt: "",
// A Cover that will not load shows the designed placeholder
// rather than the browser's broken-image glyph (#47).
onerror: (e) => e.target.replaceWith(el("div", { class: "cover ph" })),
})
? el("img", { class: "cover", src: b.cover, loading: "lazy", alt: "" })
: el("div", { class: "cover ph" }),
]),
el("div", { class: "meta" }, [
+43 -54
View File
@@ -31,9 +31,9 @@ globalThis.location = {
// og: meta tags the adapters read through meta(). Reassigned per test.
let metaTags = {};
// document.title. comix's SPA rewrites this on client routing but never
// og:title, so the comix adapter reads it instead. Reassigned per test.
let docTitle = "";
// img[alt] elements comix's coverFromPage() scans. Reassigned per test; each
// entry is {alt, src}.
let pageImages = [];
globalThis.document = {
querySelector(sel) {
const m = sel.match(/^meta\[property="([^"]+)"\]$/);
@@ -41,10 +41,13 @@ globalThis.document = {
const v = metaTags[m[1]];
return v == null ? null : { getAttribute: () => v };
},
addEventListener() {},
get title() {
return docTitle;
querySelectorAll(sel) {
if (sel !== "img[alt]") return [];
return pageImages.map((img) => ({
getAttribute: (attr) => img[attr] ?? null,
}));
},
addEventListener() {},
body: undefined,
};
@@ -91,7 +94,7 @@ test("stripBuildHash ignores suffixes that are not exactly 8 hex chars", () => {
// ============================================================
test("asura.detect reads a series page, stripping the hash from the id only", () => {
metaTags = { "og:title": "Solo Leveling | Asura Scans" };
metaTags = { "og:title": "Solo Leveling | Asura Scans", "og:image": "https://cdn.example/x.jpg" };
const p = asura.detect(loc("https://asurascans.com/comics/solo-leveling-059befe1"));
assert.equal(p.type, "series");
assert.equal(p.site, "asura");
@@ -99,11 +102,12 @@ test("asura.detect reads a series page, stripping the hash from the id only", ()
// seriesUrl keeps the hash: navigation needs the current one (stale ones 302).
assert.equal(p.seriesUrl, "https://asurascans.com/comics/solo-leveling-059befe1");
assert.equal(p.title, "Solo Leveling");
assert.equal(p.cover, "https://cdn.example/x.jpg");
assert.equal(p.chapterNum, null);
});
test("asura.detect reads a chapter page including a decimal number", () => {
metaTags = { "og:title": "Solo Leveling Chapter 12.5 - Read Online | Asura Scans" };
metaTags = { "og:title": "Solo Leveling Chapter 12.5 - Read Online | Asura Scans", "og:image": "" };
const url = "https://asurascans.com/comics/solo-leveling-059befe1/chapter/12.5";
const p = asura.detect(loc(url));
assert.equal(p.type, "chapter");
@@ -140,7 +144,7 @@ test("asura.latestChapterFromAnchors returns null when nothing matches", () => {
// ============================================================
test("demonic.detect reads a series page", () => {
metaTags = { "og:title": "The World After The Fall" };
metaTags = { "og:title": "The World After The Fall", "og:image": "https://cdn.example/y.jpg" };
const p = demonic.detect(loc("https://demonicscans.org/manga/the-world-after-the-fall"));
assert.equal(p.type, "series");
assert.equal(p.site, "demonic");
@@ -149,7 +153,7 @@ test("demonic.detect reads a series page", () => {
});
test("demonic.detect reads a chapter page and strips the suffix from the title", () => {
metaTags = { "og:title": "The World After The Fall Chapter 3" };
metaTags = { "og:title": "The World After The Fall Chapter 3", "og:image": "" };
const p = demonic.detect(loc("https://demonicscans.org/title/the-world-after-the-fall/chapter/3/1"));
assert.equal(p.type, "chapter");
assert.equal(p.chapterNum, 3);
@@ -189,15 +193,8 @@ test("comixSeriesId leaves a bare id untouched", () => {
assert.equal(comixSeriesId("n8we"), "n8we");
});
// comix is an SPA that rewrites document.title on client routing but leaves the
// server-rendered og:title untouched, so every test here pins og:title to a
// STALE value — the homepage title on first hop, the previous series after
// that. Captured live 2026-08-08.
const COMIX_STALE_HOME = "Comix - Read Comics online for free";
test("comix detects a series page", () => {
metaTags = { "og:title": COMIX_STALE_HOME };
docTitle = "Dungeons and Crayons";
metaTags = { "og:title": "Dungeons and Crayons" };
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
assert.equal(p.type, "series");
assert.equal(p.site, "comix");
@@ -207,16 +204,8 @@ test("comix detects a series page", () => {
assert.equal(p.chapterNum, null);
});
test("comix ignores a previous series' stale og:title", () => {
metaTags = { "og:title": "Full-Time Awakening" };
docTitle = "Dungeons and Crayons";
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
assert.equal(p.title, "Dungeons and Crayons");
});
test("comix detects a chapter page and strips the Ch. suffix from the title", () => {
metaTags = { "og:title": COMIX_STALE_HOME };
docTitle = "Dungeons and Crayons · Ch.80";
metaTags = { "og:title": "Dungeons and Crayons · Ch.80" };
const p = comix.detect(
loc("https://comix.to/title/n8we-dungeons-and-crayons/11139891-chapter-80")
);
@@ -229,14 +218,33 @@ test("comix detects a chapter page and strips the Ch. suffix from the title", ()
});
test("comix parses decimal chapter numbers", () => {
metaTags = {};
docTitle = "Dungeons and Crayons · Ch.80.5";
metaTags = { "og:title": "Dungeons and Crayons · Ch.80.5" };
const p = comix.detect(
loc("https://comix.to/title/n8we-dungeons-and-crayons/11139891-chapter-80.5")
);
assert.equal(p.chapterNum, 80.5);
});
test("comix.detect reads the cover from an img whose alt matches the cleaned title", () => {
metaTags = { "og:title": "Dungeons and Crayons · Ch.80" };
pageImages = [
{ alt: "Some Other Series", src: "https://cdn.example/other.jpg" },
{ alt: "Dungeons and Crayons", src: "https://cdn.example/cover.jpg" },
];
const p = comix.detect(
loc("https://comix.to/title/n8we-dungeons-and-crayons/11139891-chapter-80")
);
assert.equal(p.cover, "https://cdn.example/cover.jpg");
});
test("comix.detect leaves cover empty when no img alt matches the title", () => {
metaTags = { "og:title": "Dungeons and Crayons" };
pageImages = [{ alt: "Some Other Series", src: "https://cdn.example/other.jpg" }];
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
assert.equal(p.cover, "");
pageImages = [];
});
test("comix ignores unrelated paths", () => {
assert.equal(comix.detect(loc("https://comix.to/browse")).type, "other");
});
@@ -278,18 +286,23 @@ const KAGANE_SERIES = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b";
const KAGANE_BOOK = "019fa2e0-6dbd-73ca-b40b-fe06ab75eb0e";
test("kagane detects a series page", () => {
metaTags = { "og:title": "Infinite Decryption: The Strongest Level 0" };
metaTags = {
"og:title": "Infinite Decryption: The Strongest Level 0",
"og:image": "https://kagane.to/api/v2/image/abc/compressed",
};
const p = kagane.detect(loc("https://kagane.to/series/" + KAGANE_SERIES));
assert.equal(p.type, "series");
assert.equal(p.site, "kagane");
assert.equal(p.seriesId, KAGANE_SERIES);
assert.equal(p.title, "Infinite Decryption: The Strongest Level 0");
assert.equal(p.cover, "https://kagane.to/api/v2/image/abc/compressed");
assert.equal(p.seriesUrl, "https://kagane.to/series/" + KAGANE_SERIES);
});
test("kagane reads the chapter number out of og:title", () => {
metaTags = {
"og:title": "Infinite Decryption: The Strongest Level 0 - Chapter 41 - Episode 41",
"og:image": "https://kagane.to/api/v2/image/abc/compressed",
};
const p = kagane.detect(
loc("https://kagane.to/series/" + KAGANE_SERIES + "/reader/" + KAGANE_BOOK)
@@ -302,30 +315,6 @@ test("kagane reads the chapter number out of og:title", () => {
assert.equal(p.seriesUrl, "https://kagane.to/series/" + KAGANE_SERIES);
});
// Volume-numbered series render the suffix as "- Volume <v> Chapter <n>" with
// no episode name, because the book carries volume_no and an empty title.
// Captured live 2026-08-08 from SP Baby.
test("kagane reads through a Volume-numbered chapter suffix", () => {
metaTags = { "og:title": "SP Baby - Volume 1 Chapter 1" };
const p = kagane.detect(
loc("https://kagane.to/series/" + KAGANE_SERIES + "/reader/" + KAGANE_BOOK)
);
assert.equal(p.title, "SP Baby");
assert.equal(p.chapterNum, 1);
assert.equal(p.chapterLabel, "Chapter 1");
});
// A book with neither a volume nor an episode name ends the title right after
// the number, which the old trailing-\s regex could not match.
test("kagane reads a chapter suffix with no episode name", () => {
metaTags = { "og:title": "Some Series - Chapter 7.5" };
const p = kagane.detect(
loc("https://kagane.to/series/" + KAGANE_SERIES + "/reader/" + KAGANE_BOOK)
);
assert.equal(p.title, "Some Series");
assert.equal(p.chapterNum, 7.5);
});
test("kagane yields a null chapterNum when og:title has no chapter", () => {
metaTags = { "og:title": "Infinite Decryption: The Strongest Level 0" };
const p = kagane.detect(
+17 -27
View File
@@ -4,7 +4,8 @@ const test = require("node:test");
const assert = require("node:assert");
// ============================================================
// Minimal browser stub. Same shape as logic.test.js.
// Minimal browser stub. Same shape as logic.test.js, plus a meta[name=...]
// branch: novelfull ships no og: tags, so its cover comes from name="image".
// document.body stays UNDEFINED so the boot block waits for a DOMContentLoaded
// that never fires and no network call is ever made.
// ============================================================
@@ -19,14 +20,20 @@ globalThis.localStorage = {
globalThis.location = { href: "about:blank", hostname: "", pathname: "/", origin: "" };
let metaTags = {};
let namedMetas = {};
let elements = {};
globalThis.document = {
querySelector(sel) {
const m = sel.match(/^meta\[property="([^"]+)"\]$/);
let m = sel.match(/^meta\[property="([^"]+)"\]$/);
if (m) {
const v = metaTags[m[1]];
return v == null ? null : { getAttribute: () => v };
}
m = sel.match(/^meta\[name="([^"]+)"\]$/);
if (m) {
const v = namedMetas[m[1]];
return v == null ? null : { getAttribute: () => v };
}
const text = elements[sel];
return text == null ? null : { textContent: text };
},
@@ -51,6 +58,7 @@ function loc(href) {
function reset() {
metaTags = {};
namedMetas = {};
elements = {};
}
@@ -60,18 +68,21 @@ function reset() {
test("novelfull.detect reads a series page", () => {
reset();
namedMetas = { image: "https://novelfull.com/uploads/thumbs/ri.jpg" };
elements = { "h3.title": "Reverend Insanity" };
const p = novelfull.detect(loc("https://novelfull.com/reverend-insanity.html"));
assert.equal(p.type, "series");
assert.equal(p.site, "novelfull");
assert.equal(p.seriesId, "reverend-insanity");
assert.equal(p.title, "Reverend Insanity");
assert.equal(p.cover, "https://novelfull.com/uploads/thumbs/ri.jpg");
assert.equal(p.seriesUrl, "https://novelfull.com/reverend-insanity.html");
assert.equal(p.chapterNum, null);
});
test("novelfull.detect reads a chapter page and points seriesUrl at the series", () => {
reset();
namedMetas = { image: "https://novelfull.com/uploads/thumbs/ri.jpg" };
elements = { "a.truyen-title": "Reverend Insanity" };
const url = "https://novelfull.com/reverend-insanity/chapter-2334-fang-yuan.html";
const p = novelfull.detect(loc(url));
@@ -109,12 +120,14 @@ test("novelfull.latestChapterFromAnchors takes the max and ignores other series"
test("lightnovelworld.detect reads a series page", () => {
reset();
metaTags = { "og:image": "https://lightnovelworld.net/wp-content/uploads/awe.webp" };
elements = { "h1.entry-title": "A Will Eternal" };
const p = lightnovelworld.detect(loc("https://lightnovelworld.net/novel/a-will-eternal/"));
assert.equal(p.type, "series");
assert.equal(p.site, "lightnovelworld");
assert.equal(p.seriesId, "a-will-eternal");
assert.equal(p.title, "A Will Eternal");
assert.equal(p.cover, "https://lightnovelworld.net/wp-content/uploads/awe.webp");
});
test("lightnovelworld.detect strips the chapter suffix off the heading", () => {
@@ -128,6 +141,8 @@ test("lightnovelworld.detect strips the chapter suffix off the heading", () => {
assert.equal(p.chapterLabel, "Chapter 1298");
assert.equal(p.title, "A Will Eternal");
assert.equal(p.seriesUrl, "https://lightnovelworld.net/novel/a-will-eternal/");
// Chapter pages have no cover; the merge in bookmarkCurrent keeps the stored one.
assert.equal(p.cover, "");
});
test("lightnovelworld.detect returns other for non-series paths", () => {
@@ -165,28 +180,3 @@ test("kindOf defaults a missing kind to manga", () => {
test("kindOf passes through novel", () => {
assert.equal(kindOf({ kind: "novel" }), "novel");
});
// ============================================================
// Source guard
//
// Issue #74: the novel script was split off the manga one and lost four
// module-scope constants. The reads sit inside try/catch or a fire-and-forget
// promise, so the ReferenceError never surfaced — nothing but a static check
// catches this class.
// ============================================================
test("every SCREAMING_CASE constant the script uses is declared in it", () => {
const fs = require("node:fs");
for (const f of ["novel-bookmark.user.js", "manga-bookmark.user.js"]) {
const src = fs.readFileSync(require.resolve("../" + f), "utf8")
// comments and strings carry prose and SVG path data in the same shape
.replace(/\/\/[^\n]*|\/\*[\s\S]*?\*\/|"[^"\n]*"|'[^'\n]*'|`[\s\S]*?`/g, " ");
const declared = new Set(
[...src.matchAll(/\b(?:const|let|var|function)\s+([A-Z][A-Z0-9_]{2,})\b/g)].map((m) => m[1]),
);
for (const name of new Set(src.match(/\b[A-Z][A-Z0-9_]{2,}\b/g) || [])) {
if (name.startsWith("GM_") || name in globalThis) continue;
assert.ok(declared.has(name), `${f} uses ${name} but never declares it`);
}
}
});