Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2a3bb6922d | |||
| d1800d0707 | |||
| 84cfd1b2c1 | |||
| bfae84c5c3 | |||
| cd3a7e3d01 | |||
| 741b23322b | |||
| 2ef769d421 | |||
| c2b47eb05b | |||
| 1b1820d85a | |||
| 2cc1e69f5d | |||
| 27cf0955de | |||
| bcc6b45515 | |||
| 8cebb94b92 |
+67
-26
@@ -1,8 +1,16 @@
|
||||
# Copy to .env and fill in. Never commit the real .env.
|
||||
|
||||
# Long random secret shared with the userscript's API_TOKEN. Generate one:
|
||||
# Secret every Reader's userscript credential is derived from (issue #24):
|
||||
# the backend rebuilds install URLs from it, and only SHA-256 hashes of the
|
||||
# credentials ever touch the database. Generate one:
|
||||
# openssl rand -hex 32
|
||||
API_TOKEN=changeme-generate-a-long-random-token
|
||||
TOKEN_KEY=changeme-generate-a-long-random-token
|
||||
|
||||
# The owner's Discord user ID — seeded at startup as the first Reader, the
|
||||
# administrator (the only one who can revoke another Reader's sessions), and
|
||||
# the owner of every bookmark that predates registration. Discord snowflake,
|
||||
# e.g. 1046923170000000000.
|
||||
OWNER_DISCORD_ID=changeme-your-discord-user-id
|
||||
|
||||
# Comma-separated origins allowed to call the API (CORS). Both Asura domains
|
||||
# plus Demonic, Comix, Kagane, and the two novel sites. Add/remove as the
|
||||
@@ -26,17 +34,30 @@ POSTGRES_PASSWORD=changeme-generate-a-long-random-password
|
||||
# TRAEFIK_ENTRYPOINT=websecure
|
||||
# TRAEFIK_CERTRESOLVER=le
|
||||
|
||||
# --- Web UI ---
|
||||
# Password for the browser UI at https://$BOOKMARK_WEB_HOST. Leave unset to
|
||||
# disable the web UI entirely (the routes are not registered at all).
|
||||
# Generate one: openssl rand -base64 18
|
||||
WEB_PASSWORD=
|
||||
# --- Web UI (Discord OAuth) ---
|
||||
# Sign-in is a Discord authorization code grant (ADR-0002), and it is also
|
||||
# registration: any member of the configured guild becomes a Reader on their
|
||||
# first successful login, with their own empty library. Create the application
|
||||
# at https://discord.com/developers/applications and register the exact
|
||||
# callback URL ($BOOKMARK_WEB_HOST/auth/discord/callback) as an OAuth2
|
||||
# redirect.
|
||||
DISCORD_CLIENT_ID=
|
||||
DISCORD_CLIENT_SECRET=
|
||||
# The guild whose membership gates sign-in (Developer Mode -> right-click the
|
||||
# server -> Copy Server ID).
|
||||
DISCORD_GUILD_ID=
|
||||
# Exact callback URL, e.g. https://bookmark.example.com/auth/discord/callback.
|
||||
# Discord matches it verbatim, so it must equal the registered redirect.
|
||||
DISCORD_REDIRECT_URI=
|
||||
# Optional: a role snowflake members must hold on top of guild membership.
|
||||
# Empty (the default) means membership alone suffices.
|
||||
# DISCORD_REQUIRED_ROLE=
|
||||
|
||||
# Subdomain Traefik routes to the browser UI (required by the prod override,
|
||||
# whether or not WEB_PASSWORD is set). Left commented on purpose: an example
|
||||
# value here would be a silent wrong-hostname fallback, and Traefik would
|
||||
# publish the UI router on a domain you do not own. The same container also
|
||||
# answers on BOOKMARK_API_HOST for the userscript's API.
|
||||
# Subdomain Traefik routes to the browser UI (required by the prod override).
|
||||
# Left commented on purpose: an example value here would be a silent
|
||||
# wrong-hostname fallback, and Traefik would publish the UI router on a domain
|
||||
# you do not own. The same container also answers on BOOKMARK_API_HOST for the
|
||||
# userscript's API.
|
||||
# BOOKMARK_WEB_HOST=bookmark.example.com
|
||||
|
||||
# --- Latest-chapter poller ---
|
||||
@@ -46,13 +67,15 @@ WEB_PASSWORD=
|
||||
# Set to 0 to turn it off entirely.
|
||||
# LATEST_CHAPTER_POLL_ENABLED=1
|
||||
#
|
||||
# Two independent clocks. COOLDOWN is how long one series rests between checks;
|
||||
# INTERVAL is how often the poller wakes up and looks for series past that
|
||||
# cooldown. Shortening INTERVAL cannot shorten a COOLDOWN.
|
||||
# LATEST_CHAPTER_POLL_COOLDOWN=1h # per series, floor 15m
|
||||
# LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
|
||||
# LATEST_CHAPTER_POLL_BATCH=14 # series per wake
|
||||
# LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
|
||||
# Two independent clocks. COOLDOWN is how long a plain-TLS series rests between
|
||||
# checks; BROWSER_COOLDOWN is the longer rest for kagane and novelfull. INTERVAL
|
||||
# is how often the poller wakes up and looks for series past their cooldowns.
|
||||
# Shortening INTERVAL cannot shorten either cooldown.
|
||||
LATEST_CHAPTER_POLL_COOLDOWN=1h # plain-TLS per series, floor 15m
|
||||
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN=6h # browser-backed per series, floor 15m
|
||||
LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
|
||||
LATEST_CHAPTER_POLL_BATCH=14 # series per wake
|
||||
LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
|
||||
#
|
||||
# Uses a ticker, not an immediate first run: the first poll happens one
|
||||
# INTERVAL after startup, not at startup. A container restarting more often
|
||||
@@ -62,10 +85,28 @@ WEB_PASSWORD=
|
||||
# defaults. Beyond that the cadence stretches uniformly rather than breaking;
|
||||
# raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL.
|
||||
|
||||
# Headless-shell CDP endpoint for sites behind a JavaScript challenge (kagane).
|
||||
# Unset disables browser polling; those sites then rely on the userscript alone.
|
||||
# Leave commented — the compose files' own default (ws://172.28.0.10:9222) is
|
||||
# correct. Do NOT set this to the "headless-shell" DNS name: Chrome's DevTools
|
||||
# HTTP handler 500s any /json/version request whose Host header isn't an IP or
|
||||
# "localhost", which silently breaks every kagane poll.
|
||||
# BROWSER_WS_URL=ws://172.28.0.10:9222
|
||||
# CDP endpoint of the browser, used for the two sites behind a Cloudflare
|
||||
# JavaScript challenge (kagane, novelfull) and by the web UI's kagane cover
|
||||
# proxy. Unset disables browser polling and serves 404 for covers not already
|
||||
# stored; those sites then rely on the userscript alone. That is also exactly
|
||||
# how an unreachable browser degrades, so a home machine that is off costs
|
||||
# chapter freshness and nothing else.
|
||||
#
|
||||
# The browser does NOT run in this stack. It is its own compose unit on the
|
||||
# home machine (chrome/docker-compose.yml, chrome/.env.example) and is reached
|
||||
# over the tailnet, so set this to that machine's tailnet address:
|
||||
#
|
||||
# BROWSER_WS_URL=ws://100.x.y.z:9222
|
||||
#
|
||||
# It must be the tailnet **IP**, never a MagicDNS hostname and never the old
|
||||
# Docker service name: Chrome's DevTools HTTP handler 500s any /json/version
|
||||
# request whose Host header isn't an IP or "localhost", which silently breaks
|
||||
# every kagane poll. Left unset here on purpose — a wrong default would poll a
|
||||
# stranger's address, and "no browser" is a safe, self-announcing state.
|
||||
# BROWSER_WS_URL=ws://100.x.y.z:9222
|
||||
|
||||
# Zone the backend stamps its log lines in. Cosmetic only. Nothing else in
|
||||
# the service has a zone: bookmark timestamps are unix ms, and the two real
|
||||
# time columns are timestamptz. Defaults to Asia/Jakarta; set to UTC for the
|
||||
# conventional server default.
|
||||
# API_TZ=Asia/Jakarta
|
||||
|
||||
@@ -20,15 +20,23 @@ Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free
|
||||
- Userscript run in **isolated world**, so embedded API token safe from site's JS.
|
||||
- Cloudflare's block on manga sites **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare's bot scoring can flip previously-clean IP without notice. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test.
|
||||
- **kagane.to and novelfull.com are the exception to the above** — both sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`) and skips them entirely when that's unset. The four other sites poll fine over plain TLS.
|
||||
- **The CDP browser must look like a real browser, and stock headless images don't.** Measured 2026-08-08 against kagane.to, all from the same IP: `chromedp/headless-shell:stable` never cleared the challenge in 90s (`navigator.webdriver` true, empty plugin list, Chromium-branded client hints — suppressing `webdriver` alone changed nothing); `zenika/alpine-chrome` ships Chrome 124, refused outright; real Chrome with the default `--headless=new` UA never cleared, because the UA says `HeadlessChrome`; real Chrome with a stock UA **and** a non-UTC clock zone cleared in ~4s. Hence `chrome/` — a Debian image with `google-chrome-stable`, a version-derived UA, and `TZ`/`BROWSER_TZ`. Chrome reads the zone *name* through ICU from `/etc/localtime`'s symlink target, ignoring the file's contents, so mounting the host's `/etc/localtime` does **not** work; `/etc/timezone` is mounted instead.
|
||||
- **The browser is not in the API stack and must not be put back.** It's its own compose unit (`chrome/docker-compose.yml`) on a second machine, reached over the tailnet — it held 471 MiB on a 1974 MiB swapless VPS, and a residential egress scores better with Cloudflare anyway (ADR-0006). Consequences that constrain code: `BROWSER_WS_URL` must be a tailnet **IP** (a MagicDNS name 500s at `/json/version`, same trap as the old Docker service name); the CDP port binds to the tailnet address only, since CDP authenticates nothing and that host has a real LAN; and the browser is on-demand (ADR-0005), so an unreachable or asleep one must degrade exactly as an unset `BROWSER_WS_URL` — plain-TLS libraries unaffected, kagane/novelfull logged and skipped, stored covers still served. Never add `chromedp.NoModifyURL`: discovery per fetch is what makes a restarted Chrome invisible.
|
||||
- **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, so Cloudflare scores it as one; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one.
|
||||
- **A challenged page needs the tab kept open.** The interstitial takes seconds to solve and only then writes clearance into the browser's shared cookie jar. Navigate-read-close never clears anything; `BrowserFetcher.run` holds one tab and re-reads until the payload arrives.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Two Violentmonkey userscripts (isolated world, per-site adapters, localStorage cache)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> Postgres (volume)
|
||||
|
|
||||
| CDP over tailnet
|
||||
v
|
||||
on-demand Chrome, separate machine (chrome/)
|
||||
```
|
||||
|
||||
Backend-specific architecture (packages, endpoints, poller, config env vars) lives in `backend/AGENTS.md`. Userscript-specific structure (adapters, retry queue, UI, live URL shapes) lives in `userscript/AGENTS.md`.
|
||||
Two deployable units on two machines: the API stack (`docker-compose.yml` + `docker-compose.prod.yml`, on the VPS) and the browser (`chrome/docker-compose.yml`, on the home machine). They share nothing but `BROWSER_WS_URL` and update independently. Backend-specific architecture (packages, endpoints, poller, config env vars) lives in `backend/AGENTS.md`. Userscript-specific structure (adapters, retry queue, UI, live URL shapes) lives in `userscript/AGENTS.md`. Deploy order `DEPLOY.md` (§7 for the browser), redeploy `REDEPLOY.md` (§8 for the browser).
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -37,7 +45,12 @@ Backend (`cd backend`):
|
||||
- Single test: `go test -run TestName ./...`
|
||||
- Build static binary: `CGO_ENABLED=0 go build`
|
||||
|
||||
Local stack: `docker compose up` (bookmark-api + postgres + headless-shell; `postgres-data` named volume, `restart: unless-stopped`).
|
||||
Local stack: `docker compose up` (bookmark-api + postgres only; `postgres-data` named volume, `restart: unless-stopped`). No browser — without `BROWSER_WS_URL` the poller logs and skips kagane and novelfull. To run one: `cd chrome && BROWSER_BIND_ADDR=172.17.0.1 docker compose up -d --build`, then `BROWSER_WS_URL=ws://172.17.0.1:9222` in the root `.env` (bridge gateway, so the API container can name it by IP).
|
||||
|
||||
Live CDP proof (needs that browser and network, skipped otherwise):
|
||||
`SMOKE_BROWSER_WS_URL=ws://<ip>:<port> go test -run TestSmokeKagane ./internal/latest`
|
||||
— fetches a real kagane cover and chapter list. A red run means the challenge is
|
||||
not clearing from this IP, which is a live fact to re-check, not necessarily a defect.
|
||||
|
||||
Smoke test: `curl` endpoints with `Authorization: Bearer <token>`; confirm `OPTIONS` preflight return CORS headers and `/healthz` return 200.
|
||||
|
||||
@@ -70,7 +83,7 @@ instantly.
|
||||
|
||||
Existing guarantees — don't regress:
|
||||
|
||||
- Auth on `/bookmarks*`: require `Authorization: Bearer <API_TOKEN>`, **constant-time compare**, 401 otherwise.
|
||||
- Auth on `/bookmarks*`: require `Authorization: Bearer <credential>` — the acting Reader's credential, matched by SHA-256 against `readers.token_sha256` — **constant-time compare** (via the hash, never the secret itself), 401 otherwise.
|
||||
- CORS: reflect `Origin` only when in `ALLOWED_ORIGINS`; allow `GET,PUT,DELETE,OPTIONS` + headers `Authorization,Content-Type`; answer preflight `OPTIONS` with `204`.
|
||||
|
||||
## Secure coding rules (code you write here)
|
||||
@@ -83,18 +96,18 @@ Go backend:
|
||||
- `html/template` only for anything a browser parses, never `text/template`. Never wrap stored or fetched strings in `template.HTML`/`JS`/`URL`; that switches off the escaping every template depends on.
|
||||
- Any outbound fetch of a client-supplied URL passes `fetchableSeriesURL` (site + `https` + host check) first. `series_url` arrives in a PUT body, so without the gate the poller will probe arbitrary hosts from the server's own network position. New fetch path reuses the gate rather than re-deriving one.
|
||||
- Cap every remote body with `io.LimitReader` (`maxBodyBytes`). An unbounded read is an OOM handed to whatever is on the other end.
|
||||
- Compare secrets with `hmac.Equal` / `subtle.ConstantTimeCompare`, never `==`. Covers API token, web password, session MAC.
|
||||
- Errors: generic text to the client (`http.Error(w, "internal error", 500)`), detail to `log.Printf`. Never log `API_TOKEN`, `WEB_PASSWORD`, a session cookie value, or a whole `Authorization` header.
|
||||
- Compare secrets with `hmac.Equal` / `subtle.ConstantTimeCompare`, never `==`. A credential is matched by the SHA-256 the `readers` table holds, which is already a fixed-width equality — a new secret comparison must not regress to `==`.
|
||||
- Errors: generic text to the client (`http.Error(w, "internal error", 500)`), detail to `log.Printf`. Never log `TOKEN_KEY`, a Reader's credential, `DISCORD_CLIENT_SECRET`, a session id, or a whole `Authorization` header.
|
||||
- Proxy headers are trusted only where they already are: `X-Forwarded-Proto` for the Secure cookie flag, **rightmost** `X-Forwarded-For` for client IP (leftmost is attacker-supplied). Don't read either anywhere else.
|
||||
- Session cookies keep `HttpOnly`, `SameSite`, `Secure`-when-HTTPS, and expiry checked before signature.
|
||||
- Session cookies keep `HttpOnly`, `SameSite`, `Secure`-when-HTTPS; expiry is enforced by the `sessions` table lookup, not a signature.
|
||||
- Stdlib crypto only. No hand-rolled hashing, no MD5/SHA-1 anywhere security-bearing.
|
||||
- Validate at the handler boundary before storing: body capped by `http.MaxBytesReader` (64 KB), empty `key` and unknown `status`/`kind` rejected with `400`. A bad value that reaches the store becomes every later reader's problem.
|
||||
|
||||
Userscript:
|
||||
|
||||
- Site-derived and stored strings render via `el(..., {text})` / `textContent`. `{html}` and `innerHTML` are for author-written literal markup only (`TEMPLATE`, `CSS`) — never a title, chapter label, or API response field. The page DOM belongs to a third-party site; treat it as attacker-controlled.
|
||||
- Isolated world protects the token from the site's JS. It does not protect anything from an `innerHTML` sink you add yourself.
|
||||
- The `API_TOKEN` literal sits in both userscripts and must equal backend `API_TOKEN`. Never copy it into logs, docs, commit messages, issues, or a new file. Rotation touches three places: backend env plus both scripts.
|
||||
- Isolated world protects the credential from the site's JS. It does not protect anything from an `innerHTML` sink you add yourself.
|
||||
- The userscripts carry `__API_TOKEN__` placeholders, substituted at serve time with the requesting Reader's credential (`internal/userscript`). Never put a real credential in the repo, docs, commit messages, or issues. Rotation is a web-UI action (epoch bump, `internal/token`); `TOKEN_KEY` in backend env is what derives every credential — never log it.
|
||||
- `fetch()` targets `API_BASE` only — no dynamic origin, no site-supplied URL. `authHeaders()` goes nowhere but the backend.
|
||||
- `localStorage` is shared with the site's own JS: cache and queue live there, credentials never do.
|
||||
- Wrap every `localStorage` read/write and `JSON.parse` in try/catch (quota, private mode, corrupt entry), as the existing helpers do.
|
||||
|
||||
+299
@@ -0,0 +1,299 @@
|
||||
# SQLite → Postgres cutover runbook
|
||||
|
||||
One-way, one-time. Moves the owner's reading history out of the retired SQLite
|
||||
volume (`<compose project>_bookmarks-data`, holding `/data/bookmarks.db`) and into
|
||||
the Postgres schema the migration runner builds. There is no dual-write period:
|
||||
the old database is read once, at cutover, from a **fresh export** — anything
|
||||
written to SQLite after the export is lost, so the old API must already be down.
|
||||
|
||||
Routine deploys are `REDEPLOY.md`; first-time setup is `DEPLOY.md`. This file is
|
||||
run once and then only ever read for reference.
|
||||
|
||||
Proven end to end on 2026-08-08 against a copy of `bookmarks-20260807-213515.db`
|
||||
into a scratch Postgres: 29 Bookmarks (18 reading, 11 archived, 7 favourites),
|
||||
29 Series, all owned by the seeded Reader, and every field of every row matching
|
||||
the source exactly. Production was not touched.
|
||||
|
||||
---
|
||||
|
||||
## 0. The generator is throwaway
|
||||
|
||||
It is written at cutover, run once, and deleted. It is deliberately **not** in
|
||||
this repository and never will be:
|
||||
|
||||
- Its output is the owner's personal reading history. That does not enter
|
||||
version control.
|
||||
- It reads SQLite. The backend module dropped `modernc.org/sqlite` (ADR-0001);
|
||||
a committed generator would drag the dependency back in through the side door.
|
||||
|
||||
So §3 specifies the transformation rather than shipping a script. It is a
|
||||
twenty-line program against a sixteen-column table (fifteen after `key`, which
|
||||
is dropped) — writing it from the spec below costs less than maintaining it
|
||||
would.
|
||||
|
||||
Beyond `DEPLOY.md`'s prerequisites (Docker and Compose), this runbook needs
|
||||
`python3`: its stdlib `sqlite3` module is the whole SQLite dependency, and §5's
|
||||
read-path check uses it in place of `jq`, which the server does not have. It
|
||||
does not have to run on the server — §3 only reads the snapshot copy, so it can
|
||||
run on a laptop and the resulting `import.sql` be copied over.
|
||||
|
||||
---
|
||||
|
||||
## 1. Stop the old API and take a fresh export
|
||||
|
||||
**Order matters.** Export after the API stops, or you migrate a snapshot that is
|
||||
already stale.
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark # wherever the checkout lives
|
||||
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
|
||||
BACKUP_DIR="$(cd .. && pwd)/$(basename "$PWD")-backups"; mkdir -p "$BACKUP_DIR"
|
||||
STAMP=$(date -u +%Y%m%d-%H%M%S)
|
||||
|
||||
# The volume is <compose project>_bookmarks-data, and the project name defaults
|
||||
# to the lowercased *directory* name, not the repo name — on this host the
|
||||
# checkout is ~/mangaBookmark, so the volume is mangabookmark_bookmarks-data.
|
||||
# Derive it exactly rather than with a `--filter name=` substring match, which
|
||||
# would return every volume whose name merely contains the string.
|
||||
VOL="$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_bookmarks-data"
|
||||
docker volume inspect "$VOL" >/dev/null && echo "$VOL"
|
||||
|
||||
$COMPOSE stop bookmark-api
|
||||
|
||||
# A clean SIGTERM closes the store, which checkpoints and unlinks the -wal, so
|
||||
# bookmarks.db alone is then the whole database. But `compose stop` SIGKILLs
|
||||
# after 10s, and a surviving -wal holds writes the main file does not — assert
|
||||
# it is gone rather than assuming the shutdown was clean.
|
||||
docker run --rm -v "$VOL":/d:ro alpine ls -l /d # -> bookmarks.db, alone
|
||||
|
||||
docker run --rm -v "$VOL":/from:ro -v "$BACKUP_DIR":/to \
|
||||
alpine cp /from/bookmarks.db "/to/bookmarks-$STAMP.db"
|
||||
|
||||
ls -lh "$BACKUP_DIR/bookmarks-$STAMP.db"
|
||||
```
|
||||
|
||||
If `-wal` and `-shm` are still there, the container was killed mid-write. Copy
|
||||
all three under the same basename and let SQLite replay the log when §3 opens
|
||||
it — copying only `bookmarks.db` silently drops whatever the log still holds.
|
||||
|
||||
Work on a **copy** of that file for the rest of this runbook. The export is the
|
||||
last line of retreat; nothing below should be able to write to it.
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/cutover && cp "$BACKUP_DIR/bookmarks-$STAMP.db" /tmp/cutover/snapshot.db
|
||||
chmod 444 /tmp/cutover/snapshot.db
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Bring up Postgres with the schema and the owner Reader
|
||||
|
||||
The new stack builds its own schema and seeds exactly one Reader from
|
||||
`OWNER_DISCORD_ID` — do not hand-write either. Pull the Postgres-era commit
|
||||
first: on a server that has only ever run the SQLite build, `--build` without a
|
||||
pull silently rebuilds the old image and the checks below fail with
|
||||
"relation readers does not exist".
|
||||
|
||||
```bash
|
||||
git pull --ff-only
|
||||
git log --oneline -1
|
||||
|
||||
# .env needs the new required vars (DATABASE_URL is built from
|
||||
# POSTGRES_PASSWORD; TOKEN_KEY, OWNER_DISCORD_ID and the DISCORD_* set are
|
||||
# required). Compose fails at start for a missing one.
|
||||
git diff HEAD@{1} HEAD -- .env.example docker-compose.yml docker-compose.prod.yml
|
||||
|
||||
$COMPOSE up -d --build
|
||||
docker logs bookmark-api --tail 20 # -> "listening on :8080"
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c '\dt'
|
||||
# -> bookmarks, readers, schema_migrations, series, sessions
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks \
|
||||
-c 'select id, discord_id from readers'
|
||||
# -> exactly one row, and discord_id is the owner's
|
||||
```
|
||||
|
||||
Two rows in `readers`, or zero, means `OWNER_DISCORD_ID` is wrong or the seed
|
||||
failed. Stop here — the import attaches history to "the oldest reader row", and
|
||||
that is only unambiguous while there is one.
|
||||
|
||||
`bookmarks` and `series` are empty at this point. That is what makes the import
|
||||
a plain sequence of `INSERT`s with no conflict handling.
|
||||
|
||||
---
|
||||
|
||||
## 3. Generate the import SQL
|
||||
|
||||
Read `/tmp/cutover/snapshot.db` and emit plain SQL on stdout. The old table is
|
||||
flat and its columns map one-for-one onto the split schema — no transformation
|
||||
beyond the split itself:
|
||||
|
||||
| SQLite `bookmarks` column | lands in | notes |
|
||||
|---|---|---|
|
||||
| `site`, `series_id` | both tables | the Series key; the wire `key` column is dropped, it is re-derived as `site:series_id` on read |
|
||||
| `title`, `series_url`, `cover`, `kind` | `series` | shared facts (ADR-0003) |
|
||||
| `latest_chapter`, `latest_chapter_num`, `latest_checked_at` | `series` | `latest_chapter_num` is nullable on **both** sides and `NULL` is meaningful — never coerce it to `0` |
|
||||
| `last_chapter`, `last_chapter_num`, `last_chapter_url` | `bookmarks` | Progress |
|
||||
| `favorite`, `status`, `updated_at` | `bookmarks` | `favorite` is `0`/`1` in SQLite and a real `boolean` in Postgres — emit `true`/`false` |
|
||||
| — | `bookmarks.reader_id` | the seeded owner |
|
||||
|
||||
**`latest_chapter_num` is the only column where `NULL` survives.** The SQLite
|
||||
table declares `title`, `series_url`, `cover`, `last_chapter`,
|
||||
`last_chapter_url` as bare `TEXT` and `last_chapter_num` as bare `REAL` — all
|
||||
six nullable — while their Postgres targets are `NOT NULL DEFAULT ''` /
|
||||
`NOT NULL DEFAULT 0`. One `NULL` in any of them aborts the whole import on a
|
||||
not-null violation. Coalesce them in the `SELECT` (`ifnull(title,'')`,
|
||||
`ifnull(last_chapter_num,0)`, …) rather than discovering it at §5. The
|
||||
2026-08-07 export happened to have none; a fresh export is not promised the
|
||||
same.
|
||||
|
||||
Rules the generator must follow:
|
||||
|
||||
- **Series first, Bookmarks second.** `bookmarks` has a foreign key onto
|
||||
`series (site, series_id)`; the reverse order fails on the first row.
|
||||
- **`SELECT DISTINCT` the Series.** The old key's uniqueness already makes
|
||||
`(site, series_id)` unique, so this is belt and braces — but if it ever
|
||||
collapses two rows, the count check in §5 catches it.
|
||||
- **Never hardcode the reader id.** Emit
|
||||
`INSERT INTO bookmarks (reader_id, …) SELECT id, … FROM owner`, where `owner`
|
||||
is a temp table built once at the top:
|
||||
`CREATE TEMP TABLE owner ON COMMIT DROP AS SELECT id FROM readers ORDER BY id LIMIT 1;`
|
||||
A literal id is a number nobody verifies; this one cannot be wrong.
|
||||
- **Wrap the whole file in `BEGIN; … COMMIT;`, temp table included.** Postgres
|
||||
has transactional DDL and DML: a failure half way leaves an empty database
|
||||
rather than half a library. The ordering is load-bearing —
|
||||
`ON COMMIT DROP` outside the transaction means the temp table drops itself
|
||||
the instant it is created (psql autocommits) and every
|
||||
`SELECT … FROM owner` then fails.
|
||||
- **Quote strings by doubling `'`.** Titles contain apostrophes and the URLs
|
||||
contain `%5C%27` escapes. Emit standard SQL literals only — no `E''` strings,
|
||||
no backslash escaping (`standard_conforming_strings` is on, so a backslash is
|
||||
a literal backslash and the URLs survive verbatim).
|
||||
|
||||
```bash
|
||||
python3 gen_import.py /tmp/cutover/snapshot.db > /tmp/cutover/import.sql
|
||||
wc -l /tmp/cutover/import.sql # -> 2 header + 29 series + 29 bookmarks + framing
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Review it by eye
|
||||
|
||||
29 rows is small enough to actually read, and this is the last point at which a
|
||||
mistake is free:
|
||||
|
||||
```bash
|
||||
less /tmp/cutover/import.sql
|
||||
grep -c '^INSERT INTO series' /tmp/cutover/import.sql # -> 29
|
||||
grep -c '^INSERT INTO bookmarks' /tmp/cutover/import.sql # -> 29
|
||||
```
|
||||
|
||||
Look for: a title whose apostrophe is not doubled, a `favorite` that is still
|
||||
`0`/`1`, a `latest_chapter_num` that turned into `0`, and any `reader_id`
|
||||
written as a bare number.
|
||||
|
||||
---
|
||||
|
||||
## 5. Apply it
|
||||
|
||||
```bash
|
||||
docker cp /tmp/cutover/import.sql "$($COMPOSE ps -q postgres)":/tmp/import.sql
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -v ON_ERROR_STOP=1 \
|
||||
-f /tmp/import.sql
|
||||
```
|
||||
|
||||
`ON_ERROR_STOP=1` is not optional: without it `psql` reports the error, keeps
|
||||
going, and exits `0` on a half-imported database.
|
||||
|
||||
Then the checklist. Every number here is asserted, not eyeballed:
|
||||
|
||||
```bash
|
||||
OWNER=$(grep -E '^OWNER_DISCORD_ID=' .env | cut -d= -f2)
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -x -c "
|
||||
SELECT (SELECT count(*) FROM bookmarks) AS bookmarks_total,
|
||||
(SELECT count(*) FROM bookmarks WHERE status='reading') AS reading,
|
||||
(SELECT count(*) FROM bookmarks WHERE status='archived')AS archived,
|
||||
(SELECT count(*) FROM series) AS series_total,
|
||||
(SELECT count(*) FROM readers) AS readers_total,
|
||||
(SELECT count(*) FROM bookmarks
|
||||
WHERE reader_id <> (SELECT id FROM readers WHERE discord_id='$OWNER'))
|
||||
AS not_owned_by_owner;"
|
||||
```
|
||||
|
||||
`not_owned_by_owner` resolves the Reader by **Discord id**, not by
|
||||
`ORDER BY id LIMIT 1`. The second form is the expression §3 tells the generator
|
||||
to import with, so comparing against it is true by construction and could never
|
||||
fail; resolving by Discord id is an independent check that the rows landed on
|
||||
the identity the owner will actually log in as. If that subquery returns NULL
|
||||
the whole count comes back `0` for the wrong reason — hence `readers_total`
|
||||
beside it.
|
||||
|
||||
Expected, for the 2026-08-07 export: `29`, `18`, `11`, `29`, `1`, `0`. Against a
|
||||
different export, the invariants rather than the literals are what hold:
|
||||
|
||||
- `bookmarks_total` equals the SQLite row count.
|
||||
- `reading + archived` equals `bookmarks_total` (nothing was `finished`).
|
||||
- `series_total` equals `SELECT count(*) FROM (SELECT DISTINCT site, series_id FROM bookmarks)`
|
||||
in the source.
|
||||
- `readers_total` is `1` and `not_owned_by_owner` is `0`.
|
||||
|
||||
Then spot-check the values themselves against the source — read position,
|
||||
favourite flag and latest chapter. Take the sample from each bucket explicitly:
|
||||
`ORDER BY updated_at DESC LIMIT 5` alone returns the most recently *progressed*
|
||||
rows, which are the ones least likely to be archived.
|
||||
|
||||
```bash
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
|
||||
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
|
||||
s.latest_chapter, b.status
|
||||
FROM bookmarks b JOIN series s USING (site, series_id)
|
||||
WHERE b.status='reading' ORDER BY b.updated_at DESC LIMIT 3;"
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
|
||||
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
|
||||
s.latest_chapter, b.status
|
||||
FROM bookmarks b JOIN series s USING (site, series_id)
|
||||
WHERE b.status='archived' ORDER BY b.updated_at DESC LIMIT 2;"
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
|
||||
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
|
||||
s.latest_chapter, b.status
|
||||
FROM bookmarks b JOIN series s USING (site, series_id)
|
||||
WHERE b.favorite ORDER BY b.updated_at DESC LIMIT 2;"
|
||||
```
|
||||
|
||||
Compare each against the same row in the snapshot — the generator's own source
|
||||
is the reference, so read it back with the same `python3` you used in §3.
|
||||
|
||||
Finally, prove the **read path**, not just the tables — this is the check that
|
||||
would catch a correct import behind a broken join:
|
||||
|
||||
```bash
|
||||
API=https://bookmark-api.violetcrown.my.id
|
||||
# Your own Reader credential: sign in to the web UI and take it from the
|
||||
# Userscripts panel's install link, or read the API_TOKEN constant out of an
|
||||
# already-installed script. There is no credential in .env to grep.
|
||||
TOKEN=<your Reader credential>
|
||||
curl -s -H "Authorization: Bearer $TOKEN" $API/bookmarks |
|
||||
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' # -> 29
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Afterwards
|
||||
|
||||
- **Keep the old SQLite volume for a month.** It is already undeclared in
|
||||
compose, so `docker compose down -v` cannot take it. Remove it by hand once
|
||||
the Postgres data has been trusted for a while. That happens in a shell where
|
||||
`$VOL` from §1 is long gone, so re-derive it:
|
||||
`docker volume rm "$(basename ~/mangaBookmark | tr '[:upper:]' '[:lower:]')_bookmarks-data"`
|
||||
(see `REDEPLOY.md` §1).
|
||||
- **Delete the generator and the working copies:** `rm -rf /tmp/cutover`. The
|
||||
timestamped export in `$BACKUP_DIR` is the copy that is kept.
|
||||
- **Take the first Postgres dump immediately** — `REDEPLOY.md` §1. Until that
|
||||
exists, the only backup of the migrated data is the SQLite file it came from.
|
||||
|
||||
If the import is wrong, there is nothing to unpick: drop the rows and start
|
||||
again from §3 — `TRUNCATE bookmarks, series;` leaves the seeded Reader and the
|
||||
schema in place.
|
||||
@@ -11,7 +11,7 @@ ACME/cert resolver, and control a domain.
|
||||
- Docker + Docker Compose on the server.
|
||||
- A Traefik instance watching a Docker network (default name assumed: `proxy`).
|
||||
- DNS: an `A`/`AAAA` record for `bookmark-api.<yourdomain>` pointing at the server.
|
||||
- The repo copied to the server, e.g. `/opt/bookmarkmanager/` (needs `backend/`,
|
||||
- The repo copied to the server, e.g. `~/mangaBookmark/` (needs `backend/`,
|
||||
`docker-compose.yml`, `docker-compose.prod.yml`, `.env.example`).
|
||||
|
||||
Confirm the Traefik network exists (create if not):
|
||||
@@ -25,15 +25,22 @@ docker network ls | grep proxy || docker network create proxy
|
||||
## 1. Configure `.env`
|
||||
|
||||
```bash
|
||||
cd /opt/bookmarkmanager
|
||||
cd ~/mangaBookmark
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env`:
|
||||
|
||||
```ini
|
||||
# Required — long random secret, also goes in the userscript.
|
||||
API_TOKEN=<paste output of: openssl rand -hex 32>
|
||||
# Required — secret every Reader's userscript credential is derived from.
|
||||
# Only SHA-256 hashes of credentials are stored.
|
||||
TOKEN_KEY=<paste output of: openssl rand -hex 32>
|
||||
|
||||
# Required — the owner's Discord user ID. Seeds the first Reader: the
|
||||
# administrator, and the owner of every bookmark that predates registration.
|
||||
# The value is the snowflake in your Discord profile (Settings →
|
||||
# Advanced → Developer Mode → right-click your name → Copy User ID).
|
||||
OWNER_DISCORD_ID=<discord user id>
|
||||
|
||||
# CORS allowlist — leave as-is unless a site changes hostname.
|
||||
ALLOWED_ORIGINS=https://asuracomic.net,https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to
|
||||
@@ -47,8 +54,8 @@ POSTGRES_PASSWORD=<paste output of: openssl rand -hex 24>
|
||||
# DATABASE_URL=postgres://user:pass@host:5432/bookmarks?sslmode=require
|
||||
|
||||
# Required for the Traefik override. Both have no fallback — compose refuses
|
||||
# to start without them. BOOKMARK_WEB_HOST is required even if you never set
|
||||
# WEB_PASSWORD; see 1b.
|
||||
# to start without them. BOOKMARK_WEB_HOST is required even if the web UI
|
||||
# were unused; see 1b.
|
||||
BOOKMARK_API_HOST=bookmark-api.violetcrown.my.id
|
||||
BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
|
||||
|
||||
@@ -61,11 +68,16 @@ BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
|
||||
Generate + insert the two secrets in three lines:
|
||||
|
||||
```bash
|
||||
sed -i "s|^API_TOKEN=.*|API_TOKEN=$(openssl rand -hex 32)|" .env
|
||||
sed -i "s|^TOKEN_KEY=.*|TOKEN_KEY=$(openssl rand -hex 32)|" .env
|
||||
sed -i "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$(openssl rand -hex 24)|" .env
|
||||
grep -E '^API_TOKEN=' .env # copy this — the userscript needs the same value
|
||||
grep -E '^TOKEN_KEY=' .env
|
||||
```
|
||||
|
||||
`TOKEN_KEY` derives every Reader's userscript credential (issue #24); only
|
||||
SHA-256 hashes of the credentials are stored, so this secret is what a
|
||||
database leak alone cannot recover. Changing it invalidates every installed
|
||||
script at once.
|
||||
|
||||
`POSTGRES_PASSWORD` is read **only while the `postgres-data` volume is empty**,
|
||||
which in practice means at first boot. Changing it afterwards changes the URL
|
||||
the backend dials but not the password the database expects, and `bookmark-api`
|
||||
@@ -80,44 +92,58 @@ alone.
|
||||
|
||||
## 1b. Web UI
|
||||
|
||||
The browser UI is served by the same container on a second hostname.
|
||||
The browser UI is served by the same container on a second hostname. Sign-in
|
||||
is a Discord authorization code grant (ADR-0002): the owner's Discord account,
|
||||
gated by membership in one configured guild.
|
||||
|
||||
1. Add a DNS `A`/`AAAA` record for `bookmark.<yourdomain>` pointing at the server —
|
||||
the same address as `bookmark-api.<yourdomain>`.
|
||||
1. Add a DNS `A`/`AAAA` record for `bookmark.<yourdomain>` pointing at the
|
||||
server — the same address as `bookmark-api.<yourdomain>`.
|
||||
|
||||
2. Set both variables in `.env`:
|
||||
2. Create the Discord application at <https://discord.com/developers/applications>:
|
||||
- **OAuth2 → Redirects:** add the exact callback URL
|
||||
`https://bookmark.violetcrown.my.id/auth/discord/callback`. Discord
|
||||
matches it verbatim — a trailing slash or different hostname breaks
|
||||
sign-in.
|
||||
- **OAuth2 → General:** note the Client ID, and generate a Client Secret.
|
||||
- No scopes or bot setup are needed in the dashboard; the service requests
|
||||
`identify` and `guilds.members.read` itself, and checks the *user's*
|
||||
membership of the guild, not the application's.
|
||||
|
||||
3. Set the variables in `.env`:
|
||||
|
||||
```ini
|
||||
BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
|
||||
WEB_PASSWORD=<paste output of: openssl rand -base64 18>
|
||||
DISCORD_CLIENT_ID=<client id>
|
||||
DISCORD_CLIENT_SECRET=<client secret>
|
||||
DISCORD_GUILD_ID=<guild snowflake>
|
||||
DISCORD_REDIRECT_URI=https://bookmark.violetcrown.my.id/auth/discord/callback
|
||||
# Optional: only members holding this role may sign in.
|
||||
# DISCORD_REQUIRED_ROLE=<role snowflake>
|
||||
```
|
||||
|
||||
Generate and insert in one line:
|
||||
The guild id is in Discord's client with Developer Mode on: right-click the
|
||||
server name → Copy Server ID. The four uncommented variables are required —
|
||||
the backend refuses to start without them. Guild membership *is*
|
||||
registration: any member of `DISCORD_GUILD_ID` becomes a Reader with their
|
||||
own library on their first sign-in. `OWNER_DISCORD_ID` from §1 is only the
|
||||
administrator — the Reader who can revoke another Reader's sessions.
|
||||
|
||||
```bash
|
||||
sed -i "s|^WEB_PASSWORD=.*|WEB_PASSWORD=$(openssl rand -base64 18)|" .env
|
||||
grep -E '^WEB_PASSWORD=' .env # this is what you type into the site
|
||||
```
|
||||
|
||||
3. Redeploy and check:
|
||||
4. Redeploy and check:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build
|
||||
curl -s -o /dev/null -w '%{http_code}\n' https://bookmark.violetcrown.my.id/
|
||||
```
|
||||
|
||||
Expected `200`, serving the login page.
|
||||
Expected `200`, serving the login page with the Discord button. Signing in
|
||||
lands on the library; an account outside the guild is refused with a message
|
||||
that names neither the guild nor its id.
|
||||
|
||||
Leaving `WEB_PASSWORD` unset is safe: the web routes are not registered and `/`
|
||||
returns 404. The userscript's API on `BOOKMARK_API_HOST` is unaffected either way.
|
||||
|
||||
`BOOKMARK_WEB_HOST` itself is required by the prod override regardless — like
|
||||
`BOOKMARK_API_HOST`, its Traefik label has no fallback, so `docker compose up`
|
||||
refuses to start without it even if `WEB_PASSWORD` is unset and the web UI is
|
||||
otherwise dormant.
|
||||
|
||||
Sessions are signed with a key derived from `API_TOKEN` and `WEB_PASSWORD`, so
|
||||
rotating either one logs every browser out. The session cookie lasts 60 days.
|
||||
Sessions are rows in the database: the cookie carries only an opaque id, and
|
||||
every request looks the row up and checks its expiry. Deleting a session row —
|
||||
or the whole `sessions` table — logs the browser out immediately; nothing is
|
||||
signed, so rotating a credential does not affect browser sessions. Sessions
|
||||
last 60 days.
|
||||
|
||||
---
|
||||
|
||||
@@ -131,15 +157,17 @@ This merges the base file (build/image/env/volume) with the prod override
|
||||
(no host port, Traefik network + router labels). Always pass **both** `-f`
|
||||
flags — the prod file is not standalone.
|
||||
|
||||
Three services come up: `bookmark-api` (the backend), `postgres` (its database,
|
||||
`postgres:17-alpine`), and `headless-shell`, a CDP sidecar the poller uses to
|
||||
fetch kagane (behind a Cloudflare JS challenge). Neither of the latter two
|
||||
publishes a port: `postgres` sits alone with `bookmark-api` on an
|
||||
`internal: true` network, and `headless-shell` is reachable only over
|
||||
`BROWSER_WS_URL`. A missing headless-shell just makes the poller skip kagane and
|
||||
log it. A missing Postgres stops everything — `bookmark-api` waits for
|
||||
`pg_isready` to pass, then applies its embedded migrations, and only then
|
||||
listens. The schema is created that way; there is nothing to import by hand.
|
||||
Two services come up: `bookmark-api` (the backend) and `postgres` (its
|
||||
database, `postgres:17-alpine`). Postgres publishes no port — it sits alone
|
||||
with `bookmark-api` on an `internal: true` network — and stops everything if it
|
||||
is missing: `bookmark-api` waits for `pg_isready` to pass, then applies its
|
||||
embedded migrations, and only then listens. The schema is created that way;
|
||||
there is nothing to import by hand.
|
||||
|
||||
There is deliberately no browser here. Kagane and novelfull need one, and it
|
||||
runs on a **separate machine** over the tailnet — §7. Until you do that step,
|
||||
`BROWSER_WS_URL` is unset, the poller logs and skips those two sites, and
|
||||
everything else works normally.
|
||||
|
||||
Check it's up and healthy:
|
||||
|
||||
@@ -163,7 +191,10 @@ curl -s https://bookmark-api.violetcrown.my.id/healthz # -> ok
|
||||
curl -s -o /dev/null -w '%{http_code}\n' \
|
||||
https://bookmark-api.violetcrown.my.id/bookmarks # -> 401
|
||||
|
||||
TOKEN=$(grep -E '^API_TOKEN=' .env | cut -d= -f2)
|
||||
# A Reader's own credential. It is derived, never stored in .env — take it from
|
||||
# the Userscripts panel's install link after signing in, or from an installed
|
||||
# script's API_TOKEN constant.
|
||||
TOKEN=<your Reader credential>
|
||||
curl -s -H "Authorization: Bearer $TOKEN" \
|
||||
https://bookmark-api.violetcrown.my.id/bookmarks # -> []
|
||||
|
||||
@@ -182,29 +213,30 @@ a bad cert makes the browser block the userscript's `fetch()` (mixed content).
|
||||
|
||||
## 4. Configure the userscript
|
||||
|
||||
Edit the config block at the top of `userscript/manga-bookmark.user.js`:
|
||||
The bindmounted `userscript/*.user.js` files carry `__API_TOKEN__` placeholders
|
||||
and the deployment's `@downloadURL`/`@updateURL` lines. Check the metadata
|
||||
block — it ships hardcoded to this deployment's domain, so a deployer who
|
||||
copies the repo to another domain must edit the two lines or the script
|
||||
auto-updates from someone else's backend:
|
||||
|
||||
```js
|
||||
const API_BASE = "https://bookmark-api.yourdomain.com"; // no trailing slash
|
||||
const API_TOKEN = "<same token as .env>";
|
||||
// @downloadURL https://bookmark-api.yourdomain.com/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
// @updateURL https://bookmark-api.yourdomain.com/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
```
|
||||
|
||||
The token sits in the userscript's isolated world — the manga sites' JS can't
|
||||
read it.
|
||||
|
||||
Also edit the `@downloadURL`/`@updateURL` metadata lines near the top of the
|
||||
file — they ship hardcoded to this deployment's domain and token, so a
|
||||
deployer who skips them ends up auto-updating from someone else's backend.
|
||||
See "Installing / updating the userscript" below for how those two lines are
|
||||
used.
|
||||
The backend substitutes `__API_TOKEN__` with the requesting Reader's derived
|
||||
credential at serve time (issue #24), so no real credential ever sits in the
|
||||
file. Only the `API_BASE` constant and the metadata hostname are deployer
|
||||
edits; do not put a credential in this file.
|
||||
|
||||
---
|
||||
|
||||
## 5. Install on Bromite
|
||||
|
||||
1. Bromite → **Settings → User scripts** → enable (accept the permission prompt).
|
||||
2. Put the edited `manga-bookmark.user.js` on the device (save the file, or open
|
||||
its raw URL). Bromite detects `.user.js` and offers to install.
|
||||
2. Sign in to the web UI, open the **Userscripts** panel, and open the install
|
||||
link — Bromite detects `.user.js` and offers to install. The script already
|
||||
carries your credential; you never see or type one.
|
||||
3. Confirm install — the `@match` list covers both sites.
|
||||
4. Open a series on asurascans.com or demonicscans.org → a 📑 button appears
|
||||
bottom-right → tap → **+ Bookmark this**.
|
||||
@@ -212,6 +244,9 @@ used.
|
||||
Optional desktop test: the script is `GM_*`-free, so the same file installs in
|
||||
Tampermonkey/Violentmonkey for quick checks before going mobile.
|
||||
|
||||
Rotating the credential in the same web-UI panel invalidates every installed
|
||||
copy immediately — reinstall on all devices, or they silently stop syncing.
|
||||
|
||||
---
|
||||
|
||||
## 6. Smoke-test the full loop
|
||||
@@ -226,6 +261,116 @@ Tampermonkey/Violentmonkey for quick checks before going mobile.
|
||||
|
||||
---
|
||||
|
||||
## 7. The browser, on the home machine
|
||||
|
||||
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
|
||||
fingerprint clears, so the poller reaches them through a real Chrome over CDP.
|
||||
That browser does **not** run on the VPS: it held 471 MiB of a 1974 MiB box
|
||||
with no swap, and it scores better from a residential IP anyway (ADR-0006). It
|
||||
is its own compose unit, deployed and updated independently of everything
|
||||
above.
|
||||
|
||||
Do this after §2, on the second machine. Both machines must already be on the
|
||||
same tailnet.
|
||||
|
||||
First, on the VPS, record what you are reclaiming — this is the whole point of
|
||||
the move and there is no way to measure it afterwards:
|
||||
|
||||
```bash
|
||||
free -m | awk '/^Mem:/ {print "available before:", $NF, "MiB"}'
|
||||
```
|
||||
|
||||
Take it again after §7 is finished and the old sidecar is gone. Expect roughly
|
||||
the sidecar's former footprint back (measured at 471 MiB working set, 595 MiB
|
||||
cgroup).
|
||||
|
||||
**On the home machine:**
|
||||
|
||||
```bash
|
||||
git clone <this repo> ~/mangaBookmark && cd ~/mangaBookmark/chrome
|
||||
|
||||
tailscale ip -4 # -> 100.x.y.z, this machine's tailnet IP
|
||||
cp .env.example .env
|
||||
echo "BROWSER_BIND_ADDR=$(tailscale ip -4)" >> .env
|
||||
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
The clone is only for `chrome/`; nothing else on this machine reads the rest of
|
||||
the repo. The unit is its own compose project (`bookmark-browser`), so it shares
|
||||
no volume, network or lifecycle with an API stack that happens to sit beside it.
|
||||
|
||||
`BROWSER_BIND_ADDR` has no default on purpose. CDP authenticates nothing —
|
||||
whatever reaches port 9222 drives the browser and, through it, this host — so
|
||||
the bind address *is* the access control, backed by Tailscale device identity.
|
||||
On the VPS that job was done by Docker network membership; this machine has a
|
||||
real LAN, so `0.0.0.0` would be a hole punched into your home network. Compose
|
||||
refuses to start rather than guess.
|
||||
|
||||
**Narrow it to the one device that needs it.** The bind address keeps CDP off
|
||||
your LAN; it still leaves port 9222 open to every device on the tailnet, and
|
||||
CDP has no login. Add a rule in the Tailscale admin console's access controls
|
||||
so only the VPS can reach it — tag the two machines, then:
|
||||
|
||||
```jsonc
|
||||
// tailnet policy file
|
||||
"acls": [
|
||||
{ "action": "accept", "src": ["tag:bookmark-api"], "dst": ["tag:bookmark-browser:9222"] },
|
||||
]
|
||||
```
|
||||
|
||||
Without a rule the tailnet default is allow-all, so this step is what makes
|
||||
"Tailscale identity is the access control" true rather than aspirational.
|
||||
|
||||
Prove the bind is tight, from the home machine itself:
|
||||
|
||||
```bash
|
||||
curl -s -m 3 http://$(tailscale ip -4):9222/json/version # -> JSON
|
||||
curl -s -m 3 http://<this machine's LAN IP>:9222/json/version
|
||||
# -> curl: (7) Failed to connect ... Connection refused
|
||||
```
|
||||
|
||||
The first call is also what wakes Chrome: it is not running until something
|
||||
connects, and it is reaped again after five idle minutes. A cold first response
|
||||
takes a few seconds; that is the browser starting, not a fault.
|
||||
|
||||
**On the VPS:**
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark
|
||||
echo 'BROWSER_WS_URL=ws://100.x.y.z:9222' >> .env # the home machine's tailnet IP
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
It must be the tailnet **IP**. A MagicDNS hostname fails: Chrome's DevTools HTTP
|
||||
handler answers `/json/version` with a 500 for any `Host` header that is not an
|
||||
IP or `localhost`, and the failure looks like a broken site rather than a broken
|
||||
hostname.
|
||||
|
||||
**Prove it end to end.** This is the only check that says the challenge actually
|
||||
clears from that machine's egress — it fetches a real kagane cover and a real
|
||||
chapter list:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
SMOKE_BROWSER_WS_URL=ws://100.x.y.z:9222 go test -run TestSmokeKagane ./internal/latest
|
||||
```
|
||||
|
||||
A red run means "not clearing from this address right now", which is a live
|
||||
fact to re-check before it is a defect — Cloudflare's scoring moves. Then, from
|
||||
the web UI, open a bookmarked kagane series and confirm the cover renders. Once
|
||||
a cover is stored it is served from Postgres forever after, so the browser being
|
||||
asleep, unreachable, or mid-power-outage costs chapter freshness and nothing
|
||||
visible.
|
||||
|
||||
Finally, take the VPS `free -m` reading again and compare it against the one
|
||||
from the top of this section.
|
||||
|
||||
**Updating the browser** is independent of the API stack and has its own
|
||||
runbook — `REDEPLOY.md` §8.
|
||||
|
||||
---
|
||||
|
||||
## Updating
|
||||
|
||||
Pull new code, then rebuild:
|
||||
@@ -239,6 +384,10 @@ server predates the Postgres migration, the old SQLite volume `bookmarks-data`
|
||||
is still on disk and deliberately undeclared in compose so `down -v` cannot take
|
||||
it; see `REDEPLOY.md` §1 for when to remove it.)
|
||||
|
||||
The browser is a separate unit on a separate machine with its own update
|
||||
command — §7. Nothing above touches it, and it needs no coordination: the API
|
||||
picks up a restarted Chrome's new debugger UUID by itself.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
@@ -248,11 +397,18 @@ it; see `REDEPLOY.md` §1 for when to remove it.)
|
||||
| No cert / TLS error at the domain | `TRAEFIK_ENTRYPOINT` or `TRAEFIK_CERTRESOLVER` name wrong; or DNS not resolving yet. Check `docker logs <traefik>`. |
|
||||
| 404 from Traefik | Service not on the `proxy` network, or `BOOKMARK_API_HOST` mismatch. Confirm `docker network inspect proxy` lists `bookmark-api`. |
|
||||
| `fetch` fails in the userscript, `curl` works | Origin missing from `ALLOWED_ORIGINS`, or mixed content (backend not HTTPS). |
|
||||
| 401 with the right token | Trailing space/newline in `API_TOKEN`; regenerate and restart. |
|
||||
| 401 with the right credential | The script's credential no longer matches the stored hash — most likely a rotation happened and the device was not reinstalled. Reinstall from the web UI. |
|
||||
| 401 after rotation, even right after reinstalling | `TOKEN_KEY` changed between the rotation and the reinstall; credentials are derived from it, so changing it invalidates every credential. Keep it stable. |
|
||||
| Panel button absent | URL didn't match an adapter, or user scripts disabled in Bromite. |
|
||||
| `compose ... config` errors about `API_TOKEN` or `POSTGRES_PASSWORD` | Run compose from the dir with `.env`, or export the vars. Both are required and neither has a fallback. |
|
||||
| `compose ... config` errors about `TOKEN_KEY`, `OWNER_DISCORD_ID` or `POSTGRES_PASSWORD` | Run compose from the dir with `.env`, or export the vars. All three are required and none has a fallback. |
|
||||
| `bookmark-api` restarts in a loop, `password authentication failed for user "bookmarks"` | `POSTGRES_PASSWORD` was changed after first boot; Postgres only applies it to an empty `postgres-data`. Restore the old value, or reset the role (`REDEPLOY.md` troubleshooting). |
|
||||
| `bookmark-api` never logs `listening on :8080` | It is blocked on `postgres` passing `pg_isready`, or a migration failed. `docker compose -f docker-compose.yml -f docker-compose.prod.yml logs postgres`. |
|
||||
| kagane rows never get a `latest_chapter`; log says `browser fetcher disabled` or nothing at all | `BROWSER_WS_URL` unset. Expected before §7 is done. |
|
||||
| kagane polls all fail; log shows a 500 from `/json/version` | `BROWSER_WS_URL` names a MagicDNS hostname (or any name). Chrome's DevTools handler only accepts an IP or `localhost` — use the tailnet IP. |
|
||||
| kagane polls fail with a connection error | Home machine off, off the tailnet, or the unit is down. `tailscale ping <machine>`, then `docker compose ps` in its `chrome/`. Costs freshness only; stored covers keep serving. |
|
||||
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series (up to `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, default 6h). |
|
||||
| `compose` in `chrome/` errors `set BROWSER_BIND_ADDR to this machine's tailnet IP` | No `chrome/.env`, or the variable is empty. Deliberate — it has no default so an unset value cannot publish CDP to the LAN. |
|
||||
| browser container restarts, or is OOM-killed | `docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'`. The 512 MiB cap is sized against a measured 645 MiB untuned peak; a real breach is a Chrome regression worth reading `docker logs` for, not a number to raise reflexively. |
|
||||
|
||||
Backend config reference and endpoint list: see `README.md`.
|
||||
|
||||
@@ -261,20 +417,25 @@ Backend config reference and endpoint list: see `README.md`.
|
||||
## Installing / updating the userscript
|
||||
|
||||
The backend serves the script itself, so Violentmonkey can auto-update it.
|
||||
Complements §4 above — that step points `API_BASE`/`API_TOKEN` at your
|
||||
backend; this one points `@downloadURL`/`@updateURL` at the same place so
|
||||
auto-updates come from it too.
|
||||
Complements §4 above — the `@downloadURL`/`@updateURL` lines point at the
|
||||
credential-bearing path, so auto-updates come from the same place as the
|
||||
install.
|
||||
|
||||
Install once, on the phone (Cromite + Violentmonkey):
|
||||
Install once, on the phone (Cromite + Violentmonkey): sign in to the web UI,
|
||||
open the **Userscripts** panel, and open the install link for the library —
|
||||
the script is served with your credential already inside it. Its
|
||||
`@downloadURL`/`@updateURL` point at the same credential-bearing path for
|
||||
updates:
|
||||
|
||||
```
|
||||
https://bookmark-api.<your-domain>/u/<API_TOKEN>/manga-bookmark.user.js
|
||||
https://bookmark-api.<your-domain>/u/<your credential>/manga-bookmark.user.js
|
||||
```
|
||||
|
||||
Open that URL in Cromite; Violentmonkey offers to install it. The token is in
|
||||
the path because Violentmonkey's update poll sends no `Authorization` header,
|
||||
and the script embeds `API_TOKEN` in plain text — an open URL would leak it. A
|
||||
wrong token answers 404.
|
||||
Violentmonkey offers to install it. The credential is in the path because
|
||||
Violentmonkey's update poll sends no `Authorization` header, and the script
|
||||
embeds the credential in plain text — an open URL would leak it. A wrong
|
||||
credential answers 404. The credential is derived from `TOKEN_KEY` and never
|
||||
appears anywhere but this URL and the rendered script.
|
||||
|
||||
Updating, without a redeploy:
|
||||
|
||||
|
||||
+20
-14
@@ -8,47 +8,53 @@ web
|
||||
|
||||
## Users
|
||||
|
||||
Single user (self-hosted, no accounts, no multi-user planned). Reads manga on **asurascans.com** and **demonicscans.org** primarily via Bromite on mobile, also checks/updates from a desktop browser. The web UI is the cross-device view into progress captured by the userscript while reading.
|
||||
Members of one private Discord guild, each with their own library. Accounts exist and are created by signing in — there is no signup form, no invite code and no approval step: any member of the configured guild becomes a Reader on their first Discord login. The person running the deployment is the owner, seeded at startup, and the only Reader with an administrative capability (revoking another Reader's sessions).
|
||||
|
||||
Reading happens on **asurascans.com**, **demonicscans.org**, **comix.to** and **kagane.to** for manga and **novelfull.com** and **lightnovelworld.net** for novels, primarily via Bromite on mobile, with checks and corrections from a desktop browser. The web UI is the cross-device view into progress the userscripts capture while reading.
|
||||
|
||||
## Product Purpose
|
||||
|
||||
Tracks read-progress ("last chapter read") per manga series across two otherwise-unrelated manga sites that each have their own separate `localStorage`. A Go backend unifies bookmarks into one store; the web UI is a password-gated browser view of that store for reviewing, favouriting, correcting, or removing bookmarks, and jumping back into a series to continue reading. A background poller also refreshes each series' latest-published-chapter so the list can flag "NEW" without the user visiting the site.
|
||||
Tracks read-progress ("last chapter read") per series across sites that each have their own separate `localStorage`. A Go backend unifies bookmarks into one store; the web UI is a Discord-gated browser view of one Reader's own bookmarks, for reviewing, favouriting, correcting, shelving or removing them, and jumping back into a series to continue reading. A background poller refreshes each series' latest-published-chapter so the list can flag "NEW" without the Reader visiting the site.
|
||||
|
||||
## Positioning
|
||||
|
||||
Not a public reading tracker or social app — a private, self-hosted sync layer purpose-built for two specific scraped sites, with no server-side account system (single bearer token + one password-gated session).
|
||||
Not a public reading tracker or social app — a private, self-hosted sync layer for one Discord community, purpose-built for a fixed set of scraped sites. Multi-Reader, not multi-tenant: libraries are isolated, but the deployment belongs to one group and its membership is the whole access model.
|
||||
|
||||
## Operating Context
|
||||
|
||||
- Primary reading device: Bromite (mobile Chromium), where a userscript captures progress automatically.
|
||||
- Primary reading device: Bromite (mobile Chromium), where a userscript captures progress automatically. Each Reader installs their own copy, rendered with their own credential.
|
||||
- Web UI is a secondary surface: checking list state, correcting a wrong chapter number, removing dead bookmarks, jumping to "continue reading."
|
||||
- Manga cover art and titles come from the source sites' `og:image`/`og:title` — real content, not placeholders.
|
||||
- List order is driven by `updated_at`, which moves only on real reading progress (not favouriting, not a newly detected chapter) — a UI constraint the redesign must not break.
|
||||
- Cover art and titles come from the source sites' `og:image`/`og:title` — real content, not placeholders. They are facts about the series, so they are shared between Readers who track it; progress is not.
|
||||
- List order is driven by `updated_at`, which moves only on real reading progress (not favouriting, not a newly detected chapter) — a UI constraint the design must not break.
|
||||
|
||||
## Capabilities and Constraints
|
||||
|
||||
- Two tabs: All / Favourites. Search-filter by title (client-side, `filter.js`).
|
||||
- Card actions: continue (opens source site), toggle favourite, manual chapter override, delete (with confirm).
|
||||
- "Continue reading" horizontal strip for recently-progressed series.
|
||||
- htmx-driven partial updates (card re-render on favourite/chapter/delete), no client-side framework/build step — templates are Go `html/template`, `go:embed`-ed.
|
||||
- Two libraries (manga, novels) with lifecycle tabs: All / Updated / Favourites / Archived / Finished. Search-filter by title (client-side, `filter.js`).
|
||||
- Card actions: continue (opens source site), toggle favourite, manual chapter override, archive, finish, remove — each move out of the list confirm-gated.
|
||||
- "Continue reading" horizontal strip for series with an unread chapter.
|
||||
- A Reader with no bookmarks at all sees a deliberate empty library offering both userscript install links, not an error and not a blank page.
|
||||
- Isolation is the load-bearing invariant: two Readers cannot see or change each other's bookmarks. A series both track is one shared row polled once, with independent progress on each side.
|
||||
- The owner can revoke a specific Reader's sessions; nothing else in the UI differs by Reader.
|
||||
- htmx-driven partial updates, no client-side framework or build step — templates are Go `html/template`, `go:embed`-ed.
|
||||
- Mobile-first is a hard functional constraint (primary device is a phone), not just a starting breakpoint.
|
||||
|
||||
## Brand Commitments
|
||||
|
||||
- Name: **BookmarkManager**.
|
||||
- **Dark-first is binding**: current dark-by-default / light-follows-system-preference behavior must be preserved as a design constraint, not just a starting default, because reading happens at night.
|
||||
- **Dark-first is binding**: dark-by-default / light-follows-system-preference must be preserved as a design constraint, not just a starting default, because reading happens at night.
|
||||
|
||||
## Evidence on Hand
|
||||
|
||||
- Live templates/CSS at `backend/templates/*.html`, `backend/static/style.css` — current implemented UI, functional but not yet treated as an intentional design system.
|
||||
- No logo, screenshots, or marketing copy exist; none should be fabricated.
|
||||
- Live templates/CSS at `backend/internal/web/templates/*.html`, `backend/internal/web/static/style.css`, governed by the Cinder design system (`docs/design-system.md`).
|
||||
- No logo beyond the wordmark, no screenshots, no marketing copy; none should be fabricated.
|
||||
|
||||
## Product Principles
|
||||
|
||||
- Dark-first, night-reading-optimized — never regress to a light-default or high-glare surface.
|
||||
- Mobile is the primary target; desktop is an enhancement, not the design center.
|
||||
- Progress data integrity over visual flourish: `updated_at`/list-ordering behavior is a correctness constraint the UI must respect, not decorate over.
|
||||
- No accounts, no multi-tenant chrome — the whole product is for one reader.
|
||||
- A leak between Readers fails silently and looks like working software — isolation is asserted from both directions, never inferred from counting one Reader's rows.
|
||||
- No roles, no org chrome: the owner's Readers panel is one list with one button (revoke someone's sessions), not an admin console, and otherwise every Reader's view is the same.
|
||||
- Prefer native platform affordances (system dark/light, native touch targets) over custom widgetry — this is a lean self-hosted tool, not a product to demo.
|
||||
|
||||
## Accessibility & Inclusion
|
||||
|
||||
@@ -15,8 +15,21 @@ Two parts:
|
||||
```
|
||||
Bromite userscript (isolated world, Shadow DOM UI, localStorage cache)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> Postgres (volume)
|
||||
|
|
||||
| CDP over the tailnet
|
||||
v
|
||||
headless Chrome, on-demand,
|
||||
on a separate machine
|
||||
(chrome/, ADR-0006)
|
||||
```
|
||||
|
||||
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
|
||||
fingerprint clears, so the poller reaches those two through a real Chrome over
|
||||
CDP. That browser is **not** part of the API stack: it is its own compose unit
|
||||
on a second machine, spawned on the first connection and reaped when idle. The
|
||||
API needs it only to discover new chapters and to fetch a kagane cover once —
|
||||
covers are stored, so the library renders in full with the browser switched off.
|
||||
|
||||
---
|
||||
|
||||
## 1. Backend
|
||||
@@ -25,21 +38,46 @@ Bromite userscript (isolated world, Shadow DOM UI, localStorage cache)
|
||||
|
||||
| Var | Default | Notes |
|
||||
|-----|---------|-------|
|
||||
| `API_TOKEN` | *(required)* | Bearer token shared with the userscript. |
|
||||
| `TOKEN_KEY` | *(required)* | Secret every Reader's userscript credential is derived from (issue #24); only SHA-256 hashes of credentials are stored. |
|
||||
| `OWNER_DISCORD_ID` | *(required)* | Discord user ID of the owner: seeded as the first Reader, owns every pre-registration bookmark, and is the only Reader who can revoke another's sessions. |
|
||||
| `ALLOWED_ORIGINS` | Asura + Demonic + Comix + Kagane origins | Comma-separated CORS allowlist. |
|
||||
| `DATABASE_URL` | *(required)* | Postgres connection URL, e.g. `postgres://bookmarks:…@postgres:5432/bookmarks?sslmode=disable`. Compose builds it from `POSTGRES_PASSWORD`. |
|
||||
| `PORT` | `8080` | Plain HTTP; TLS terminated by the proxy. |
|
||||
| `BROWSER_WS_URL` | `ws://172.28.0.10:9222` | Headless-shell CDP endpoint used to poll Kagane past its JS challenge. Must be an IP or `localhost` — Chrome's DevTools handler 500s any other Host header. |
|
||||
| `BROWSER_WS_URL` | empty | CDP endpoint of the remote browser (`ws://<tailnet IP>:9222`), used to poll Kagane/Novelfull past their JS challenge and to fetch uncached Kagane covers. Must be an IP or `localhost` — Chrome's DevTools handler 500s any other Host header, MagicDNS names included. Unset disables both; stored covers still serve. |
|
||||
| `DISCORD_CLIENT_ID` | *(required)* | Discord application credentials for the browser sign-in (ADR-0002). |
|
||||
| `DISCORD_CLIENT_SECRET` | *(required)* | As above. Never logged, never echoed in an error. |
|
||||
| `DISCORD_GUILD_ID` | *(required)* | The one guild whose membership gates sign-in, checked at login only. Membership *is* registration: any member becomes a Reader on first login. |
|
||||
| `DISCORD_REDIRECT_URI` | *(required)* | Exact callback URL; Discord matches it verbatim against the registered redirect. |
|
||||
| `DISCORD_REQUIRED_ROLE` | empty | Role snowflake a member must additionally hold. Empty means guild membership alone suffices. |
|
||||
| `DISCORD_API_BASE` | `https://discord.com/api/v10` | Test seam — tests point it at a local stub so the real token exchange runs. |
|
||||
| `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. |
|
||||
| `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. |
|
||||
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. |
|
||||
| `LATEST_CHAPTER_POLL_COOLDOWN` | `1h` | Rest between checks of one plain-TLS series; floor `15m`. |
|
||||
| `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN` | `6h` | Rest between checks of one browser-backed series; floor `15m`. |
|
||||
| `LATEST_CHAPTER_POLL_INTERVAL` | `10m` | How often the poller wakes. Cannot shorten either cooldown. |
|
||||
| `LATEST_CHAPTER_POLL_BATCH` | `14` | Series per wake. Keep `BATCH × STAGGER` under `INTERVAL`. |
|
||||
| `LATEST_CHAPTER_POLL_STAGGER` | `20s` | Delay between fetches in a batch — this is the outbound request rate. |
|
||||
|
||||
Compose reads a few more from the same `.env` that the backend never sees:
|
||||
`POSTGRES_PASSWORD` (required — `DATABASE_URL` is built from it, and Postgres
|
||||
only applies it while `postgres-data` is empty), `BOOKMARK_API_HOST` and
|
||||
`BOOKMARK_WEB_HOST` (required by the prod override), and the optional
|
||||
`PROXY_NETWORK` / `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER`. The browser
|
||||
unit has its own `chrome/.env` on its own machine — `BROWSER_BIND_ADDR`
|
||||
(required, the tailnet IP the CDP port is published on) and the optional
|
||||
`BROWSER_TZ`. Full commentary is in `.env.example` and `chrome/.env.example`;
|
||||
deployment order is `DEPLOY.md`.
|
||||
|
||||
### Endpoints
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/bookmarks` | Bearer | All bookmarks (single-user). |
|
||||
| `GET` | `/bookmarks` | Bearer | All bookmarks of the acting Reader. |
|
||||
| `PUT` | `/bookmarks/{key}` | Bearer | Upsert one series; returns the row as stored. |
|
||||
| `DELETE` | `/bookmarks/{key}` | Bearer | Remove one. |
|
||||
| `GET` | `/healthz` | none | `200 ok`. |
|
||||
| `GET` | `/u/{token}/manga-bookmark.user.js` | token in path | Serves the userscript with an mtime-derived `@version`. |
|
||||
| `GET` | `/u/{token}/manga-bookmark.user.js` | credential in path | Serves the userscript with the requesting Reader's credential substituted in and an mtime-derived `@version`. |
|
||||
|
||||
`key` is `<site>:<series_id>` — e.g. `asura:trash-of-the-counts-family-f886a8af`,
|
||||
`demonic:Infinite-Level-Up-in-Murim`, `comix:12345`, or
|
||||
@@ -69,16 +107,36 @@ and nothing reaches the network beyond the local Docker daemon.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# edit .env: set API_TOKEN (openssl rand -hex 32) and
|
||||
# edit .env: set TOKEN_KEY (openssl rand -hex 32) and
|
||||
# POSTGRES_PASSWORD (openssl rand -hex 24)
|
||||
|
||||
docker compose up -d --build # binds 127.0.0.1:8080
|
||||
```
|
||||
|
||||
That brings up two services — the API and Postgres. The browser is deliberately
|
||||
not one of them; without `BROWSER_WS_URL` the poller logs and skips kagane and
|
||||
novelfull, and everything else works. To run one locally, publish it on the
|
||||
Docker bridge gateway so the API container can name it by IP:
|
||||
|
||||
```bash
|
||||
cd chrome
|
||||
echo 'BROWSER_BIND_ADDR=172.17.0.1' > .env
|
||||
docker compose up -d --build
|
||||
# then in the repo's own .env: BROWSER_WS_URL=ws://172.17.0.1:9222
|
||||
```
|
||||
|
||||
Bind it to `127.0.0.1` instead if you only want to drive it from the host, e.g.
|
||||
`SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeKagane ./internal/latest`.
|
||||
In production that address is the home machine's tailnet IP and nothing else —
|
||||
see `DEPLOY.md` §7 and ADR-0006.
|
||||
|
||||
Smoke test:
|
||||
|
||||
```bash
|
||||
TOKEN=$(grep '^API_TOKEN=' .env | cut -d= -f2)
|
||||
# The credential is per Reader and derived, so there is no token in .env to
|
||||
# grep. Take yours from the Userscripts panel's install link after signing in,
|
||||
# or read it out of an installed script's API_TOKEN constant.
|
||||
TOKEN=<your Reader credential>
|
||||
curl -s localhost:8080/healthz # ok
|
||||
curl -s localhost:8080/bookmarks # 401
|
||||
curl -s -H "Authorization: Bearer $TOKEN" localhost:8080/bookmarks # []
|
||||
@@ -113,17 +171,18 @@ CORS headers.
|
||||
|
||||
## 2. Userscript
|
||||
|
||||
### Configure
|
||||
### Install
|
||||
|
||||
Edit the config block at the top of `userscript/manga-bookmark.user.js`:
|
||||
Sign in to the web UI and open the **Userscripts** panel: it offers one
|
||||
install link per library. Each link serves a script rendered with your own
|
||||
credential already inside it — you never see, type or copy a credential. The
|
||||
served script carries `@downloadURL`/`@updateURL` pointing at its
|
||||
credential-bearing path, so Violentmonkey keeps auto-updating it.
|
||||
|
||||
```js
|
||||
const API_BASE = "https://bookmark-api.<domain>"; // no trailing slash
|
||||
const API_TOKEN = "<same token as backend>";
|
||||
```
|
||||
|
||||
The token lives in the userscript's **isolated world** — the manga sites' own
|
||||
JS cannot read it.
|
||||
The bindmounted files carry `__API_TOKEN__` placeholders; the backend
|
||||
substitutes the requesting Reader's credential at serve time, so no real
|
||||
credential is ever committed. Rotating the credential (same panel) invalidates
|
||||
every installed copy immediately — reinstall on all devices.
|
||||
|
||||
### Install on Bromite (mobile)
|
||||
|
||||
@@ -131,8 +190,8 @@ Bromite runs Chromium's native userscript engine (no Tampermonkey needed):
|
||||
|
||||
1. Bromite → **Settings → User scripts** → enable user scripts (allow the
|
||||
permission prompt).
|
||||
2. Save the configured `manga-bookmark.user.js` to the device (or open its raw
|
||||
URL). Bromite detects the `.user.js` and offers to install it.
|
||||
2. Open the install link from the web UI — Bromite detects the `.user.js` and
|
||||
offers to install it.
|
||||
3. Confirm the install; the `@match` list covers both sites.
|
||||
4. Open a series on either site — a 📑 button appears bottom-right.
|
||||
|
||||
|
||||
+87
-21
@@ -9,16 +9,16 @@ Whole thing is ~5 minutes, most of it waiting on `docker build`. Order matters:
|
||||
**back up before you pull.** A backup taken after a bad migration is a backup of
|
||||
the damage.
|
||||
|
||||
Paths below assume the checkout is at `/opt/bookmarkmanager`; substitute your own. The
|
||||
one absolute rule about paths: **backups live in `../bookmarkmanager-backups/`**, a
|
||||
sibling of the project directory (`/opt/bookmarkmanager-backups`), never inside it. It
|
||||
Paths below assume the checkout is at `~/mangaBookmark`, which is where it lives
|
||||
on this deployment; substitute your own. The one absolute rule about paths:
|
||||
**backups live in a `-backups` sibling of the checkout**, never inside it. It
|
||||
sits outside the repo so `git pull`, `git clean -fd` and a bad `rm -rf` inside
|
||||
the checkout cannot take the backups with them.
|
||||
|
||||
```
|
||||
/opt/
|
||||
├── bookmarkmanager/ <- the checkout (this repo)
|
||||
└── bookmarkmanager-backups/ <- bookmarks-YYYYmmdd-HHMMSS.dump
|
||||
~/
|
||||
├── mangaBookmark/ <- the checkout (this repo)
|
||||
└── mangaBookmark-backups/ <- bookmarks-YYYYmmdd-HHMMSS.dump
|
||||
```
|
||||
|
||||
---
|
||||
@@ -26,7 +26,7 @@ the checkout cannot take the backups with them.
|
||||
## 0. Preflight
|
||||
|
||||
```bash
|
||||
cd /opt/bookmarkmanager
|
||||
cd ~/mangaBookmark
|
||||
|
||||
# Both -f flags, every time. The prod override is not standalone.
|
||||
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
|
||||
@@ -44,9 +44,9 @@ dirty tree fails halfway and leaves you in a worse spot than either.
|
||||
Create the backup directory once, and make sure it is a sibling, not a child:
|
||||
|
||||
```bash
|
||||
mkdir -p ../bookmarkmanager-backups
|
||||
BACKUP_DIR="$(cd .. && pwd)/bookmarkmanager-backups" # absolute — Docker needs it
|
||||
echo "$BACKUP_DIR" # -> /opt/bookmarkmanager-backups
|
||||
BACKUP_DIR="$(cd .. && pwd)/$(basename "$PWD")-backups" # absolute — Docker needs it
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
echo "$BACKUP_DIR" # -> /home/sulthan/mangaBookmark-backups
|
||||
```
|
||||
|
||||
---
|
||||
@@ -59,7 +59,7 @@ network can reach it — so every command below goes in through the container:
|
||||
|
||||
```bash
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c '\dt'
|
||||
# -> bookmarks, schema_migrations, series
|
||||
# -> bookmarks, covers, readers, schema_migrations, series, sessions
|
||||
```
|
||||
|
||||
Inside the container that connects over the local socket as the `bookmarks`
|
||||
@@ -99,8 +99,11 @@ you will act as though you have one:
|
||||
docker run --rm -v "$BACKUP_DIR":/backup postgres:17-alpine \
|
||||
pg_restore --list "/backup/bookmarks-$STAMP.dump" | grep 'TABLE DATA'
|
||||
# -> 1234; 0 0 TABLE DATA public bookmarks bookmarks
|
||||
# -> 1235; 0 0 TABLE DATA public schema_migrations bookmarks
|
||||
# -> 1236; 0 0 TABLE DATA public series series
|
||||
# -> 1235; 0 0 TABLE DATA public covers bookmarks
|
||||
# -> 1236; 0 0 TABLE DATA public readers bookmarks
|
||||
# -> 1237; 0 0 TABLE DATA public schema_migrations bookmarks
|
||||
# -> 1238; 0 0 TABLE DATA public series bookmarks
|
||||
# -> 1239; 0 0 TABLE DATA public sessions bookmarks
|
||||
|
||||
# 2. Sanity-check the live row count you just captured.
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks \
|
||||
@@ -129,8 +132,11 @@ container stopped the shutdown checkpoint has already flushed everything and a
|
||||
plain archive of the volume is consistent.
|
||||
|
||||
```bash
|
||||
VOL=$(docker volume ls --filter name=postgres-data -q | head -1)
|
||||
echo "$VOL" # -> bookmarkmanager_postgres-data
|
||||
# Derived exactly, not with a `--filter name=` substring match plus `head -1`:
|
||||
# that quietly picks the first of however many volumes happen to contain the
|
||||
# string, and archiving the wrong data directory is not a visible failure.
|
||||
VOL="$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_postgres-data"
|
||||
docker volume inspect "$VOL" >/dev/null && echo "$VOL" # -> mangabookmark_postgres-data
|
||||
|
||||
$COMPOSE stop
|
||||
docker run --rm -v "$VOL":/from:ro -v "$BACKUP_DIR":/to alpine \
|
||||
@@ -160,11 +166,13 @@ ls -1t "$BACKUP_DIR"/bookmarks-*.dump | tail -n +31 | xargs -r rm -v
|
||||
`bookmarks-data` is the **pre-migration SQLite volume**. It is deliberately not
|
||||
declared in `docker-compose.yml` any more, which is what keeps `docker compose
|
||||
down -v` from taking it with the rest of the stack. It is not the live database
|
||||
and nothing reads it. Once the Postgres data has been trusted for a while,
|
||||
remove it by hand — nothing else will:
|
||||
and nothing reads it — the one-way move out of it is `CUTOVER.md`. Once the
|
||||
Postgres data has been trusted for a while, remove it by hand — nothing else will.
|
||||
Its full name is `<compose project>_bookmarks-data`, and the project name is the
|
||||
lowercased directory name of the checkout:
|
||||
|
||||
```bash
|
||||
docker volume rm bookmarkmanager_bookmarks-data
|
||||
docker volume rm "$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_bookmarks-data"
|
||||
```
|
||||
|
||||
---
|
||||
@@ -224,7 +232,10 @@ Same four API checks as `DEPLOY.md` §3, plus the web UI. Set the host names onc
|
||||
```bash
|
||||
API=https://bookmark-api.violetcrown.my.id
|
||||
WEB=https://bookmark.violetcrown.my.id
|
||||
TOKEN=$(grep -E '^API_TOKEN=' .env | cut -d= -f2)
|
||||
# Your own Reader credential - derived, never stored in .env. Take it from the
|
||||
# Userscripts panel's install link after signing in, or from an installed
|
||||
# script's API_TOKEN constant.
|
||||
TOKEN=<your Reader credential>
|
||||
|
||||
curl -s $API/healthz # -> ok
|
||||
curl -s -o /dev/null -w '%{http_code}\n' $API/bookmarks # -> 401
|
||||
@@ -377,6 +388,55 @@ panel works on the phone.
|
||||
|
||||
---
|
||||
|
||||
## 8. The browser unit (separate machine, separate cadence)
|
||||
|
||||
Everything above is the API stack on the VPS. The headless browser is its own
|
||||
compose unit on the home machine (ADR-0006, `DEPLOY.md` §7) and is redeployed
|
||||
on its own schedule — it holds no data you can lose, so there is nothing to
|
||||
back up and no ordering constraint against the API.
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark/chrome
|
||||
git pull --ff-only
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Then confirm it answers, and that a stopped-and-restarted Chrome is invisible
|
||||
to the API:
|
||||
|
||||
```bash
|
||||
curl -s -m 15 http://$(tailscale ip -4):9222/json/version | head -c 120
|
||||
# -> {"Browser":"Chrome/1xx...","webSocketDebuggerUrl":"ws://...<new uuid>"}
|
||||
```
|
||||
|
||||
The first call takes a few seconds: Chrome is not running until something
|
||||
connects, and it is reaped again after five idle minutes. The debugger UUID
|
||||
changes on every start and the API does not care — chromedp re-runs
|
||||
`/json/version` discovery per fetch, which is exactly why `chromedp.NoModifyURL`
|
||||
must never be added to `browser.go`.
|
||||
|
||||
**Rebuild is the Chrome upgrade path.** The image installs
|
||||
`google-chrome-stable` unpinned on purpose: a stale browser is what Cloudflare
|
||||
turns away, and the pinned Chrome 124 in `zenika/alpine-chrome` is the worked
|
||||
example. The `chrome-profile` volume survives `--build`, so clearance cookies
|
||||
are reused rather than re-solved.
|
||||
|
||||
Two things worth a glance after several days, both from the acceptance criteria
|
||||
of the move:
|
||||
|
||||
```bash
|
||||
docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'
|
||||
# -> 0 false
|
||||
free -m # the Gitea runner should still have its headroom
|
||||
```
|
||||
|
||||
Nothing here needs doing during an API redeploy. The API stack does not
|
||||
`depends_on` the browser, and an unreachable one degrades exactly as an unset
|
||||
`BROWSER_WS_URL`: plain-TLS libraries unaffected, kagane and novelfull logged
|
||||
and skipped, stored covers still served.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Cause / fix |
|
||||
@@ -385,7 +445,7 @@ panel works on the phone.
|
||||
| UI looks like plain Georgia / system sans | `static/fonts/` missing from the image, or the browser cached an old `style.css`. `/static/*` is served `max-age=3600`, so hard-reload or wait an hour. |
|
||||
| CSS or template change did not appear | You restarted without `--build`. Assets are `//go:embed`ed. |
|
||||
| Font answers `application/octet-stream` | Old binary — the `.woff2` MIME registration is in `web.go`. Rebuild. |
|
||||
| Everyone logged out of the web UI | `API_TOKEN` or `WEB_PASSWORD` changed; sessions are derived from both. Expected, just log in again. |
|
||||
| Everyone logged out of the web UI | The `sessions` table was wiped; sessions are database rows, not signed cookies. Expected after a deliberate revoke. |
|
||||
| `compose` errors about `BOOKMARK_WEB_HOST` | Run from the directory holding `.env`. Both host vars are required even when the web UI is unused. |
|
||||
| Userscript did not update on the phone | Violentmonkey polls on its own schedule; force a check. `@version` comes from the file's mtime, so confirm the pull actually touched it. |
|
||||
| `bookmark-api` crash-loops, log says `password authentication failed for user "bookmarks"` | `POSTGRES_PASSWORD` in `.env` no longer matches the one burned into `postgres-data` at first init — Postgres reads that variable only when initialising an empty volume. Put the old value back, or reset the role: `$COMPOSE exec postgres psql -U bookmarks -d bookmarks -c '\password bookmarks'` (prompts, so nothing lands in shell history) and then match `.env` to it. |
|
||||
@@ -393,6 +453,12 @@ panel works on the phone.
|
||||
| `postgres` never leaves `starting`; `bookmark-api` never starts either | The healthcheck (`pg_isready`) is failing and `bookmark-api` waits on it. `$COMPOSE logs postgres` — usually `postgres-data` was initialised by a different major version ("database files are incompatible with server"), or the disk is full. |
|
||||
| `pg_restore`: `cannot drop … other objects depend on it` / `being accessed by other users` | Live connections block `--clean`. `$COMPOSE stop bookmark-api` first (§6). If they persist: `$COMPOSE exec -T postgres psql -U bookmarks -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='bookmarks' and pid <> pg_backend_pid()"`. |
|
||||
| Dump is 0 bytes, or `pg_restore`: `did not find magic string in file header` | You ran `exec` without `-T`. The allocated TTY rewrites newlines in the binary stream and corrupts the archive in flight (§1). |
|
||||
| `git pull`: `could not read Username for 'https://…'` | The checkout's remote is the HTTPS clone URL and the server has no credential helper, so the pull prompts into a closed stdin. Switch it to SSH once — `git remote set-url origin ssh://git@gitea.violetcrown.my.id:2222/sulthan/mangaBookmark.git`. Gitea's SSH listens on **2222**, not 22; port 22 is the host's own sshd and answers `Permission denied (publickey)` no matter which key is registered. |
|
||||
| kagane rows stopped updating after a redeploy | Check `BROWSER_WS_URL` survived the `.env` edit and still names the home machine's tailnet **IP**. A hostname 500s at `/json/version`; an empty value disables the browser silently. Plain-TLS sites keep working either way, which is why this is easy to miss. |
|
||||
| kagane covers went blank in the web UI | They should not — covers are rows in `covers`, not an in-process cache. `$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c 'select count(*) from covers'`. Zero after a restore means the dump predates the covers table; they refill on the next poll of each series. |
|
||||
| Browser unit will not start: `set BROWSER_BIND_ADDR to this machine's tailnet IP` | `chrome/.env` is missing or the variable is empty. It has no default on purpose — an unset value must fail the deploy rather than publish an unauthenticated CDP port to the LAN. |
|
||||
| `bookmark-browser` shows `OOMKilled true` | The cap did its job. Read `docker logs bookmark-browser` before raising it — the sizing and what the cap protects are in ADR-0006. |
|
||||
|
||||
Full first-time setup: `DEPLOY.md`. Config reference and endpoints: `README.md`.
|
||||
Full first-time setup: `DEPLOY.md`. The one-off SQLite→Postgres move:
|
||||
`CUTOVER.md`. Config reference and endpoints: `README.md`.
|
||||
UI conventions: `docs/design-system.md`.
|
||||
|
||||
+71
-19
@@ -21,24 +21,42 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
|
||||
container per test binary (`TestMain` -> `pgtest.Main`) and hands each test
|
||||
its own database (`pgtest.URL(t)`). A package whose tests touch the store
|
||||
must have that `TestMain`.
|
||||
- **Single-user store, two tables.** `series` keyed `(site, series_id)`
|
||||
- **Reader-owned store, four tables.** `readers` is keyed by Discord user ID
|
||||
and carries the SHA-256 of the Reader's userscript credential plus a
|
||||
`token_epoch` (issue #24). Credentials are derived, never stored: `token.Token(TOKEN_KEY, discord_id, epoch)` (HMAC, `internal/token`), and only its SHA-256 sits in `readers.token_sha256`, so install URLs can be rebuilt after any restart while a database leak yields nothing but hashes. The seed creates the **owner** row at startup; its epoch-0 hash is refreshed on every start **only while the row has never been rotated**, so a restart can never resurrect a rotated-away credential. Every other row is created by that Reader's own first login (`Store.EnsureReader`, idempotent on `discord_id`, and it never rewrites an existing row's hash). Rotation is `Store.RotateToken` (epoch bump + hash rewrite in one transaction), driven by the web UI.
|
||||
`series` keyed `(site, series_id)`
|
||||
(`asura`|`demonic`|`comix`|`kagane`|`novelfull`|`lightnovelworld`) owns the
|
||||
shared facts — title, cover, canonical URL, `kind` (`manga`|`novel`),
|
||||
Latest Chapter, `latest_checked_at` — and `bookmarks` holds only what
|
||||
differs between readers: progress, favourite, lifecycle bucket,
|
||||
`updated_at`. Sync **last-write-wins**; the wire format stays flat
|
||||
(ADR-0004). `Store.Upsert` decomposes one flat body across both tables and
|
||||
enforces the ownership rule: client `title`/`series_url`/`cover` are
|
||||
written only when the series row is new (ADR-0003).
|
||||
`updated_at`. A bookmark is keyed `(reader_id, site, series_id)` — no
|
||||
surrogate id; the wire `key` is derived as `site:series_id` on read — and
|
||||
every store read/write is scoped to the reader it names. Auth resolves the
|
||||
acting Reader from the presented credential (`httpmw.Auth`) and nothing
|
||||
else — there is no unauthenticated-by-Reader route and no global token; the
|
||||
reader id travels in the request context. Sync **last-write-wins**; the wire format
|
||||
stays flat (ADR-0004). `Store.Upsert` decomposes one flat body across two
|
||||
tables and enforces the ownership rule: client `title`/`series_url`/`cover`
|
||||
are written only when the series row is new (ADR-0003).
|
||||
- **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth).
|
||||
- **Web UI:** same binary serve password-gated browser UI on second
|
||||
- **Web UI:** same binary serve the browser UI on a second
|
||||
hostname — `GET /` (list, or login page when no session),
|
||||
`POST /login`, `POST /logout`, `GET /static/*`, htmx fragment endpoints
|
||||
`GET /auth/discord` + `GET /auth/discord/callback` (Discord OAuth,
|
||||
ADR-0002), `POST /logout`, `GET /static/*`, htmx fragment endpoints
|
||||
under `/ui/*`. Templates + assets `go:embed`-ed under
|
||||
`backend/internal/web/`, so `backend/Dockerfile` must copy the whole
|
||||
`internal/` tree, not just `*.go`. Sessions stateless
|
||||
HMAC cookies keyed off `API_TOKEN`; `WEB_PASSWORD` gates them, and when empty,
|
||||
web routes not registered at all. UI mutations read-modify-write
|
||||
`internal/` tree, not just `*.go`. Sessions are rows in the `sessions`
|
||||
table: the cookie carries only an opaque id, looked up (and expiry-
|
||||
checked) on every request, and deleting the row revokes the session.
|
||||
Guild membership *is* registration (issue #27): `discordCallback` gates on
|
||||
membership (and `DISCORD_REQUIRED_ROLE` when set) and then calls
|
||||
`Store.EnsureReader`, so a refusal creates nothing and a returning Reader
|
||||
reuses their row. The owner is the only Reader with administrative reach:
|
||||
`POST /readers/{id}/revoke` (404 for anyone else) drops that Reader's
|
||||
sessions, and the `readers` panel renders only on the owner's page.
|
||||
A Reader with no bookmarks at all sees `listView.Fresh`, whose empty state
|
||||
offers both install links instead of describing a filter.
|
||||
UI mutations read-modify-write
|
||||
through `Store.Get` + `Store.Upsert` so `updated_at` rule stays one
|
||||
place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`.
|
||||
**Design-tool caveat:** templates link `/static/style.css` root-absolutely
|
||||
@@ -91,16 +109,50 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
|
||||
`excluded.*` is post-evaluation row and default applied there would
|
||||
wipe bucket on every PUT from client that predates column. See
|
||||
`docs/superpowers/specs/2026-07-27-status-buckets-design.md`.
|
||||
- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list),
|
||||
- **Config via env:** `TOKEN_KEY` (derives every Reader's userscript credential;
|
||||
required), `OWNER_DISCORD_ID` (seeds the owner Reader — the administrator and
|
||||
the owner of every pre-registration bookmark; required),
|
||||
`ALLOWED_ORIGINS` (comma list),
|
||||
`DATABASE_URL` (Postgres connection URL, required — no default),
|
||||
`PORT` (default `8080`), `WEB_PASSWORD`
|
||||
(gates browser UI; unset disable it),
|
||||
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER`
|
||||
(background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`).
|
||||
`PORT` (default `8080`), `DISCORD_CLIENT_ID`/`_CLIENT_SECRET`/`_GUILD_ID`/
|
||||
`_REDIRECT_URI` (required; Discord OAuth for the browser UI),
|
||||
`DISCORD_REQUIRED_ROLE` (optional role gate, empty by default),
|
||||
`DISCORD_API_BASE` (default `https://discord.com/api/v10`),
|
||||
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/
|
||||
`_BATCH`/`_STAGGER` (background latest-chapter poller; defaults on,
|
||||
`1h` plain-TLS cooldown, `6h` browser cooldown, `10m`/`14`/`20s`; both
|
||||
cooldowns have a `15m` floor).
|
||||
`USERSCRIPT_PATH` and `NOVEL_USERSCRIPT_PATH` (files served at
|
||||
`/u/{token}/manga-bookmark.user.js` and `/u/{token}/novel-bookmark.user.js`,
|
||||
defaults `/userscript/manga-bookmark.user.js` and
|
||||
`/userscript/novel-bookmark.user.js`, both supplied by bindmount).
|
||||
`BROWSER_WS_URL` (headless-shell CDP endpoint for kagane and novelfull;
|
||||
unset disables browser polling and leaves those sites to the userscript
|
||||
alone).
|
||||
`/userscript/novel-bookmark.user.js`, both supplied by bindmount; the
|
||||
`__API_TOKEN__` placeholder inside them is substituted with the requesting
|
||||
Reader's credential at serve time).
|
||||
`BROWSER_WS_URL` (CDP endpoint of the browser, which runs on a **separate
|
||||
machine** and is reached over the tailnet — ADR-0006, `chrome/docker-compose.yml`.
|
||||
Used by the poller for kagane and novelfull *and* by the web UI's kagane
|
||||
cover proxy; unset — the default — disables browser polling and serves 404
|
||||
for covers not already stored, leaving those sites to the userscript alone.
|
||||
Must be a tailnet IP, never a hostname: Chrome's DevTools handler 500s
|
||||
`/json/version` for any Host that isn't an IP or `localhost`).
|
||||
- **kagane covers are proxied, not hot-linked:** kagane serves cover images
|
||||
behind the same challenge as its pages and with
|
||||
`cross-origin-resource-policy: same-origin`, so no `<img>` on the web UI's
|
||||
origin can load one — not even from a browser holding the clearance cookie
|
||||
`og:image` to `/img/kagane/{id}`. `internal/web/cover.go` reads the persistent
|
||||
`covers` table first, then fetches a miss through `latest.BrowserFetcher.Image`.
|
||||
The templates render `.CoverURL`, never `.Cover`. The id is matched against a
|
||||
UUID regex before it reaches the browser: the stored value is client-supplied,
|
||||
so an unchecked one is an SSRF primitive pointed at the deployment's own
|
||||
network.
|
||||
- **Web UI also owns:** session-gated `GET /install/{manga,novel}-bookmark.user.js`
|
||||
(renders the bindmounted script with the acting Reader's derived credential
|
||||
substituted in — the credential never appears in page markup, the address
|
||||
bar, or a redirect; `?download=1` adds `Content-Disposition: attachment` for
|
||||
mobile Violentmonkey, which ignores a `.user.js` navigation) and
|
||||
`POST /rotate-token` (atomic epoch bump + hash
|
||||
rewrite; invalidates every installed copy, so the panel warns to reinstall
|
||||
on all devices).
|
||||
Owner-only `POST /readers/{id}/revoke` (drops one Reader's session rows and
|
||||
re-renders the `readers` panel; 404 for any non-owner) is the only route that
|
||||
reaches across Readers.
|
||||
|
||||
+69
-25
@@ -14,18 +14,32 @@ import (
|
||||
|
||||
"bookmarkmanager/backend/internal/pgtest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
const testToken = "s3cret-token"
|
||||
// testTokenKey derives every test Reader's credential; it must match the key
|
||||
// newTestStoreURL seeds the owner with, or derived credentials authenticate
|
||||
// nothing.
|
||||
const testTokenKey = "test-token-key"
|
||||
|
||||
// testDiscordID is the owner row's discord_id (newTestStoreURL); the derived
|
||||
// credential is a function of it.
|
||||
const testDiscordID = "test-owner"
|
||||
|
||||
func testConfig() Config {
|
||||
return Config{
|
||||
Token: testToken,
|
||||
TokenKey: testTokenKey,
|
||||
AllowedOrigins: []string{"https://asuracomic.net", "https://demonicscans.org"},
|
||||
Port: "8080",
|
||||
}
|
||||
}
|
||||
|
||||
// ownerCredential is the owner's epoch-0 derived credential: the string the
|
||||
// install links carry and the userscript routes authenticate.
|
||||
func ownerCredential() string {
|
||||
return token.Token([]byte(testTokenKey), testDiscordID, 0)
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
|
||||
func newTestServer(t *testing.T) http.Handler {
|
||||
@@ -35,16 +49,29 @@ func newTestServer(t *testing.T) http.Handler {
|
||||
|
||||
func newTestStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
s, err := store.Open(pgtest.URL(t))
|
||||
s, _ := newTestStoreURL(t)
|
||||
return s
|
||||
}
|
||||
|
||||
// newTestStoreURL is newTestStore plus the database URL, for tests that need
|
||||
// to reach the same database directly.
|
||||
func newTestStoreURL(t *testing.T) (*store.Store, string) {
|
||||
t.Helper()
|
||||
url := pgtest.URL(t)
|
||||
s, err := store.Open(url, store.Owner{
|
||||
DiscordID: testDiscordID, TokenHash: token.Hash(ownerCredential()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("store.Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s
|
||||
return s, url
|
||||
}
|
||||
|
||||
// auth authenticates a request as the owner Reader, whose derived credential
|
||||
// is the only thing the API accepts.
|
||||
func auth(req *http.Request) *http.Request {
|
||||
req.Header.Set("Authorization", "Bearer "+testToken)
|
||||
req.Header.Set("Authorization", "Bearer "+ownerCredential())
|
||||
return req
|
||||
}
|
||||
|
||||
@@ -58,7 +85,7 @@ func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key,
|
||||
Site: site,
|
||||
SeriesID: seriesID,
|
||||
@@ -105,7 +132,7 @@ func TestAuthRequired(t *testing.T) {
|
||||
}{
|
||||
{"no header", ""},
|
||||
{"bad token", "Bearer wrong"},
|
||||
{"not bearer", "Basic " + testToken},
|
||||
{"not bearer", "Basic " + ownerCredential()},
|
||||
{"empty bearer", "Bearer "},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
@@ -535,16 +562,29 @@ func TestLatestChapterNullable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigWebPassword(t *testing.T) {
|
||||
t.Setenv("API_TOKEN", "token-abc")
|
||||
t.Setenv("WEB_PASSWORD", "hunter2")
|
||||
if got := loadConfig().WebPassword; got != "hunter2" {
|
||||
t.Fatalf("WebPassword = %q, want hunter2", got)
|
||||
func TestLoadConfigDiscord(t *testing.T) {
|
||||
t.Setenv("DISCORD_CLIENT_ID", "client-1")
|
||||
t.Setenv("DISCORD_CLIENT_SECRET", "client-secret-1")
|
||||
t.Setenv("DISCORD_GUILD_ID", "guild-1")
|
||||
t.Setenv("DISCORD_REQUIRED_ROLE", "role-9")
|
||||
t.Setenv("DISCORD_REDIRECT_URI", "https://bm.example.com/auth/discord/callback")
|
||||
t.Setenv("DISCORD_API_BASE", "https://stub.example/api")
|
||||
if got := loadConfig().Discord; got.ClientID != "client-1" || got.ClientSecret != "client-secret-1" ||
|
||||
got.GuildID != "guild-1" || got.RequiredRole != "role-9" ||
|
||||
got.RedirectURI != "https://bm.example.com/auth/discord/callback" ||
|
||||
got.APIBase != "https://stub.example/api" {
|
||||
t.Fatalf("Discord config = %+v, want every field set", got)
|
||||
}
|
||||
|
||||
t.Setenv("WEB_PASSWORD", "")
|
||||
if got := loadConfig().WebPassword; got != "" {
|
||||
t.Fatalf("WebPassword = %q with the variable unset, want empty", got)
|
||||
// API base falls back to the Discord default; the role is optional.
|
||||
t.Setenv("DISCORD_REQUIRED_ROLE", "")
|
||||
t.Setenv("DISCORD_API_BASE", "")
|
||||
got := loadConfig().Discord
|
||||
if got.RequiredRole != "" {
|
||||
t.Fatalf("RequiredRole = %q, want empty by default", got.RequiredRole)
|
||||
}
|
||||
if got.APIBase != "https://discord.com/api/v10" {
|
||||
t.Fatalf("APIBase = %q, want the Discord default", got.APIBase)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -562,7 +602,7 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
|
||||
"series_url":"https://asurascans.com/comics/x",
|
||||
"last_chapter":"Chapter 5","last_chapter_num":5}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body))
|
||||
req.Header.Set("Authorization", "Bearer "+testToken)
|
||||
req.Header.Set("Authorization", "Bearer "+ownerCredential())
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
@@ -575,29 +615,33 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The userscript route is registered outside the `if cfg.WebPassword != ""`
|
||||
// block in newRouter, so it must keep working on a deployment that never set
|
||||
// WEB_PASSWORD — see internal/userscript for the handler's own behaviour.
|
||||
// The userscript route is registered outside the web UI's Discord auth, so it
|
||||
// must keep working whatever the web config — see internal/userscript for the
|
||||
// handler's own behaviour. The credential in the path is the owner's derived
|
||||
// one, and the served script carries it substituted in.
|
||||
func TestUserscriptServedWithWebUIDisabled(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "manga-bookmark.user.js")
|
||||
if err := os.WriteFile(path, []byte("console.log(1);\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(path, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
s := newTestStore(t)
|
||||
cfg := testConfig() // WebPassword empty
|
||||
cfg := testConfig() // no Discord config needed for the userscript route
|
||||
cfg.UserscriptPath = path
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/u/"+testToken+"/manga-bookmark.user.js", nil)
|
||||
req := httptest.NewRequest(http.MethodGet, "/u/"+ownerCredential()+"/manga-bookmark.user.js", nil)
|
||||
newRouter(s, cfg).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+ownerCredential()+`"`) {
|
||||
t.Fatalf("served script does not carry the requesting Reader's credential:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Both scripts are served from the same handler on the same token, outside the
|
||||
// WEB_PASSWORD gate — a wrong token is a 404, never a 401.
|
||||
// Both scripts are served from the same handler, outside the web UI's auth —
|
||||
// a wrong credential is a 404, never a 401.
|
||||
func TestNovelUserscriptServed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
novelPath := filepath.Join(dir, "novel-bookmark.user.js")
|
||||
@@ -613,7 +657,7 @@ func TestNovelUserscriptServed(t *testing.T) {
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet,
|
||||
"/u/"+testToken+"/novel-bookmark.user.js", nil))
|
||||
"/u/"+ownerCredential()+"/novel-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"bookmarkmanager/backend/internal/pgtest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// fakeCovers stands in for the headless browser. It counts calls so the test
|
||||
// can prove the store spares the browser after the first navigation.
|
||||
type fakeCovers struct {
|
||||
body []byte
|
||||
contentType string
|
||||
err error
|
||||
calls atomic.Int32
|
||||
lastID atomic.Value
|
||||
}
|
||||
|
||||
func (f *fakeCovers) Image(_ context.Context, imageID string) ([]byte, string, error) {
|
||||
f.calls.Add(1)
|
||||
f.lastID.Store(imageID)
|
||||
if f.err != nil {
|
||||
return nil, "", f.err
|
||||
}
|
||||
return f.body, f.contentType, nil
|
||||
}
|
||||
|
||||
const testCoverID = "019fe11a-84c3-7fc3-a84b-88787374b617"
|
||||
|
||||
func getCover(t *testing.T, srv http.Handler, path string, cookie *http.Cookie) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
if cookie != nil {
|
||||
req.AddCookie(cookie)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
// kagane serves its covers behind a Cloudflare challenge and with
|
||||
// cross-origin-resource-policy: same-origin, so the UI can only show one by
|
||||
// re-serving the bytes from its own origin.
|
||||
func TestKaganeCoverPersistsAndReusesStoredBytes(t *testing.T) {
|
||||
cf := &fakeCovers{body: []byte("\x00webp-bytes"), contentType: "image/webp"}
|
||||
cfg := testConfig()
|
||||
cfg.Covers = cf
|
||||
srv, st := newWebTestServer(t, cfg)
|
||||
cookie := sessionCookie(t, st)
|
||||
|
||||
rr := getCover(t, srv, "/img/kagane/"+testCoverID, cookie)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("first request: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); got != string(cf.body) {
|
||||
t.Fatalf("first request: body = %q, want %q", got, cf.body)
|
||||
}
|
||||
|
||||
// A new Handler has no process-local state from the first request. The same
|
||||
// store must still answer without navigating the browser again.
|
||||
srv = newRouter(st, cfg)
|
||||
rr = getCover(t, srv, "/img/kagane/"+testCoverID, cookie)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("stored request: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); got != string(cf.body) {
|
||||
t.Fatalf("stored request: body = %q, want %q", got, cf.body)
|
||||
}
|
||||
if got := cf.calls.Load(); got != 1 {
|
||||
t.Fatalf("fetcher called %d times, want 1 — stored bytes must survive a new handler", got)
|
||||
}
|
||||
if got := cf.lastID.Load(); got != testCoverID {
|
||||
t.Fatalf("fetched image id = %v, want %s", got, testCoverID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKaganeCoverServesStoredBytesWithoutBrowser(t *testing.T) {
|
||||
cf := &fakeCovers{err: errors.New("browser must not be called")}
|
||||
cfg := testConfig()
|
||||
cfg.Covers = cf
|
||||
srv, st := newWebTestServer(t, cfg)
|
||||
if err := st.PutKaganeCover(testCoverID, []byte("already-stored"), "image/png"); err != nil {
|
||||
t.Fatalf("PutKaganeCover: %v", err)
|
||||
}
|
||||
|
||||
rr := getCover(t, srv, "/img/kagane/"+testCoverID, sessionCookie(t, st))
|
||||
if rr.Code != http.StatusOK || rr.Body.String() != "already-stored" {
|
||||
t.Fatalf("stored request = (%d, %q), want (200, already-stored)", rr.Code, rr.Body.String())
|
||||
}
|
||||
if got := cf.calls.Load(); got != 0 {
|
||||
t.Fatalf("fetcher called %d times for a stored cover, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKaganeCoverServesPersistedBytesAfterRestart(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
owner := store.Owner{
|
||||
DiscordID: "cover-owner",
|
||||
TokenHash: sha256.Sum256([]byte("cover-owner-token")),
|
||||
}
|
||||
first, err := store.Open(url, owner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
if err := first.PutKaganeCover(testCoverID, []byte("survives-restart"), "image/jpeg"); err != nil {
|
||||
first.Close()
|
||||
t.Fatalf("PutKaganeCover: %v", err)
|
||||
}
|
||||
if err := first.Close(); err != nil {
|
||||
t.Fatalf("close first store: %v", err)
|
||||
}
|
||||
|
||||
second, err := store.Open(url, owner)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer second.Close()
|
||||
cf := &fakeCovers{err: errors.New("browser must not be called after restart")}
|
||||
cfg := testConfig()
|
||||
cfg.Covers = cf
|
||||
rr := getCover(t, newRouter(second, cfg), "/img/kagane/"+testCoverID, sessionCookie(t, second))
|
||||
if rr.Code != http.StatusOK || rr.Body.String() != "survives-restart" {
|
||||
t.Fatalf("restarted request = (%d, %q), want (200, survives-restart)", rr.Code, rr.Body.String())
|
||||
}
|
||||
if got := cf.calls.Load(); got != 0 {
|
||||
t.Fatalf("fetcher called %d times after restart, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The proxy reaches a headless browser, so it is not open to the internet.
|
||||
func TestKaganeCoverRequiresSession(t *testing.T) {
|
||||
cf := &fakeCovers{body: []byte("x"), contentType: "image/webp"}
|
||||
cfg := testConfig()
|
||||
cfg.Covers = cf
|
||||
srv, _ := newWebTestServer(t, cfg)
|
||||
|
||||
rr := getCover(t, srv, "/img/kagane/"+testCoverID, nil)
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", rr.Code)
|
||||
}
|
||||
if got := cf.calls.Load(); got != 0 {
|
||||
t.Fatalf("fetcher called %d times for an unauthenticated request, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKaganeCoverRejectsBadInput(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
id string
|
||||
fetch *fakeCovers
|
||||
}{
|
||||
{
|
||||
"an id that is not a uuid never reaches the browser",
|
||||
"solo-leveling",
|
||||
&fakeCovers{body: []byte("x"), contentType: "image/webp"},
|
||||
},
|
||||
{
|
||||
"a uuid-shaped id with a trailing segment is rejected whole",
|
||||
testCoverID + "x",
|
||||
&fakeCovers{body: []byte("x"), contentType: "image/webp"},
|
||||
},
|
||||
{
|
||||
"a challenged fetch is a missing cover",
|
||||
testCoverID,
|
||||
&fakeCovers{err: errors.New("challenge held")},
|
||||
},
|
||||
{
|
||||
"a content type outside the image set is not echoed back",
|
||||
testCoverID,
|
||||
&fakeCovers{body: []byte("<script>"), contentType: "text/html"},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
cfg.Covers = tc.fetch
|
||||
srv, st := newWebTestServer(t, cfg)
|
||||
rr := getCover(t, srv, "/img/kagane/"+tc.id, sessionCookie(t, st))
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want 404", rr.Code)
|
||||
}
|
||||
_, _, ok, err := st.GetKaganeCover(tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetKaganeCover after rejection: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatal("rejected cover was persisted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ServeMux path-cleans a traversal into a redirect before the handler runs, so
|
||||
// the guarantee to pin down is that no request shaped like one ever gets bytes.
|
||||
func TestKaganeCoverTraversalServesNothing(t *testing.T) {
|
||||
cf := &fakeCovers{body: []byte("secret"), contentType: "image/webp"}
|
||||
cfg := testConfig()
|
||||
cfg.Covers = cf
|
||||
srv, st := newWebTestServer(t, cfg)
|
||||
|
||||
rr := getCover(t, srv, "/img/kagane/../../etc/passwd", sessionCookie(t, st))
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatalf("status = 200, want anything but a served body")
|
||||
}
|
||||
if got := cf.calls.Load(); got != 0 {
|
||||
t.Fatalf("fetcher called %d times for a traversal, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Without BROWSER_WS_URL there is no fetcher, and the endpoint must answer
|
||||
// rather than reach for a nil one.
|
||||
func TestKaganeCoverWithoutFetcher(t *testing.T) {
|
||||
srv, st := newWebTestServer(t, testConfig())
|
||||
rr := getCover(t, srv, "/img/kagane/"+testCoverID, sessionCookie(t, st))
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want 404", rr.Code)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/httpmw"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
@@ -25,9 +26,9 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
}
|
||||
}
|
||||
|
||||
// List returns all bookmarks. GET /bookmarks
|
||||
// List returns all bookmarks of the acting Reader. GET /bookmarks
|
||||
func (h *Handler) List(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := h.Store.List()
|
||||
items, err := h.Store.List(httpmw.ReaderID(r))
|
||||
if err != nil {
|
||||
log.Printf("list: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
@@ -91,7 +92,7 @@ func (h *Handler) Put(w http.ResponseWriter, r *http.Request) {
|
||||
// reading progress actually moved. Any client value is ignored.
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
|
||||
stored, err := h.Store.Upsert(b)
|
||||
stored, err := h.Store.Upsert(httpmw.ReaderID(r), b)
|
||||
if err != nil {
|
||||
log.Printf("upsert: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
@@ -109,7 +110,7 @@ func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.Store.Delete(key); err != nil {
|
||||
if err := h.Store.Delete(httpmw.ReaderID(r), key); err != nil {
|
||||
log.Printf("delete: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
|
||||
@@ -2,28 +2,56 @@ package httpmw
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"crypto/subtle"
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
const bearerPrefix = "Bearer "
|
||||
|
||||
// Auth guards a handler with a constant-time bearer-token check.
|
||||
func Auth(token string, next http.Handler) http.Handler {
|
||||
want := []byte(token)
|
||||
type ctxKey int
|
||||
|
||||
// readerCtxKey is where Auth stashes the authenticated Reader id.
|
||||
const readerCtxKey ctxKey = iota
|
||||
|
||||
// ReaderID returns the Reader id Auth authenticated, for handlers that take
|
||||
// the acting Reader from the request rather than from a fixed field.
|
||||
func ReaderID(r *http.Request) int64 { return r.Context().Value(readerCtxKey).(int64) }
|
||||
|
||||
// ResolveReader maps a presented credential to a Reader. The credential is
|
||||
// hashed and matched against readers.token_sha256 — an equality on 32-byte
|
||||
// values, never a comparison of the credential itself. The same resolution
|
||||
// backs the API bearer header and the userscript download path, so a Reader
|
||||
// has exactly one credential with one blast radius.
|
||||
func ResolveReader(s *store.Store, cred string) (int64, bool) {
|
||||
readerID, ok, err := s.ReaderIDForTokenHash(token.Hash(cred))
|
||||
if err != nil {
|
||||
log.Printf("auth: reader lookup: %v", err)
|
||||
return 0, false
|
||||
}
|
||||
return readerID, ok
|
||||
}
|
||||
|
||||
// Auth guards a handler with a per-Reader bearer credential. The acting
|
||||
// Reader travels in the request context, so a handler scopes every store call
|
||||
// to exactly the Reader that authenticated.
|
||||
func Auth(s *store.Store, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(h, bearerPrefix) {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
got := []byte(strings.TrimPrefix(h, bearerPrefix))
|
||||
if subtle.ConstantTimeCompare(got, want) != 1 {
|
||||
readerID, ok := ResolveReader(s, strings.TrimPrefix(h, bearerPrefix))
|
||||
if !ok {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), readerCtxKey, readerID)))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@ package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
@@ -22,6 +24,12 @@ const challengeTimeout = 45 * time.Second
|
||||
|
||||
var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
||||
|
||||
// kaganeImageIDRe pins the only path segment Image interpolates into an
|
||||
// outbound URL. The id arrives from a stored cover URL, which a client
|
||||
// supplied, so it is matched rather than trusted: a headless browser is a
|
||||
// strong SSRF primitive.
|
||||
var kaganeImageIDRe = regexp.MustCompile(`^[0-9a-f-]{36}$`)
|
||||
|
||||
// BrowserFetcher retrieves pages through a remote headless Chrome over the
|
||||
// DevTools Protocol.
|
||||
//
|
||||
@@ -45,19 +53,22 @@ var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
||||
type BrowserFetcher struct {
|
||||
allocCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
// One page at a time: caps the sidecar's memory and keeps series from
|
||||
// sharing page state.
|
||||
// One page at a time: caps the browser's memory — it runs under a hard
|
||||
// cgroup cap on a shared machine — and keeps series from sharing page state.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
var _ Fetcher = (*BrowserFetcher)(nil)
|
||||
|
||||
// NewBrowserFetcher connects to a headless-shell over CDP. wsURL must name the
|
||||
// sidecar by IP, e.g. ws://172.28.0.10:9222 — not by Docker DNS name. Chrome's
|
||||
// DevTools HTTP handler 500s any /json/version request whose Host header
|
||||
// isn't an IP or "localhost" (confirmed 2026-08-03 against
|
||||
// chromedp/headless-shell:stable), so the compose network pins the sidecar's
|
||||
// address for this to resolve at all.
|
||||
// NewBrowserFetcher connects to a Chrome over CDP. The browser is not a
|
||||
// sidecar: it runs on a separate machine and is reached over the tailnet
|
||||
// (ADR-0006), so wsURL is that machine's tailnet address, e.g.
|
||||
// ws://100.64.0.5:9222.
|
||||
//
|
||||
// It must be an IP, never a hostname — not MagicDNS, not a Docker service
|
||||
// name. Chrome's DevTools HTTP handler 500s any /json/version request whose
|
||||
// Host header isn't an IP or "localhost" (confirmed 2026-08-03), so a name
|
||||
// fails at discovery and surfaces as a dead site rather than a bad URL.
|
||||
//
|
||||
// Do not add chromedp.NoModifyURL here: that option skips the /json/version
|
||||
// discovery request entirely and dials wsURL as if it were already the full
|
||||
@@ -65,8 +76,10 @@ var _ Fetcher = (*BrowserFetcher)(nil)
|
||||
// /devtools/browser/<uuid>, a path chosen fresh at every Chrome start — dialing
|
||||
// the bare host:port 404s. The default (discovery) path works precisely
|
||||
// because Chrome's /json/version response echoes back the Host header of the
|
||||
// discovery request in webSocketDebuggerUrl, so as long as wsURL is a
|
||||
// container-reachable IP, the URL chromedp gets back already points at it.
|
||||
// discovery request in webSocketDebuggerUrl, so as long as wsURL is an IP this
|
||||
// process can reach, the URL chromedp gets back already points at it. That is
|
||||
// also why a Chrome restarted behind a stable endpoint needs no reconnect
|
||||
// here: the fresh UUID arrives with the next discovery.
|
||||
func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
|
||||
if wsURL == "" {
|
||||
return nil, fmt.Errorf("empty browser websocket url")
|
||||
@@ -91,23 +104,6 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int
|
||||
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
|
||||
defer cancel()
|
||||
// A fresh tab per fetch, closed on return, so one wedged page cannot
|
||||
// poison later polls.
|
||||
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
|
||||
defer cancelTab()
|
||||
// Bind the caller's deadline to the tab.
|
||||
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
|
||||
defer cancelDeadline()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cancelDeadline()
|
||||
}()
|
||||
|
||||
var body string
|
||||
// kagane's chapter list is only in its JSON API, which must be called from
|
||||
// inside the page so the request carries the clearance cookie. novelfull
|
||||
@@ -123,24 +119,171 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int
|
||||
)
|
||||
}
|
||||
|
||||
err := chromedp.Run(tabCtx,
|
||||
chromedp.Navigate(seriesURL),
|
||||
// The challenge reloads the page itself when it passes; waiting for the
|
||||
// site's own root element is what tells us we are through it.
|
||||
chromedp.WaitReady("body", chromedp.ByQuery),
|
||||
read,
|
||||
)
|
||||
if err != nil {
|
||||
// novelfull's payload is the DOM itself, and the interstitial has a DOM
|
||||
// too, so "we have an answer" has to exclude it explicitly. kagane's
|
||||
// in-page fetch just fails while challenged, which is already the signal.
|
||||
done := func() bool { return body != "" && (isKagane || !isInterstitial(body)) }
|
||||
if err := f.run(ctx, seriesURL, read, done); err != nil {
|
||||
// Challenge never cleared, or the API refused. Indistinguishable from
|
||||
// here and handled identically by the caller.
|
||||
if errors.Is(err, errChallengeHeld) {
|
||||
return "", 403, nil
|
||||
}
|
||||
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
|
||||
}
|
||||
if body == "" {
|
||||
// Challenge still up, or the API refused. Indistinguishable from here
|
||||
// and handled identically by the caller.
|
||||
return "", 403, nil
|
||||
}
|
||||
return body, 200, nil
|
||||
}
|
||||
|
||||
// Image retrieves one kagane cover as raw bytes and its content type.
|
||||
//
|
||||
// It exists because kagane serves covers behind the same challenge as its
|
||||
// pages *and* with `cross-origin-resource-policy: same-origin`, so an <img> on
|
||||
// the web UI's origin cannot load one even from a browser that already holds
|
||||
// the clearance cookie (verified 2026-08-08). Proxying is the only route.
|
||||
//
|
||||
// The image URL is navigated to rather than fetched from some other kagane
|
||||
// page: the challenge only runs on a top-level navigation, and once it clears
|
||||
// the document *is* the image, so a same-origin fetch of location.href reads
|
||||
// it straight back out of the cache.
|
||||
//
|
||||
// The challenge is not solved by the first read: WaitReady("body") is satisfied
|
||||
// by the interstitial too. run holds the tab open until the in-page fetch
|
||||
// succeeds, which is what gives the challenge script the seconds it needs.
|
||||
func (f *BrowserFetcher) Image(ctx context.Context, imageID string) ([]byte, string, error) {
|
||||
if !kaganeImageIDRe.MatchString(imageID) {
|
||||
return nil, "", fmt.Errorf("not a kagane image id: %q", imageID)
|
||||
}
|
||||
var dataURL string
|
||||
err := f.run(ctx, "https://kagane.to/api/v2/image/"+imageID+"/compressed",
|
||||
chromedp.Evaluate(`fetch(location.href).then(r => r.ok
|
||||
? r.blob().then(b => new Promise(res => {
|
||||
const fr = new FileReader();
|
||||
fr.onload = () => res(fr.result);
|
||||
fr.readAsDataURL(b);
|
||||
}))
|
||||
: "")`, &dataURL, awaitPromise),
|
||||
func() bool { return dataURL != "" })
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageID, err)
|
||||
}
|
||||
// "data:image/webp;base64,<payload>".
|
||||
head, payload, ok := strings.Cut(dataURL, ";base64,")
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("browser image %s: not a data url", imageID)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(payload)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageID, err)
|
||||
}
|
||||
return raw, strings.TrimPrefix(head, "data:"), nil
|
||||
}
|
||||
|
||||
// errChallengeHeld reports that the budget ran out with the interstitial still
|
||||
// up. Distinct from a transport failure: it means "this site said no", which
|
||||
// the poller answers with a 403 and its ordinary cooldown.
|
||||
var errChallengeHeld = errors.New("challenge held")
|
||||
|
||||
// errBrowserInterrupted distinguishes a remote Chrome restart from the
|
||||
// caller's own deadline. chromedp reports both as context.Canceled.
|
||||
var errBrowserInterrupted = errors.New("browser interrupted")
|
||||
|
||||
func classifyBrowserError(ctx context.Context, browserLost bool, err error) error {
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
if !browserLost {
|
||||
return err
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", errBrowserInterrupted, err)
|
||||
}
|
||||
|
||||
func browserConnectionLost(ctx context.Context) bool {
|
||||
c := chromedp.FromContext(ctx)
|
||||
if c == nil || c.Browser == nil {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case <-c.Browser.LostConnection:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// challengePollInterval paces re-reads while a challenge solves itself.
|
||||
const challengePollInterval = 2 * time.Second
|
||||
|
||||
// isInterstitial reports whether html is Cloudflare's challenge page rather
|
||||
// than the site's own. Matched on the challenge runtime's script path, which is
|
||||
// stable across the interstitial's wording and locale — the visible "Just a
|
||||
// moment..." title is neither.
|
||||
func isInterstitial(html string) bool {
|
||||
return strings.Contains(html, "/cdn-cgi/challenge-platform/")
|
||||
}
|
||||
|
||||
// run navigates to target and re-reads until done reports an answer, bounded by
|
||||
// challengeTimeout and by the caller's own deadline, in a tab that is closed on
|
||||
// return so one wedged page cannot poison later calls.
|
||||
//
|
||||
// Holding the tab open across re-reads is the whole point. A Cloudflare
|
||||
// interstitial needs several seconds of a live page to solve itself and write
|
||||
// clearance into the browser's shared cookie jar; reading once and closing the
|
||||
// tab — which is what this did before 2026-08-08 — never gives it that window,
|
||||
// so every fetch lands on the interstitial and the clearance that would have
|
||||
// unblocked all the later ones is never obtained.
|
||||
func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.Action, done func() bool) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
callerCtx := ctx
|
||||
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
|
||||
defer cancel()
|
||||
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
|
||||
defer cancelTab()
|
||||
// Bind the caller's deadline to the tab.
|
||||
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
|
||||
defer cancelDeadline()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cancelDeadline()
|
||||
}()
|
||||
|
||||
if err := chromedp.Run(tabCtx,
|
||||
chromedp.Navigate(target),
|
||||
chromedp.WaitReady("body", chromedp.ByQuery),
|
||||
); err != nil {
|
||||
return classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
|
||||
}
|
||||
var lastErr error
|
||||
for {
|
||||
// The challenge reloads the page when it passes, which tears down the
|
||||
// execution context mid-read. That is a retry, not a failure.
|
||||
if err := chromedp.Run(tabCtx, read); err != nil {
|
||||
err = classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
|
||||
if errors.Is(err, errBrowserInterrupted) {
|
||||
return err
|
||||
}
|
||||
lastErr = err
|
||||
} else if done() {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if err := callerCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if lastErr != nil {
|
||||
return fmt.Errorf("%w (last read: %v)", errChallengeHeld, lastErr)
|
||||
}
|
||||
return errChallengeHeld
|
||||
case <-time.After(challengePollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
|
||||
// chapter list. Returning false for anything else is a second line of defence
|
||||
// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package latest
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKaganeAPIURL(t *testing.T) {
|
||||
const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
@@ -57,3 +61,17 @@ func TestNovelfullSeriesURL(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestClassifyBrowserInterruption(t *testing.T) {
|
||||
if err := classifyBrowserError(context.Background(), true, context.Canceled); !errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("classifyBrowserError(context.Canceled) = %v, want browser interruption", err)
|
||||
}
|
||||
if err := classifyBrowserError(context.Background(), false, context.Canceled); errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("ordinary cancellation misclassified as browser interruption: %v", err)
|
||||
}
|
||||
|
||||
caller, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := classifyBrowserError(caller, true, context.Canceled); errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("caller cancellation misclassified as browser interruption: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
@@ -15,6 +16,11 @@ type Fetcher interface {
|
||||
Get(ctx context.Context, url string) (body string, status int, err error)
|
||||
}
|
||||
|
||||
// CoverFetcher retrieves one kagane cover through the browser-backed path.
|
||||
type CoverFetcher interface {
|
||||
Image(ctx context.Context, imageID string) (body []byte, contentType string, err error)
|
||||
}
|
||||
|
||||
// Poller re-checks each bookmarked series' newest published chapter on a
|
||||
// schedule, independent of the userscript's own in-browser checks. The two run
|
||||
// in parallel and report the same observable fact, so whichever writes last wins
|
||||
@@ -23,24 +29,56 @@ type Fetcher interface {
|
||||
// Two clocks, deliberately independent:
|
||||
//
|
||||
// - Interval is how often this goroutine wakes up and looks.
|
||||
// - Cooldown is how long one series rests since its own last check.
|
||||
// - Cooldowns are how long a series rests since its own last check. Browser-
|
||||
// backed sites use the longer BrowserCooldown.
|
||||
//
|
||||
// Only the cooldown is per series, and it is enforced by the WHERE clause in
|
||||
// DueForLatestCheck rather than by any timer. Shortening Interval therefore
|
||||
// cannot shorten anyone's cooldown; it only makes the poller wake up and find
|
||||
// nothing due more often.
|
||||
// Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
|
||||
// by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
|
||||
// it only makes the poller wake up and find nothing due more often.
|
||||
type Poller struct {
|
||||
Store *store.Store
|
||||
Fetch Fetcher
|
||||
Store *store.Store
|
||||
Fetch Fetcher
|
||||
// BrowserFetch handles sites behind a JavaScript challenge that Fetch
|
||||
// cannot clear. Nil disables those sites entirely rather than falling back
|
||||
// to Fetch, which would only ever retrieve a challenge page.
|
||||
BrowserFetch Fetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
Cooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
// CoverFetch is optional: it prefetches kagane covers without changing the
|
||||
// chapter poll's result when the browser or persistence fails.
|
||||
CoverFetch CoverFetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
Cooldown time.Duration
|
||||
BrowserCooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
}
|
||||
|
||||
var browserBackedSites = []string{"kagane", "novelfull"}
|
||||
|
||||
func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) {
|
||||
if sr.Site != "kagane" || p.CoverFetch == nil {
|
||||
return
|
||||
}
|
||||
imageID, ok := store.KaganeImageID(sr.Cover)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
_, _, found, err := p.Store.GetKaganeCover(imageID)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
return
|
||||
}
|
||||
body, contentType, err := p.CoverFetch.Image(ctx, imageID)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil {
|
||||
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
|
||||
@@ -49,8 +87,7 @@ type Poller struct {
|
||||
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
|
||||
// profile TLSFetcher uses — so they are browser-only or nothing.
|
||||
func (p *Poller) fetcherFor(site string) Fetcher {
|
||||
switch site {
|
||||
case "kagane", "novelfull":
|
||||
if slices.Contains(browserBackedSites, site) {
|
||||
return p.BrowserFetch
|
||||
}
|
||||
return p.Fetch
|
||||
@@ -63,8 +100,8 @@ func (p *Poller) fetcherFor(site string) Fetcher {
|
||||
// failure mode for a misconfigured batch x stagger: a slower cadence, never
|
||||
// concurrent fetch storms.
|
||||
func (p *Poller) Run(ctx context.Context) {
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s",
|
||||
p.Interval, p.Cooldown, p.Batch, p.Stagger)
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
|
||||
p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
|
||||
t := time.NewTicker(p.Interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
@@ -80,8 +117,10 @@ func (p *Poller) Run(ctx context.Context) {
|
||||
|
||||
// runOnce processes one batch of due series.
|
||||
func (p *Poller) runOnce(ctx context.Context) {
|
||||
cutoff := p.Now().Add(-p.Cooldown).UnixMilli()
|
||||
due, err := p.Store.DueForLatestCheck(cutoff, p.Batch)
|
||||
now := p.Now()
|
||||
cutoff := now.Add(-p.Cooldown).UnixMilli()
|
||||
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
|
||||
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
|
||||
if err != nil {
|
||||
log.Printf("latest poll: due query: %v", err)
|
||||
return
|
||||
@@ -152,6 +191,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
||||
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||
return
|
||||
}
|
||||
p.prefetchCover(ctx, sr)
|
||||
|
||||
body, status, err := f.Get(ctx, sr.SeriesURL)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,7 +2,9 @@ package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -15,15 +17,22 @@ import (
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
|
||||
// newTestStore opens a store on a Postgres database of this test's own.
|
||||
func newTestStore(t *testing.T) *store.Store {
|
||||
// testOwner is the owner every test store seeds. A second reader, where a
|
||||
// test needs one, is created by opening the same database as a second owner.
|
||||
var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}
|
||||
|
||||
// newTestStore opens a store on a Postgres database of this test's own and
|
||||
// returns the URL, for helpers that need a second connection to the same
|
||||
// database (see TestRunOnceFetchesSharedSeriesOnce).
|
||||
func newTestStore(t *testing.T) (*store.Store, string) {
|
||||
t.Helper()
|
||||
s, err := store.Open(pgtest.URL(t))
|
||||
url := pgtest.URL(t)
|
||||
s, err := store.Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s
|
||||
return s, url
|
||||
}
|
||||
|
||||
// seedForCheck inserts a bookmark (and with it its series) and forces the
|
||||
@@ -34,7 +43,7 @@ func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key,
|
||||
Site: site,
|
||||
SeriesID: seriesID,
|
||||
@@ -95,23 +104,48 @@ func (f *fakeFetcher) callCount() int {
|
||||
return len(f.calls)
|
||||
}
|
||||
|
||||
type fakeCoverFetcher struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
body []byte
|
||||
contentType string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCoverFetcher) Image(_ context.Context, imageID string) ([]byte, string, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, imageID)
|
||||
f.mu.Unlock()
|
||||
if f.err != nil {
|
||||
return nil, "", f.err
|
||||
}
|
||||
return f.body, f.contentType, nil
|
||||
}
|
||||
|
||||
func (f *fakeCoverFetcher) callCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.calls)
|
||||
}
|
||||
|
||||
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
|
||||
// instantly and deterministically.
|
||||
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
|
||||
t.Helper()
|
||||
return &Poller{
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: func() time.Time { return at },
|
||||
Cooldown: time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 0,
|
||||
Batch: 14,
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: func() time.Time { return at },
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 0,
|
||||
Batch: 14,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
|
||||
|
||||
@@ -119,7 +153,7 @@ func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
b, ok, err := s.Get("asura:chronicles-of-the-demon-faction-f886a8af")
|
||||
b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get: %v ok=%v", err, ok)
|
||||
}
|
||||
@@ -137,19 +171,19 @@ func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
||||
// The whole point of the updated_at CASE in Upsert: a newly published chapter is
|
||||
// not reading progress and must not move the series up the list.
|
||||
func TestRunOnceDoesNotReorderList(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
const key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
|
||||
// "other" is the most recently read, so it must stay at the top of List().
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: "asura:other", Site: "asura", SeriesID: "other",
|
||||
SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed other: %v", err)
|
||||
}
|
||||
seedForCheck(t, s, key, url, 0)
|
||||
before, _, err := s.Get(key)
|
||||
before, _, err := s.Get(s.OwnerID(), key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get before: %v", err)
|
||||
}
|
||||
@@ -157,7 +191,7 @@ func TestRunOnceDoesNotReorderList(t *testing.T) {
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
|
||||
|
||||
after, _, err := s.Get(key)
|
||||
after, _, err := s.Get(s.OwnerID(), key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after: %v", err)
|
||||
}
|
||||
@@ -165,7 +199,7 @@ func TestRunOnceDoesNotReorderList(t *testing.T) {
|
||||
t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
|
||||
before.UpdatedAt, after.UpdatedAt)
|
||||
}
|
||||
list, err := s.List()
|
||||
list, err := s.List(s.OwnerID())
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
@@ -188,7 +222,7 @@ func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/x"
|
||||
seedForCheck(t, s, "asura:x", url, 0)
|
||||
|
||||
@@ -199,7 +233,7 @@ func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
||||
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
||||
}
|
||||
b, _, err := s.Get("asura:x")
|
||||
b, _, err := s.Get(s.OwnerID(), "asura:x")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
@@ -211,7 +245,7 @@ func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunOnceRespectsBatchLimit(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
for i := 0; i < 20; i++ {
|
||||
key := "asura:s" + string(rune('a'+i))
|
||||
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
|
||||
@@ -228,18 +262,25 @@ func TestRunOnceRespectsBatchLimit(t *testing.T) {
|
||||
}
|
||||
|
||||
// The point of the split (ADR-0003): a series referenced by several bookmarks
|
||||
// is fetched once per due cycle, not once per bookmark. Today the bookmark key
|
||||
// is <site>:<series_id>, so the second bookmark only exists once keys stop
|
||||
// being derived from the series identity (issue #22).
|
||||
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
|
||||
// series when two readers track it (issue #22).
|
||||
func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, url := newTestStore(t)
|
||||
// The slug must match the fixture's own anchors: asura's parser scopes
|
||||
// chapter links to the stored slug.
|
||||
const slug = "chronicles-of-the-demon-faction-f886a8af"
|
||||
const url = "https://asurascans.com/comics/" + slug
|
||||
seedForCheck(t, s, "asura:"+slug, url, 0)
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
Key: "asura:" + slug + ":2", Site: "asura", SeriesID: slug, UpdatedAt: 2000,
|
||||
const seriesURL = "https://asurascans.com/comics/" + slug
|
||||
seedForCheck(t, s, "asura:"+slug, seriesURL, 0)
|
||||
// A second reader tracks the same series. The seed is the only
|
||||
// reader-creation path, so a second Open as a different owner is how a
|
||||
// test gets a second reader on the same database.
|
||||
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))})
|
||||
if err != nil {
|
||||
t.Fatalf("Open second reader: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { other.Close() })
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
@@ -251,20 +292,20 @@ func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
|
||||
t.Fatalf("fetched shared series %d times, want 1", got)
|
||||
}
|
||||
// Both bookmarks join to the same updated series row.
|
||||
for _, key := range []string{"asura:" + slug, "asura:" + slug + ":2"} {
|
||||
b, ok, err := s.Get(key)
|
||||
for _, st := range []*store.Store{s, other} {
|
||||
b, ok, err := st.Get(st.OwnerID(), "asura:"+slug)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get %s: %v ok=%v", key, err, ok)
|
||||
t.Fatalf("Get: %v ok=%v", err, ok)
|
||||
}
|
||||
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
||||
t.Fatalf("%s LatestChapterNum = %v, want 181", key, b.LatestChapterNum)
|
||||
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One unreachable series must not abandon the rest of the batch.
|
||||
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
|
||||
for _, k := range keys {
|
||||
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
||||
@@ -289,10 +330,30 @@ func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLogsCooldowns(t *testing.T) {
|
||||
var logs strings.Builder
|
||||
previous := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(previous) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
(&Poller{
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
Interval: time.Hour,
|
||||
}).Run(ctx)
|
||||
|
||||
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
|
||||
!strings.Contains(got, "browser-cooldown=6h") {
|
||||
t.Fatalf("startup log = %q, want both cooldowns", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The cooldown is enforced by the due query, so a second immediate pass must do
|
||||
// nothing at all — this is what makes the tick interval independent of it.
|
||||
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/x"
|
||||
seedForCheck(t, s, "asura:x", url, 0)
|
||||
|
||||
@@ -319,15 +380,53 @@ func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
|
||||
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
|
||||
|
||||
hour := time.Hour
|
||||
now := time.Unix(2*int64(hour/time.Second), 0)
|
||||
tls := &fakeFetcher{status: 200}
|
||||
browser := &fakeFetcher{status: 200}
|
||||
p := &Poller{
|
||||
Store: s,
|
||||
Fetch: tls,
|
||||
BrowserFetch: browser,
|
||||
Now: func() time.Time { return now },
|
||||
Cooldown: hour,
|
||||
BrowserCooldown: 6 * hour,
|
||||
Batch: 10,
|
||||
}
|
||||
|
||||
p.runOnce(context.Background())
|
||||
if got := tls.callCount(); got != 1 {
|
||||
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
|
||||
}
|
||||
if got := browser.callCount(); got != 0 {
|
||||
t.Fatalf("browser fetches after 2h = %d, want 0", got)
|
||||
}
|
||||
|
||||
now = time.Unix(7*int64(hour/time.Second), 0)
|
||||
p.runOnce(context.Background())
|
||||
if got := tls.callCount(); got != 2 {
|
||||
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
|
||||
}
|
||||
if got := browser.callCount(); got != 1 {
|
||||
t.Fatalf("browser fetches after 7h = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A site that retracts a chapter should correct the stored number downward,
|
||||
// mirroring the userscript's equality check (L427) rather than a >.
|
||||
func TestRunOnceCorrectsDownward(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
||||
const key = "demonic:Catastrophic-Necromancer"
|
||||
|
||||
high := 400.0
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
|
||||
SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
|
||||
UpdatedAt: 1000,
|
||||
@@ -338,7 +437,7 @@ func TestRunOnceCorrectsDownward(t *testing.T) {
|
||||
f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
||||
|
||||
b, _, err := s.Get(key)
|
||||
b, _, err := s.Get(s.OwnerID(), key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
@@ -364,9 +463,9 @@ func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
key := tt.site + ":x"
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
||||
UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
@@ -391,7 +490,7 @@ func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
||||
|
||||
// A cancelled context must abandon the batch rather than run it to completion.
|
||||
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
|
||||
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
||||
}
|
||||
@@ -440,8 +539,8 @@ func TestFetchableSeriesURL(t *testing.T) {
|
||||
// receive a challenge page, and the browser fetcher is the whole reason kagane
|
||||
// is pollable at all.
|
||||
func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
s, _ := newTestStore(t)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
Site: "kagane",
|
||||
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
@@ -450,12 +549,14 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
||||
|
||||
p := &Poller{
|
||||
Store: s, Fetch: f,
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
||||
Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
|
||||
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
||||
Interval: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
@@ -466,9 +567,9 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
||||
|
||||
// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
|
||||
func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s, _ := newTestStore(t)
|
||||
key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
if _, err := s.Upsert(store.Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key,
|
||||
Site: "kagane",
|
||||
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
@@ -483,7 +584,8 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
||||
p := &Poller{
|
||||
Store: s, Fetch: tlsF, BrowserFetch: browserF,
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
||||
Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
|
||||
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
||||
Interval: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
@@ -493,7 +595,7 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
||||
if len(browserF.calls) != 1 {
|
||||
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
|
||||
}
|
||||
got, found, err := s.Get(key)
|
||||
got, found, err := s.Get(s.OwnerID(), key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: %v found=%v", err, found)
|
||||
}
|
||||
@@ -502,6 +604,171 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
Cover: coverURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
||||
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
||||
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
body, contentType, ok, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetKaganeCover: %v found=%v", err, ok)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/webp" {
|
||||
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "kagane", SeriesID: seriesID,
|
||||
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
at := time.UnixMilli(5_000_000)
|
||||
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
||||
p := &Poller{
|
||||
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
||||
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
at = at.Add(2 * time.Hour)
|
||||
p.runOnce(context.Background())
|
||||
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "kagane", SeriesID: seriesID,
|
||||
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
now := time.UnixMilli(5_000_000)
|
||||
p := &Poller{
|
||||
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
||||
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
|
||||
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
got, found, err := s.Get(s.OwnerID(), key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: %v found=%v", err, found)
|
||||
}
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
||||
}
|
||||
if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "kagane", SeriesID: seriesID,
|
||||
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
p := &Poller{
|
||||
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
||||
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found {
|
||||
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const (
|
||||
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
||||
)
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "kagane", SeriesID: seriesID,
|
||||
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
p := &Poller{
|
||||
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found {
|
||||
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceDoesNotPrefetchNonKaganeCover(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
const key = "asura:solo"
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
|
||||
Cover: "https://asurascans.com/covers/solo.jpg", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
covers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
|
||||
p := &Poller{
|
||||
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: covers,
|
||||
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
|
||||
if got := covers.callCount(); got != 0 {
|
||||
t.Fatalf("cover fetch calls for asura = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherForRoutesNovelSites(t *testing.T) {
|
||||
tls := &fakeFetcher{}
|
||||
browser := &fakeFetcher{}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSmokeKaganeImage is the live proof that the cover proxy's fetch actually
|
||||
// clears Cloudflare and returns image bytes. It needs the real browser unit
|
||||
// with outbound network, so it runs only when SMOKE_BROWSER_WS_URL is set:
|
||||
//
|
||||
// cd chrome && BROWSER_BIND_ADDR=127.0.0.1 docker compose up -d --build
|
||||
// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeKaganeImage ./internal/latest
|
||||
//
|
||||
// Not chromedp/headless-shell: its challenge never clears (see chrome/Dockerfile),
|
||||
// so a red run there proves nothing about kagane.
|
||||
func TestSmokeKaganeImage(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
const imageID = "019fe11a-84c3-7fc3-a84b-88787374b617" // SP Baby's cover
|
||||
|
||||
// The same URL through a plain client is what the web UI's <img> gets.
|
||||
// Asserting on it keeps the test honest about why the browser is needed.
|
||||
req, err := http.NewRequest(http.MethodGet,
|
||||
"https://kagane.to/api/v2/image/"+imageID+"/compressed", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res, err := (&http.Client{Timeout: 15 * time.Second}).Do(req); err == nil {
|
||||
res.Body.Close()
|
||||
if res.StatusCode == http.StatusOK {
|
||||
t.Log("note: kagane answered a plain request 200 — the challenge is not up right now")
|
||||
}
|
||||
}
|
||||
|
||||
f, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
body, contentType, err := f.Image(ctx, imageID)
|
||||
if err != nil {
|
||||
t.Fatalf("Image: %v", err)
|
||||
}
|
||||
if len(body) < 1000 {
|
||||
t.Fatalf("body is %d bytes, want a real image", len(body))
|
||||
}
|
||||
if contentType != "image/webp" {
|
||||
t.Fatalf("content type = %q, want image/webp", contentType)
|
||||
}
|
||||
// WebP files start with "RIFF....WEBP".
|
||||
if string(body[:4]) != "RIFF" || string(body[8:12]) != "WEBP" {
|
||||
t.Fatalf("body is not a WebP: % x", body[:12])
|
||||
}
|
||||
t.Logf("fetched %d bytes of %s", len(body), contentType)
|
||||
|
||||
if _, _, err := f.Image(ctx, "not-a-uuid"); err == nil {
|
||||
t.Fatal("Image accepted a non-uuid id")
|
||||
}
|
||||
}
|
||||
|
||||
// Control for the test above: the poller's own kagane path, same sidecar. If
|
||||
// this fails too, the sidecar is not clearing the challenge at all and the
|
||||
// image result says nothing about Image itself.
|
||||
func TestSmokeKaganeGet(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
f, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
body, status, err := f.Get(ctx, "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
t.Logf("status=%d bytes=%d head=%.80q", status, len(body), body)
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status)
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,10 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -16,49 +13,18 @@ import (
|
||||
const (
|
||||
CookieName = "bmgr_session"
|
||||
// 60 days: long enough that a phone stays logged in between reading spells.
|
||||
sessionTTL = 60 * 24 * time.Hour
|
||||
// Domain separation, so the session key can never collide with any other
|
||||
// use of the secrets it is derived from. Changing this string logs
|
||||
// everyone out.
|
||||
sessionKeyPurpose = "bmgr-web-session-v1"
|
||||
SessionTTL = 60 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// Key derives the cookie-signing key from both secrets. Sessions are
|
||||
// stateless — there is no session table — so rotating either API_TOKEN or
|
||||
// WEB_PASSWORD invalidates every outstanding cookie at once. The \x00
|
||||
// separator prevents the concatenation ambiguity a bare apiToken+webPassword
|
||||
// would have (e.g. "ab"+"c" colliding with "a"+"bc").
|
||||
func Key(apiToken, webPassword string) []byte {
|
||||
sum := sha256.Sum256([]byte(apiToken + "\x00" + webPassword + sessionKeyPurpose))
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
// Sign encodes "<expiryMs>.<base64url HMAC(expiryMs)>".
|
||||
func Sign(key []byte, expiryMs int64) string {
|
||||
payload := strconv.FormatInt(expiryMs, 10)
|
||||
return payload + "." + sessionMAC(key, payload)
|
||||
}
|
||||
|
||||
func sessionMAC(key []byte, payload string) string {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(payload))
|
||||
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// Verify checks shape, then expiry, then the signature — in that order.
|
||||
// The signature comparison is constant-time; the checks before it only look at
|
||||
// data the holder already supplied, so their timing leaks nothing.
|
||||
func Verify(key []byte, value string, nowMs int64) bool {
|
||||
payload, sig, ok := strings.Cut(value, ".")
|
||||
if !ok {
|
||||
return false
|
||||
// NewID returns an opaque session id: 32 random bytes, hex-encoded. The id is
|
||||
// all the cookie carries and all the sessions table keys on, so its entropy is
|
||||
// what stops a guessed id from being someone else's session.
|
||||
func NewID() string {
|
||||
var b [32]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
panic("session id: " + err.Error())
|
||||
}
|
||||
expiry, err := strconv.ParseInt(payload, 10, 64)
|
||||
if err != nil || expiry <= nowMs {
|
||||
return false
|
||||
}
|
||||
want := sessionMAC(key, payload)
|
||||
return subtle.ConstantTimeCompare([]byte(sig), []byte(want)) == 1
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// isHTTPS reports whether the browser's connection is encrypted. Behind Traefik
|
||||
@@ -69,12 +35,14 @@ func isHTTPS(r *http.Request) bool {
|
||||
return r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"
|
||||
}
|
||||
|
||||
func SetCookie(w http.ResponseWriter, r *http.Request, key []byte) {
|
||||
// SetCookie writes the session cookie. The value is the session id and nothing
|
||||
// else; the row behind it is looked up on every request.
|
||||
func SetCookie(w http.ResponseWriter, r *http.Request, id string) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: CookieName,
|
||||
Value: Sign(key, time.Now().Add(sessionTTL).UnixMilli()),
|
||||
Value: id,
|
||||
Path: "/",
|
||||
MaxAge: int(sessionTTL / time.Second),
|
||||
MaxAge: int(SessionTTL / time.Second),
|
||||
HttpOnly: true,
|
||||
Secure: isHTTPS(r),
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
@@ -120,14 +88,14 @@ func ClientIP(r *http.Request) string {
|
||||
return host
|
||||
}
|
||||
|
||||
// LoginLimiter throttles password guessing: MaxFailures failures inside a
|
||||
// rolling Window blocks further attempts from that IP until the oldest one
|
||||
// LoginLimiter throttles failed sign-in attempts: MaxFailures failures inside
|
||||
// a rolling Window blocks further attempts from that IP until the oldest one
|
||||
// ages out. There is no permanent ban and no unlock step.
|
||||
//
|
||||
// Behind carrier-grade NAT this budget is shared with every other subscriber on
|
||||
// the same public address, so a stranger can lock the owner out for up to one
|
||||
// window. That is accepted: the block self-heals, and ten attempts is generous
|
||||
// for a mistyped password.
|
||||
// for the occasional fumbled sign-in.
|
||||
//
|
||||
// State is in memory and per-process, so a restart clears it. Entries are
|
||||
// pruned lazily on access; for a single-user deployment the map cannot grow
|
||||
|
||||
@@ -9,66 +9,19 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSessionRoundTrip(t *testing.T) {
|
||||
key := Key("token-abc", "pw-abc")
|
||||
now := time.Now().UnixMilli()
|
||||
value := Sign(key, now+60_000)
|
||||
if !Verify(key, value, now) {
|
||||
t.Fatal("Verify = false for a freshly signed cookie, want true")
|
||||
func TestNewID(t *testing.T) {
|
||||
a := NewID()
|
||||
b := NewID()
|
||||
if a == b {
|
||||
t.Fatal("NewID returned the same value twice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionRejects(t *testing.T) {
|
||||
key := Key("token-abc", "pw-abc")
|
||||
now := time.Now().UnixMilli()
|
||||
valid := Sign(key, now+60_000)
|
||||
payload, sig, _ := strings.Cut(valid, ".")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{"empty", ""},
|
||||
{"no separator", payload + sig},
|
||||
{"unparseable expiry", "notanumber." + sig},
|
||||
{"expired", Sign(key, now-1)},
|
||||
{"tampered signature", payload + "." + flipLastChar(sig)},
|
||||
{"tampered expiry", "99999999999999." + sig},
|
||||
{"signed with another key", Sign(Key("other-token", "pw-abc"), now+60_000)},
|
||||
if len(a) != 64 { // 32 random bytes, hex
|
||||
t.Fatalf("NewID() length = %d, want 64", len(a))
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if Verify(key, tc.value, now) {
|
||||
t.Fatalf("Verify(%q) = true, want false", tc.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func flipLastChar(s string) string {
|
||||
if s == "" {
|
||||
return "x"
|
||||
}
|
||||
last := s[len(s)-1]
|
||||
if last == 'A' {
|
||||
return s[:len(s)-1] + "B"
|
||||
}
|
||||
return s[:len(s)-1] + "A"
|
||||
}
|
||||
|
||||
func TestSessionKeyDependsOnToken(t *testing.T) {
|
||||
a := Key("token-a", "pw-abc")
|
||||
b := Key("token-b", "pw-abc")
|
||||
if string(a) == string(b) {
|
||||
t.Fatal("Key collided for different API tokens")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionKeyDependsOnWebPassword(t *testing.T) {
|
||||
a := Key("token-abc", "pw-a")
|
||||
b := Key("token-abc", "pw-b")
|
||||
if string(a) == string(b) {
|
||||
t.Fatal("Key collided for different web passwords with the same API token")
|
||||
for _, r := range a {
|
||||
if !strings.ContainsRune("0123456789abcdef", r) {
|
||||
t.Fatalf("NewID() = %q, want hex", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,7 +39,7 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/login", nil)
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
if tc.tls {
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
}
|
||||
@@ -94,7 +47,7 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
r.Header.Set("X-Forwarded-Proto", tc.forwarded)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
SetCookie(rr, r, Key("token-abc", "pw-abc"))
|
||||
SetCookie(rr, r, "abc123")
|
||||
|
||||
cookies := rr.Result().Cookies()
|
||||
if len(cookies) != 1 {
|
||||
@@ -104,6 +57,9 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
if c.Name != CookieName {
|
||||
t.Fatalf("cookie name = %q, want %q", c.Name, CookieName)
|
||||
}
|
||||
if c.Value != "abc123" {
|
||||
t.Fatalf("cookie value = %q, want the session id verbatim", c.Value)
|
||||
}
|
||||
if !c.HttpOnly {
|
||||
t.Fatal("cookie HttpOnly = false, want true")
|
||||
}
|
||||
@@ -116,8 +72,8 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
if c.Secure != tc.wantSecure {
|
||||
t.Fatalf("cookie Secure = %v, want %v", c.Secure, tc.wantSecure)
|
||||
}
|
||||
if c.MaxAge != int(sessionTTL/time.Second) {
|
||||
t.Fatalf("cookie MaxAge = %d, want %d", c.MaxAge, int(sessionTTL/time.Second))
|
||||
if c.MaxAge != int(SessionTTL/time.Second) {
|
||||
t.Fatalf("cookie MaxAge = %d, want %d", c.MaxAge, int(SessionTTL/time.Second))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -163,7 +119,7 @@ func TestClientIP(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/login", nil)
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r.RemoteAddr = tc.remoteAddr
|
||||
for _, v := range tc.xff {
|
||||
r.Header.Add("X-Forwarded-For", v)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
-- One row per person. Keyed by their Discord user ID; carries the SHA-256 of
|
||||
-- their userscript token and when they were created. Hashed because a token
|
||||
-- in the database is a token anyone with the database can replay; SHA-256 is
|
||||
-- enough because the tokens are high-entropy random values with nothing to
|
||||
-- brute-force. No one can register yet, so this table holds exactly the one
|
||||
-- owner row the seed creates at startup (see Store.Open).
|
||||
CREATE TABLE readers (
|
||||
id bigserial PRIMARY KEY,
|
||||
discord_id text NOT NULL UNIQUE,
|
||||
token_sha256 bytea NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Every bookmark now belongs to a reader. Added nullable: rows created before
|
||||
-- this migration have no owner yet — 0004 attaches them to the seeded owner
|
||||
-- before NOT NULL and the composite key land.
|
||||
ALTER TABLE bookmarks ADD COLUMN reader_id bigint;
|
||||
@@ -0,0 +1,19 @@
|
||||
-- Attach every pre-existing bookmark to the owner reader, seeded between the
|
||||
-- two migrate passes (Store.Open). The oldest reader is the owner by
|
||||
-- construction: only the seed creates readers, and it runs once per database.
|
||||
-- Run-once via the version table, like every migration.
|
||||
UPDATE bookmarks SET reader_id = (SELECT id FROM readers ORDER BY id LIMIT 1);
|
||||
|
||||
-- Ownership lands structurally: reader_id becomes part of the key, so a
|
||||
-- bookmark is one Reader's progress on one Series and a duplicate for the
|
||||
-- same pair is impossible at the database level. Deleting a Reader takes
|
||||
-- their bookmarks with them. The old text key is gone — the wire "key" is
|
||||
-- derived as site:series_id on read, and nothing references the column.
|
||||
-- Dropping it drops the primary key it carried; the composite key replaces
|
||||
-- it, and the FK index the series constraint needs is created automatically.
|
||||
ALTER TABLE bookmarks
|
||||
ALTER COLUMN reader_id SET NOT NULL,
|
||||
DROP COLUMN key,
|
||||
ADD PRIMARY KEY (reader_id, site, series_id),
|
||||
ADD CONSTRAINT bookmarks_reader_fk
|
||||
FOREIGN KEY (reader_id) REFERENCES readers (id) ON DELETE CASCADE;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- One row per browser session. The id is an opaque random value the cookie
|
||||
-- carries verbatim; a request is authenticated by looking the row up, and
|
||||
-- deleting the row is how a session is revoked. Expired rows are removed
|
||||
-- lazily on lookup and swept by the next login, so nothing runs a background
|
||||
-- cleanup.
|
||||
CREATE TABLE sessions (
|
||||
id text PRIMARY KEY,
|
||||
reader_id bigint NOT NULL REFERENCES readers (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Rotation is an epoch bump: a Reader's credential is derived from the
|
||||
-- deployment secret, their Discord id and this epoch, so bumping it issues a
|
||||
-- new credential and the rewritten token_sha256 invalidates the old one the
|
||||
-- moment the transaction commits. The seed's ON CONFLICT refresh (Store.Open)
|
||||
-- is gated on this being 0, so a restart can never undo a rotation by
|
||||
-- restoring the epoch-0 hash.
|
||||
ALTER TABLE readers ADD COLUMN token_epoch bigint NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Kagane cover bytes belong in their own table so image blobs never enter the
|
||||
-- series queries that drive the latest-chapter poller.
|
||||
CREATE TABLE covers (
|
||||
image_id text PRIMARY KEY,
|
||||
body bytea NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
fetched_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,80 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Session is one browser login: an opaque id the cookie carries verbatim,
|
||||
// the Reader it belongs to, and when it stops being valid.
|
||||
type Session struct {
|
||||
ID string
|
||||
ReaderID int64
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// CreateSession stores a new session row for reader. The id is generated by
|
||||
// the caller (session.NewID) — the store only persists it. Expired rows that
|
||||
// were never looked up are swept in the same transaction: this is the one
|
||||
// write every login makes, so the table stays bounded without a background
|
||||
// job.
|
||||
func (s *Store) CreateSession(id string, readerID int64, ttl time.Duration) (Session, error) {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
expires := time.Now().Add(ttl)
|
||||
if _, err := tx.Exec(`INSERT INTO sessions (id, reader_id, expires_at) VALUES ($1, $2, $3)`,
|
||||
id, readerID, expires); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM sessions WHERE expires_at < now()`); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
return Session{ID: id, ReaderID: readerID, ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
// GetSession returns the live session row for id, or ok=false when the id is
|
||||
// unknown or expired. An expired row is deleted on the way out, so the table
|
||||
// never grows past sessions that are still valid.
|
||||
func (s *Store) GetSession(id string, now time.Time) (Session, bool, error) {
|
||||
var sess Session
|
||||
err := s.db.QueryRow(
|
||||
`SELECT id, reader_id, expires_at FROM sessions WHERE id = $1`, id,
|
||||
).Scan(&sess.ID, &sess.ReaderID, &sess.ExpiresAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return Session{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Session{}, false, err
|
||||
}
|
||||
if !sess.ExpiresAt.After(now) {
|
||||
// Best-effort: the row is dead either way; failing the request over a
|
||||
// cleanup delete would only hide the real error. CreateSession's
|
||||
// sweep catches anything this misses.
|
||||
_, _ = s.db.Exec(`DELETE FROM sessions WHERE id = $1`, id)
|
||||
return Session{}, false, nil
|
||||
}
|
||||
return sess, true, nil
|
||||
}
|
||||
|
||||
// DeleteSession revokes one session. Deleting an unknown id is not an error.
|
||||
func (s *Store) DeleteSession(id string) error {
|
||||
_, err := s.db.Exec(`DELETE FROM sessions WHERE id = $1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteReaderSessions revokes every session one Reader holds — the owner's
|
||||
// remedy when a Reader's browser must be logged out everywhere at once. The
|
||||
// next request carrying any of those cookies finds no row and is rejected.
|
||||
func (s *Store) DeleteReaderSessions(readerID int64) error {
|
||||
if _, err := s.db.Exec(`DELETE FROM sessions WHERE reader_id = $1`, readerID); err != nil {
|
||||
return fmt.Errorf("delete sessions for reader %d: %w", readerID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCreateAndGetSession(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
owner := s.OwnerID()
|
||||
|
||||
sess, err := s.CreateSession("sess-1", owner, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
if sess.ID != "sess-1" || sess.ReaderID != owner {
|
||||
t.Fatalf("CreateSession returned %+v, want id sess-1 reader %d", sess, owner)
|
||||
}
|
||||
|
||||
got, ok, err := s.GetSession("sess-1", time.Now())
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetSession: ok=%v err=%v, want ok", ok, err)
|
||||
}
|
||||
if got.ReaderID != owner {
|
||||
t.Fatalf("session reader = %d, want %d", got.ReaderID, owner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSessionUnknownID(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if _, ok, err := s.GetSession("nope", time.Now()); err != nil || ok {
|
||||
t.Fatalf("GetSession(unknown) = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiredSessionIsGone(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
owner := s.OwnerID()
|
||||
if _, err := s.CreateSession("sess-exp", owner, -time.Minute); err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if _, ok, err := s.GetSession("sess-exp", now); err != nil || ok {
|
||||
t.Fatalf("GetSession(expired) = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
// The expired row is deleted on lookup, so the next call cannot revive it.
|
||||
if _, ok, err := s.GetSession("sess-exp", now.Add(-time.Hour)); err != nil || ok {
|
||||
t.Fatalf("GetSession(expired again) = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSessionRevokes(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
owner := s.OwnerID()
|
||||
if _, err := s.CreateSession("sess-del", owner, time.Hour); err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
if err := s.DeleteSession("sess-del"); err != nil {
|
||||
t.Fatalf("DeleteSession: %v", err)
|
||||
}
|
||||
if _, ok, err := s.GetSession("sess-del", time.Now()); err != nil || ok {
|
||||
t.Fatalf("GetSession after delete = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
// Deleting twice is not an error.
|
||||
if err := s.DeleteSession("sess-del"); err != nil {
|
||||
t.Fatalf("DeleteSession twice: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSessionIsPerReader(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
other := secondReader(t, s)
|
||||
if _, err := s.CreateSession("sess-other", other, time.Hour); err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
got, ok, err := s.GetSession("sess-other", time.Now())
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetSession: ok=%v err=%v, want ok", ok, err)
|
||||
}
|
||||
if got.ReaderID != other {
|
||||
t.Fatalf("session reader = %d, want %d", got.ReaderID, other)
|
||||
}
|
||||
}
|
||||
+346
-45
@@ -12,6 +12,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
@@ -137,6 +138,43 @@ func (b Bookmark) Initial() string {
|
||||
return "?"
|
||||
}
|
||||
|
||||
// kaganeCoverRe matches the cover URL kagane's og:image carries, which is what
|
||||
// the userscript stores for that site.
|
||||
var kaganeCoverRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`)
|
||||
|
||||
// KaganeImageID extracts the validated image id from the cover URL recorded by
|
||||
// the userscript.
|
||||
func KaganeImageID(cover string) (string, bool) {
|
||||
m := kaganeCoverRe.FindStringSubmatch(cover)
|
||||
if m == nil {
|
||||
return "", false
|
||||
}
|
||||
return m[1], true
|
||||
}
|
||||
|
||||
// IsKaganeCoverContentType reports whether a fetched response is safe to store
|
||||
// and serve as a cover.
|
||||
func IsKaganeCoverContentType(contentType string) bool {
|
||||
switch contentType {
|
||||
case "image/webp", "image/jpeg", "image/png", "image/avif", "image/gif":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// CoverURL is the src the web UI puts in an <img>. For every site but kagane
|
||||
// that is Cover as stored. kagane serves its images behind a Cloudflare
|
||||
// challenge *and* with `cross-origin-resource-policy: same-origin`, so no page
|
||||
// on another origin can load one however it asks (verified 2026-08-08); those
|
||||
// go through the backend's own proxy instead.
|
||||
func (b Bookmark) CoverURL() string {
|
||||
if imageID, ok := KaganeImageID(b.Cover); ok {
|
||||
return "/img/kagane/" + imageID
|
||||
}
|
||||
return b.Cover
|
||||
}
|
||||
|
||||
// Library buckets. A bookmark is in exactly one. This cannot be derived from
|
||||
// Site: asurascans serves manga and novels from the same /comics/ path, so the
|
||||
// userscript that recorded the page is the only party that knows which.
|
||||
@@ -159,7 +197,7 @@ var migrations embed.FS
|
||||
// compile-time constant; every request value is bound as a parameter. The
|
||||
// series-owned fields are joined in from the series table, in scanBookmark
|
||||
// order, so the flat Bookmark reads back whole despite the split (ADR-0004).
|
||||
const bookmarkColumns = `b.key, b.site, b.series_id, s.title, s.series_url, s.cover,
|
||||
const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover,
|
||||
b.last_chapter, b.last_chapter_num, b.last_chapter_url,
|
||||
b.favorite, s.latest_chapter, s.latest_chapter_num, b.updated_at, b.status, s.kind`
|
||||
|
||||
@@ -169,32 +207,235 @@ const bookmarkColumns = `b.key, b.site, b.series_id, s.title, s.series_url, s.co
|
||||
const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at`
|
||||
|
||||
// Owner is the person running the service: the first Reader, seeded at startup
|
||||
// so a fresh deployment has a library before anyone logs in. The seed makes
|
||||
// sure exactly one readers row matches their Discord ID, carrying the SHA-256
|
||||
// of their epoch-0 userscript credential (derived by internal/token). Every
|
||||
// other Reader is created by their own first login (EnsureReader).
|
||||
type Owner struct {
|
||||
DiscordID string
|
||||
// TokenHash is the SHA-256 of the epoch-0 credential; the array shape
|
||||
// makes it a compile error to store anything that is not a hash.
|
||||
TokenHash [32]byte
|
||||
}
|
||||
|
||||
// Store is the Postgres-backed bookmark store.
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
// ownerID is the seeded owner Reader (issue #22) — the only Reader with
|
||||
// administrative reach (revoking another Reader's sessions). Every store
|
||||
// method takes a reader id explicitly, so ownership is never implicit.
|
||||
ownerID int64
|
||||
}
|
||||
|
||||
// OwnerID returns the seeded owner Reader's id: the administrator, and the
|
||||
// Reader every pre-registration bookmark belongs to.
|
||||
func (s *Store) OwnerID() int64 { return s.ownerID }
|
||||
|
||||
// ReaderIDForTokenHash resolves the Reader whose stored credential hash
|
||||
// matches, reporting absence with ok=false. The comparison is an equality on
|
||||
// the 32-byte SHA-256 of the presented credential — never on the credential
|
||||
// itself — and the indexed lookup reveals only whether some Reader matches,
|
||||
// which the 401/200 split has to reveal anyway. An attacker's probe is the
|
||||
// hash of their guess, so even the index's prefix comparisons leak nothing
|
||||
// about the real credential.
|
||||
func (s *Store) ReaderIDForTokenHash(hash [32]byte) (int64, bool, error) {
|
||||
var id int64
|
||||
err := s.db.QueryRow(
|
||||
`SELECT id FROM readers WHERE token_sha256 = $1`, hash[:]).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("reader by token hash: %w", err)
|
||||
}
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// ReaderTokenInfo returns the identity halves a Reader's credential is
|
||||
// derived from (internal/token.Token): their Discord id and token epoch. The
|
||||
// web UI needs these to rebuild the install URL — the only place a credential
|
||||
// is ever produced in plaintext.
|
||||
func (s *Store) ReaderTokenInfo(readerID int64) (string, int64, error) {
|
||||
var (
|
||||
discordID string
|
||||
epoch int64
|
||||
)
|
||||
err := s.db.QueryRow(
|
||||
`SELECT discord_id, token_epoch FROM readers WHERE id = $1`, readerID).
|
||||
Scan(&discordID, &epoch)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("reader %d token info: %w", readerID, err)
|
||||
}
|
||||
return discordID, epoch, nil
|
||||
}
|
||||
|
||||
// RotateToken bumps a Reader's token epoch and rewrites the stored hash in
|
||||
// one statement, so the new hash always matches the new epoch. expectedEpoch
|
||||
// is the epoch the caller derived newHash for (ReaderTokenInfo + 1); a
|
||||
// concurrent rotation — or an unknown reader — leaves the row untouched and
|
||||
// is reported as an error rather than silently succeeding.
|
||||
func (s *Store) RotateToken(readerID, expectedEpoch int64, newHash [32]byte) error {
|
||||
var epoch int64
|
||||
err := s.db.QueryRow(`
|
||||
UPDATE readers SET token_epoch = token_epoch + 1, token_sha256 = $3
|
||||
WHERE id = $1 AND token_epoch = $2
|
||||
RETURNING token_epoch`, readerID, expectedEpoch, newHash[:]).Scan(&epoch)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fmt.Errorf("rotate token for reader %d: concurrent rotation or unknown reader", readerID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("rotate token for reader %d: %w", readerID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureReader returns the Reader registered to discordID, creating the row on
|
||||
// first sight. Registration is open to every guild member (issue #27), and the
|
||||
// Discord identity is the only thing that decides which Reader a login is: one
|
||||
// code path serves the first login and every later one, so a returning Reader
|
||||
// can never end up with a second library.
|
||||
//
|
||||
// epochZeroHash is only used for a brand-new row. An existing row keeps its
|
||||
// stored hash untouched, or a login would silently undo a rotation and revive
|
||||
// the credential the Reader rotated away from.
|
||||
func (s *Store) EnsureReader(discordID string, epochZeroHash [32]byte) (int64, error) {
|
||||
var id int64
|
||||
// DO UPDATE rather than DO NOTHING because only an updated row is
|
||||
// returned by RETURNING; assigning the column to itself is the no-op that
|
||||
// makes the existing id come back.
|
||||
err := s.db.QueryRow(`
|
||||
INSERT INTO readers (discord_id, token_sha256) VALUES ($1, $2)
|
||||
ON CONFLICT (discord_id) DO UPDATE SET discord_id = readers.discord_id
|
||||
RETURNING id`, discordID, epochZeroHash[:]).Scan(&id)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ensure reader: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// ReaderSummary is one Reader as the owner's administration panel sees them:
|
||||
// who they are and how many live sessions they hold. No credential material,
|
||||
// hashed or otherwise, is exposed.
|
||||
type ReaderSummary struct {
|
||||
ID int64
|
||||
DiscordID string
|
||||
// Sessions counts unexpired session rows — what the owner revokes.
|
||||
Sessions int
|
||||
}
|
||||
|
||||
// Readers lists every Reader with their live session count, oldest first, so
|
||||
// the owner row (always the oldest) heads the list.
|
||||
func (s *Store) Readers() ([]ReaderSummary, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT r.id, r.discord_id,
|
||||
count(sess.id) FILTER (WHERE sess.expires_at > now()) AS sessions
|
||||
FROM readers r
|
||||
LEFT JOIN sessions sess ON sess.reader_id = r.id
|
||||
GROUP BY r.id, r.discord_id
|
||||
ORDER BY r.id`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query readers: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []ReaderSummary{}
|
||||
for rows.Next() {
|
||||
var r ReaderSummary
|
||||
if err := rows.Scan(&r.ID, &r.DiscordID, &r.Sessions); err != nil {
|
||||
return nil, fmt.Errorf("scan reader: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// readersMigration is the version that creates the readers table. The owner
|
||||
// seed runs between two migrate passes, so that the run-once migration which
|
||||
// attaches existing bookmarks (0004) finds the owner row.
|
||||
const readersMigration = 3
|
||||
|
||||
// allMigrations is the migrate() cap that applies every pending version.
|
||||
const allMigrations = 0
|
||||
|
||||
// Open connects to Postgres at url — a libpq connection URL such as
|
||||
// "postgres://user:pass@host:5432/bookmarks?sslmode=disable" — and brings its
|
||||
// schema up to date.
|
||||
func Open(url string) (*Store, error) {
|
||||
// "postgres://user:pass@host:5432/bookmarks?sslmode=disable" — brings its
|
||||
// schema up to date, and seeds the owner Reader.
|
||||
func Open(url string, owner Owner) (*Store, error) {
|
||||
db, err := sql.Open("pgx", url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open postgres: %w", err)
|
||||
}
|
||||
if err := migrate(db); err != nil {
|
||||
// Schema runs in two passes with the seed between: 0003 creates the
|
||||
// readers table, the owner row must exist before 0004 attaches the
|
||||
// existing bookmarks to it. Anything past 0004 is applied by the second
|
||||
// pass.
|
||||
if err := migrate(db, readersMigration); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("migrate schema: %w", err)
|
||||
}
|
||||
// The owner row must exist before 0004 attaches the existing bookmarks to
|
||||
// it. The hash refresh is a separate statement after all migrations: the
|
||||
// token_epoch column 0006 adds does not exist yet at this point, and the
|
||||
// refresh only ever concerns rows that have never been rotated.
|
||||
if err := seedOwner(db, owner); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("seed owner: %w", err)
|
||||
}
|
||||
if err := migrate(db, allMigrations); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("migrate: %w", err)
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
if err := refreshOwnerToken(db, owner); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("refresh owner token: %w", err)
|
||||
}
|
||||
var ownerID int64
|
||||
if err := db.QueryRow(
|
||||
`SELECT id FROM readers WHERE discord_id = $1`, owner.DiscordID).Scan(&ownerID); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("resolve owner: %w", err)
|
||||
}
|
||||
return &Store{db: db, ownerID: ownerID}, nil
|
||||
}
|
||||
|
||||
// seedOwner makes sure the configured owner exists as exactly one readers row.
|
||||
// The hash is only ever written here for a brand-new row; existing rows keep
|
||||
// what they have until refreshOwnerToken decides otherwise, so the seed can
|
||||
// never clobber a rotation.
|
||||
func seedOwner(db *sql.DB, o Owner) error {
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO readers (discord_id, token_sha256) VALUES ($1, $2)
|
||||
ON CONFLICT (discord_id) DO NOTHING`,
|
||||
o.DiscordID, o.TokenHash[:]); err != nil {
|
||||
return fmt.Errorf("seed owner: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// refreshOwnerToken brings a never-rotated owner row's hash current with the
|
||||
// configured credential. That is the cutover path: a database seeded under
|
||||
// the retired global token still carries its hash at epoch 0, and the
|
||||
// epoch-0 derivation is the caller's TokenHash. A rotated row (epoch > 0) is
|
||||
// left alone — a restart must not resurrect the old credential by
|
||||
// overwriting the hash a rotation wrote.
|
||||
func refreshOwnerToken(db *sql.DB, o Owner) error {
|
||||
if _, err := db.Exec(`
|
||||
UPDATE readers SET token_sha256 = $2
|
||||
WHERE discord_id = $1 AND token_epoch = 0`,
|
||||
o.DiscordID, o.TokenHash[:]); err != nil {
|
||||
return fmt.Errorf("refresh owner token: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrate applies every embedded migration this database has not recorded, in
|
||||
// filename order, each in its own transaction. Files are named
|
||||
// "<version>_<name>.sql" and are append-only: editing an applied file changes
|
||||
// nothing, because schema_migrations is how a database remembers what it ran.
|
||||
// Runs on every start and is a no-op once current.
|
||||
func migrate(db *sql.DB) error {
|
||||
// filename order, each in its own transaction. upto caps the highest version
|
||||
// applied; 0 means all. Files are named "<version>_<name>.sql" and are
|
||||
// append-only: editing an applied file changes nothing, because
|
||||
// schema_migrations is how a database remembers what it ran. Runs on every
|
||||
// start and is a no-op once current.
|
||||
func migrate(db *sql.DB, upto int64) error {
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version bigint PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now())`); err != nil {
|
||||
@@ -212,6 +453,9 @@ func migrate(db *sql.DB) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration %q: filename must start with a version number", name)
|
||||
}
|
||||
if upto > 0 && version > upto {
|
||||
continue
|
||||
}
|
||||
body, err := migrations.ReadFile(name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -261,7 +505,7 @@ func scanBookmark(scan func(...any) error) (Bookmark, error) {
|
||||
latestChapterNum sql.NullFloat64
|
||||
)
|
||||
if err := scan(
|
||||
&b.Key, &b.Site, &b.SeriesID, &b.Title, &b.SeriesURL, &b.Cover,
|
||||
&b.Site, &b.SeriesID, &b.Title, &b.SeriesURL, &b.Cover,
|
||||
&b.LastChapter, &b.LastChapterNum, &b.LastChapterURL,
|
||||
&b.Favorite, &b.LatestChapter, &latestChapterNum, &b.UpdatedAt, &b.Status, &b.Kind,
|
||||
); err != nil {
|
||||
@@ -270,6 +514,9 @@ func scanBookmark(scan func(...any) error) (Bookmark, error) {
|
||||
if latestChapterNum.Valid {
|
||||
b.LatestChapterNum = &latestChapterNum.Float64
|
||||
}
|
||||
// The wire identity is derived: there is no stored key column, the
|
||||
// bookmark is keyed (reader_id, site, series_id) (issue #22).
|
||||
b.Key = b.Site + ":" + b.SeriesID
|
||||
// An unrecognised bucket (a hand-edited row) would leave the row in no list
|
||||
// at all, so anything outside the three known buckets reads as the default
|
||||
// rather than being passed through.
|
||||
@@ -303,13 +550,49 @@ func scanSeries(scan func(...any) error) (Series, error) {
|
||||
// Close releases the underlying database handle.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// List returns every bookmark, newest activity first. Series-owned fields are
|
||||
// joined in, so each Bookmark reads back whole and flat (ADR-0004).
|
||||
func (s *Store) List() ([]Bookmark, error) {
|
||||
rows, err := s.db.Query(`SELECT ` + bookmarkColumns + `
|
||||
// GetKaganeCover returns one persisted cover. Missing covers are reported with
|
||||
// ok=false rather than as an error so the web handler can fetch them once.
|
||||
func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) {
|
||||
var (
|
||||
body []byte
|
||||
contentType string
|
||||
)
|
||||
err := s.db.QueryRow(
|
||||
`SELECT body, content_type FROM covers WHERE image_id = $1`, imageID,
|
||||
).Scan(&body, &contentType)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("get kagane cover %q: %w", imageID, err)
|
||||
}
|
||||
return body, contentType, true, nil
|
||||
}
|
||||
|
||||
// PutKaganeCover persists one fetched cover. Image ids are immutable, so a
|
||||
// later fetch cannot replace the bytes already made durable.
|
||||
func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error {
|
||||
if !IsKaganeCoverContentType(contentType) {
|
||||
return fmt.Errorf("put kagane cover %q: unsupported content type %q", imageID, contentType)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO covers (image_id, body, content_type)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (image_id) DO NOTHING`, imageID, body, contentType); err != nil {
|
||||
return fmt.Errorf("put kagane cover %q: %w", imageID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns every bookmark of one reader, newest activity first.
|
||||
// Series-owned fields are joined in, so each Bookmark reads back whole and
|
||||
// flat (ADR-0004).
|
||||
func (s *Store) List(readerID int64) ([]Bookmark, error) {
|
||||
rows, err := s.db.Query(`SELECT `+bookmarkColumns+`
|
||||
FROM bookmarks b
|
||||
JOIN series s ON s.site = b.site AND s.series_id = b.series_id
|
||||
ORDER BY b.updated_at DESC`)
|
||||
WHERE b.reader_id = $1
|
||||
ORDER BY b.updated_at DESC`, readerID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query bookmarks: %w", err)
|
||||
}
|
||||
@@ -326,14 +609,19 @@ func (s *Store) List() ([]Bookmark, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Get returns one bookmark by key. A missing key is not an error: ok is false
|
||||
// and err is nil. UI mutations read-modify-write through this so they preserve
|
||||
// the fields they do not touch.
|
||||
func (s *Store) Get(key string) (Bookmark, bool, error) {
|
||||
// Get returns one bookmark of one reader by key. A missing key is not an
|
||||
// error: ok is false and err is nil. UI mutations read-modify-write through
|
||||
// this so they preserve the fields they do not touch.
|
||||
func (s *Store) Get(readerID int64, key string) (Bookmark, bool, error) {
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
return Bookmark{}, false, nil
|
||||
}
|
||||
b, err := scanBookmark(s.db.QueryRow(
|
||||
`SELECT `+bookmarkColumns+` FROM bookmarks b
|
||||
JOIN series s ON s.site = b.site AND s.series_id = b.series_id
|
||||
WHERE b.key = $1`, key).Scan)
|
||||
WHERE b.reader_id = $1 AND b.site = $2 AND b.series_id = $3`,
|
||||
readerID, site, seriesID).Scan)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Bookmark{}, false, nil
|
||||
}
|
||||
@@ -343,9 +631,11 @@ func (s *Store) Get(key string) (Bookmark, bool, error) {
|
||||
return b, true, nil
|
||||
}
|
||||
|
||||
// Upsert inserts or replaces a bookmark by key (last-write-wins) and returns
|
||||
// the row as actually stored — one flat object with the series-owned fields
|
||||
// joined in, exactly as GET reports it (ADR-0004).
|
||||
// Upsert inserts or replaces one reader's bookmark by key (last-write-wins)
|
||||
// and returns the row as actually stored — one flat object with the
|
||||
// series-owned fields joined in, exactly as GET reports it (ADR-0004). A
|
||||
// bookmark is keyed (reader_id, site, series_id), so the same key upserts two
|
||||
// independent rows for two readers.
|
||||
//
|
||||
// The flat body is decomposed across two tables in one transaction. The series
|
||||
// row is written first (the bookmarks FK requires it to exist), then the
|
||||
@@ -359,7 +649,7 @@ func (s *Store) Get(key string) (Bookmark, bool, error) {
|
||||
// order their list by updated_at, so favoriting a series or recording a newly
|
||||
// published chapter must not disturb that order — only real reading progress
|
||||
// does. Callers must therefore use the returned bookmark, not the argument.
|
||||
func (s *Store) Upsert(b Bookmark) (Bookmark, error) {
|
||||
func (s *Store) Upsert(readerID int64, b Bookmark) (Bookmark, error) {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return Bookmark{}, fmt.Errorf("begin %q: %w", b.Key, err)
|
||||
@@ -404,13 +694,12 @@ func (s *Store) Upsert(b Bookmark) (Bookmark, error) {
|
||||
// stored row and excluded.* is the incoming one; a brand-new key never
|
||||
// reaches this clause, so it keeps the fresh timestamp from VALUES.
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO bookmarks (key, site, series_id, last_chapter, last_chapter_num,
|
||||
INSERT INTO bookmarks (reader_id, site, series_id, last_chapter, last_chapter_num,
|
||||
last_chapter_url, favorite, status, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7,
|
||||
COALESCE(NULLIF($8::text, ''), (SELECT status FROM bookmarks WHERE key = $1), 'reading'),
|
||||
COALESCE(NULLIF($8::text, ''), (SELECT status FROM bookmarks WHERE reader_id = $1 AND site = $2 AND series_id = $3), 'reading'),
|
||||
$9)
|
||||
ON CONFLICT (key) DO UPDATE SET
|
||||
site=excluded.site, series_id=excluded.series_id,
|
||||
ON CONFLICT (reader_id, site, series_id) DO UPDATE SET
|
||||
last_chapter=excluded.last_chapter, last_chapter_num=excluded.last_chapter_num,
|
||||
last_chapter_url=excluded.last_chapter_url,
|
||||
favorite=excluded.favorite,
|
||||
@@ -420,7 +709,7 @@ func (s *Store) Upsert(b Bookmark) (Bookmark, error) {
|
||||
THEN excluded.updated_at
|
||||
ELSE bookmarks.updated_at
|
||||
END`,
|
||||
b.Key, b.Site, b.SeriesID,
|
||||
readerID, b.Site, b.SeriesID,
|
||||
b.LastChapter, b.LastChapterNum, b.LastChapterURL,
|
||||
b.Favorite, b.Status, b.UpdatedAt); err != nil {
|
||||
return Bookmark{}, fmt.Errorf("upsert %q: %w", b.Key, err)
|
||||
@@ -429,7 +718,8 @@ func (s *Store) Upsert(b Bookmark) (Bookmark, error) {
|
||||
stored, err := scanBookmark(tx.QueryRow(
|
||||
`SELECT `+bookmarkColumns+` FROM bookmarks b
|
||||
JOIN series s ON s.site = b.site AND s.series_id = b.series_id
|
||||
WHERE b.key = $1`, b.Key).Scan)
|
||||
WHERE b.reader_id = $1 AND b.site = $2 AND b.series_id = $3`,
|
||||
readerID, b.Site, b.SeriesID).Scan)
|
||||
if err != nil {
|
||||
return Bookmark{}, fmt.Errorf("read back %q: %w", b.Key, err)
|
||||
}
|
||||
@@ -439,25 +729,33 @@ func (s *Store) Upsert(b Bookmark) (Bookmark, error) {
|
||||
return stored, nil
|
||||
}
|
||||
|
||||
// Delete removes a bookmark by key. Deleting a missing key is not an error.
|
||||
func (s *Store) Delete(key string) error {
|
||||
if _, err := s.db.Exec(`DELETE FROM bookmarks WHERE key = $1`, key); err != nil {
|
||||
// Delete removes one reader's bookmark by key. Deleting a missing key is not
|
||||
// an error.
|
||||
func (s *Store) Delete(readerID int64, key string) error {
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.db.Exec(
|
||||
`DELETE FROM bookmarks WHERE reader_id = $1 AND site = $2 AND series_id = $3`,
|
||||
readerID, site, seriesID); err != nil {
|
||||
return fmt.Errorf("delete %q: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DueForLatestCheck returns series whose server-side latest-chapter check has
|
||||
// aged past cutoffMs, ordered by how many bookmarks reference them (descending)
|
||||
// then least-recently-checked first, at most limit of them.
|
||||
// aged past the appropriate cutoff, ordered by how many bookmarks reference
|
||||
// them (descending) then least-recently-checked first, at most limit of them.
|
||||
// Browser-backed sites use browserCutoffMs; every other site uses cutoffMs.
|
||||
//
|
||||
// The reader_count ordering is the point of the split (ADR-0003): a series
|
||||
// shared by several readers is fetched once per due cycle, and the popular
|
||||
// ones stay freshest while the long tail absorbs any shortfall. Within one
|
||||
// reader count, oldest-first keeps the poll fair when the backlog outgrows its
|
||||
// reader count, oldest-first keeps the poll fair when the backlog outgrows
|
||||
// throughput: the most neglected series is always next, so a large collection
|
||||
// refreshes uniformly slower rather than leaving a tail that never refreshes at
|
||||
// all. The userscript sorts its own queue the same way (L453).
|
||||
// refreshes uniformly slower rather than leaving a tail that never refreshes
|
||||
// at all. The userscript sorts its own queue the same way (L453).
|
||||
//
|
||||
// Series with no series_url are skipped — there is nothing to fetch, which is
|
||||
// the same filter the userscript applies at L452. Series whose only bookmarks
|
||||
@@ -465,17 +763,20 @@ func (s *Store) Delete(key string) error {
|
||||
// burns requests. Archived bookmarks still count — knowing what a shelved
|
||||
// series is up to is the whole reason for archiving instead of deleting.
|
||||
// A series with no bookmarks at all never appears: the join excludes it.
|
||||
func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Series, error) {
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(b.key) AS reader_count
|
||||
func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites []string, limit int) ([]Series, error) {
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
|
||||
FROM series s
|
||||
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
WHERE s.series_url <> ''
|
||||
AND s.latest_checked_at <= $1
|
||||
AND s.latest_checked_at <= CASE
|
||||
WHEN s.site = ANY($3::text[]) THEN $2::bigint
|
||||
ELSE $1::bigint
|
||||
END
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
|
||||
HAVING COUNT(b.key) FILTER (WHERE b.status <> 'finished') > 0
|
||||
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
ORDER BY reader_count DESC, s.latest_checked_at ASC
|
||||
LIMIT $2`, cutoffMs, limit)
|
||||
LIMIT $4`, cutoffMs, browserCutoffMs, pgtype.FlatArray[string](browserSites), limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query due series: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -12,9 +15,13 @@ import (
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
|
||||
// testOwner is the owner every test store seeds. Tests that need a second
|
||||
// reader register one (see secondReader).
|
||||
var testOwner = Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}
|
||||
|
||||
func newTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
store, err := Open(pgtest.URL(t))
|
||||
store, err := Open(pgtest.URL(t), testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
@@ -22,29 +29,41 @@ func newTestStore(t *testing.T) *Store {
|
||||
return store
|
||||
}
|
||||
|
||||
// secondReader registers an extra reader through the same path a first login
|
||||
// takes, and returns its id.
|
||||
func secondReader(t *testing.T, s *Store) int64 {
|
||||
t.Helper()
|
||||
discordID := "second-" + strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
id, err := s.EnsureReader(discordID, sha256.Sum256([]byte("token-"+discordID)))
|
||||
if err != nil {
|
||||
t.Fatalf("register second reader: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// The migration runner runs on every start, so a second Open against a
|
||||
// database it already built must be a no-op rather than a duplicate-table
|
||||
// error, and must leave the rows alone.
|
||||
func TestOpenIsIdempotent(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
first, err := Open(url)
|
||||
first, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
if _, err := first.Upsert(Bookmark{
|
||||
if _, err := first.Upsert(first.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
first.Close()
|
||||
|
||||
second, err := Open(url)
|
||||
second, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { second.Close() })
|
||||
|
||||
list, err := second.List()
|
||||
list, err := second.List(second.OwnerID())
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
@@ -53,16 +72,102 @@ func TestOpenIsIdempotent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The hash lookup is the whole authentication path: the store resolves a
|
||||
// Reader from the SHA-256 of their presented credential, and nothing else.
|
||||
func TestReaderIDForTokenHash(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
ownerHash := sha256.Sum256([]byte("owner-token-hash"))
|
||||
|
||||
id, ok, err := store.ReaderIDForTokenHash(ownerHash)
|
||||
if err != nil {
|
||||
t.Fatalf("ReaderIDForTokenHash: %v", err)
|
||||
}
|
||||
if !ok || id != store.OwnerID() {
|
||||
t.Fatalf("owner lookup = (%d, %v), want (%d, true)", id, ok, store.OwnerID())
|
||||
}
|
||||
|
||||
if _, ok, err := store.ReaderIDForTokenHash(sha256.Sum256([]byte("nope"))); err != nil {
|
||||
t.Fatalf("miss: %v", err)
|
||||
} else if ok {
|
||||
t.Fatal("unknown hash resolved to a Reader")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderTokenInfo(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
discordID, epoch, err := store.ReaderTokenInfo(store.OwnerID())
|
||||
if err != nil {
|
||||
t.Fatalf("ReaderTokenInfo: %v", err)
|
||||
}
|
||||
if discordID != testOwner.DiscordID || epoch != 0 {
|
||||
t.Fatalf("ReaderTokenInfo = (%q, %d), want (%q, 0)", discordID, epoch, testOwner.DiscordID)
|
||||
}
|
||||
}
|
||||
|
||||
// Rotation swaps the stored hash and bumps the epoch in one step, and the
|
||||
// seed must not undo it: a restart re-runs seedOwner, which refreshes the
|
||||
// epoch-0 hash only while the row has never been rotated.
|
||||
func TestRotateTokenInvalidatesOldAndSurvivesRestart(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
store, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
|
||||
oldHash := sha256.Sum256([]byte("owner-token-hash"))
|
||||
newHash := sha256.Sum256([]byte("rotated-token-hash"))
|
||||
if err := store.RotateToken(store.OwnerID(), 0, newHash); err != nil {
|
||||
t.Fatalf("RotateToken: %v", err)
|
||||
}
|
||||
// A second rotation against the stale epoch is refused: the stored hash
|
||||
// must never describe a different epoch than the column says.
|
||||
if err := store.RotateToken(store.OwnerID(), 0, sha256.Sum256([]byte("third-hash"))); err == nil {
|
||||
t.Fatal("stale-epoch rotation succeeded, want error")
|
||||
}
|
||||
if _, ok, err := store.ReaderIDForTokenHash(oldHash); err != nil {
|
||||
t.Fatalf("old lookup: %v", err)
|
||||
} else if ok {
|
||||
t.Fatal("old hash still resolves after rotation")
|
||||
}
|
||||
if id, ok, err := store.ReaderIDForTokenHash(newHash); err != nil {
|
||||
t.Fatalf("new lookup: %v", err)
|
||||
} else if !ok || id != store.OwnerID() {
|
||||
t.Fatalf("new hash resolved to (%d, %v), want owner", id, ok)
|
||||
}
|
||||
if _, epoch, err := store.ReaderTokenInfo(store.OwnerID()); err != nil {
|
||||
t.Fatalf("ReaderTokenInfo: %v", err)
|
||||
} else if epoch != 1 {
|
||||
t.Fatalf("epoch = %d after rotation, want 1", epoch)
|
||||
}
|
||||
store.Close()
|
||||
|
||||
reopened, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { reopened.Close() })
|
||||
if _, ok, err := reopened.ReaderIDForTokenHash(oldHash); err != nil {
|
||||
t.Fatalf("old lookup after reopen: %v", err)
|
||||
} else if ok {
|
||||
t.Fatal("restart resurrected the pre-rotation hash")
|
||||
}
|
||||
if _, ok, err := reopened.ReaderIDForTokenHash(newHash); err != nil {
|
||||
t.Fatalf("new lookup after reopen: %v", err)
|
||||
} else if !ok {
|
||||
t.Fatal("restart dropped the rotated hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreGet(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
if _, err := store.Upsert(Bookmark{
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", LastChapterNum: 45, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
|
||||
got, ok, err := store.Get("asura:solo")
|
||||
got, ok, err := store.Get(store.OwnerID(), "asura:solo")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
@@ -76,7 +181,7 @@ func TestStoreGet(t *testing.T) {
|
||||
|
||||
func TestStoreGetMissing(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
_, ok, err := store.Get("asura:nope")
|
||||
_, ok, err := store.Get(store.OwnerID(), "asura:nope")
|
||||
if err != nil {
|
||||
t.Fatalf("Get missing returned error %v, want nil", err)
|
||||
}
|
||||
@@ -151,7 +256,7 @@ func seedForCheck(t *testing.T, s *Store, key, seriesURL string, checkedAt int64
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
if _, err := s.Upsert(Bookmark{
|
||||
if _, err := s.Upsert(s.OwnerID(), Bookmark{
|
||||
Key: key,
|
||||
Site: site,
|
||||
SeriesID: seriesID,
|
||||
@@ -186,7 +291,7 @@ func TestDueForLatestCheck(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
|
||||
|
||||
due, err := s.DueForLatestCheck(now-hour, 10)
|
||||
due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -204,7 +309,7 @@ func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
|
||||
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
|
||||
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 2)
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -239,12 +344,12 @@ func TestUpsertPreservesLatestCheckedAt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 999)
|
||||
|
||||
b, ok, err := s.Get("asura:x")
|
||||
b, ok, err := s.Get(s.OwnerID(), "asura:x")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get: %v ok=%v", err, ok)
|
||||
}
|
||||
b.Title = "changed"
|
||||
if _, err := s.Upsert(b); err != nil {
|
||||
if _, err := s.Upsert(s.OwnerID(), b); err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
if got := readLatestCheckedAt(t, s, "asura:x"); got != 999 {
|
||||
@@ -254,7 +359,7 @@ func TestUpsertPreservesLatestCheckedAt(t *testing.T) {
|
||||
|
||||
func TestUpsertDefaultsStatusToReading(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
stored, err := store.Upsert(Bookmark{
|
||||
stored, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
UpdatedAt: time.Now().UnixMilli(),
|
||||
})
|
||||
@@ -274,13 +379,13 @@ func TestUpsertEmptyStatusPreservesStored(t *testing.T) {
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Status: StatusArchived, UpdatedAt: time.Now().UnixMilli(),
|
||||
}
|
||||
if _, err := store.Upsert(base); err != nil {
|
||||
if _, err := store.Upsert(store.OwnerID(), base); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
base.Status = ""
|
||||
base.LastChapterNum = 12
|
||||
stored, err := store.Upsert(base)
|
||||
stored, err := store.Upsert(store.OwnerID(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
@@ -299,11 +404,11 @@ func TestLatestPollRoundTripPreservesArchived(t *testing.T) {
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Status: StatusArchived, UpdatedAt: time.Now().UnixMilli(),
|
||||
}
|
||||
if _, err := store.Upsert(base); err != nil {
|
||||
if _, err := store.Upsert(store.OwnerID(), base); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
cur, found, err := store.Get(base.Key)
|
||||
cur, found, err := store.Get(store.OwnerID(), base.Key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: found=%v err=%v", found, err)
|
||||
}
|
||||
@@ -313,7 +418,7 @@ func TestLatestPollRoundTripPreservesArchived(t *testing.T) {
|
||||
cur.LatestChapterNum = &num
|
||||
cur.UpdatedAt = time.Now().UnixMilli()
|
||||
|
||||
stored, err := store.Upsert(cur)
|
||||
stored, err := store.Upsert(store.OwnerID(), cur)
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
@@ -328,12 +433,12 @@ func TestUpsertReplacesStatusWhenGiven(t *testing.T) {
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Status: StatusArchived, UpdatedAt: time.Now().UnixMilli(),
|
||||
}
|
||||
if _, err := store.Upsert(base); err != nil {
|
||||
if _, err := store.Upsert(store.OwnerID(), base); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
base.Status = StatusReading
|
||||
stored, err := store.Upsert(base)
|
||||
stored, err := store.Upsert(store.OwnerID(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
@@ -350,14 +455,14 @@ func TestUpsertStatusChangeKeepsUpdatedAt(t *testing.T) {
|
||||
LastChapter: "45", LastChapterNum: 45,
|
||||
UpdatedAt: time.Now().UnixMilli(),
|
||||
}
|
||||
first, err := store.Upsert(base)
|
||||
first, err := store.Upsert(store.OwnerID(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
base.Status = StatusArchived
|
||||
base.UpdatedAt = first.UpdatedAt + 60_000
|
||||
stored, err := store.Upsert(base)
|
||||
stored, err := store.Upsert(store.OwnerID(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
@@ -375,7 +480,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
{"asura:archived", StatusArchived},
|
||||
{"asura:finished", StatusFinished},
|
||||
} {
|
||||
if _, err := store.Upsert(Bookmark{
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: tc.key, Site: "asura", SeriesID: strings.TrimPrefix(tc.key, "asura:"),
|
||||
SeriesURL: "https://asurascans.com/comics/" + tc.key,
|
||||
Status: tc.status, UpdatedAt: time.Now().UnixMilli(),
|
||||
@@ -384,7 +489,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), 10)
|
||||
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -438,9 +543,41 @@ func TestDisplayChapter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
cover string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"kagane routes through the proxy",
|
||||
"https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
||||
"/img/kagane/019fe11a-84c3-7fc3-a84b-88787374b617",
|
||||
},
|
||||
{
|
||||
"another site is served as stored",
|
||||
"https://gg.asuracomic.net/storage/media/1/conversions/cover.webp",
|
||||
"https://gg.asuracomic.net/storage/media/1/conversions/cover.webp",
|
||||
},
|
||||
{
|
||||
"a lookalike host is not rewritten",
|
||||
"https://evil.example/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
||||
"https://evil.example/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
||||
},
|
||||
{"no cover stays empty", "", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := (Bookmark{Cover: tc.cover}).CoverURL(); got != tc.want {
|
||||
t.Errorf("CoverURL() = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertKindDefaultsToManga(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
got, err := store.Upsert(Bookmark{
|
||||
got, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -453,7 +590,7 @@ func TestUpsertKindDefaultsToManga(t *testing.T) {
|
||||
|
||||
func TestUpsertKindRoundTrips(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
got, err := store.Upsert(Bookmark{
|
||||
got, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "lightnovelworld:a-will-eternal", Site: "lightnovelworld",
|
||||
SeriesID: "a-will-eternal", Kind: KindNovel, UpdatedAt: 1000,
|
||||
})
|
||||
@@ -469,14 +606,14 @@ func TestUpsertKindRoundTrips(t *testing.T) {
|
||||
// must keep the stored library, not silently demote a novel to manga.
|
||||
func TestUpsertEmptyKindKeepsStoredValue(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
if _, err := store.Upsert(Bookmark{
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "lightnovelworld:a-will-eternal", Site: "lightnovelworld",
|
||||
SeriesID: "a-will-eternal", Kind: KindNovel, LastChapterNum: 10, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.Upsert(Bookmark{
|
||||
got, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "lightnovelworld:a-will-eternal", Site: "lightnovelworld",
|
||||
SeriesID: "a-will-eternal", Kind: "", LastChapterNum: 11, UpdatedAt: 2000,
|
||||
})
|
||||
@@ -528,10 +665,14 @@ func TestMigration0002BackfillsExistingBookmarks(t *testing.T) {
|
||||
t.Fatalf("seed legacy row: %v", err)
|
||||
}
|
||||
|
||||
// Bring it current: 0002 must backfill the series row, not lose data.
|
||||
if err := migrate(db); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
// Bring it current through the production path: Open runs the schema to
|
||||
// 0003, seeds the owner, then applies 0004 which attaches this row. 0002
|
||||
// must have backfilled the series row, not lost data.
|
||||
st, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open after migrate: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
var (
|
||||
title string
|
||||
checked int64
|
||||
@@ -545,20 +686,15 @@ func TestMigration0002BackfillsExistingBookmarks(t *testing.T) {
|
||||
if title != "Solo Leveling" || checked != 123456 {
|
||||
t.Fatalf("series = (%q, %d), want backfilled title and latest_checked_at", title, checked)
|
||||
}
|
||||
// The key column is gone; the bookmark is read by its composite key.
|
||||
if err := db.QueryRow(`SELECT favorite, last_chapter_num FROM bookmarks
|
||||
WHERE key = 'asura:solo'`).Scan(&fav, &lastNum); err != nil {
|
||||
WHERE reader_id = $1 AND site = 'asura' AND series_id = 'solo'`,
|
||||
st.OwnerID()).Scan(&fav, &lastNum); err != nil {
|
||||
t.Fatalf("bookmark row missing after migration: %v", err)
|
||||
}
|
||||
if !fav || lastNum != 10 {
|
||||
t.Fatalf("bookmark = (%v, %v), want favorite and progress kept", fav, lastNum)
|
||||
}
|
||||
|
||||
// The migrated database opens as a normal store.
|
||||
st, err := Open(url)
|
||||
if err != nil {
|
||||
t.Fatalf("Open after migrate: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
}
|
||||
|
||||
// readSeries reads the series row directly, for asserting on what Upsert
|
||||
@@ -578,7 +714,7 @@ func readSeries(t *testing.T, s *Store, site, seriesID string) Series {
|
||||
// and URL — there is no other source for them (ADR-0003).
|
||||
func TestUpsertCreatesSeriesFromClient(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
if _, err := store.Upsert(Bookmark{
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", SeriesURL: "https://asurascans.com/comics/solo",
|
||||
Cover: "https://asurascans.com/covers/solo.jpg", Kind: KindManga,
|
||||
@@ -604,7 +740,7 @@ func TestUpsertExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
|
||||
Cover: "https://asurascans.com/covers/solo.jpg", LastChapterNum: 10,
|
||||
UpdatedAt: 1000,
|
||||
}
|
||||
if _, err := store.Upsert(base); err != nil {
|
||||
if _, err := store.Upsert(store.OwnerID(), base); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
@@ -613,7 +749,7 @@ func TestUpsertExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
|
||||
base.SeriesURL = "https://evil.example/solo"
|
||||
base.Cover = "https://evil.example/solo.jpg"
|
||||
base.LastChapterNum = 11
|
||||
got, err := store.Upsert(base)
|
||||
got, err := store.Upsert(store.OwnerID(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
@@ -635,7 +771,7 @@ func TestUpsertExistingSeriesAcceptsKindAndLatest(t *testing.T) {
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", Kind: KindManga,
|
||||
UpdatedAt: 1000,
|
||||
}
|
||||
if _, err := store.Upsert(base); err != nil {
|
||||
if _, err := store.Upsert(store.OwnerID(), base); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
@@ -643,7 +779,7 @@ func TestUpsertExistingSeriesAcceptsKindAndLatest(t *testing.T) {
|
||||
base.Kind = KindNovel
|
||||
base.LatestChapter = "Chapter 12"
|
||||
base.LatestChapterNum = &num
|
||||
got, err := store.Upsert(base)
|
||||
got, err := store.Upsert(store.OwnerID(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
@@ -657,14 +793,14 @@ func TestUpsertExistingSeriesAcceptsKindAndLatest(t *testing.T) {
|
||||
// re-bookmark shows title and cover immediately instead of waiting for a poll.
|
||||
func TestDeleteKeepsSeriesRow(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
if _, err := store.Upsert(Bookmark{
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", Cover: "https://asurascans.com/covers/solo.jpg",
|
||||
UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := store.Delete("asura:solo"); err != nil {
|
||||
if err := store.Delete(store.OwnerID(), "asura:solo"); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
|
||||
@@ -674,7 +810,7 @@ func TestDeleteKeepsSeriesRow(t *testing.T) {
|
||||
}
|
||||
|
||||
// Re-bookmark with nothing but progress: the stored title/cover come back.
|
||||
stored, err := store.Upsert(Bookmark{
|
||||
stored, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
LastChapterNum: 5, UpdatedAt: 2000,
|
||||
})
|
||||
@@ -686,13 +822,12 @@ func TestDeleteKeepsSeriesRow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// seedSecondReader inserts an extra bookmark on an existing series. Today the
|
||||
// bookmark key is <site>:<series_id>, so two bookmarks can share a series only
|
||||
// once keys stop being derived from the series identity (issue #22); the due
|
||||
// queue's reader-count ordering must already be right for that world.
|
||||
// seedSecondReader inserts an extra bookmark on an existing series, owned by a
|
||||
// second reader. Two bookmarks can share a series only across readers now
|
||||
// (issue #22); the due queue's reader-count ordering counts them all.
|
||||
func seedSecondReader(t *testing.T, s *Store, key, site, seriesID string, updatedAt int64) {
|
||||
t.Helper()
|
||||
if _, err := s.Upsert(Bookmark{
|
||||
if _, err := s.Upsert(secondReader(t, s), Bookmark{
|
||||
Key: key, Site: site, SeriesID: seriesID, UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader %q: %v", key, err)
|
||||
@@ -709,7 +844,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
|
||||
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 10)
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -735,7 +870,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
|
||||
t.Fatalf("seed orphan series: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 10)
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -751,3 +886,375 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
|
||||
t.Fatalf("orphan series count = %d, want 1 (never deleted)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The seed must never multiply the owner row: reopening the same database with
|
||||
// a different token hash refreshes the stored hash, not the row. That is what
|
||||
// keeps the readers table at exactly one row across restarts and token
|
||||
// rotations.
|
||||
func TestSeedOwnerIdempotentAndRefreshesTokenHash(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
first, err := Open(url, Owner{DiscordID: "owner", TokenHash: sha256.Sum256([]byte("hash-v1"))})
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
ownerID := first.OwnerID()
|
||||
first.Close()
|
||||
|
||||
second, err := Open(url, Owner{DiscordID: "owner", TokenHash: sha256.Sum256([]byte("hash-v2"))})
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer second.Close()
|
||||
if second.OwnerID() != ownerID {
|
||||
t.Fatalf("owner id = %d after reopen, want %d (same row)", second.OwnerID(), ownerID)
|
||||
}
|
||||
var (
|
||||
n int
|
||||
hash []byte
|
||||
)
|
||||
if err := second.db.QueryRow(`SELECT count(*), (SELECT token_sha256 FROM readers LIMIT 1) FROM readers`).Scan(&n, &hash); err != nil {
|
||||
t.Fatalf("read readers: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("readers count = %d, want 1", n)
|
||||
}
|
||||
want := sha256.Sum256([]byte("hash-v2"))
|
||||
if !bytes.Equal(hash, want[:]) {
|
||||
t.Fatalf("token hash = %x, want the refreshed sha256", hash)
|
||||
}
|
||||
}
|
||||
|
||||
// The upgrade path for a deployed database: bookmarks created before readers
|
||||
// existed must all land on the seeded owner, the key column must be gone, and
|
||||
// the same database must be able to hold two readers' bookmarks for one series.
|
||||
func TestMigration0004AttachesBookmarksToOwner(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
db, err := sql.Open("pgx", url)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
// A database at the state before #21 shipped: 0001 applied, bookmarks
|
||||
// keyed by <site>:<series_id>, no series table. Rows land before 0002, the
|
||||
// way a real deployment's data did.
|
||||
if err := migrate(db, 1); err != nil {
|
||||
t.Fatalf("migrate to 0001: %v", err)
|
||||
}
|
||||
for _, key := range []string{"asura:solo", "demonic:catastrophic-necromancer"} {
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO bookmarks (key, site, series_id, updated_at)
|
||||
VALUES ($1, $2, $3, 1000)`, key, site, seriesID); err != nil {
|
||||
t.Fatalf("seed legacy row %q: %v", key, err)
|
||||
}
|
||||
}
|
||||
// 0002 backfills the series rows, as it did in the real upgrade.
|
||||
if err := migrate(db, 2); err != nil {
|
||||
t.Fatalf("migrate to 0002: %v", err)
|
||||
}
|
||||
|
||||
st, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
var (
|
||||
attached int
|
||||
readers int
|
||||
)
|
||||
if err := st.db.QueryRow(
|
||||
`SELECT count(*) FROM bookmarks WHERE reader_id = $1`, st.OwnerID()).Scan(&attached); err != nil {
|
||||
t.Fatalf("count attached bookmarks: %v", err)
|
||||
}
|
||||
if attached != 2 {
|
||||
t.Fatalf("bookmarks attached to owner = %d, want all 2", attached)
|
||||
}
|
||||
if err := st.db.QueryRow(`SELECT count(*) FROM readers`).Scan(&readers); err != nil {
|
||||
t.Fatalf("count readers: %v", err)
|
||||
}
|
||||
if readers != 1 {
|
||||
t.Fatalf("readers = %d, want 1", readers)
|
||||
}
|
||||
// The surrogate key column is gone; only the composite key remains.
|
||||
if _, err := st.db.Query(`SELECT key FROM bookmarks`); err == nil {
|
||||
t.Fatal("bookmarks.key still exists after the migration")
|
||||
}
|
||||
}
|
||||
|
||||
// One Reader and Series pair must admit at most one bookmark, enforced by the
|
||||
// primary key itself — a raw INSERT that skips the upsert must fail.
|
||||
func TestBookmarkDuplicateImpossibleAtDatabaseLevel(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
// A series row on its own, no bookmark: the raw inserts below must only
|
||||
// ever collide on the bookmark primary key.
|
||||
if _, err := st.db.Exec(
|
||||
`INSERT INTO series (site, series_id) VALUES ('asura', 'solo')`); err != nil {
|
||||
t.Fatalf("seed series: %v", err)
|
||||
}
|
||||
insert := func() error {
|
||||
_, err := st.db.Exec(`
|
||||
INSERT INTO bookmarks (reader_id, site, series_id, updated_at)
|
||||
VALUES ($1, 'asura', 'solo', 1000)`, st.OwnerID())
|
||||
return err
|
||||
}
|
||||
if err := insert(); err != nil {
|
||||
t.Fatalf("first insert: %v", err)
|
||||
}
|
||||
if err := insert(); err == nil {
|
||||
t.Fatal("duplicate bookmark for the same reader and series was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Every read and write is scoped to the reader it names: a second reader sees
|
||||
// an empty list, cannot read or delete the owner's row, and a delete by the
|
||||
// wrong reader leaves the row alone.
|
||||
func TestStoreScopesBookmarksToReader(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
other := secondReader(t, st)
|
||||
if _, err := st.Upsert(st.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed owner bookmark: %v", err)
|
||||
}
|
||||
|
||||
otherList, err := st.List(other)
|
||||
if err != nil {
|
||||
t.Fatalf("List(other): %v", err)
|
||||
}
|
||||
if len(otherList) != 0 {
|
||||
t.Fatalf("other reader's list = %+v, want empty", otherList)
|
||||
}
|
||||
if _, ok, err := st.Get(other, "asura:solo"); err != nil || ok {
|
||||
t.Fatalf("Get(other, asura:solo) = ok:%v err:%v, want not found", ok, err)
|
||||
}
|
||||
if err := st.Delete(other, "asura:solo"); err != nil {
|
||||
t.Fatalf("Delete(other): %v", err)
|
||||
}
|
||||
ownerList, err := st.List(st.OwnerID())
|
||||
if err != nil {
|
||||
t.Fatalf("List(owner): %v", err)
|
||||
}
|
||||
if len(ownerList) != 1 || ownerList[0].Key != "asura:solo" {
|
||||
t.Fatalf("owner's list after other's delete = %+v, want the row intact", ownerList)
|
||||
}
|
||||
|
||||
// The same key under a second reader is an independent bookmark.
|
||||
if _, err := st.Upsert(other, Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert other's bookmark: %v", err)
|
||||
}
|
||||
if got, err := st.List(other); err != nil || len(got) != 1 {
|
||||
t.Fatalf("other's list after own upsert = %+v err:%v, want 1 row", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting a reader must take their bookmarks with them (ON DELETE CASCADE)
|
||||
// while leaving the shared series row behind.
|
||||
func TestDeleteReaderCascadesToBookmarks(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
other := secondReader(t, st)
|
||||
if _, err := st.Upsert(other, Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed other's bookmark: %v", err)
|
||||
}
|
||||
|
||||
if _, err := st.db.Exec(`DELETE FROM readers WHERE id = $1`, other); err != nil {
|
||||
t.Fatalf("delete reader: %v", err)
|
||||
}
|
||||
var n int
|
||||
if err := st.db.QueryRow(`SELECT count(*) FROM bookmarks`).Scan(&n); err != nil {
|
||||
t.Fatalf("count bookmarks: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("bookmarks after reader delete = %d, want 0 (cascade)", n)
|
||||
}
|
||||
sr := readSeries(t, st, "asura", "solo")
|
||||
if sr.Title != "Solo Leveling" {
|
||||
t.Fatalf("series = %+v, want it kept after its only reader was deleted", sr)
|
||||
}
|
||||
}
|
||||
|
||||
// Registration is one code path: the first sight of a Discord identity creates
|
||||
// the Reader, every later one returns the same row. The epoch-0 hash argument
|
||||
// is for creation only — a returning Reader who has rotated must not have that
|
||||
// rotation undone by logging in again.
|
||||
func TestEnsureReaderCreatesOnceAndNeverClobbersARotation(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
first, err := s.EnsureReader("new-member", sha256.Sum256([]byte("cred-epoch-0")))
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureReader: %v", err)
|
||||
}
|
||||
if first == s.OwnerID() {
|
||||
t.Fatal("a new Discord identity resolved to the owner Reader")
|
||||
}
|
||||
if id, ok, err := s.ReaderIDForTokenHash(sha256.Sum256([]byte("cred-epoch-0"))); err != nil || !ok || id != first {
|
||||
t.Fatalf("new Reader's credential resolved to (%d, %v, %v), want (%d, true, nil)", id, ok, err, first)
|
||||
}
|
||||
|
||||
rotated := sha256.Sum256([]byte("cred-epoch-1"))
|
||||
if err := s.RotateToken(first, 0, rotated); err != nil {
|
||||
t.Fatalf("RotateToken: %v", err)
|
||||
}
|
||||
|
||||
again, err := s.EnsureReader("new-member", sha256.Sum256([]byte("cred-epoch-0")))
|
||||
if err != nil {
|
||||
t.Fatalf("second EnsureReader: %v", err)
|
||||
}
|
||||
if again != first {
|
||||
t.Fatalf("second login returned Reader %d, want the existing %d", again, first)
|
||||
}
|
||||
if _, ok, err := s.ReaderIDForTokenHash(sha256.Sum256([]byte("cred-epoch-0"))); err != nil {
|
||||
t.Fatalf("stale lookup: %v", err)
|
||||
} else if ok {
|
||||
t.Fatal("logging in again revived the pre-rotation credential")
|
||||
}
|
||||
if id, ok, err := s.ReaderIDForTokenHash(rotated); err != nil || !ok || id != first {
|
||||
t.Fatalf("rotated credential resolved to (%d, %v, %v), want the same Reader", id, ok, err)
|
||||
}
|
||||
|
||||
// Signing in as the owner's own Discord identity reuses the seeded row
|
||||
// rather than minting a duplicate library.
|
||||
if id, err := s.EnsureReader(testOwner.DiscordID, sha256.Sum256([]byte("ignored"))); err != nil {
|
||||
t.Fatalf("EnsureReader(owner): %v", err)
|
||||
} else if id != s.OwnerID() {
|
||||
t.Fatalf("owner login returned Reader %d, want the seeded owner %d", id, s.OwnerID())
|
||||
}
|
||||
}
|
||||
|
||||
// The owner's administration view: who exists and how many live sessions each
|
||||
// holds. Revocation drops all of one Reader's sessions and nobody else's.
|
||||
func TestReadersAndSessionRevocation(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
other := secondReader(t, s)
|
||||
for _, id := range []string{"own-1", "own-2"} {
|
||||
if _, err := s.CreateSession(id, s.OwnerID(), time.Hour); err != nil {
|
||||
t.Fatalf("CreateSession(%s): %v", id, err)
|
||||
}
|
||||
}
|
||||
if _, err := s.CreateSession("other-1", other, time.Hour); err != nil {
|
||||
t.Fatalf("CreateSession(other): %v", err)
|
||||
}
|
||||
// An expired row must not be counted as a session the owner can revoke.
|
||||
if _, err := s.CreateSession("other-dead", other, -time.Minute); err != nil {
|
||||
t.Fatalf("CreateSession(expired): %v", err)
|
||||
}
|
||||
|
||||
readers, err := s.Readers()
|
||||
if err != nil {
|
||||
t.Fatalf("Readers: %v", err)
|
||||
}
|
||||
if len(readers) != 2 || readers[0].ID != s.OwnerID() || readers[1].ID != other {
|
||||
t.Fatalf("readers = %+v, want the owner then the second Reader", readers)
|
||||
}
|
||||
if readers[0].DiscordID != testOwner.DiscordID {
|
||||
t.Fatalf("owner discord id = %q, want %q", readers[0].DiscordID, testOwner.DiscordID)
|
||||
}
|
||||
if readers[0].Sessions != 2 || readers[1].Sessions != 1 {
|
||||
t.Fatalf("session counts = %d, %d; want 2 and 1 live", readers[0].Sessions, readers[1].Sessions)
|
||||
}
|
||||
|
||||
if err := s.DeleteReaderSessions(other); err != nil {
|
||||
t.Fatalf("DeleteReaderSessions: %v", err)
|
||||
}
|
||||
if _, ok, err := s.GetSession("other-1", time.Now()); err != nil || ok {
|
||||
t.Fatalf("revoked session still resolves: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if _, ok, err := s.GetSession("own-1", time.Now()); err != nil || !ok {
|
||||
t.Fatalf("owner's session was collateral: ok=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Two Readers on one Series: one series row, two independent progresses. The
|
||||
// second Reader starts at zero however far the first has read, and the shared
|
||||
// row is still due exactly once.
|
||||
func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
other := secondReader(t, s)
|
||||
if _, err := s.Upsert(s.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", SeriesURL: "https://asurascans.com/comics/solo",
|
||||
LastChapter: "Chapter 200", LastChapterNum: 200, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed owner: %v", err)
|
||||
}
|
||||
theirs, err := s.Upsert(other, Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 2000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seed other: %v", err)
|
||||
}
|
||||
if theirs.LastChapterNum != 0 || theirs.LastChapter != "" {
|
||||
t.Fatalf("second Reader's progress = %+v, want zero regardless of the first's 200", theirs)
|
||||
}
|
||||
// The shared facts are still shared: the series row it joined to is the
|
||||
// one the first Reader created.
|
||||
if theirs.Title != "Solo Leveling" {
|
||||
t.Fatalf("second Reader's title = %q, want the shared series title", theirs.Title)
|
||||
}
|
||||
var series int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM series`).Scan(&series); err != nil {
|
||||
t.Fatalf("count series: %v", err)
|
||||
}
|
||||
if series != 1 {
|
||||
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 || due[0].Key() != "asura:solo" {
|
||||
t.Fatalf("due = %+v, want the shared series exactly once per cycle", due)
|
||||
}
|
||||
|
||||
// One Reader dropping their bookmark leaves the other's intact and the
|
||||
// series still polled.
|
||||
if err := s.Delete(other, "asura:solo"); err != nil {
|
||||
t.Fatalf("Delete(other): %v", err)
|
||||
}
|
||||
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
|
||||
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck after delete: %v", err)
|
||||
}
|
||||
if len(due) != 1 || due[0].Key() != "asura:solo" {
|
||||
t.Fatalf("due after one Reader left = %+v, want the series still polled", due)
|
||||
}
|
||||
}
|
||||
func TestKaganeCoverPersistsAcrossReopen(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
first, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
body := []byte("stored-cover")
|
||||
if err := first.PutKaganeCover("019fe11a-84c3-7fc3-a84b-88787374b617", body, "image/webp"); err != nil {
|
||||
t.Fatalf("PutKaganeCover: %v", err)
|
||||
}
|
||||
if err := first.Close(); err != nil {
|
||||
t.Fatalf("close first store: %v", err)
|
||||
}
|
||||
|
||||
second, err := Open(url, testOwner)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer second.Close()
|
||||
got, contentType, ok, err := second.GetKaganeCover("019fe11a-84c3-7fc3-a84b-88787374b617")
|
||||
if err != nil {
|
||||
t.Fatalf("GetKaganeCover: %v", err)
|
||||
}
|
||||
if !ok || !bytes.Equal(got, body) || contentType != "image/webp" {
|
||||
t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package token
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Token derives one Reader's userscript credential from the deployment
|
||||
// secret, the Reader's Discord id and their token epoch.
|
||||
//
|
||||
// The credential is deterministic rather than stored random because the
|
||||
// server must be able to rebuild the install URL after a restart while the
|
||||
// database holds only hashes: a random token with no plaintext copy anywhere
|
||||
// would be unreconstructible, and keeping plaintext in memory would break
|
||||
// every install link on restart. HMAC output is high-entropy, indistinguishable
|
||||
// from random to anyone without the secret, and changes whenever the epoch
|
||||
// does — which is what rotation is. The stored form is Hash of this value,
|
||||
// so a database leak yields nothing but hashes of unguessable strings.
|
||||
func Token(key []byte, discordID string, epoch int64) string {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
// The separator is unambiguous: discord ids are decimal snowflakes and
|
||||
// epochs are plain integers, so no two (id, epoch) pairs can collide.
|
||||
mac.Write([]byte(discordID))
|
||||
mac.Write([]byte{0})
|
||||
mac.Write([]byte(strconv.FormatInt(epoch, 10)))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// Hash is the SHA-256 of a credential — the only form that ever touches the
|
||||
// database (readers.token_sha256). SHA-256 rather than a password hash is
|
||||
// deliberate: these are unguessable values with nothing to brute-force, so a
|
||||
// slow hash would only add per-request cost.
|
||||
func Hash(cred string) [32]byte {
|
||||
return sha256.Sum256([]byte(cred))
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package token
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTokenDeterministicPerReaderAndEpoch(t *testing.T) {
|
||||
key := []byte("deployment-secret")
|
||||
a := Token(key, "reader-1", 0)
|
||||
b := Token(key, "reader-1", 0)
|
||||
if a != b {
|
||||
t.Fatal("same (reader, epoch) derived different credentials")
|
||||
}
|
||||
if a == Token(key, "reader-2", 0) {
|
||||
t.Fatal("different readers derived the same credential")
|
||||
}
|
||||
if a == Token(key, "reader-1", 1) {
|
||||
t.Fatal("rotation epoch derived the same credential")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenChangesWithSecret(t *testing.T) {
|
||||
a := Token([]byte("key-1"), "reader-1", 0)
|
||||
b := Token([]byte("key-2"), "reader-1", 0)
|
||||
if a == b {
|
||||
t.Fatal("different secrets derived the same credential")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenFormat(t *testing.T) {
|
||||
cred := Token([]byte("key"), "reader-1", 0)
|
||||
// 32 bytes of HMAC-SHA256, hex-encoded: the length the install URL and
|
||||
// the committed placeholder both assume.
|
||||
if len(cred) != 64 {
|
||||
t.Fatalf("credential length = %d, want 64", len(cred))
|
||||
}
|
||||
for _, c := range cred {
|
||||
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
|
||||
t.Fatalf("credential contains non-hex byte %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashIsSha256OfCredential(t *testing.T) {
|
||||
cred := Token([]byte("key"), "reader-1", 0)
|
||||
got := Hash(cred)
|
||||
want := sha256.Sum256([]byte(cred))
|
||||
if !bytes.Equal(got[:], want[:]) {
|
||||
t.Fatal("Hash is not the SHA-256 of the credential")
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,24 @@
|
||||
package userscript
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"bytes"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/httpmw"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// tokenPlaceholder is what the bindmounted userscript carries where the
|
||||
// Reader's credential goes: in the API_TOKEN constant and in the @downloadURL
|
||||
// and @updateURL metadata lines. The handler substitutes the requesting
|
||||
// Reader's credential for it at serve time, so no credential literal is ever
|
||||
// committed or deployed, and each Reader's copy carries exactly their own.
|
||||
var tokenPlaceholder = []byte("__API_TOKEN__")
|
||||
|
||||
// versionLine matches the userscript metadata block's @version directive.
|
||||
var versionLine = regexp.MustCompile(`(?m)^// @version[ \t]+.*$`)
|
||||
|
||||
@@ -26,35 +36,65 @@ func stampVersion(src []byte, mod time.Time) []byte {
|
||||
return versionLine.ReplaceAll(src, []byte("// @version "+mod.UTC().Format("2006.01.02.1504")))
|
||||
}
|
||||
|
||||
// userscriptHandler serves the userscript to Violentmonkey's updater.
|
||||
// substituteToken replaces every tokenPlaceholder with the Reader's
|
||||
// credential. A file without the placeholder is returned unchanged so Render
|
||||
// can warn about it rather than silently serving a credential-less script.
|
||||
func substituteToken(src []byte, credential string) []byte {
|
||||
return bytes.ReplaceAll(src, tokenPlaceholder, []byte(credential))
|
||||
}
|
||||
|
||||
// Render writes one userscript file with the credential substituted and the
|
||||
// mtime-derived version stamped. Shared by the download path (Handler) and
|
||||
// the web UI's install endpoints, so both serve byte-identical scripts.
|
||||
//
|
||||
// The token lives in the path because the update poll sends no Authorization
|
||||
// header, and the file embeds API_TOKEN in plain text, so an open path would
|
||||
// hand that token to anyone who guessed the URL. A mismatch answers 404 rather
|
||||
// than 401: a prober learns nothing about whether the route exists.
|
||||
// The file is read per request — that is what lets a bindmounted copy be
|
||||
// edited on the host without a restart. It is ~50 KB and polled about once a
|
||||
// day.
|
||||
func Render(w http.ResponseWriter, r *http.Request, path, credential string) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
log.Printf("userscript: stat %s: %v", path, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
log.Printf("userscript: read %s: %v", path, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
rendered := substituteToken(src, credential)
|
||||
if bytes.Equal(rendered, src) {
|
||||
// The bindmounted file was not built for per-Reader rendering. Serving
|
||||
// it as written is the operator's freedom, but a credential-less copy
|
||||
// is a deployment bug worth one log line — the symptom (silent 401s on
|
||||
// every device) is otherwise indistinguishable from a network fault.
|
||||
log.Printf("userscript: %s has no %s placeholder; serving as written", path, tokenPlaceholder)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Write(stampVersion(rendered, info.ModTime()))
|
||||
}
|
||||
|
||||
// Handler serves the userscript to Violentmonkey's updater, rendered for the
|
||||
// Reader whose credential is in the path.
|
||||
//
|
||||
// The file is read per request — that is what lets a bindmounted copy be edited
|
||||
// on the host without a restart. It is ~50 KB and polled about once a day.
|
||||
func Handler(token, path string) http.HandlerFunc {
|
||||
// The credential lives in the path because the update poll sends no
|
||||
// Authorization header, and the rendered file embeds the credential in
|
||||
// plaintext, so an open path would hand it to anyone who guessed the URL. A
|
||||
// mismatch answers 404 rather than 401: a prober learns nothing about whether
|
||||
// the route exists. The same credential authenticates the API bearer header,
|
||||
// so the two are one secret with one blast radius.
|
||||
//
|
||||
// The path segment is the credential itself, so once it resolves it is also
|
||||
// exactly what the served copy must carry — no re-derivation needed.
|
||||
func Handler(s *store.Store, path string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if subtle.ConstantTimeCompare([]byte(r.PathValue("token")), []byte(token)) != 1 {
|
||||
cred := r.PathValue("token")
|
||||
if _, ok := httpmw.ResolveReader(s, cred); !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
log.Printf("userscript: stat %s: %v", path, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
log.Printf("userscript: read %s: %v", path, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Write(stampVersion(src, info.ModTime()))
|
||||
Render(w, r, path, cred)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,115 +1,67 @@
|
||||
package userscript
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const testToken = "s3cret-token"
|
||||
|
||||
// sampleScript is a stand-in for the real userscript: a metadata block with a
|
||||
// @version line, plus a body that must survive the rewrite untouched.
|
||||
// @version line, the credential placeholder in its metadata and body, plus
|
||||
// content that must survive the rewrites untouched.
|
||||
const sampleScript = `// ==UserScript==
|
||||
// @name Manga Bookmark Sync
|
||||
// @version 1.5.0
|
||||
// @downloadURL https://api.example/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
// @match https://asurascans.com/*
|
||||
// ==/UserScript==
|
||||
(function () { "use strict"; })();
|
||||
(function () { "use strict";
|
||||
const API_TOKEN = "__API_TOKEN__";
|
||||
})();
|
||||
`
|
||||
|
||||
// writeScript drops a userscript in a temp dir with a known mtime and returns
|
||||
// its path plus the version string the handler is expected to stamp.
|
||||
func writeScript(t *testing.T, body string) (path, wantVersion string) {
|
||||
t.Helper()
|
||||
path = filepath.Join(t.TempDir(), "manga-bookmark.user.js")
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
func TestStampVersionReplacesVersionLineOnly(t *testing.T) {
|
||||
mod := time.Date(2026, 7, 28, 16, 42, 0, 0, time.UTC)
|
||||
if err := os.Chtimes(path, mod, mod); err != nil {
|
||||
t.Fatalf("chtimes: %v", err)
|
||||
}
|
||||
return path, "2026.07.28.1642"
|
||||
}
|
||||
got := string(stampVersion([]byte(sampleScript), mod))
|
||||
|
||||
// newTestMux registers Handler the same way main.go's router does, without
|
||||
// pulling in the store or the rest of the app.
|
||||
func newTestMux(token, path string) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js", Handler(token, path))
|
||||
return mux
|
||||
}
|
||||
|
||||
func getScript(t *testing.T, srv http.Handler, token string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+token+"/manga-bookmark.user.js", nil))
|
||||
return rr
|
||||
}
|
||||
|
||||
func TestUserscriptServedWithStampedVersion(t *testing.T) {
|
||||
path, wantVersion := writeScript(t, sampleScript)
|
||||
rr := getScript(t, newTestMux(testToken, path), testToken)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
if !strings.Contains(got, "// @version "+mod.UTC().Format("2006.01.02.1504")) {
|
||||
t.Errorf("body has no stamped version:\n%s", got)
|
||||
}
|
||||
if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/javascript") {
|
||||
t.Errorf("Content-Type = %q, want text/javascript", ct)
|
||||
if strings.Contains(got, "1.5.0") {
|
||||
t.Errorf("body still carries the file's own version:\n%s", got)
|
||||
}
|
||||
if cc := rr.Header().Get("Cache-Control"); cc != "no-cache" {
|
||||
t.Errorf("Cache-Control = %q, want no-cache", cc)
|
||||
}
|
||||
body := rr.Body.String()
|
||||
if !strings.Contains(body, "// @version "+wantVersion) {
|
||||
t.Errorf("body has no stamped version %q:\n%s", wantVersion, body)
|
||||
}
|
||||
if strings.Contains(body, "1.5.0") {
|
||||
t.Errorf("body still carries the file's own version:\n%s", body)
|
||||
}
|
||||
// Everything outside the @version line is served verbatim.
|
||||
if !strings.Contains(body, `(function () { "use strict"; })();`) {
|
||||
t.Errorf("body was altered beyond the version line:\n%s", body)
|
||||
}
|
||||
if !strings.Contains(body, "// @name Manga Bookmark Sync") {
|
||||
t.Errorf("metadata block was altered:\n%s", body)
|
||||
// Everything outside the @version line is served verbatim, including the
|
||||
// placeholder — stamping must not do the substitution's job.
|
||||
if !strings.Contains(got, `const API_TOKEN = "__API_TOKEN__";`) {
|
||||
t.Errorf("body was altered beyond the version line:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The empty-token case ("/u//manga-bookmark.user.js") is covered at the
|
||||
// router level (see backend's guardEmptyUserscriptToken): ServeMux 307s it to
|
||||
// "/u/manga-bookmark.user.js" before this handler's own token check ever runs.
|
||||
func TestUserscriptWrongTokenIs404(t *testing.T) {
|
||||
path, _ := writeScript(t, sampleScript)
|
||||
srv := newTestMux(testToken, path)
|
||||
for _, tok := range []string{"wrong", testToken + "x", testToken[:3]} {
|
||||
if got := getScript(t, srv, tok).Code; got != http.StatusNotFound {
|
||||
t.Errorf("token %q: status = %d, want 404", tok, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserscriptMissingFileIs404(t *testing.T) {
|
||||
srv := newTestMux(testToken, filepath.Join(t.TempDir(), "absent.user.js"))
|
||||
if got := getScript(t, srv, testToken).Code; got != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want 404", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserscriptWithoutVersionLineServedUnmodified(t *testing.T) {
|
||||
func TestStampVersionWithoutVersionLineServedUnmodified(t *testing.T) {
|
||||
const noVersion = "// ==UserScript==\n// @name x\n// ==/UserScript==\nconsole.log(1);\n"
|
||||
path, _ := writeScript(t, noVersion)
|
||||
rr := getScript(t, newTestMux(testToken, path), testToken)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if rr.Body.String() != noVersion {
|
||||
t.Fatalf("body = %q, want it unmodified", rr.Body.String())
|
||||
if got := string(stampVersion([]byte(noVersion), time.Now())); got != noVersion {
|
||||
t.Errorf("stampVersion altered a file with no @version line:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubstituteTokenReplacesEveryPlaceholder(t *testing.T) {
|
||||
got := string(substituteToken([]byte(sampleScript), "abc123"))
|
||||
|
||||
if strings.Contains(got, "__API_TOKEN__") {
|
||||
t.Errorf("placeholder survived substitution:\n%s", got)
|
||||
}
|
||||
// The credential lands in the constant and in both metadata lines.
|
||||
if want := `const API_TOKEN = "abc123";`; !strings.Contains(got, want) {
|
||||
t.Errorf("no substituted constant %q:\n%s", want, got)
|
||||
}
|
||||
if want := "https://api.example/u/abc123/manga-bookmark.user.js"; !strings.Contains(got, want) {
|
||||
t.Errorf("no substituted download URL %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubstituteTokenWithoutPlaceholderServedUnmodified(t *testing.T) {
|
||||
const noPlaceholder = "// ==UserScript==\n// @name x\n// ==/UserScript==\n"
|
||||
if got := string(substituteToken([]byte(noPlaceholder), "abc123")); got != noPlaceholder {
|
||||
t.Errorf("substituteToken altered a file without the placeholder:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CoverFetcher retrieves one kagane cover by image id. Satisfied by
|
||||
// latest.BrowserFetcher. It is nil when BROWSER_WS_URL is unset; uncached
|
||||
// covers are then unavailable, while covers already stored by the backend
|
||||
// remain available without a browser.
|
||||
type CoverFetcher interface {
|
||||
Image(ctx context.Context, imageID string) (body []byte, contentType string, err error)
|
||||
}
|
||||
|
||||
// coverIDRe matches the request path segment that becomes part of an outbound
|
||||
// URL. The proxy is session-gated, but the id still reaches a headless browser,
|
||||
// so it is validated at the boundary rather than passed through.
|
||||
var coverIDRe = regexp.MustCompile(`^[0-9a-f-]{36}$`)
|
||||
|
||||
// coverTimeout bounds one proxied cover. Shorter than the fetcher's own
|
||||
// challenge budget on purpose: a browser page is waiting on this, and a cover
|
||||
// that has not arrived by now is better left as a broken slot than as a request
|
||||
// holding a connection open.
|
||||
const coverTimeout = 20 * time.Second
|
||||
|
||||
// kaganeCover serves a kagane cover from the backend's own origin.
|
||||
//
|
||||
// kagane answers image requests with a Cloudflare challenge and
|
||||
// `cross-origin-resource-policy: same-origin`, so the web UI cannot render one
|
||||
// directly under any combination of referrer policy or crossorigin attribute
|
||||
// (verified 2026-08-08). Fetching it through the headless browser that already
|
||||
// clears the challenge, and re-serving it here, is what puts the bytes on an
|
||||
// origin the page may load from.
|
||||
func (h *Handler) kaganeCover(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if !coverIDRe.MatchString(id) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
body, contentType, ok, err := h.store.GetKaganeCover(id)
|
||||
if err != nil {
|
||||
log.Printf("read kagane cover %s: %v", id, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
writeCover(w, body, contentType)
|
||||
return
|
||||
}
|
||||
if h.covers == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), coverTimeout)
|
||||
defer cancel()
|
||||
body, contentType, err = h.covers.Image(ctx, id)
|
||||
if err != nil {
|
||||
log.Printf("kagane cover %s: %v", id, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if !store.IsKaganeCoverContentType(contentType) {
|
||||
log.Printf("kagane cover %s: unexpected content type %q", id, contentType)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.PutKaganeCover(id, body, contentType); err != nil {
|
||||
log.Printf("persist kagane cover %s: %v", id, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeCover(w, body, contentType)
|
||||
}
|
||||
|
||||
// writeCover sends the bytes with a long cache life: an image id names one
|
||||
// immutable rendering, so a client that has it never needs to ask again.
|
||||
func writeCover(w http.ResponseWriter, body []byte, contentType string) {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Cache-Control", "private, max-age=604800, immutable")
|
||||
w.Write(body)
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/session"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
const (
|
||||
// oauthStateTTL bounds how long a started sign-in stays valid. Ten
|
||||
// minutes is generous for Discord's round trip and short enough that a
|
||||
// captured state is stale before it is worth replaying.
|
||||
oauthStateTTL = 10 * time.Minute
|
||||
// maxStates caps the state map so a flood of /auth/discord hits cannot
|
||||
// grow memory; past the cap the oldest state is evicted, which at worst
|
||||
// invalidates an in-flight sign-in.
|
||||
maxStates = 256
|
||||
// maxResponseBytes caps Discord API bodies; they are small, and an
|
||||
// unbounded read is an OOM handed to Discord's CDN.
|
||||
maxResponseBytes = 1 << 20
|
||||
// discordTimeout keeps a hung Discord request from hanging the login
|
||||
// callback forever.
|
||||
discordTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// DiscordConfig is the OAuth application this service registers as, plus the
|
||||
// guild that gates access.
|
||||
type DiscordConfig struct {
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
GuildID string
|
||||
// RequiredRole, when non-empty, is a role ID a member must hold on top of
|
||||
// guild membership. Empty by default: membership alone suffices.
|
||||
RequiredRole string
|
||||
// APIBase is the Discord API root; configurable so tests run the whole
|
||||
// flow against a local stub.
|
||||
APIBase string
|
||||
// RedirectURI is the full public URL of the callback — Discord requires
|
||||
// the exact string, so it is configured, never derived from headers.
|
||||
RedirectURI string
|
||||
}
|
||||
|
||||
// oauthStates stores one-time sign-in states. A state is generated at
|
||||
// /auth/discord, echoed back by Discord at the callback, and consumed there.
|
||||
type oauthStates struct {
|
||||
mu sync.Mutex
|
||||
expiry map[string]time.Time
|
||||
}
|
||||
|
||||
func newOAuthStates() *oauthStates {
|
||||
return &oauthStates{expiry: make(map[string]time.Time)}
|
||||
}
|
||||
|
||||
func (s *oauthStates) put(state string, expires time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now()
|
||||
for k, at := range s.expiry {
|
||||
if !at.After(now) {
|
||||
delete(s.expiry, k)
|
||||
}
|
||||
}
|
||||
// Evict the state closest to expiring when full, so a flood of starts
|
||||
// cannot grow memory; at worst it invalidates an in-flight sign-in.
|
||||
if len(s.expiry) >= maxStates {
|
||||
var oldest string
|
||||
var oldestAt time.Time
|
||||
for k, at := range s.expiry {
|
||||
if oldest == "" || at.Before(oldestAt) {
|
||||
oldest, oldestAt = k, at
|
||||
}
|
||||
}
|
||||
delete(s.expiry, oldest)
|
||||
}
|
||||
s.expiry[state] = expires
|
||||
}
|
||||
|
||||
// take validates and consumes a state in one step: a state works exactly
|
||||
// once, which is what makes a replayed callback useless.
|
||||
func (s *oauthStates) take(state string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
expires, ok := s.expiry[state]
|
||||
if !ok || !expires.After(time.Now()) {
|
||||
return false
|
||||
}
|
||||
delete(s.expiry, state)
|
||||
return true
|
||||
}
|
||||
|
||||
// discordStart begins the authorization code grant: a fresh state, then a
|
||||
// redirect to Discord's authorize page.
|
||||
func (h *Handler) discordStart(w http.ResponseWriter, r *http.Request) {
|
||||
state := session.NewID()
|
||||
h.states.put(state, time.Now().Add(oauthStateTTL))
|
||||
u := h.discord.APIBase + "/oauth2/authorize?" + url.Values{
|
||||
"client_id": {h.discord.ClientID},
|
||||
"redirect_uri": {h.discord.RedirectURI},
|
||||
"response_type": {"code"},
|
||||
"scope": {"identify guilds.members.read"},
|
||||
"state": {state},
|
||||
}.Encode()
|
||||
http.Redirect(w, r, u, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// discordCallback completes the grant: exchange the code, verify identity,
|
||||
// membership and role, then mint a session. Every failure path renders the
|
||||
// login page with an author-written message — nothing Discord supplied is
|
||||
// ever interpolated into a page, and no secret reaches a log line.
|
||||
func (h *Handler) discordCallback(w http.ResponseWriter, r *http.Request) {
|
||||
ip := session.ClientIP(r)
|
||||
if wait := h.limiter.RetryAfter(ip, time.Now()); wait > 0 {
|
||||
secs := int(wait.Seconds()) + 1
|
||||
w.Header().Set("Retry-After", strconv.Itoa(secs))
|
||||
h.renderLogin(w, http.StatusTooManyRequests,
|
||||
"Too many attempts. Try again in "+strconv.Itoa((secs+59)/60)+" min.")
|
||||
return
|
||||
}
|
||||
|
||||
// Discord refuses the grant (the reader hit cancel, or the application
|
||||
// was misconfigured). The state is consumed so the flow is cleanly over;
|
||||
// this makes no Discord calls, so it is not a failure the limiter counts.
|
||||
if oerr := r.URL.Query().Get("error"); oerr != "" {
|
||||
h.states.take(r.URL.Query().Get("state"))
|
||||
h.renderLogin(w, http.StatusBadRequest, "Sign-in was cancelled.")
|
||||
return
|
||||
}
|
||||
|
||||
code := r.URL.Query().Get("code")
|
||||
if code == "" || !h.states.take(r.URL.Query().Get("state")) {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
h.renderLogin(w, http.StatusBadRequest,
|
||||
"This sign-in link was invalid or already used. Start again.")
|
||||
return
|
||||
}
|
||||
|
||||
tok, err := h.exchangeToken(r.Context(), code)
|
||||
if err != nil {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
log.Printf("discord token exchange: %v", err)
|
||||
h.renderLogin(w, http.StatusBadGateway,
|
||||
"Discord sign-in is unavailable right now. Try again in a moment.")
|
||||
return
|
||||
}
|
||||
|
||||
userID, err := h.discordUserID(r.Context(), tok.AccessToken)
|
||||
if err != nil {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
log.Printf("discord users/@me: %v", err)
|
||||
h.renderLogin(w, http.StatusBadGateway,
|
||||
"Discord sign-in is unavailable right now. Try again in a moment.")
|
||||
return
|
||||
}
|
||||
member, isMember, err := h.discordMember(r.Context(), tok.AccessToken)
|
||||
if err != nil {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
log.Printf("discord member check: %v", err)
|
||||
h.renderLogin(w, http.StatusBadGateway,
|
||||
"Discord sign-in is unavailable right now. Try again in a moment.")
|
||||
return
|
||||
}
|
||||
// The refusal is the same for a non-member and a member without the
|
||||
// required role, and it names neither the guild nor its id: an outsider
|
||||
// cannot tell whether the guild exists, let alone which one gates.
|
||||
//
|
||||
// It also returns before EnsureReader, so a refused sign-in leaves no
|
||||
// Reader row behind — the gate is the only thing standing between guild
|
||||
// membership and a library.
|
||||
if !isMember || (h.discord.RequiredRole != "" && !slices.Contains(member.Roles, h.discord.RequiredRole)) {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
h.renderLogin(w, http.StatusForbidden,
|
||||
"This Discord account is not a member of this community.")
|
||||
return
|
||||
}
|
||||
|
||||
// Registration is the login (issue #27): first sight of a guild member
|
||||
// creates their Reader, every later sight returns the same one. Their
|
||||
// userscript credential is derived at epoch 0 the way the owner's is, so
|
||||
// the install links work before they have read anything.
|
||||
readerID, err := h.store.EnsureReader(userID, token.Hash(token.Token(h.tokenKey, userID, 0)))
|
||||
if err != nil {
|
||||
log.Printf("register reader: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
h.limiter.Reset(ip)
|
||||
sess, err := h.store.CreateSession(session.NewID(), readerID, session.SessionTTL)
|
||||
if err != nil {
|
||||
log.Printf("create session: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
session.SetCookie(w, r, sess.ID)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// exchangeToken trades an authorization code for an access token. The body is
|
||||
// form-encoded because that is what Discord accepts — it rejects a JSON
|
||||
// payload — so the wire format is fixed here, not in a client library.
|
||||
func (h *Handler) exchangeToken(ctx context.Context, code string) (discordToken, error) {
|
||||
form := url.Values{
|
||||
"client_id": {h.discord.ClientID},
|
||||
"client_secret": {h.discord.ClientSecret},
|
||||
"grant_type": {"authorization_code"},
|
||||
"code": {code},
|
||||
"redirect_uri": {h.discord.RedirectURI},
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
h.discord.APIBase+"/oauth2/token", strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return discordToken{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := h.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return discordToken{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return discordToken{}, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
var tok discordToken
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&tok); err != nil {
|
||||
return discordToken{}, err
|
||||
}
|
||||
if tok.AccessToken == "" {
|
||||
return discordToken{}, errors.New("empty access token")
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
// discordUserID fetches the signed-in user's id via the identify scope.
|
||||
func (h *Handler) discordUserID(ctx context.Context, accessToken string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
h.discord.APIBase+"/users/@me", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+accessToken)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := h.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
var u struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&u); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if u.ID == "" {
|
||||
return "", errors.New("empty user id")
|
||||
}
|
||||
return u.ID, nil
|
||||
}
|
||||
|
||||
type discordMember struct {
|
||||
Roles []string `json:"roles"`
|
||||
}
|
||||
|
||||
// discordMember fetches the current user's membership in the configured guild.
|
||||
//
|
||||
// This is the OAuth endpoint (Get Current User Guild Member), the one the
|
||||
// guilds.members.read scope grants. Its bot-side twin, GET /guilds/{id}/
|
||||
// members/{user}, reads almost identically and is the wrong one: it wants a
|
||||
// Bot token and the application present in the guild, and answers a user
|
||||
// Bearer token with 401 — which fails as an outage rather than a refusal, so
|
||||
// nobody could sign in at all.
|
||||
//
|
||||
// A 404 or 403 (not in the guild, or the token lacks the scope) is a
|
||||
// non-member, not an error.
|
||||
func (h *Handler) discordMember(ctx context.Context, accessToken string) (discordMember, bool, error) {
|
||||
u := h.discord.APIBase + "/users/@me/guilds/" +
|
||||
url.PathEscape(h.discord.GuildID) + "/member"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return discordMember{}, false, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+accessToken)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := h.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return discordMember{}, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden {
|
||||
return discordMember{}, false, nil
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return discordMember{}, false, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
var m discordMember
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&m); err != nil {
|
||||
return discordMember{}, false, err
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
type discordToken struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOAuthStateSingleUse(t *testing.T) {
|
||||
s := newOAuthStates()
|
||||
s.put("st", time.Now().Add(time.Minute))
|
||||
if !s.take("st") {
|
||||
t.Fatal("take of a fresh state = false, want true")
|
||||
}
|
||||
if s.take("st") {
|
||||
t.Fatal("take of a consumed state = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOAuthStateUnknownOrExpired(t *testing.T) {
|
||||
s := newOAuthStates()
|
||||
if s.take("never-seen") {
|
||||
t.Fatal("take of an unknown state = true, want false")
|
||||
}
|
||||
s.put("stale", time.Now().Add(-time.Minute))
|
||||
if s.take("stale") {
|
||||
t.Fatal("take of an expired state = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// The map is capped: a flood of starts evicts older states instead of growing,
|
||||
// and consumed states must not change that.
|
||||
func TestOAuthStateEviction(t *testing.T) {
|
||||
s := newOAuthStates()
|
||||
key := func(i, salt int) string {
|
||||
return string(rune('a'+i%26)) + string(rune('0'+i/26+salt*16))
|
||||
}
|
||||
now := time.Now().Add(time.Hour)
|
||||
for i := 0; i < maxStates*2; i++ {
|
||||
s.put(key(i, 0), now)
|
||||
}
|
||||
if got := len(s.expiry); got != maxStates {
|
||||
t.Fatalf("states after a flood = %d, want %d", got, maxStates)
|
||||
}
|
||||
|
||||
// Consume everything, then flood again: the map stays bounded.
|
||||
for state := range s.expiry {
|
||||
s.take(state)
|
||||
}
|
||||
for i := 0; i < maxStates; i++ {
|
||||
s.put(key(i, 1), now)
|
||||
}
|
||||
if got := len(s.expiry); got != maxStates {
|
||||
t.Fatalf("states after consume+flood = %d, want %d", got, maxStates)
|
||||
}
|
||||
}
|
||||
@@ -243,6 +243,80 @@ button { cursor: pointer; }
|
||||
/* The label is 15px tall by design; the thumb gets 44 without moving it. */
|
||||
.ghost::after { content: ""; position: absolute; inset: -15px -12px; }
|
||||
|
||||
/* ---- userscript setup: collapsed by default, one hairline, no card ---- */
|
||||
.setup {
|
||||
margin: 0 20px;
|
||||
padding: 12px 0 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
color: var(--mute);
|
||||
}
|
||||
.setup summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-height: 44px;
|
||||
padding: 0;
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
cursor: pointer;
|
||||
list-style: none;
|
||||
}
|
||||
.setup summary::-webkit-details-marker { display: none; }
|
||||
.setup summary:hover { color: var(--paper); }
|
||||
.setup[open] { padding-bottom: 16px; }
|
||||
.setup-copy {
|
||||
margin: 0;
|
||||
padding: 4px 0 12px;
|
||||
font: 14px/1.55 var(--font-body);
|
||||
color: var(--mute);
|
||||
}
|
||||
.setup-links {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px 20px;
|
||||
margin: 0 0 14px;
|
||||
}
|
||||
.setup-links .ghost { font-size: 11px; }
|
||||
.setup-rotate { margin: 0; }
|
||||
/* Rotation confirmation: the one hot state the panel wears, and it is
|
||||
destruction, not new-chapter signal — danger, never ember. */
|
||||
.setup-warn {
|
||||
margin: 0;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--danger);
|
||||
color: var(--danger);
|
||||
font: 500 12px/1.5 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
}
|
||||
|
||||
/* ---- reader roster (owner only): same hairline panel, one row per Reader ---- */
|
||||
.readerlist { margin: 0; padding: 0; list-style: none; }
|
||||
.readerlist li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 16px;
|
||||
min-height: 44px;
|
||||
border-top: 1px solid var(--rule);
|
||||
}
|
||||
.readerlist form { margin: 0 0 0 auto; }
|
||||
.reader-id {
|
||||
font: 500 13px/1.4 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
color: var(--paper);
|
||||
}
|
||||
.reader-sessions {
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute);
|
||||
}
|
||||
/* Revocation cuts someone off, so it wears --danger. Ember stays reserved for
|
||||
the new-chapter signal. */
|
||||
.ghost.danger { color: var(--danger); }
|
||||
.ghost.danger:hover { color: var(--danger); border-bottom-color: var(--danger); }
|
||||
|
||||
.chrome { display: flex; flex-direction: column; }
|
||||
|
||||
.searchbar {
|
||||
@@ -774,26 +848,6 @@ button { cursor: pointer; }
|
||||
filter: drop-shadow(0 0 34px var(--ember-wash)) drop-shadow(0 18px 24px rgba(0,0,0,.5));
|
||||
}
|
||||
.login-card form { display: flex; flex-direction: column; gap: 18px; }
|
||||
.login-card label {
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .16em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute);
|
||||
}
|
||||
.login-card input {
|
||||
width: 100%;
|
||||
height: 54px;
|
||||
margin-top: 9px;
|
||||
padding: 0 2px;
|
||||
border: none;
|
||||
border-bottom: 1px solid var(--field-line);
|
||||
background: transparent;
|
||||
color: var(--paper);
|
||||
font: 500 20px var(--font-mono);
|
||||
letter-spacing: .16em;
|
||||
outline: none;
|
||||
}
|
||||
.login-card input:focus { border-bottom-color: var(--paper); }
|
||||
.login-card .error {
|
||||
margin: 0;
|
||||
min-height: 20px;
|
||||
@@ -810,7 +864,13 @@ button { cursor: pointer; }
|
||||
.login-card button:hover {
|
||||
background: var(--ember);
|
||||
border-color: var(--ember);
|
||||
color: #fff;
|
||||
color: var(--ember-ink);
|
||||
}
|
||||
.login-card .login-note {
|
||||
margin: 14px 0 0;
|
||||
text-align: center;
|
||||
font: 400 12px/1.4 var(--font-body);
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
/* ---- laptop and up: the whole sheet is drawn 20% larger, which is what
|
||||
|
||||
@@ -74,6 +74,10 @@
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
{{template "setup" .}}
|
||||
|
||||
{{if .Owner}}{{template "readers" .}}{{end}}
|
||||
|
||||
{{template "keyrow" .}}
|
||||
|
||||
{{template "recent" .}}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<div class="row">
|
||||
<a class="cover" href="{{.ContinueURL}}" target="_blank" rel="noopener noreferrer"
|
||||
tabindex="-1" aria-hidden="true">
|
||||
{{if .Cover}}<img src="{{.Cover}}" alt="" loading="lazy">
|
||||
{{if .CoverURL}}<img src="{{.CoverURL}}" alt="" loading="lazy">
|
||||
{{/* aria-hidden on the cover link is not enough — Chromium still exposes
|
||||
the letter because the link is programmatically focusable — so the
|
||||
monogram carries its own, same as the recent strip's. */}}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<a class="recent-card {{if .HasNewChapter}}is-new{{end}}" href="{{.ContinueURL}}"
|
||||
target="_blank" rel="noopener noreferrer">
|
||||
<span class="recent-cover">
|
||||
{{if .Cover}}<img src="{{.Cover}}" alt="" loading="lazy">
|
||||
{{if .CoverURL}}<img src="{{.CoverURL}}" alt="" loading="lazy">
|
||||
{{else}}<span class="monogram" aria-hidden="true">{{.Initial}}</span>{{end}}
|
||||
{{if .HasNewChapter}}<span class="foot-rule"></span>
|
||||
{{else if .Favorite}}<span class="foot-rule brass"></span>{{end}}
|
||||
|
||||
@@ -16,6 +16,17 @@
|
||||
<div class="empty"><strong>Nothing archived.</strong><p>Shelve a series to park it here — it keeps getting checked for new chapters.</p></div>
|
||||
{{else if eq .Tab "finished"}}
|
||||
<div class="empty"><strong>Nothing finished yet.</strong><p>Mark a series finished and it moves out of your reading list.</p></div>
|
||||
{{else if .EmptyLibrary}}
|
||||
{{/* Nothing in either library, so the links are the only thing this page can
|
||||
usefully say. Both scripts: the two libraries are separate installs. */}}
|
||||
<div class="empty">
|
||||
<strong>Nothing here yet.</strong>
|
||||
<p>Install the userscripts, then open a series and read a chapter — bookmarks arrive on their own.</p>
|
||||
<p class="setup-links">
|
||||
<a class="ghost" href="/install/manga-bookmark.user.js">Install Manga script</a>
|
||||
<a class="ghost" href="/install/novel-bookmark.user.js">Install Novels script</a>
|
||||
</p>
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="empty"><strong>Nothing here yet.</strong><p>Bookmarks appear once the userscript records a chapter.</p></div>
|
||||
{{end}}
|
||||
|
||||
@@ -19,17 +19,13 @@
|
||||
<figure class="login-art" aria-hidden="true">
|
||||
<img src="/static/login-art.png" alt="">
|
||||
</figure>
|
||||
<form method="post" action="/login">
|
||||
<div>
|
||||
<label for="password">Password</label>
|
||||
<input id="password" name="password" type="password"
|
||||
autocomplete="current-password" autofocus required>
|
||||
</div>
|
||||
<form method="get" action="/auth/discord">
|
||||
{{/* The page reloads on a failed sign-in, so the message is present from
|
||||
the start; role=alert is what gets it announced anyway. */}}
|
||||
<p class="error" role="alert">{{.Error}}</p>
|
||||
<button type="submit">Sign in</button>
|
||||
<button type="submit">Continue with Discord</button>
|
||||
</form>
|
||||
<p class="login-note">Guild membership is required to sign in.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
{{/* The owner's Reader roster. Rendered only for the owner (listView.Owner),
|
||||
and re-rendered whole as the response to a revocation so the session
|
||||
counts cannot describe the state before the tap. Revocation is
|
||||
confirm-gated: it signs someone out of every device at once. */}}
|
||||
{{define "readers"}}
|
||||
<details class="setup" id="readers">
|
||||
<summary>Readers</summary>
|
||||
<p class="setup-copy">Everyone who has signed in through Discord. Revoking
|
||||
signs a Reader out of every device; their library and bookmarks are
|
||||
untouched, and they can sign in again.</p>
|
||||
<ul class="readerlist">
|
||||
{{range .Readers}}
|
||||
<li>
|
||||
<span class="reader-id">{{.DiscordID}}</span>
|
||||
<span class="reader-sessions">{{.Sessions}} session{{if ne .Sessions 1}}s{{end}}</span>
|
||||
{{/* The owner's own row never offers Revoke: it is the one row where the
|
||||
button would sign the tapping browser out, and the endpoint refuses
|
||||
it anyway. Logout is the deliberate way to do that. */}}
|
||||
{{if and .Sessions (ne .ID $.OwnerID)}}
|
||||
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
|
||||
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
|
||||
<button type="submit" class="ghost danger">Revoke sessions</button>
|
||||
</form>
|
||||
{{end}}
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
</details>
|
||||
{{end}}
|
||||
@@ -0,0 +1,34 @@
|
||||
{{/* The userscript install panel. Each link serves the script rendered
|
||||
with the acting Reader's credential inside it, so the credential never
|
||||
appears in this page's markup, the address bar, or a redirect. Rotation
|
||||
is confirm-gated because it invalidates every installed copy at once;
|
||||
the response swaps this same panel open with the reinstall warning. */}}
|
||||
{{define "setup"}}
|
||||
<details class="setup" id="setup"{{if .Rotated}} open{{end}}>
|
||||
<summary>Userscripts</summary>
|
||||
<p class="setup-copy">Install each script once per device. They keep your
|
||||
bookmarks in sync across every site and update themselves from here.</p>
|
||||
<p class="setup-links">
|
||||
<a class="ghost" href="/install/manga-bookmark.user.js">Install Manga script</a>
|
||||
<a class="ghost" href="/install/novel-bookmark.user.js">Install Novels script</a>
|
||||
</p>
|
||||
<p class="setup-copy">On mobile, Violentmonkey does not pick up the install
|
||||
links — the script opens as text. Download the file instead, then add it
|
||||
from Violentmonkey's own menu.</p>
|
||||
<p class="setup-links">
|
||||
<a class="ghost" href="/install/manga-bookmark.user.js?download=1">Download Manga script</a>
|
||||
<a class="ghost" href="/install/novel-bookmark.user.js?download=1">Download Novels script</a>
|
||||
</p>
|
||||
{{if .Rotated}}
|
||||
<p class="setup-warn" role="status">Credential rotated — the old one no
|
||||
longer works. Reinstall both scripts on every device now, or they will
|
||||
silently stop syncing.</p>
|
||||
{{else}}
|
||||
<form class="setup-rotate" hx-post="/rotate-token" hx-target="#setup"
|
||||
hx-swap="outerHTML"
|
||||
hx-confirm="Rotation invalidates the current credential on every device immediately. You will have to reinstall both scripts everywhere. Rotate?">
|
||||
<button type="submit" class="ghost">Rotate credential</button>
|
||||
</form>
|
||||
{{end}}
|
||||
</details>
|
||||
{{end}}
|
||||
+220
-57
@@ -1,7 +1,7 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"context"
|
||||
"embed"
|
||||
"html/template"
|
||||
"io/fs"
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
|
||||
"bookmarkmanager/backend/internal/session"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
"bookmarkmanager/backend/internal/userscript"
|
||||
)
|
||||
|
||||
//go:embed templates
|
||||
@@ -31,11 +33,26 @@ const RecentCount = 5
|
||||
// It is a separate handler from api.Handler because the two speak different
|
||||
// representations (HTML versus JSON) to different clients under different auth.
|
||||
type Handler struct {
|
||||
store *store.Store
|
||||
tmpl *template.Template
|
||||
key []byte
|
||||
password string
|
||||
limiter *session.LoginLimiter
|
||||
store *store.Store
|
||||
// tokenKey derives Readers' userscript credentials (internal/token): the
|
||||
// install endpoints render the scripts with the credential inside, which
|
||||
// is the one place the UI needs the secret.
|
||||
tokenKey []byte
|
||||
// mangaUserscriptPath / novelUserscriptPath are the bindmounted script
|
||||
// files the install endpoints render — the same files the /u/ download
|
||||
// paths serve.
|
||||
mangaUserscriptPath string
|
||||
novelUserscriptPath string
|
||||
tmpl *template.Template
|
||||
discord DiscordConfig
|
||||
states *oauthStates
|
||||
limiter *session.LoginLimiter
|
||||
// httpClient is the plain stdlib client that talks to Discord. It is not
|
||||
// an injected interface: tests point APIBase at a stub server instead.
|
||||
httpClient *http.Client
|
||||
// covers proxies kagane cover images, which no browser can load directly.
|
||||
// Nil disables the endpoint — see CoverFetcher.
|
||||
covers CoverFetcher
|
||||
}
|
||||
|
||||
// listView is what every list-rendering template receives.
|
||||
@@ -43,7 +60,7 @@ type listView struct {
|
||||
// Lib is the library this view renders: store.KindManga or store.KindNovel.
|
||||
// Manga is the default and carries no query parameter, so every pre-novel
|
||||
// URL keeps meaning exactly what it did.
|
||||
Lib string
|
||||
Lib string
|
||||
Tab string // "all", "fav", or "new"
|
||||
Recent []store.Bookmark
|
||||
Items []store.Bookmark
|
||||
@@ -55,6 +72,22 @@ type listView struct {
|
||||
// OOB marks a render of the chrome partials as an out-of-band swap rather
|
||||
// than the inline copy app.html lays out.
|
||||
OOB bool
|
||||
// Rotated marks the setup panel as having just rotated the credential:
|
||||
// it swaps the reinstall warning in over the button row.
|
||||
Rotated bool
|
||||
// EmptyLibrary means this Reader holds no bookmarks in either library, so
|
||||
// the empty state can offer the installs instead of reporting on a filter.
|
||||
// It is not "newly registered": a Reader who deletes their last bookmark is
|
||||
// in the same position and needs the same links.
|
||||
EmptyLibrary bool
|
||||
// Owner marks the acting Reader as the deployment's owner, which unlocks
|
||||
// the Readers panel. Nothing else in the UI differs.
|
||||
Owner bool
|
||||
// Readers is the owner's roster, populated only for the owner's own page
|
||||
// render and the revocation fragment. OwnerID travels with it so the roster
|
||||
// can tell the owner's own row apart from the Readers they may revoke.
|
||||
Readers []store.ReaderSummary
|
||||
OwnerID int64
|
||||
}
|
||||
|
||||
// PageURL and ListURL are the two link shapes every tab needs. Building them
|
||||
@@ -81,23 +114,29 @@ type loginView struct {
|
||||
|
||||
// New parses every template up front so a broken one kills the process at
|
||||
// startup rather than the first request that touches it.
|
||||
func New(s *store.Store, apiToken, webPassword string) (*Handler, error) {
|
||||
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string, covers CoverFetcher) (*Handler, error) {
|
||||
tmpl, err := template.ParseFS(templateFS, "templates/*.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Handler{
|
||||
store: s,
|
||||
tmpl: tmpl,
|
||||
key: session.Key(apiToken, webPassword),
|
||||
password: webPassword,
|
||||
limiter: session.NewLoginLimiter(),
|
||||
store: s,
|
||||
tokenKey: tokenKey,
|
||||
mangaUserscriptPath: mangaPath,
|
||||
novelUserscriptPath: novelPath,
|
||||
tmpl: tmpl,
|
||||
discord: discord,
|
||||
states: newOAuthStates(),
|
||||
limiter: session.NewLoginLimiter(),
|
||||
httpClient: &http.Client{Timeout: discordTimeout},
|
||||
covers: covers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /{$}", h.index)
|
||||
mux.HandleFunc("POST /login", h.login)
|
||||
mux.HandleFunc("GET /auth/discord", h.discordStart)
|
||||
mux.HandleFunc("GET /auth/discord/callback", h.discordCallback)
|
||||
mux.HandleFunc("POST /logout", h.logout)
|
||||
mux.Handle("GET /static/", staticHandler())
|
||||
|
||||
@@ -106,6 +145,21 @@ func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /ui/bookmarks/{key}/status", h.requireSession(h.uiStatus))
|
||||
mux.HandleFunc("POST /ui/bookmarks/{key}/chapter", h.requireSession(h.uiChapter))
|
||||
mux.HandleFunc("DELETE /ui/bookmarks/{key}", h.requireSession(h.uiDelete))
|
||||
|
||||
// Session-gated like every other UI route: the deployment proxies kagane's
|
||||
// images for its own Readers, not for the internet.
|
||||
mux.HandleFunc("GET /img/kagane/{id}", h.requireSession(h.kaganeCover))
|
||||
|
||||
// Install endpoints render the script directly under the session: the
|
||||
// credential travels inside the served bytes, never in the address bar or
|
||||
// the page markup. Updates after install use the credential-bearing /u/
|
||||
// path the script embeds, which needs no session.
|
||||
mux.HandleFunc("GET /install/manga-bookmark.user.js", h.requireSession(h.installUserscript("manga-bookmark.user.js")))
|
||||
mux.HandleFunc("GET /install/novel-bookmark.user.js", h.requireSession(h.installUserscript("novel-bookmark.user.js")))
|
||||
mux.HandleFunc("POST /rotate-token", h.requireSession(h.rotateToken))
|
||||
|
||||
// Owner-only: the one place the UI crosses the Reader boundary.
|
||||
mux.HandleFunc("POST /readers/{id}/revoke", h.requireSession(h.revokeReaderSessions))
|
||||
}
|
||||
|
||||
// staticHandler serves the embedded assets. An hour, not longer: assets are
|
||||
@@ -131,10 +185,26 @@ func staticHandler() http.Handler {
|
||||
}))
|
||||
}
|
||||
|
||||
// authed reports whether the request carries a valid session cookie.
|
||||
func (h *Handler) authed(r *http.Request) bool {
|
||||
type ctxKey int
|
||||
|
||||
// readerCtxKey is where requireSession stashes the authenticated Reader id.
|
||||
const readerCtxKey ctxKey = iota
|
||||
|
||||
// sessionReader reports whether the request carries a live session, and for
|
||||
// whom. The cookie holds only the session id; the row behind it is looked up
|
||||
// on every request, so deleting a session takes effect immediately. Expiry is
|
||||
// enforced here, in the store, which also removes rows that have lapsed.
|
||||
func (h *Handler) sessionReader(r *http.Request) (int64, bool) {
|
||||
c, err := r.Cookie(session.CookieName)
|
||||
return err == nil && session.Verify(h.key, c.Value, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
sess, ok, err := h.store.GetSession(c.Value, time.Now())
|
||||
if err != nil {
|
||||
log.Printf("session lookup: %v", err)
|
||||
return 0, false
|
||||
}
|
||||
return sess.ReaderID, ok
|
||||
}
|
||||
|
||||
// requireSession guards the fragment endpoints. It answers 401 rather than
|
||||
@@ -142,14 +212,18 @@ func (h *Handler) authed(r *http.Request) bool {
|
||||
// redirected login page would be spliced into the card list.
|
||||
func (h *Handler) requireSession(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.authed(r) {
|
||||
readerID, ok := h.sessionReader(r)
|
||||
if !ok {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
next(w, r.WithContext(context.WithValue(r.Context(), readerCtxKey, readerID)))
|
||||
}
|
||||
}
|
||||
|
||||
// readerOf returns the authenticated Reader id requireSession stashed.
|
||||
func readerOf(r *http.Request) int64 { return r.Context().Value(readerCtxKey).(int64) }
|
||||
|
||||
func (h *Handler) render(w http.ResponseWriter, status int, name string, data any) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
@@ -163,16 +237,25 @@ func (h *Handler) render(w http.ResponseWriter, status int, name string, data an
|
||||
// page is served at / with status 200 rather than as a redirect to a separate
|
||||
// URL: one page, no redirect loop to reason about.
|
||||
func (h *Handler) index(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.authed(r) {
|
||||
readerID, ok := h.sessionReader(r)
|
||||
if !ok {
|
||||
h.render(w, http.StatusOK, "login", loginView{})
|
||||
return
|
||||
}
|
||||
view, err := h.buildListView(libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
|
||||
view, err := h.buildListView(readerID, libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
|
||||
if err != nil {
|
||||
log.Printf("index: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if readerID == h.store.OwnerID() {
|
||||
view.Owner, view.OwnerID = true, readerID
|
||||
if view.Readers, err = h.store.Readers(); err != nil {
|
||||
log.Printf("index readers: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
h.render(w, http.StatusOK, "app", view)
|
||||
}
|
||||
|
||||
@@ -208,18 +291,22 @@ func libOf(q string) string {
|
||||
return store.KindManga
|
||||
}
|
||||
|
||||
// buildListView loads the list once and derives both the tab-filtered items and
|
||||
// the recent strip from it.
|
||||
// buildListView loads one reader's list once and derives both the tab-filtered
|
||||
// items and the recent strip from it.
|
||||
//
|
||||
// Archived and finished series appear in their own tab and nowhere else — not
|
||||
// in All, not in Updated, not in Favourites, and not in the recent strip. An
|
||||
// archived favourite therefore shows only under Archived: Favourites means
|
||||
// "favourites I am currently reading".
|
||||
func (h *Handler) buildListView(lib, tab string) (listView, error) {
|
||||
all, err := h.store.List() // already ordered updated_at DESC
|
||||
func (h *Handler) buildListView(readerID int64, lib, tab string) (listView, error) {
|
||||
all, err := h.store.List(readerID) // already ordered updated_at DESC
|
||||
if err != nil {
|
||||
return listView{}, err
|
||||
}
|
||||
// Taken before the filter narrows the slice: a Reader with novels but no
|
||||
// manga has a working install already, and does not need to be told to go
|
||||
// and get one.
|
||||
emptyLibrary := len(all) == 0
|
||||
// Narrow to one library first: reading, withNew and recent all derive from
|
||||
// this slice, so doing it later would let the other library's rows into the
|
||||
// strip and the Updated badge.
|
||||
@@ -263,11 +350,12 @@ func (h *Handler) buildListView(lib, tab string) (listView, error) {
|
||||
recent = recent[:RecentCount]
|
||||
}
|
||||
}
|
||||
return listView{Lib: lib, Tab: tab, Recent: recent, Items: items, NewCount: len(withNew)}, nil
|
||||
return listView{Lib: lib, Tab: tab, Recent: recent, Items: items,
|
||||
NewCount: len(withNew), EmptyLibrary: emptyLibrary}, nil
|
||||
}
|
||||
|
||||
func (h *Handler) uiList(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.buildListView(libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
|
||||
view, err := h.buildListView(readerOf(r), libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
|
||||
if err != nil {
|
||||
log.Printf("ui list: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
@@ -325,7 +413,7 @@ func (h *Handler) writeChromeOOB(w http.ResponseWriter, view listView) {
|
||||
// refreshChrome rebuilds the chrome for the reader's current tab after a
|
||||
// mutation and appends it to the response.
|
||||
func (h *Handler) refreshChrome(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.buildListView(currentLib(r), currentTab(r))
|
||||
view, err := h.buildListView(readerOf(r), currentLib(r), currentTab(r))
|
||||
if err != nil {
|
||||
log.Printf("ui chrome: %v", err)
|
||||
return
|
||||
@@ -333,35 +421,21 @@ func (h *Handler) refreshChrome(w http.ResponseWriter, r *http.Request) {
|
||||
h.writeChromeOOB(w, view)
|
||||
}
|
||||
|
||||
func (h *Handler) login(w http.ResponseWriter, r *http.Request) {
|
||||
ip := session.ClientIP(r)
|
||||
if wait := h.limiter.RetryAfter(ip, time.Now()); wait > 0 {
|
||||
secs := int(wait.Seconds()) + 1
|
||||
w.Header().Set("Retry-After", strconv.Itoa(secs))
|
||||
h.render(w, http.StatusTooManyRequests, "login", loginView{
|
||||
Error: "Too many attempts. Try again in " +
|
||||
strconv.Itoa((secs+59)/60) + " min.",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
got := r.PostFormValue("password")
|
||||
if subtle.ConstantTimeCompare([]byte(got), []byte(h.password)) != 1 {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
h.render(w, http.StatusUnauthorized, "login", loginView{Error: "Wrong password."})
|
||||
return
|
||||
}
|
||||
|
||||
h.limiter.Reset(ip)
|
||||
session.SetCookie(w, r, h.key)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
// renderLogin renders the login page with an error message, for refused or
|
||||
// failed sign-ins. Every message is author-written text — nothing Discord
|
||||
// supplied is ever interpolated into a page.
|
||||
func (h *Handler) renderLogin(w http.ResponseWriter, status int, msg string) {
|
||||
h.render(w, status, "login", loginView{Error: msg})
|
||||
}
|
||||
|
||||
// logout revokes the session row and clears the cookie in one step: the next
|
||||
// request finds no row and is rejected.
|
||||
func (h *Handler) logout(w http.ResponseWriter, r *http.Request) {
|
||||
if c, err := r.Cookie(session.CookieName); err == nil {
|
||||
if err := h.store.DeleteSession(c.Value); err != nil {
|
||||
log.Printf("delete session: %v", err)
|
||||
}
|
||||
}
|
||||
session.ClearCookie(w, r)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
@@ -374,7 +448,7 @@ func (h *Handler) loadForMutation(w http.ResponseWriter, r *http.Request) (store
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return store.Bookmark{}, false
|
||||
}
|
||||
b, ok, err := h.store.Get(key)
|
||||
b, ok, err := h.store.Get(readerOf(r), key)
|
||||
if err != nil {
|
||||
log.Printf("ui get %q: %v", key, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
@@ -397,7 +471,7 @@ func (h *Handler) loadForMutation(w http.ResponseWriter, r *http.Request) (store
|
||||
// describe the whole library, so they are rebuilt out of band on every
|
||||
// mutation, at the cost of one extra list read per toggle.
|
||||
func (h *Handler) saveAndRenderCard(w http.ResponseWriter, r *http.Request, b store.Bookmark) {
|
||||
stored, err := h.store.Upsert(b)
|
||||
stored, err := h.store.Upsert(readerOf(r), b)
|
||||
if err != nil {
|
||||
log.Printf("ui upsert %q: %v", b.Key, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
@@ -489,7 +563,7 @@ func (h *Handler) uiDelete(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.Delete(key); err != nil {
|
||||
if err := h.store.Delete(readerOf(r), key); err != nil {
|
||||
log.Printf("ui delete %q: %v", key, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
@@ -500,3 +574,92 @@ func (h *Handler) uiDelete(w http.ResponseWriter, r *http.Request) {
|
||||
// the library got smaller.
|
||||
h.refreshChrome(w, r)
|
||||
}
|
||||
|
||||
// installUserscript renders the bindmounted script with the acting Reader's
|
||||
// derived credential substituted in. The credential is derived, not stored,
|
||||
// so installs work after any restart; the Reader never types or copies it —
|
||||
// clicking Install is the whole setup.
|
||||
//
|
||||
// ?download=1 forces a save instead. Mobile Violentmonkey (Chromium) does not
|
||||
// intercept navigation to a .user.js URL, so the Install link only renders the
|
||||
// source as text there; the Reader needs the file on disk to add it by hand.
|
||||
func (h *Handler) installUserscript(name string) http.HandlerFunc {
|
||||
path := h.mangaUserscriptPath
|
||||
if name == "novel-bookmark.user.js" {
|
||||
path = h.novelUserscriptPath
|
||||
}
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
discordID, epoch, err := h.store.ReaderTokenInfo(readerOf(r))
|
||||
if err != nil {
|
||||
log.Printf("install %s: %v", name, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if r.URL.Query().Has("download") {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`)
|
||||
}
|
||||
userscript.Render(w, r, path, token.Token(h.tokenKey, discordID, epoch))
|
||||
}
|
||||
}
|
||||
|
||||
// rotateToken issues the acting Reader a new credential: the epoch bumps and
|
||||
// the stored hash is rewritten, so the old credential stops authenticating
|
||||
// the moment the statement commits. Every device must reinstall, or its
|
||||
// script keeps failing silently — the setup panel states that warning next
|
||||
// to the button, and the response repeats it as confirmation.
|
||||
func (h *Handler) rotateToken(w http.ResponseWriter, r *http.Request) {
|
||||
readerID := readerOf(r)
|
||||
discordID, epoch, err := h.store.ReaderTokenInfo(readerID)
|
||||
if err != nil {
|
||||
log.Printf("rotate token: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// The hash is computed for epoch+1 and guarded by it in the store, so a
|
||||
// concurrent rotation cannot leave the stored hash describing another
|
||||
// epoch.
|
||||
if err := h.store.RotateToken(readerID, epoch, token.Hash(token.Token(h.tokenKey, discordID, epoch+1))); err != nil {
|
||||
log.Printf("rotate token: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
view := listView{Lib: store.KindManga, Rotated: true}
|
||||
h.render(w, http.StatusOK, "setup", view)
|
||||
}
|
||||
|
||||
// revokeReaderSessions logs one Reader out of every browser they are signed
|
||||
// in on. Owner-only: it reaches across the Reader boundary every other handler
|
||||
// respects, so the guard is a comparison against the seeded owner rather than
|
||||
// a role a Reader could acquire. A non-owner gets 404 — the panel does not
|
||||
// exist for them, so neither should the endpoint.
|
||||
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
|
||||
if readerOf(r) != h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
target, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "bad reader id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// The owner is not one of the Readers this endpoint reaches: revoking
|
||||
// themselves would sign out the browser making the request, which is what
|
||||
// logout is for. The roster hides the button; this refuses the hand-rolled
|
||||
// POST behind it.
|
||||
if target == h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.DeleteReaderSessions(target); err != nil {
|
||||
log.Printf("revoke sessions: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
log.Printf("revoke sessions: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "readers", listView{Owner: true, Readers: readers, OwnerID: h.store.OwnerID()})
|
||||
}
|
||||
|
||||
+136
-65
@@ -16,20 +16,28 @@ import (
|
||||
"bookmarkmanager/backend/internal/httpmw"
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
"bookmarkmanager/backend/internal/userscript"
|
||||
"bookmarkmanager/backend/internal/web"
|
||||
)
|
||||
|
||||
// Config holds all runtime settings, sourced from environment variables.
|
||||
type Config struct {
|
||||
Token string
|
||||
// TokenKey derives every Reader's userscript credential (internal/token).
|
||||
// Required: without it no install URL can ever be built.
|
||||
TokenKey string
|
||||
AllowedOrigins []string
|
||||
// DatabaseURL is the Postgres connection URL; required, no default,
|
||||
// because a wrong guess would silently start on an empty database.
|
||||
DatabaseURL string
|
||||
Port string
|
||||
// WebPassword gates the browser UI. Empty disables the web routes entirely.
|
||||
WebPassword string
|
||||
// OwnerDiscordID identifies the seeded owner Reader (issue #22). Required:
|
||||
// bookmarks are scoped to a Reader, and a fresh deployment needs one
|
||||
// before anybody logs in. The owner is also the only Reader who can revoke
|
||||
// another Reader's sessions.
|
||||
OwnerDiscordID string
|
||||
// Discord is the OAuth application the browser UI signs in with.
|
||||
Discord web.DiscordConfig
|
||||
// UserscriptPath is the file served at /u/{token}/manga-bookmark.user.js.
|
||||
// Supplied by a bindmount so the script can be edited without a rebuild.
|
||||
UserscriptPath string
|
||||
@@ -39,6 +47,10 @@ type Config struct {
|
||||
NovelUserscriptPath string
|
||||
// LatestPoll configures the background latest-chapter fetcher.
|
||||
LatestPoll LatestPoll
|
||||
// Covers proxies kagane cover images for the web UI. Not from the
|
||||
// environment: it is the shared headless browser, wired in main once it
|
||||
// connects, and nil in every test router.
|
||||
Covers web.CoverFetcher
|
||||
}
|
||||
|
||||
// LatestPoll configures the background latest-chapter poller.
|
||||
@@ -48,18 +60,20 @@ type Config struct {
|
||||
// deployment. Past that nothing breaks; the effective cadence stretches to
|
||||
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
|
||||
type LatestPoll struct {
|
||||
Enabled bool
|
||||
Cooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
Enabled bool
|
||||
Cooldown time.Duration
|
||||
BrowserCooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
}
|
||||
|
||||
const (
|
||||
defaultPollCooldown = time.Hour
|
||||
defaultPollInterval = 10 * time.Minute
|
||||
defaultPollStagger = 20 * time.Second
|
||||
defaultPollBatch = 14
|
||||
defaultPollCooldown = time.Hour
|
||||
defaultBrowserPollCooldown = 6 * time.Hour
|
||||
defaultPollInterval = 10 * time.Minute
|
||||
defaultPollStagger = 20 * time.Second
|
||||
defaultPollBatch = 14
|
||||
// minPollCooldown keeps a typo from turning a polite background check into
|
||||
// a hammer against sites that are already bot-scoring us.
|
||||
minPollCooldown = 15 * time.Minute
|
||||
@@ -117,20 +131,27 @@ func envInt(key string, def int) int {
|
||||
return n
|
||||
}
|
||||
|
||||
func clampPollCooldown(name string, d time.Duration) time.Duration {
|
||||
if d < minPollCooldown {
|
||||
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
|
||||
return minPollCooldown
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// loadLatestPoll reads the poller's settings, clamping anything that would make
|
||||
// it antisocial.
|
||||
func loadLatestPoll() LatestPoll {
|
||||
p := LatestPoll{
|
||||
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
|
||||
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
|
||||
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
|
||||
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
|
||||
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
|
||||
}
|
||||
if p.Cooldown < minPollCooldown {
|
||||
log.Printf("config: cooldown %s is below the %s floor, clamping", p.Cooldown, minPollCooldown)
|
||||
p.Cooldown = minPollCooldown
|
||||
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
|
||||
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
|
||||
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
|
||||
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
|
||||
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
|
||||
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
|
||||
}
|
||||
p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
|
||||
p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
|
||||
// batch x stagger has to fit inside one tick or a batch is still running
|
||||
// when the next one is due. Run() serialises them, so this degrades to a
|
||||
// slower cadence rather than to overlapping fetches — worth a warning, not
|
||||
@@ -144,14 +165,22 @@ func loadLatestPoll() LatestPoll {
|
||||
|
||||
func loadConfig() Config {
|
||||
c := Config{
|
||||
Token: os.Getenv("API_TOKEN"),
|
||||
TokenKey: os.Getenv("TOKEN_KEY"),
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
Port: envOr("PORT", "8080"),
|
||||
WebPassword: os.Getenv("WEB_PASSWORD"),
|
||||
OwnerDiscordID: os.Getenv("OWNER_DISCORD_ID"),
|
||||
UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"),
|
||||
NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"),
|
||||
LatestPoll: loadLatestPoll(),
|
||||
}
|
||||
c.Discord = web.DiscordConfig{
|
||||
ClientID: os.Getenv("DISCORD_CLIENT_ID"),
|
||||
ClientSecret: os.Getenv("DISCORD_CLIENT_SECRET"),
|
||||
GuildID: os.Getenv("DISCORD_GUILD_ID"),
|
||||
RequiredRole: os.Getenv("DISCORD_REQUIRED_ROLE"),
|
||||
APIBase: envOr("DISCORD_API_BASE", "https://discord.com/api/v10"),
|
||||
RedirectURI: os.Getenv("DISCORD_REDIRECT_URI"),
|
||||
}
|
||||
for _, o := range strings.Split(os.Getenv("ALLOWED_ORIGINS"), ",") {
|
||||
if o = strings.TrimSpace(o); o != "" {
|
||||
c.AllowedOrigins = append(c.AllowedOrigins, o)
|
||||
@@ -168,10 +197,14 @@ func newRouter(s *store.Store, cfg Config) http.Handler {
|
||||
mux.HandleFunc("GET /healthz", api.Healthz)
|
||||
|
||||
// Outside httpmw.Auth (the updater sends no Authorization header) and
|
||||
// outside the WEB_PASSWORD gate (the script must be installable either
|
||||
// way). The path segment carries the token instead.
|
||||
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js", userscript.Handler(cfg.Token, cfg.UserscriptPath))
|
||||
mux.HandleFunc("GET /u/{token}/novel-bookmark.user.js", userscript.Handler(cfg.Token, cfg.NovelUserscriptPath))
|
||||
// outside the web UI's Discord auth (the script must be installable
|
||||
// without a browser session). The path segment carries the credential
|
||||
// instead, and the script is rendered with the resolved Reader's
|
||||
// credential substituted in.
|
||||
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js",
|
||||
userscript.Handler(s, cfg.UserscriptPath))
|
||||
mux.HandleFunc("GET /u/{token}/novel-bookmark.user.js",
|
||||
userscript.Handler(s, cfg.NovelUserscriptPath))
|
||||
|
||||
h := &api.Handler{Store: s}
|
||||
protected := http.NewServeMux()
|
||||
@@ -179,20 +212,18 @@ func newRouter(s *store.Store, cfg Config) http.Handler {
|
||||
protected.HandleFunc("PUT /bookmarks/{key}", h.Put)
|
||||
protected.HandleFunc("DELETE /bookmarks/{key}", h.Delete)
|
||||
|
||||
auth := httpmw.Auth(cfg.Token, protected)
|
||||
auth := httpmw.Auth(s, protected)
|
||||
mux.Handle("/bookmarks", auth)
|
||||
mux.Handle("/bookmarks/", auth)
|
||||
|
||||
// The browser UI is registered only when a password is configured, so a
|
||||
// deployment that forgets WEB_PASSWORD exposes nothing rather than
|
||||
// exposing an unprotected list.
|
||||
if cfg.WebPassword != "" {
|
||||
wh, err := web.New(s, cfg.Token, cfg.WebPassword)
|
||||
if err != nil {
|
||||
log.Fatalf("web handler: %v", err)
|
||||
}
|
||||
wh.Register(mux)
|
||||
// The browser UI is always registered; signing in is Discord OAuth, so
|
||||
// there is no password to forget and no gate to leave unset.
|
||||
wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey),
|
||||
cfg.UserscriptPath, cfg.NovelUserscriptPath, cfg.Covers)
|
||||
if err != nil {
|
||||
log.Fatalf("web handler: %v", err)
|
||||
}
|
||||
wh.Register(mux)
|
||||
|
||||
return httpmw.CORS(cfg.AllowedOrigins, httpmw.Gzip(guardEmptyUserscriptToken(mux)))
|
||||
}
|
||||
@@ -214,14 +245,36 @@ func guardEmptyUserscriptToken(next http.Handler) http.Handler {
|
||||
|
||||
func main() {
|
||||
cfg := loadConfig()
|
||||
if cfg.Token == "" {
|
||||
log.Fatal("API_TOKEN is required")
|
||||
if cfg.TokenKey == "" {
|
||||
log.Fatal("TOKEN_KEY is required")
|
||||
}
|
||||
if cfg.OwnerDiscordID == "" {
|
||||
log.Fatal("OWNER_DISCORD_ID is required")
|
||||
}
|
||||
if cfg.DatabaseURL == "" {
|
||||
log.Fatal("DATABASE_URL is required")
|
||||
}
|
||||
// The web UI signs in through Discord, so a deployment without the OAuth
|
||||
// application is misconfigured rather than passwordless.
|
||||
for key, v := range map[string]string{
|
||||
"DISCORD_CLIENT_ID": cfg.Discord.ClientID,
|
||||
"DISCORD_CLIENT_SECRET": cfg.Discord.ClientSecret,
|
||||
"DISCORD_GUILD_ID": cfg.Discord.GuildID,
|
||||
"DISCORD_REDIRECT_URI": cfg.Discord.RedirectURI,
|
||||
} {
|
||||
if v == "" {
|
||||
log.Fatalf("%s is required", key)
|
||||
}
|
||||
}
|
||||
// The owner's userscript credential is derived from TOKEN_KEY at epoch 0
|
||||
// (internal/token); the readers row carries its SHA-256, not the
|
||||
// credential itself.
|
||||
owner := store.Owner{
|
||||
DiscordID: cfg.OwnerDiscordID,
|
||||
TokenHash: token.Hash(token.Token([]byte(cfg.TokenKey), cfg.OwnerDiscordID, 0)),
|
||||
}
|
||||
|
||||
s, err := store.Open(cfg.DatabaseURL)
|
||||
s, err := store.Open(cfg.DatabaseURL, owner)
|
||||
if err != nil {
|
||||
log.Fatalf("open store: %v", err)
|
||||
}
|
||||
@@ -230,9 +283,26 @@ func main() {
|
||||
// The poller is off the request path entirely: if it cannot start, the
|
||||
// service still serves bookmarks and the userscript still captures latest
|
||||
// chapters on its own.
|
||||
//
|
||||
// One headless browser serves both consumers that need a Cloudflare
|
||||
// challenge cleared: the poller's kagane/novelfull fetches and the web
|
||||
// UI's kagane cover proxy. Optional — unset leaves both degraded to what
|
||||
// they were before the sidecar existed.
|
||||
var browser latest.Fetcher
|
||||
pollCtx, stopPoll := context.WithCancel(context.Background())
|
||||
defer stopPoll()
|
||||
startLatestPoller(pollCtx, s, cfg.LatestPoll)
|
||||
if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" {
|
||||
bf, err := latest.NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
log.Printf("browser fetcher disabled: %v", err)
|
||||
} else {
|
||||
browser = bf
|
||||
cfg.Covers = bf
|
||||
context.AfterFunc(pollCtx, bf.Close)
|
||||
log.Printf("browser fetcher at %s", ws)
|
||||
}
|
||||
}
|
||||
startLatestPoller(pollCtx, s, cfg.LatestPoll, browser)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: ":" + cfg.Port,
|
||||
@@ -263,11 +333,31 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
|
||||
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
|
||||
var covers latest.CoverFetcher
|
||||
if f, ok := browser.(latest.CoverFetcher); ok {
|
||||
covers = f
|
||||
}
|
||||
return &latest.Poller{
|
||||
Store: s,
|
||||
Fetch: fetch,
|
||||
BrowserFetch: browser,
|
||||
CoverFetch: covers,
|
||||
Now: time.Now,
|
||||
Cooldown: cfg.Cooldown,
|
||||
BrowserCooldown: cfg.BrowserCooldown,
|
||||
Interval: cfg.Interval,
|
||||
Stagger: cfg.Stagger,
|
||||
Batch: cfg.Batch,
|
||||
}
|
||||
}
|
||||
|
||||
// startLatestPoller launches the background poller unless it is disabled or its
|
||||
// HTTP client cannot be built. Any problem here is logged and skipped: this
|
||||
// feature going missing degrades the service to userscript-only latest-chapter
|
||||
// tracking, which is exactly how it behaved before.
|
||||
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll) {
|
||||
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) {
|
||||
if !cfg.Enabled {
|
||||
log.Println("latest-chapter poller: disabled by config")
|
||||
return
|
||||
@@ -277,29 +367,10 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll) {
|
||||
log.Printf("latest-chapter poller: disabled, cannot build client: %v", err)
|
||||
return
|
||||
}
|
||||
p := &latest.Poller{
|
||||
Store: s,
|
||||
Fetch: f,
|
||||
Now: time.Now,
|
||||
Cooldown: cfg.Cooldown,
|
||||
Interval: cfg.Interval,
|
||||
Stagger: cfg.Stagger,
|
||||
Batch: cfg.Batch,
|
||||
}
|
||||
|
||||
// Optional: without it, sites behind a JavaScript challenge are simply not
|
||||
// polled, and their latest_chapter comes from the userscript alone — which
|
||||
// is how the service behaved before the sidecar existed.
|
||||
if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" {
|
||||
bf, err := latest.NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
log.Printf("latest-chapter poller: browser fetcher disabled: %v", err)
|
||||
} else {
|
||||
p.BrowserFetch = bf
|
||||
context.AfterFunc(ctx, bf.Close)
|
||||
log.Printf("latest-chapter poller: browser fetcher at %s", ws)
|
||||
}
|
||||
}
|
||||
// Nil browser: sites behind a JavaScript challenge are simply not polled,
|
||||
// and their latest_chapter comes from the userscript alone — which is how
|
||||
// the service behaved before the sidecar existed.
|
||||
p := newLatestPoller(s, cfg, f, browser)
|
||||
|
||||
go p.Run(ctx)
|
||||
}
|
||||
|
||||
+43
-8
@@ -17,19 +17,20 @@ import (
|
||||
func TestLoadLatestPollDefaults(t *testing.T) {
|
||||
for _, k := range []string{
|
||||
"LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN",
|
||||
"LATEST_CHAPTER_POLL_INTERVAL", "LATEST_CHAPTER_POLL_STAGGER",
|
||||
"LATEST_CHAPTER_POLL_BATCH",
|
||||
"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", "LATEST_CHAPTER_POLL_INTERVAL",
|
||||
"LATEST_CHAPTER_POLL_STAGGER", "LATEST_CHAPTER_POLL_BATCH",
|
||||
} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
|
||||
got := loadLatestPoll()
|
||||
want := LatestPoll{
|
||||
Enabled: true,
|
||||
Cooldown: time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 20 * time.Second,
|
||||
Batch: 14,
|
||||
Enabled: true,
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 20 * time.Second,
|
||||
Batch: 14,
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, want)
|
||||
@@ -74,6 +75,30 @@ func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
|
||||
wantFrom: func(p LatestPoll) any { return p.Cooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown below the floor is clamped up",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "1m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown at the floor is kept",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "15m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown override is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "8h"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 8 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown unparseable value falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "six hours"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 6 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "a valid override is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"},
|
||||
@@ -123,6 +148,16 @@ func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLatestPollerWiresCooldowns(t *testing.T) {
|
||||
p := newLatestPoller(nil, LatestPoll{
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
}, nil, nil)
|
||||
if p.Cooldown != time.Hour || p.BrowserCooldown != 6*time.Hour {
|
||||
t.Fatalf("poller cooldowns = %s/%s, want 1h/6h", p.Cooldown, p.BrowserCooldown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutStatusValidation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -202,7 +237,7 @@ func TestPutOmittedStatusPreservesArchivedAndAppliesProgress(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGzipCompressesTextNotFonts(t *testing.T) {
|
||||
srv, _ := newWebTestServer(t, webConfig())
|
||||
srv, _ := newWebTestServer(t, testConfig())
|
||||
|
||||
cases := []struct {
|
||||
path string
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
// registerReader creates an extra Reader the way a first login does and
|
||||
// returns its id. The credential is derived the same way the owner's is, so it
|
||||
// authenticates through the real router.
|
||||
func registerReader(t *testing.T, s *store.Store, discordID string) int64 {
|
||||
t.Helper()
|
||||
id, err := s.EnsureReader(discordID, token.Hash(readerCredential(discordID)))
|
||||
if err != nil {
|
||||
t.Fatalf("register reader %q: %v", discordID, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// credRequest builds a request authenticated as the Reader whose credential
|
||||
// is passed.
|
||||
func credRequest(method, target, cred string) *http.Request {
|
||||
req := httptest.NewRequest(method, target, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+cred)
|
||||
return req
|
||||
}
|
||||
|
||||
// readerCredential is the epoch-0 derived credential of an arbitrary Reader.
|
||||
func readerCredential(discordID string) string {
|
||||
return token.Token([]byte(testTokenKey), discordID, 0)
|
||||
}
|
||||
|
||||
// withBody attaches a request body, for PUTs that carry a JSON payload.
|
||||
func withBody(req *http.Request, body string) *http.Request {
|
||||
req.Body = io.NopCloser(strings.NewReader(body))
|
||||
req.ContentLength = int64(len(body))
|
||||
return req
|
||||
}
|
||||
|
||||
// A refused credential is refused however plausible it looks: only a hash the
|
||||
// readers table holds authenticates anything.
|
||||
func TestUnknownCredentialRejected(t *testing.T) {
|
||||
srv := newRouter(newTestStore(t), testConfig())
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("never-registered")))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unregistered Reader's credential: status = %d, want 401", rr.Code)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", ownerCredential()))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("owner's derived credential: status = %d, want 200", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A Reader's credential authenticates exactly that Reader: rows written under
|
||||
// one credential are invisible to the other, on the same key.
|
||||
func TestPerReaderIsolation(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
registerReader(t, s, "other-reader")
|
||||
srv := newRouter(s, testConfig())
|
||||
|
||||
ownerKey := "asura:solo"
|
||||
putBookmark(t, srv, ownerKey, store.Bookmark{
|
||||
Key: ownerKey, Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", UpdatedAt: 1,
|
||||
})
|
||||
|
||||
// The other Reader's list is empty even though the owner holds the key.
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other reader list: status = %d, want 200", rr.Code)
|
||||
}
|
||||
var theirs []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &theirs); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(theirs) != 0 {
|
||||
t.Fatalf("other reader sees %d bookmarks, want 0 (owner's rows leaked)", len(theirs))
|
||||
}
|
||||
|
||||
// The other Reader writes the same key; both rows coexist, each visible
|
||||
// only to its owner. The series title is shared (ADR-0003) — the
|
||||
// reader-owned fields are progress and updated_at.
|
||||
req := credRequest(http.MethodPut, "/bookmarks/"+ownerKey, readerCredential("other-reader"))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
body := `{"key":"asura:solo","site":"asura","series_id":"solo","title":"Theirs","last_chapter_num":3}`
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, withBody(req, body))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other reader put: status = %d, want 200", rr.Code)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
|
||||
var theirs2 []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &theirs2); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(theirs2) != 1 || theirs2[0].LastChapterNum != 3 {
|
||||
t.Fatalf("other reader list = %+v, want their own row with their progress", theirs2)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", ownerCredential()))
|
||||
var owners []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &owners); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(owners) != 1 || owners[0].Title != "Solo Leveling" || owners[0].LastChapterNum != 0 {
|
||||
t.Fatalf("owner list = %+v, want their own row at their own progress", owners)
|
||||
}
|
||||
|
||||
// The mirror: the owner's write does not move the other Reader's progress
|
||||
// either. Without it, isolation is only asserted in one direction.
|
||||
req = credRequest(http.MethodPut, "/bookmarks/"+ownerKey, ownerCredential())
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, withBody(req, `{"key":"asura:solo","site":"asura","series_id":"solo","title":"Solo Leveling","last_chapter_num":9}`))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("owner put: status = %d, want 200", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
|
||||
var theirs3 []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &theirs3); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(theirs3) != 1 || theirs3[0].LastChapterNum != 3 {
|
||||
t.Fatalf("other reader list = %+v, want progress 3 after the owner's write", theirs3)
|
||||
}
|
||||
|
||||
// DELETE is scoped to its caller too, asserted in both directions: each
|
||||
// Reader's delete on the shared key takes only their own row.
|
||||
list := func(cred string) []store.Bookmark {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", cred))
|
||||
var got []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
del := func(cred string) {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodDelete, "/bookmarks/"+ownerKey, cred))
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("delete: status = %d, want 204", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
del(readerCredential("other-reader"))
|
||||
if got := list(ownerCredential()); len(got) != 1 {
|
||||
t.Fatalf("owner's row was deletable by the other Reader: %+v", got)
|
||||
}
|
||||
if got := list(readerCredential("other-reader")); len(got) != 0 {
|
||||
t.Fatalf("other Reader's own delete left %+v behind", got)
|
||||
}
|
||||
|
||||
// The mirror: the other Reader takes the key again, the owner deletes
|
||||
// theirs, and the other's row is untouched.
|
||||
req = credRequest(http.MethodPut, "/bookmarks/"+ownerKey, readerCredential("other-reader"))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, withBody(req, body))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other reader re-put: status = %d, want 200", rr.Code)
|
||||
}
|
||||
del(ownerCredential())
|
||||
if got := list(readerCredential("other-reader")); len(got) != 1 {
|
||||
t.Fatalf("other Reader's row was deletable by the owner: %+v", got)
|
||||
}
|
||||
if got := list(ownerCredential()); len(got) != 0 {
|
||||
t.Fatalf("owner's own delete left %+v behind", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The install endpoints are session-gated and render the script directly
|
||||
// with the Reader's credential inside: the credential never appears in the
|
||||
// address bar, the page markup, or any Location header.
|
||||
func TestInstallServesScriptWithCredential(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "manga-bookmark.user.js")
|
||||
novelPath := filepath.Join(dir, "novel-bookmark.user.js")
|
||||
for _, p := range []string{path, novelPath} {
|
||||
if err := os.WriteFile(p, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
}
|
||||
cfg.UserscriptPath = path
|
||||
cfg.NovelUserscriptPath = novelPath
|
||||
srv, st := newWebTestServer(t, cfg)
|
||||
|
||||
for _, script := range []string{"manga-bookmark.user.js", "novel-bookmark.user.js"} {
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/install/"+script, nil))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("%s without session: status = %d, want 401", script, rr.Code)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/install/"+script, nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("%s with session: status = %d, want 200", script, rr.Code)
|
||||
}
|
||||
body := rr.Body.String()
|
||||
if strings.Contains(body, "__API_TOKEN__") {
|
||||
t.Fatalf("%s served with an unsubstituted placeholder", script)
|
||||
}
|
||||
// The credential rides inside the served script — nowhere visible in
|
||||
// the UI — and is the session holder's own.
|
||||
if !strings.Contains(body, `API_TOKEN = "`+ownerCredential()+`"`) {
|
||||
t.Fatalf("%s does not carry the owner's credential:\n%s", script, body)
|
||||
}
|
||||
if loc := rr.Header().Get("Location"); loc != "" {
|
||||
t.Fatalf("%s answered with a redirect, credential in Location %q", script, loc)
|
||||
}
|
||||
// The plain link must stay inline: Violentmonkey's updater polls the
|
||||
// /u/ path and an attachment disposition there would break updates.
|
||||
if cd := rr.Header().Get("Content-Disposition"); cd != "" {
|
||||
t.Fatalf("%s served as %q, want inline", script, cd)
|
||||
}
|
||||
|
||||
// ?download=1 is the mobile path: Violentmonkey on Chromium ignores a
|
||||
// .user.js navigation, so the Reader saves the file and adds it by hand.
|
||||
req = httptest.NewRequest(http.MethodGet, "/install/"+script+"?download=1", nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("%s?download=1: status = %d, want 200", script, rr.Code)
|
||||
}
|
||||
if got, want := rr.Header().Get("Content-Disposition"), `attachment; filename="`+script+`"`; got != want {
|
||||
t.Fatalf("%s?download=1: Content-Disposition = %q, want %q", script, got, want)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), `API_TOKEN = "`+ownerCredential()+`"`) {
|
||||
t.Fatalf("%s?download=1 does not carry the owner's credential", script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rotation through the web UI invalidates the old credential immediately,
|
||||
// mints one that authenticates the API and the script path, and warns that
|
||||
// every device must reinstall.
|
||||
func TestRotateCredentialViaWebUI(t *testing.T) {
|
||||
s, _ := newTestStoreURL(t)
|
||||
path := filepath.Join(t.TempDir(), "manga-bookmark.user.js")
|
||||
if err := os.WriteFile(path, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
cfg := testConfig()
|
||||
cfg.UserscriptPath = path
|
||||
srv := newRouter(s, cfg)
|
||||
|
||||
oldCred := ownerCredential()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", oldCred))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("old credential before rotation: status = %d, want 200", rr.Code)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/rotate-token", nil)
|
||||
req.AddCookie(sessionCookie(t, s))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("rotate: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "Credential rotated") {
|
||||
t.Fatalf("rotation response does not warn about reinstall:\n%s", rr.Body.String())
|
||||
}
|
||||
|
||||
// The old credential is dead on the API and on the script path.
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", oldCred))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("old credential after rotation: status = %d, want 401", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+oldCred+"/manga-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("old credential script path after rotation: status = %d, want 404", rr.Code)
|
||||
}
|
||||
|
||||
// The new credential authenticates the API and the script path, and is
|
||||
// substituted into the served script.
|
||||
newCred := token.Token([]byte(testTokenKey), testDiscordID, 1)
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", newCred))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("new credential after rotation: status = %d, want 200", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+newCred+"/manga-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("new credential script path: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+newCred+`"`) {
|
||||
t.Fatalf("served script does not carry the rotated credential:\n%s", got)
|
||||
}
|
||||
|
||||
// The install link now renders the script with the new credential.
|
||||
req = httptest.NewRequest(http.MethodGet, "/install/manga-bookmark.user.js", nil)
|
||||
req.AddCookie(sessionCookie(t, s))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("install after rotation: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+newCred+`"`) {
|
||||
t.Fatalf("install after rotation does not carry the new credential:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The app page offers the install links; the credential never appears in its
|
||||
// markup.
|
||||
func TestIndexShowsSetupPanelWithoutCredential(t *testing.T) {
|
||||
srv, st := newWebTestServer(t, testConfig())
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
for _, want := range []string{
|
||||
`href="/install/manga-bookmark.user.js"`,
|
||||
`href="/install/novel-bookmark.user.js"`,
|
||||
`href="/install/manga-bookmark.user.js?download=1"`,
|
||||
`href="/install/novel-bookmark.user.js?download=1"`,
|
||||
"Rotate credential",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("app page lacks %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, ownerCredential()) {
|
||||
t.Fatal("app page leaks the credential")
|
||||
}
|
||||
}
|
||||
+725
-137
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
# Copy to chrome/.env on the home machine. Never commit the real .env.
|
||||
#
|
||||
# This file configures the browser unit only. It is separate from the API
|
||||
# stack's ../.env on purpose: the two run on different machines.
|
||||
|
||||
# The address the CDP port is published on — required, no default.
|
||||
#
|
||||
# Use this machine's **tailnet IP**, e.g. 100.x.y.z (`tailscale ip -4`). Not
|
||||
# 0.0.0.0, not the LAN address: CDP has no authentication of its own, so
|
||||
# anything that can reach this port has full control of the browser and a
|
||||
# foothold on this host. Tailscale device identity plus an ACL is the access
|
||||
# control; the bind address is what enforces it.
|
||||
#
|
||||
# For a throwaway local test, 127.0.0.1 is fine — but then only this machine
|
||||
# can reach it, so the API must run here too.
|
||||
# Left commented so `cp .env.example .env && docker compose up` fails with the
|
||||
# variable's own message telling you what to set, rather than Docker rejecting
|
||||
# "100.x.y.z" as an invalid IP.
|
||||
# BROWSER_BIND_ADDR=100.x.y.z
|
||||
|
||||
# Clock zone the browser reports. Any real zone works and it need not match
|
||||
# the egress IP's country — but it must not be UTC, which is itself the bot
|
||||
# signal that stops the challenge clearing. The measurement is in entrypoint.sh.
|
||||
#
|
||||
# Unset falls back to the host's /etc/timezone, which is a real zone whenever
|
||||
# the host clock is set to local time. Set this when the host runs UTC — a UTC
|
||||
# server is exactly the case that fails.
|
||||
# BROWSER_TZ=Asia/Jakarta
|
||||
@@ -0,0 +1,45 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# Real Google Chrome for the latest-chapter poller and the kagane cover proxy.
|
||||
#
|
||||
# Not chromedp/headless-shell, which this replaces. headless-shell is a stripped
|
||||
# Chrome build and Cloudflare's managed challenge on kagane.to never clears for
|
||||
# it: measured 2026-08-08, 60s of a held-open tab still served the interstitial,
|
||||
# while stock Chrome from the same IP cleared in ~4s. The tells are structural
|
||||
# rather than a header — navigator.webdriver true, an empty plugin list, and
|
||||
# Chromium- rather than Chrome-branded client hints. Overriding webdriver alone
|
||||
# was tried and did not move it, so the browser build itself is the fix.
|
||||
#
|
||||
# zenika/alpine-chrome was also tried: its Chrome is 124 (2024), old enough that
|
||||
# Cloudflare refuses it outright and old enough to break chromedp's CDP structs.
|
||||
FROM debian:trixie-slim
|
||||
|
||||
# Chrome is deliberately unpinned, against the usual rule. A pinned build goes
|
||||
# stale, and a stale browser is exactly what Cloudflare turns away — the 124 in
|
||||
# alpine-chrome is the worked example. Rebuild is the upgrade path.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates wget gnupg util-linux \
|
||||
&& wget -qO- https://dl.google.com/linux/linux_signing_key.pub \
|
||||
| gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg \
|
||||
&& echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \
|
||||
> /etc/apt/sources.list.d/google-chrome.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends google-chrome-stable socat \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Keep the profile path present so Docker initializes the named volume with
|
||||
# the unprivileged user's ownership.
|
||||
|
||||
RUN useradd --create-home --shell /usr/sbin/nologin chrome \
|
||||
&& mkdir -p /home/chrome/profile /home/chrome/state \
|
||||
&& chown -R chrome:chrome /home/chrome
|
||||
|
||||
# Unprivileged: Chrome refuses to run as root, and the CDP endpoint is a shell
|
||||
# on whatever user owns it.
|
||||
USER chrome
|
||||
WORKDIR /home/chrome
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
EXPOSE 9222
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,62 @@
|
||||
# The browser, as its own deployable unit.
|
||||
#
|
||||
# This does NOT run beside the API. It runs on the home machine, reached from
|
||||
# the VPS over the tailnet, and is updated without touching the API stack:
|
||||
#
|
||||
# cd chrome && docker compose up -d --build
|
||||
#
|
||||
# Set BROWSER_BIND_ADDR in chrome/.env to this machine's tailnet IP. See
|
||||
# ../DEPLOY.md §7 for the full first-time procedure and ../docs/adr/
|
||||
# 0006-browser-on-the-home-machine.md for why the browser lives here at all.
|
||||
name: bookmark-browser
|
||||
|
||||
services:
|
||||
browser:
|
||||
build: .
|
||||
image: bookmarkmanager-chrome:latest
|
||||
container_name: bookmark-browser
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Any real zone works, but a UTC clock is itself the bot signal and the
|
||||
# challenge then never clears — measurement in entrypoint.sh. Unset falls
|
||||
# back to the host's /etc/timezone below, which is a real zone whenever
|
||||
# the host clock is local; set BROWSER_TZ when the host runs UTC.
|
||||
TZ: ${BROWSER_TZ:-}
|
||||
volumes:
|
||||
# The zone *name*, which is what Chrome's ICU needs — see entrypoint.sh.
|
||||
# Absent on a non-Debian host, which the entrypoint handles by falling back to UTC.
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
# Cloudflare clearance must survive Chrome reaping and image recreation.
|
||||
- chrome-profile:/home/chrome/profile
|
||||
# Bound to the tailnet address only, never 0.0.0.0. CDP authenticates
|
||||
# nothing: whatever reaches this port drives the browser and, through it,
|
||||
# this host. On the VPS the safety was Docker network membership; here the
|
||||
# machine has a real LAN, so the bind address *is* the access control,
|
||||
# backed by Tailscale device identity. No default — an unset variable must
|
||||
# fail the deploy rather than silently publish CDP to the LAN.
|
||||
ports:
|
||||
- "${BROWSER_BIND_ADDR:?set BROWSER_BIND_ADDR to this machine's tailnet IP}:9222:9222"
|
||||
# Reaps zombie renderer processes, which otherwise accumulate for the
|
||||
# container's lifetime.
|
||||
init: true
|
||||
# Chrome allocates shared memory per tab and dies on Docker's 64MB default.
|
||||
# 128MB against a measured 19MB peak: the old 1GB reservation was sized by
|
||||
# superstition, and this box has 1.8GB total.
|
||||
shm_size: '128mb'
|
||||
# The browser is the newcomer on a machine where a Gitea runner already
|
||||
# holds ~1.2GiB of 1.8GiB. Load-bearing, not decorative: untuned Chrome
|
||||
# peaked at 645MiB cgroup, which is more than is free here.
|
||||
#
|
||||
# memswap_limit is memory+swap combined, so this allows 512MiB of swap —
|
||||
# Chrome reclaims its own cold pages onto this box's 5.9GiB of SATA swap
|
||||
# instead of taking resident memory from the runner.
|
||||
mem_limit: 512m
|
||||
memswap_limit: 1g
|
||||
# If the box does run out, the kernel takes the browser and never CI.
|
||||
oom_score_adj: 800
|
||||
# A challenge solve yields to a running build. Cold start degrades to ~3s
|
||||
# at half a CPU, immaterial against a 45-second challenge budget.
|
||||
cpu_shares: 512
|
||||
|
||||
volumes:
|
||||
chrome-profile:
|
||||
Executable
+212
@@ -0,0 +1,212 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# A UTC clock is itself the bot signal — Cloudflare treats it as the datacenter
|
||||
# default — and kagane's challenge then never clears. Measured 2026-08-08 with
|
||||
# an identical container on one Indonesian egress IP: UTC never cleared in 60s
|
||||
# (twice), while Asia/Jakarta and America/New_York both cleared in 4s. Any real
|
||||
# zone will do; the zone does not have to match the IP's country, it just must
|
||||
# not be UTC. It does have to be right the way Chrome reads it.
|
||||
#
|
||||
# TZ must carry the zone *name*. Chrome resolves the zone through ICU, which
|
||||
# takes the name from /etc/localtime's symlink target and ignores the file's
|
||||
# contents; bind-mounting the host's /etc/localtime therefore lands on the
|
||||
# image's own symlink to Etc/UTC and leaves glibc reporting +07 while Chrome
|
||||
# still reports UTC. /etc/timezone, mounted by docker-compose.yml, is the name.
|
||||
[ -n "${TZ:-}" ] || TZ=$(cat /etc/timezone 2>/dev/null || echo UTC)
|
||||
export TZ
|
||||
|
||||
state=/home/chrome/state
|
||||
profile=/home/chrome/profile
|
||||
lock_file=$state/lock
|
||||
pid_file=$state/chrome.pid
|
||||
connections_dir=$state/connections
|
||||
last_use_file=$state/last-use
|
||||
idle_seconds=300
|
||||
|
||||
mkdir -p "$state" "$profile" "$connections_dir"
|
||||
exec 9>>"$lock_file"
|
||||
|
||||
# Chrome's own UA advertises "HeadlessChrome" under --headless=new, and that
|
||||
# one token is the difference between kagane.to's challenge clearing in ~4s and
|
||||
# never clearing at all. Read the installed major version so client hints and
|
||||
# the UA stay aligned after an image rebuild.
|
||||
major=$(google-chrome-stable --version | sed -E 's/[^0-9]*([0-9]+)\..*/\1/')
|
||||
ua="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36"
|
||||
|
||||
lock() {
|
||||
flock 9
|
||||
}
|
||||
|
||||
unlock() {
|
||||
flock -u 9
|
||||
}
|
||||
|
||||
browser_alive() {
|
||||
[ -s "$pid_file" ] || return 1
|
||||
pid=$(cat "$pid_file")
|
||||
[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null
|
||||
}
|
||||
|
||||
has_connections() {
|
||||
for marker in "$connections_dir"/*; do
|
||||
[ -e "$marker" ] || continue
|
||||
pid=${marker##*/}
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
# A SIGKILLed helper cannot run its cleanup trap. Reconcile its marker
|
||||
# here so one dead client cannot pin Chrome forever.
|
||||
rm -f "$marker"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
start_browser() {
|
||||
# No --enable-automation: it sets navigator.webdriver, the first thing a
|
||||
# bot check reads. setsid gives Chrome a process group so the reaper can
|
||||
# terminate its renderer children with the browser.
|
||||
# --no-sandbox avoids granting SYS_ADMIN solely for Docker's unavailable
|
||||
# user namespaces; containment is the unprivileged user and private network.
|
||||
setsid google-chrome-stable \
|
||||
--headless=new \
|
||||
--no-sandbox \
|
||||
--remote-debugging-port=9223 \
|
||||
--user-agent="$ua" \
|
||||
--user-data-dir="$profile" \
|
||||
--no-first-run \
|
||||
--no-default-browser-check \
|
||||
--disable-gpu \
|
||||
about:blank >/dev/null 2>&1 &
|
||||
printf '%s\n' "$!" >"$pid_file"
|
||||
}
|
||||
|
||||
stop_browser() {
|
||||
pid=$(cat "$pid_file")
|
||||
kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
|
||||
i=0
|
||||
while kill -0 "$pid" 2>/dev/null && [ "$i" -lt 100 ]; do
|
||||
i=$((i + 1))
|
||||
sleep 0.1
|
||||
done
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
}
|
||||
|
||||
wait_for_browser() {
|
||||
i=0
|
||||
while [ "$i" -lt 300 ]; do
|
||||
if wget -qO /dev/null http://127.0.0.1:9223/json/version; then
|
||||
return 0
|
||||
fi
|
||||
browser_alive || return 1
|
||||
i=$((i + 1))
|
||||
sleep 0.1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
finish_connection() {
|
||||
lock
|
||||
rm -f "$connection_marker"
|
||||
date +%s >"$last_use_file"
|
||||
unlock
|
||||
}
|
||||
|
||||
connection_signal() {
|
||||
trap - INT TERM HUP
|
||||
finish_connection
|
||||
exit 143
|
||||
}
|
||||
|
||||
connection() {
|
||||
connection_marker=$connections_dir/$$
|
||||
lock
|
||||
: >"$connection_marker"
|
||||
if ! browser_alive; then
|
||||
rm -f "$pid_file"
|
||||
start_browser
|
||||
fi
|
||||
date +%s >"$last_use_file"
|
||||
unlock
|
||||
|
||||
trap connection_signal INT TERM HUP
|
||||
if wait_for_browser; then
|
||||
if socat STDIO TCP:127.0.0.1:9223; then
|
||||
result=0
|
||||
else
|
||||
result=$?
|
||||
fi
|
||||
else
|
||||
result=1
|
||||
fi
|
||||
finish_connection
|
||||
return "$result"
|
||||
}
|
||||
|
||||
reaper() {
|
||||
while :; do
|
||||
sleep 10
|
||||
lock
|
||||
if ! has_connections && browser_alive; then
|
||||
now=$(date +%s)
|
||||
last=$(cat "$last_use_file" 2>/dev/null || printf '%s' "$now")
|
||||
if [ $((now - last)) -ge "$idle_seconds" ]; then
|
||||
stop_browser
|
||||
fi
|
||||
fi
|
||||
unlock
|
||||
done
|
||||
}
|
||||
|
||||
if [ "${1:-}" = connection ]; then
|
||||
connection
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# The files are process state, not the Chrome profile. The profile is a named
|
||||
# volume in Compose, so clearance survives both a reap and a container rebuild.
|
||||
for marker in "$connections_dir"/*; do
|
||||
[ -e "$marker" ] || continue
|
||||
rm -f "$marker"
|
||||
done
|
||||
rm -f "$pid_file" "$last_use_file"
|
||||
|
||||
# Chrome binds DevTools to loopback and silently ignores
|
||||
# --remote-debugging-address. socat remains the network front-end, but each
|
||||
# accepted connection now starts a browser on demand and is tracked by a
|
||||
# per-helper marker. A connection held by Go's transport delays reap by its
|
||||
# idle timeout; the 300-second threshold starts once the last connection closes.
|
||||
reaper &
|
||||
reaper_pid=$!
|
||||
socat TCP-LISTEN:9222,reuseaddr,fork EXEC:'/entrypoint.sh connection',nofork &
|
||||
front_pid=$!
|
||||
|
||||
stop_browser_gracefully() {
|
||||
lock
|
||||
if browser_alive; then
|
||||
# Chrome is a separate session, so stop its process group explicitly;
|
||||
# this gives its cookie batch time to flush before the container exits.
|
||||
stop_browser
|
||||
fi
|
||||
unlock
|
||||
}
|
||||
|
||||
shutdown() {
|
||||
trap - INT TERM HUP
|
||||
stop_browser_gracefully
|
||||
kill "$front_pid" "$reaper_pid" 2>/dev/null || true
|
||||
exit 143
|
||||
}
|
||||
trap shutdown INT TERM HUP
|
||||
|
||||
if wait "$front_pid"; then
|
||||
status=0
|
||||
else
|
||||
status=$?
|
||||
fi
|
||||
stop_browser_gracefully
|
||||
kill "$reaper_pid" 2>/dev/null || true
|
||||
exit "$status"
|
||||
+10
-20
@@ -17,24 +17,12 @@ services:
|
||||
bookmark-api:
|
||||
# Traffic arrives over the Traefik network, not a published port.
|
||||
ports: !reset []
|
||||
environment:
|
||||
# Must be an IP, not the DNS name — see the base file's comment on this
|
||||
# same key: Chrome's DevTools HTTP handler 500s any Host header that
|
||||
# isn't an IP or "localhost".
|
||||
BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://172.28.0.10:9222}
|
||||
depends_on:
|
||||
headless-shell:
|
||||
condition: service_started
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
# `networks:` here replaces the base file's list entirely, so all three must
|
||||
# be named: `proxy` for Traefik routing, and `browser` / `db` (defined in
|
||||
# the base file) to keep reaching headless-shell and Postgres without
|
||||
# putting either on `proxy`.
|
||||
# Compose *merges* this list with the base file's, so the service ends up on
|
||||
# `default`, `db` and `proxy` — only the addition is named here. Do not
|
||||
# "tidy" the base file down to `db` on the strength of `proxy` being present:
|
||||
# `db` is `internal: true`, and egress comes from `default`.
|
||||
networks:
|
||||
- proxy
|
||||
- browser
|
||||
- db
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.docker.network=${PROXY_NETWORK:-proxy}"
|
||||
@@ -52,10 +40,12 @@ services:
|
||||
- "traefik.http.routers.bmweb.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
|
||||
- "traefik.http.routers.bmweb.service=bmapi"
|
||||
|
||||
# headless-shell is untouched here: it keeps its `browser` network membership
|
||||
# from the base file and must never join `proxy` — that network is shared
|
||||
# with whatever else sits behind Traefik on this host, and an exposed
|
||||
# CDP endpoint on it would be remote code execution for any of them.
|
||||
# No browser service here. It runs on the home machine as its own unit
|
||||
# (chrome/docker-compose.yml) and is reached over the tailnet — see
|
||||
# docs/adr/0006-browser-on-the-home-machine.md. It must never be given a
|
||||
# service on this host: `proxy` is shared with whatever else sits behind
|
||||
# Traefik, and an unauthenticated CDP endpoint on it is remote code
|
||||
# execution for any of them.
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
|
||||
+43
-50
@@ -5,6 +5,9 @@
|
||||
# If your proxy runs in Docker on its own network, use the prod override which
|
||||
# attaches to that network instead of publishing a port:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
#
|
||||
# The browser is not here. It is its own unit on the home machine —
|
||||
# chrome/docker-compose.yml — reached over the tailnet via BROWSER_WS_URL.
|
||||
|
||||
services:
|
||||
bookmark-api:
|
||||
@@ -13,15 +16,33 @@ services:
|
||||
container_name: bookmark-api
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# API_TOKEN is required — compose refuses to start without it.
|
||||
API_TOKEN: ${API_TOKEN:?set API_TOKEN in .env}
|
||||
# TOKEN_KEY derives every Reader's userscript credential (issue #24) —
|
||||
# compose refuses to start without it.
|
||||
TOKEN_KEY: ${TOKEN_KEY:?set TOKEN_KEY in .env}
|
||||
# Owner's Discord user ID — required. Seeds the owner Reader (the
|
||||
# administrator); every other Reader registers on their first login.
|
||||
OWNER_DISCORD_ID: ${OWNER_DISCORD_ID:?set OWNER_DISCORD_ID in .env}
|
||||
ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-https://asuracomic.net,https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to,https://novelfull.com,https://lightnovelworld.net}
|
||||
# The bookmarks database. Host is the compose service name; the password
|
||||
# comes from .env so it is never committed.
|
||||
DATABASE_URL: ${DATABASE_URL:-postgres://bookmarks:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/bookmarks?sslmode=disable}
|
||||
PORT: "8080"
|
||||
# Gates the browser UI. Unset means the web routes are not served at all.
|
||||
WEB_PASSWORD: ${WEB_PASSWORD:-}
|
||||
# Log timestamps only. Go's `log` stamps lines in local time, and this
|
||||
# service has no other use for a zone: bookmark timestamps are unix ms
|
||||
# and the two real time columns are timestamptz, both absolute instants.
|
||||
# Purely so these lines read on the same clock as the browser's. Named
|
||||
# API_TZ rather than TZ so an operator's exported shell TZ cannot leak
|
||||
# in; distroless already carries tzdata, so the name just resolves.
|
||||
TZ: ${API_TZ:-Asia/Jakarta}
|
||||
# Discord OAuth for the browser UI (ADR-0002). The first four are
|
||||
# required; DISCORD_REQUIRED_ROLE is optional and empty by default.
|
||||
# Guild membership is the whole gate: any member becomes a Reader.
|
||||
DISCORD_CLIENT_ID: ${DISCORD_CLIENT_ID:?set DISCORD_CLIENT_ID in .env}
|
||||
DISCORD_CLIENT_SECRET: ${DISCORD_CLIENT_SECRET:?set DISCORD_CLIENT_SECRET in .env}
|
||||
DISCORD_GUILD_ID: ${DISCORD_GUILD_ID:?set DISCORD_GUILD_ID in .env}
|
||||
DISCORD_REQUIRED_ROLE: ${DISCORD_REQUIRED_ROLE:-}
|
||||
DISCORD_API_BASE: ${DISCORD_API_BASE:-https://discord.com/api/v10}
|
||||
DISCORD_REDIRECT_URI: ${DISCORD_REDIRECT_URI:?set DISCORD_REDIRECT_URI in .env}
|
||||
# Path inside the container; matches the bindmount above.
|
||||
USERSCRIPT_PATH: ${USERSCRIPT_PATH:-/userscript/manga-bookmark.user.js}
|
||||
# Second script from the same bindmount; the novel library is a separate
|
||||
@@ -31,21 +52,24 @@ services:
|
||||
# switch; it only takes effect because these are listed here.
|
||||
LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1}
|
||||
LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h}
|
||||
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN: ${LATEST_CHAPTER_POLL_BROWSER_COOLDOWN:-6h}
|
||||
LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m}
|
||||
LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14}
|
||||
LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s}
|
||||
# CDP endpoint for sites behind a JavaScript challenge (kagane). Unset
|
||||
# disables browser polling for those sites; the userscript still covers them.
|
||||
# Must be an IP, not the "headless-shell" DNS name: Chrome's DevTools HTTP
|
||||
# handler rejects the discovery request (GET /json/version) with a 500
|
||||
# unless the Host header is an IP address or "localhost" — confirmed
|
||||
# 2026-08-03 against chromedp/headless-shell:stable, independent of
|
||||
# chromedp's own dial logic. The sidecar's static address below exists so
|
||||
# this URL survives container recreation.
|
||||
BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://172.28.0.10:9222}
|
||||
# CDP endpoint for sites behind a JavaScript challenge (kagane,
|
||||
# novelfull). The browser is not part of this stack — it runs on the home
|
||||
# machine as its own unit (chrome/docker-compose.yml) and is reached over
|
||||
# the tailnet. Unset disables browser polling for those sites and serves
|
||||
# 404 from the cover proxy for covers not already stored; the userscript
|
||||
# still covers them. Set it in .env to ws://<home machine tailnet IP>:9222.
|
||||
#
|
||||
# Must be an IP, not a MagicDNS hostname: Chrome's DevTools HTTP handler
|
||||
# rejects the discovery request (GET /json/version) with a 500 unless the
|
||||
# Host header is an IP address or "localhost" — confirmed 2026-08-03,
|
||||
# independent of chromedp's own dial logic. The same trap that used to
|
||||
# force a pinned Docker IP now forbids the tailnet name.
|
||||
BROWSER_WS_URL: ${BROWSER_WS_URL:-}
|
||||
depends_on:
|
||||
headless-shell:
|
||||
condition: service_started
|
||||
# The migration runner is the first thing the binary does, so a Postgres
|
||||
# that is still initialising means a crash-loop until it is not.
|
||||
postgres:
|
||||
@@ -60,8 +84,11 @@ services:
|
||||
# the public internet does not.
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
# `default` is not decoration: `db` is `internal: true`, and a container on
|
||||
# nothing but an internal network gets neither a published port nor egress
|
||||
# — which would silently kill every poller fetch.
|
||||
networks:
|
||||
- browser
|
||||
- default
|
||||
- db
|
||||
|
||||
postgres:
|
||||
@@ -83,33 +110,6 @@ services:
|
||||
networks:
|
||||
- db
|
||||
|
||||
headless-shell:
|
||||
image: chromedp/headless-shell:stable
|
||||
restart: unless-stopped
|
||||
# Chrome allocates shared memory per tab and dies on Docker's 64MB default.
|
||||
shm_size: '1gb'
|
||||
# Reaps zombie renderer processes, which otherwise accumulate for the
|
||||
# container's lifetime.
|
||||
init: true
|
||||
# Deliberately no `ports:` — an exposed CDP endpoint is remote code
|
||||
# execution. Only bookmark-api, via the `browser` network below, may reach it.
|
||||
# Don't pass --remote-debugging-address/--remote-debugging-port here: the
|
||||
# image's own entrypoint (/headless-shell/run.sh) already starts Chrome on
|
||||
# 127.0.0.1:9223 and fronts it with a socat proxy listening on 0.0.0.0:9222.
|
||||
# Redeclaring the port flag here overrides Chrome's, so it binds 9222
|
||||
# directly (IPv6 loopback only) instead of 9223 — collides with socat's own
|
||||
# bind on 9222 and leaves nothing listening on 9223, so every external
|
||||
# connection to headless-shell:9222 fails with EOF. Only pass flags the
|
||||
# entrypoint doesn't already set.
|
||||
command:
|
||||
- --disable-gpu
|
||||
- --no-sandbox
|
||||
networks:
|
||||
browser:
|
||||
# Pinned so BROWSER_WS_URL can name an IP (required, see above) that
|
||||
# survives `docker compose up` recreating this container.
|
||||
ipv4_address: 172.28.0.10
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
# The pre-Postgres SQLite volume (bookmarks-data) is deliberately no longer
|
||||
@@ -118,13 +118,6 @@ volumes:
|
||||
# removes it by hand.
|
||||
|
||||
networks:
|
||||
# Not `internal: true`: headless Chrome still needs outbound access to reach
|
||||
# kagane.to. Isolation here comes from membership (only bookmark-api and
|
||||
# headless-shell join it), not from cutting egress.
|
||||
browser:
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.28.0.0/24
|
||||
# Postgres needs no egress and nothing outside bookmark-api needs to reach
|
||||
# it, so this one really can be cut off from the outside world.
|
||||
db:
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# ADR-0005: On-demand browser sidecar
|
||||
|
||||
Date: 2026-08-09
|
||||
Status: accepted
|
||||
|
||||
Superseded in part by ADR-0006: the lifecycle below is unchanged, but the
|
||||
service no longer lives in the API stack and the name `headless-shell` is gone.
|
||||
|
||||
## Decision
|
||||
|
||||
Keep the `headless-shell` service and its CDP port alive, but start Google Chrome
|
||||
only when the first CDP connection arrives. The entrypoint supervises a `socat`
|
||||
front-end, serializes browser start/reap state with `flock`, and tracks each
|
||||
connection with a marker named for its helper PID. A reaper stops Chrome after
|
||||
300 seconds with no live markers. Marker reconciliation covers a helper killed
|
||||
before its cleanup trap runs.
|
||||
|
||||
Chrome runs in its own process group so reap sends the termination signal to
|
||||
Chrome and its renderer children. The explicit `/home/chrome/profile` user-data
|
||||
directory remains: Chrome remaps remote debugging to loopback on modern builds,
|
||||
and Chrome ignores remote-debugging flags on a default profile. `socat` therefore
|
||||
continues to front Chrome's loopback CDP port.
|
||||
|
||||
The profile is a named Compose volume. Clearance cookies survive both a reap and
|
||||
`docker compose up --build`; the browser still starts with a fresh debugger UUID,
|
||||
so chromedp must keep endpoint discovery enabled and must not use
|
||||
`chromedp.NoModifyURL`.
|
||||
|
||||
The socat front-end and explicit profile are retained because Chromium remaps a
|
||||
non-loopback debugging address to loopback since M113, while Chrome ignores the
|
||||
remote-debugging flags on a default profile since Chrome 136. Flag tuning is
|
||||
deliberately not adopted: its roughly 30% idle-footprint saving is irrelevant
|
||||
to a browser that exists for seconds per wake and risks an untested fingerprint.
|
||||
|
||||
## Constraints
|
||||
|
||||
The 300-second floor is deliberate. Chromium batches cookie persistence on a
|
||||
roughly 31-second timer, and Go's default HTTP transport can keep the discovery
|
||||
connection parked for about 90 seconds after use. Reaping only with zero live
|
||||
connections holds Chrome through both windows and through the poller's staggered
|
||||
batch plus cover prefetch.
|
||||
|
||||
The anti-bot properties remain unchanged: a plausible non-UTC timezone, a
|
||||
Chrome-version-derived User-Agent without `HeadlessChrome`, and no automation
|
||||
flag. A remote browser restart can surface as `context.Canceled`, the same error
|
||||
as a caller deadline, so the backend wraps cancellation observed with a closed
|
||||
CDP connection as `browser interrupted`; the focused test asserts that
|
||||
classification without killing a real browser.
|
||||
The same process-group stop runs during supervisor shutdown, not only during
|
||||
idle reap, so Chrome can flush its cookie batch before a container rebuild or
|
||||
graceful stop.
|
||||
@@ -0,0 +1,74 @@
|
||||
# ADR-0006: The browser runs on the home machine, over the tailnet
|
||||
|
||||
Date: 2026-08-09
|
||||
Status: accepted
|
||||
|
||||
## Decision
|
||||
|
||||
The headless browser is no longer part of the API stack. It is its own compose
|
||||
unit (`chrome/docker-compose.yml`), deployed on the home machine, and the API on
|
||||
the VPS reaches it over the existing tailnet through `BROWSER_WS_URL`. No
|
||||
fallback sidecar remains on the VPS.
|
||||
|
||||
The backend needs no code change for this. The CDP endpoint was already a
|
||||
configuration seam and the fetcher only ever holds the endpoint URL, so
|
||||
relocation — and reversal — is one environment variable.
|
||||
|
||||
## Why
|
||||
|
||||
The sidecar held 471 MiB working set (645 MiB peak) on a 1974 MiB VPS with no
|
||||
swap, which also hosts Traefik, Gitea and its Postgres. That is 24% of the host
|
||||
and 86% of this project's memory, for a service that at the time answered zero
|
||||
requests: the poller's due query joins bookmarks, production held four kagane
|
||||
series and no bookmarks on any of them, and with no kagane bookmark the web UI
|
||||
never rendered a kagane cover either.
|
||||
|
||||
The home machine has 5.9 GiB of swap and a residential egress, which Cloudflare
|
||||
scores better than a datacenter IP. Both machines were already on the tailnet.
|
||||
|
||||
This move is only safe because covers are persisted (ADR-0005's sibling work,
|
||||
issue #43/#45) and the browser is on-demand (ADR-0005). Without stored covers a
|
||||
sleeping home machine would blank the library; without on-demand start the CI
|
||||
runner that already holds ~1.2 GiB of that box's 1.8 GiB would be squeezed
|
||||
around the clock.
|
||||
|
||||
## Constraints
|
||||
|
||||
**`BROWSER_WS_URL` must be the tailnet IP, never a MagicDNS hostname.** Chrome's
|
||||
DevTools HTTP handler answers `/json/version` with a 500 for any `Host` header
|
||||
that is not an IP or `localhost`. This is the same trap that previously forced a
|
||||
pinned Docker IP; the pinned subnet is gone, the constraint is not.
|
||||
|
||||
**The CDP port binds to the tailnet address only, never `0.0.0.0`.** CDP
|
||||
authenticates nothing: whatever reaches the port drives the browser and, through
|
||||
it, the host. On the VPS the safety came from Docker network membership; the
|
||||
home machine has a real LAN, so a `0.0.0.0` bind is a hole punched into it. The
|
||||
bind address is the enforcement and Tailscale device identity plus a per-device
|
||||
ACL is the policy. `BROWSER_BIND_ADDR` deliberately has no default, so an unset
|
||||
value fails the deploy instead of publishing CDP to the LAN.
|
||||
|
||||
No bearer-token proxy is added in front of CDP. It would only defend against a
|
||||
device already inside the tailnet, and it would be one more thing between the
|
||||
poller and a browser that is already hard enough to keep clearing challenges.
|
||||
|
||||
**Resource limits are load-bearing, not decorative.** The browser is the
|
||||
newcomer on that box, not the incumbent. A hard 512 MiB cap with 1 GiB
|
||||
memory+swap makes Chrome reclaim its own cold pages onto the machine's SATA swap
|
||||
instead of taking resident memory from the runner; untuned Chrome peaked at
|
||||
645 MiB cgroup, which is more than is free there. `oom_score_adj` biases the
|
||||
kernel to kill the browser first and never CI. Reduced CPU weight makes a
|
||||
challenge solve yield to a running build — cold start degrades to about 3 s at
|
||||
half a CPU, immaterial against a 45-second challenge budget. The shared-memory
|
||||
reservation drops from 1 GiB to 128 MiB against a measured 19 MiB peak.
|
||||
|
||||
## Consequences
|
||||
|
||||
An unreachable browser degrades exactly as an unset `BROWSER_WS_URL` already
|
||||
does: plain-TLS libraries are unaffected, kagane and novelfull log and skip, the
|
||||
series waits out its cooldown, and stored covers keep serving. A power outage at
|
||||
home costs chapter freshness on two sites, never the appearance of the library.
|
||||
|
||||
The two units are deployed and updated independently. `REDEPLOY.md` §8 covers
|
||||
the browser; everything before it covers the API stack. A local `docker compose
|
||||
up` now brings up two services, not three, and polls kagane only if
|
||||
`BROWSER_WS_URL` is pointed somewhere.
|
||||
@@ -44,6 +44,25 @@ Guidance for OpenCode (and Claude Code) working under `userscript/`. See root `A
|
||||
Encodings (incl. triple-encoded punctuation like `%25252D`) identical
|
||||
on /manga/ and /title/ pages, so decode-once seriesIds match — verified
|
||||
2026-07-28.
|
||||
- **comix.to**: series `/title/<id>-<slug>`, chapter
|
||||
`/title/<id>-<slug>/<uploadId>-chapter-<n>`. Only the leading `<id>` is
|
||||
identity — the slug re-renders when a series is renamed (`comixSeriesId`).
|
||||
An SPA that **never rewrites `og:title`**: the server-rendered head keeps
|
||||
whatever document loaded first, so on a cold load `og:title` is the homepage's
|
||||
"Comix — Read Comics online for free" and after an in-page hop it is the
|
||||
*previous* series' name. `document.title` is the one thing client routing does
|
||||
update, so titles come from there, with the chapter page's `" · Ch.<n>"` tail
|
||||
stripped. Covers likewise: `og:image` is absent, so the cover is the `img`
|
||||
whose `alt` matches the cleaned title — verified live 2026-08-08.
|
||||
- **kagane.to**: series `/series/<uuid>`, reader
|
||||
`/series/<uuid>/reader/<bookUuid>`. Reader URLs carry no chapter number, so
|
||||
the number comes out of `og:title`. Two shapes exist: `"<Series> - Chapter
|
||||
<n>[ - Episode <n>]"` and, for volume-numbered series, `"<Series> - Volume <v>
|
||||
Chapter <n>"` with no episode name — both must yield a bare series title, or
|
||||
the volume tail lands in the bookmark's title. Its covers are challenge- and
|
||||
CORP-protected, so the web UI proxies them; the userscript still stores the
|
||||
raw `og:image`. Behind a Cloudflare JS challenge, so the backend polls it
|
||||
through the headless browser.
|
||||
- **novelfull.com** (novel script): series `/<slug>.html`, chapter
|
||||
`/<slug>/chapter-<n>[-<title-slug>].html`. No `og:*` tags at all — title from
|
||||
`h3.title` (series) or `a.truyen-title` (chapter), cover from
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
// @version 1.6.0
|
||||
// @description Track read progress on Asura, Demonic, Comix & Kagane and sync to a self-hosted backend. Bromite-compatible (no GM_* APIs).
|
||||
// @author you
|
||||
// @downloadURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/manga-bookmark.user.js
|
||||
// @updateURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/manga-bookmark.user.js
|
||||
// @downloadURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
// @updateURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
// @match https://asuracomic.net/*
|
||||
// @match https://asurascans.com/*
|
||||
// @match https://demonicscans.org/*
|
||||
@@ -22,7 +22,7 @@
|
||||
// CONFIG — fill these in before installing.
|
||||
// ============================================================
|
||||
const API_BASE = "https://bookmark-api.violetcrown.my.id"; // your backend origin, no trailing slash
|
||||
const API_TOKEN = "40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df"; // must equal backend API_TOKEN
|
||||
const API_TOKEN = "__API_TOKEN__"; // substituted by the backend at serve time (issue #24)
|
||||
const WEB_BASE = "https://bookmark.violetcrown.my.id"; // the browser UI, for the panel's nav chips
|
||||
|
||||
// This script owns the manga library; the novel script is a separate install
|
||||
@@ -242,6 +242,12 @@
|
||||
matches: (loc) => /(^|\.)comix\.to$/.test(loc.hostname),
|
||||
detect(loc) {
|
||||
const path = loc.pathname;
|
||||
// comix client-routes without ever rewriting og:title — the head keeps
|
||||
// whatever the first server-rendered document carried, so a bookmark
|
||||
// taken after a client route got the homepage's title, then the
|
||||
// previous series'. document.title is the one thing its router does
|
||||
// update. Verified live 2026-08-08; do not "restore" meta("og:title").
|
||||
const pageTitle = cleanTitle(document.title);
|
||||
// /title/<id>-<slug>/<uploadId>-chapter-<n>. Several uploads (different
|
||||
// groups or languages) share one chapter number; the number is the
|
||||
// progress identity, the upload id is not.
|
||||
@@ -252,8 +258,8 @@
|
||||
type: "chapter",
|
||||
site: this.site,
|
||||
seriesId: comixSeriesId(m[1]),
|
||||
title: cleanTitle(meta("og:title")),
|
||||
cover: coverFromPage(),
|
||||
title: pageTitle,
|
||||
cover: coverFromPage(pageTitle),
|
||||
seriesUrl: loc.origin + "/title/" + m[1],
|
||||
chapterLabel: "Chapter " + m[2],
|
||||
chapterNum: isNaN(num) ? null : num,
|
||||
@@ -267,8 +273,8 @@
|
||||
type: "series",
|
||||
site: this.site,
|
||||
seriesId: comixSeriesId(m[1]),
|
||||
title: cleanTitle(meta("og:title")),
|
||||
cover: coverFromPage(),
|
||||
title: pageTitle,
|
||||
cover: coverFromPage(pageTitle),
|
||||
seriesUrl: loc.origin + "/title/" + m[1],
|
||||
chapterLabel: null,
|
||||
chapterNum: null,
|
||||
@@ -277,7 +283,7 @@
|
||||
}
|
||||
return { type: "other" };
|
||||
|
||||
// comix chapter og:title is "<Title> · Ch.<n>"; series is clean.
|
||||
// comix chapter document.title is "<Title> · Ch.<n>"; series is clean.
|
||||
function cleanTitle(t) {
|
||||
if (!t) return "";
|
||||
return t.replace(/\s*·\s*Ch\.[\d.]+\s*$/i, "").trim();
|
||||
@@ -287,8 +293,7 @@
|
||||
// the DOM for a cover. Matching on alt rather than a class keeps it off
|
||||
// the site's styling: the cover is the image whose alt is the title.
|
||||
// Do not "simplify" this into meta("og:image") — that returns null.
|
||||
function coverFromPage() {
|
||||
const title = cleanTitle(meta("og:title"));
|
||||
function coverFromPage(title) {
|
||||
if (!title || !document.querySelectorAll) return "";
|
||||
for (const img of document.querySelectorAll("img[alt]")) {
|
||||
if (img.getAttribute("alt") === title) return img.getAttribute("src") || "";
|
||||
@@ -313,6 +318,14 @@
|
||||
},
|
||||
};
|
||||
|
||||
// Kagane builds the reader og:title suffix out of the book's metadata, so
|
||||
// every combination occurs: the volume part appears only when the book has a
|
||||
// volume_no, the episode part only when it has a non-empty title. All four
|
||||
// shapes captured live 2026-08-08 — "SP Baby - Volume 1 Chapter 1" is the one
|
||||
// the old trailing-space regex missed, which left both the number and the
|
||||
// series title wrong.
|
||||
const KAGANE_CHAPTER_SUFFIX = /\s-\s(?:Volume\s[\d.]+\s)?Chapter\s([\d.]+)(?:\s-\s.*)?$/i;
|
||||
|
||||
const kagane = {
|
||||
site: "kagane",
|
||||
matches: (loc) => /(^|\.)kagane\.to$/.test(loc.hostname),
|
||||
@@ -353,9 +366,8 @@
|
||||
}
|
||||
return { type: "other" };
|
||||
|
||||
// Reader og:title is "<Title> - Chapter <n> - <episode name>".
|
||||
function chapterNumFromTitle(t) {
|
||||
const m = t && t.match(/\s-\sChapter\s([\d.]+)\s/);
|
||||
const m = t && t.match(KAGANE_CHAPTER_SUFFIX);
|
||||
if (!m) return null;
|
||||
const num = parseFloat(m[1]);
|
||||
return isNaN(num) ? null : num;
|
||||
@@ -363,7 +375,7 @@
|
||||
|
||||
function cleanTitle(t) {
|
||||
if (!t) return "";
|
||||
return t.replace(/\s-\sChapter\s[\d.]+\s-\s.*$/i, "").trim();
|
||||
return t.replace(KAGANE_CHAPTER_SUFFIX, "").trim();
|
||||
}
|
||||
},
|
||||
// Reader hrefs are uuids with no number in them, so no maximum can be taken
|
||||
@@ -1450,8 +1462,15 @@
|
||||
}
|
||||
|
||||
let lastUrl = location.href;
|
||||
function onNavigate() {
|
||||
let lastPageSig = "";
|
||||
|
||||
function setPage() {
|
||||
state.page = detect();
|
||||
lastPageSig = JSON.stringify(state.page);
|
||||
}
|
||||
|
||||
function onNavigate() {
|
||||
setPage();
|
||||
render();
|
||||
maybeAutoUpdate();
|
||||
maybeCaptureLatestOnSeriesPage();
|
||||
@@ -1484,7 +1503,14 @@
|
||||
wrap("pushState");
|
||||
wrap("replaceState");
|
||||
window.addEventListener("popstate", fire);
|
||||
setInterval(fire, 1500); // catch routes that bypass history
|
||||
// Also catches routes that bypass history — and comix, which fills
|
||||
// document.title a beat after the route changes, so the 300ms snapshot
|
||||
// above can still hold the previous page's title. Re-detect whenever what
|
||||
// we would read has changed, not only when the URL has.
|
||||
setInterval(() => {
|
||||
fire();
|
||||
if (JSON.stringify(detect()) !== lastPageSig) onNavigate();
|
||||
}, 1500);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
@@ -1563,7 +1589,7 @@
|
||||
|
||||
function init() {
|
||||
buildUI();
|
||||
state.page = detect();
|
||||
setPage();
|
||||
render();
|
||||
installNavWatcher();
|
||||
installLongPress();
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
// @version 1.0.0
|
||||
// @description Track read progress on NovelFull & LightNovelWorld and sync to a self-hosted backend. Bromite-compatible (no GM_* APIs).
|
||||
// @author you
|
||||
// @downloadURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/novel-bookmark.user.js
|
||||
// @updateURL https://bookmark-api.violetcrown.my.id/u/40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df/novel-bookmark.user.js
|
||||
// @downloadURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/novel-bookmark.user.js
|
||||
// @updateURL https://bookmark-api.violetcrown.my.id/u/__API_TOKEN__/novel-bookmark.user.js
|
||||
// @match https://novelfull.com/*
|
||||
// @match https://lightnovelworld.net/*
|
||||
// @run-at document-idle
|
||||
@@ -19,7 +19,7 @@
|
||||
// CONFIG — fill these in before installing.
|
||||
// ============================================================
|
||||
const API_BASE = "https://bookmark-api.violetcrown.my.id"; // your backend origin, no trailing slash
|
||||
const API_TOKEN = "40d79969b5442f90df4fe306a092c7c50e7b4a7a98099f98cc398f4fb374b1df"; // must equal backend API_TOKEN
|
||||
const API_TOKEN = "__API_TOKEN__"; // substituted by the backend at serve time (issue #24)
|
||||
const WEB_BASE = "https://bookmark.violetcrown.my.id"; // the browser UI, for the panel's nav chips
|
||||
|
||||
// This script owns the novel library; the manga script is a separate install
|
||||
|
||||
@@ -31,6 +31,9 @@ globalThis.location = {
|
||||
|
||||
// og: meta tags the adapters read through meta(). Reassigned per test.
|
||||
let metaTags = {};
|
||||
// document.title. comix's SPA rewrites this on client routing but never
|
||||
// og:title, so the comix adapter reads it instead. Reassigned per test.
|
||||
let docTitle = "";
|
||||
// img[alt] elements comix's coverFromPage() scans. Reassigned per test; each
|
||||
// entry is {alt, src}.
|
||||
let pageImages = [];
|
||||
@@ -48,6 +51,9 @@ globalThis.document = {
|
||||
}));
|
||||
},
|
||||
addEventListener() {},
|
||||
get title() {
|
||||
return docTitle;
|
||||
},
|
||||
body: undefined,
|
||||
};
|
||||
|
||||
@@ -193,8 +199,15 @@ test("comixSeriesId leaves a bare id untouched", () => {
|
||||
assert.equal(comixSeriesId("n8we"), "n8we");
|
||||
});
|
||||
|
||||
// comix is an SPA that rewrites document.title on client routing but leaves the
|
||||
// server-rendered og:title untouched, so every test here pins og:title to a
|
||||
// STALE value — the homepage title on first hop, the previous series after
|
||||
// that. Captured live 2026-08-08.
|
||||
const COMIX_STALE_HOME = "Comix - Read Comics online for free";
|
||||
|
||||
test("comix detects a series page", () => {
|
||||
metaTags = { "og:title": "Dungeons and Crayons" };
|
||||
metaTags = { "og:title": COMIX_STALE_HOME };
|
||||
docTitle = "Dungeons and Crayons";
|
||||
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
|
||||
assert.equal(p.type, "series");
|
||||
assert.equal(p.site, "comix");
|
||||
@@ -204,8 +217,16 @@ test("comix detects a series page", () => {
|
||||
assert.equal(p.chapterNum, null);
|
||||
});
|
||||
|
||||
test("comix ignores a previous series' stale og:title", () => {
|
||||
metaTags = { "og:title": "Full-Time Awakening" };
|
||||
docTitle = "Dungeons and Crayons";
|
||||
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
|
||||
assert.equal(p.title, "Dungeons and Crayons");
|
||||
});
|
||||
|
||||
test("comix detects a chapter page and strips the Ch. suffix from the title", () => {
|
||||
metaTags = { "og:title": "Dungeons and Crayons · Ch.80" };
|
||||
metaTags = { "og:title": COMIX_STALE_HOME };
|
||||
docTitle = "Dungeons and Crayons · Ch.80";
|
||||
const p = comix.detect(
|
||||
loc("https://comix.to/title/n8we-dungeons-and-crayons/11139891-chapter-80")
|
||||
);
|
||||
@@ -218,7 +239,8 @@ test("comix detects a chapter page and strips the Ch. suffix from the title", ()
|
||||
});
|
||||
|
||||
test("comix parses decimal chapter numbers", () => {
|
||||
metaTags = { "og:title": "Dungeons and Crayons · Ch.80.5" };
|
||||
metaTags = {};
|
||||
docTitle = "Dungeons and Crayons · Ch.80.5";
|
||||
const p = comix.detect(
|
||||
loc("https://comix.to/title/n8we-dungeons-and-crayons/11139891-chapter-80.5")
|
||||
);
|
||||
@@ -226,19 +248,19 @@ test("comix parses decimal chapter numbers", () => {
|
||||
});
|
||||
|
||||
test("comix.detect reads the cover from an img whose alt matches the cleaned title", () => {
|
||||
metaTags = { "og:title": "Dungeons and Crayons · Ch.80" };
|
||||
metaTags = { "og:title": COMIX_STALE_HOME };
|
||||
docTitle = "Dungeons and Crayons";
|
||||
pageImages = [
|
||||
{ alt: "Some Other Series", src: "https://cdn.example/other.jpg" },
|
||||
{ alt: "Dungeons and Crayons", src: "https://cdn.example/cover.jpg" },
|
||||
];
|
||||
const p = comix.detect(
|
||||
loc("https://comix.to/title/n8we-dungeons-and-crayons/11139891-chapter-80")
|
||||
);
|
||||
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
|
||||
assert.equal(p.cover, "https://cdn.example/cover.jpg");
|
||||
});
|
||||
|
||||
test("comix.detect leaves cover empty when no img alt matches the title", () => {
|
||||
metaTags = { "og:title": "Dungeons and Crayons" };
|
||||
metaTags = {};
|
||||
docTitle = "Dungeons and Crayons";
|
||||
pageImages = [{ alt: "Some Other Series", src: "https://cdn.example/other.jpg" }];
|
||||
const p = comix.detect(loc("https://comix.to/title/n8we-dungeons-and-crayons"));
|
||||
assert.equal(p.cover, "");
|
||||
@@ -315,6 +337,33 @@ test("kagane reads the chapter number out of og:title", () => {
|
||||
assert.equal(p.seriesUrl, "https://kagane.to/series/" + KAGANE_SERIES);
|
||||
});
|
||||
|
||||
// Volume-numbered series render the suffix as "- Volume <v> Chapter <n>" with
|
||||
// no episode name, because the book carries volume_no and an empty title.
|
||||
// Captured live 2026-08-08 from SP Baby.
|
||||
test("kagane reads through a Volume-numbered chapter suffix", () => {
|
||||
metaTags = {
|
||||
"og:title": "SP Baby - Volume 1 Chapter 1",
|
||||
"og:image": "https://kagane.to/api/v2/image/abc/compressed",
|
||||
};
|
||||
const p = kagane.detect(
|
||||
loc("https://kagane.to/series/" + KAGANE_SERIES + "/reader/" + KAGANE_BOOK)
|
||||
);
|
||||
assert.equal(p.title, "SP Baby");
|
||||
assert.equal(p.chapterNum, 1);
|
||||
assert.equal(p.chapterLabel, "Chapter 1");
|
||||
});
|
||||
|
||||
// A book with neither a volume nor an episode name ends the title right after
|
||||
// the number, which the old trailing-\s regex could not match.
|
||||
test("kagane reads a chapter suffix with no episode name", () => {
|
||||
metaTags = { "og:title": "Some Series - Chapter 7.5" };
|
||||
const p = kagane.detect(
|
||||
loc("https://kagane.to/series/" + KAGANE_SERIES + "/reader/" + KAGANE_BOOK)
|
||||
);
|
||||
assert.equal(p.title, "Some Series");
|
||||
assert.equal(p.chapterNum, 7.5);
|
||||
});
|
||||
|
||||
test("kagane yields a null chapterNum when og:title has no chapter", () => {
|
||||
metaTags = { "og:title": "Infinite Decryption: The Strongest Level 0" };
|
||||
const p = kagane.detect(
|
||||
|
||||
Reference in New Issue
Block a user