Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sulthan
|
a6d245c25e | ||
|
sulthan
|
8b7bf27796 | ||
|
sulthan
|
b00b532635 | ||
|
sulthan
|
9fca8eb0d8 | ||
|
sulthan
|
95c374976b | ||
|
sulthan
|
f22db44c35 | ||
|
sulthan
|
264cf21960 | ||
|
sulthan
|
aacf57170c | ||
|
sulthan
|
cb00c1b306 | ||
|
sulthan
|
f739f6acbe | ||
|
sulthan
|
d16789226c | ||
|
sulthan
|
9cb9b51b41 | ||
|
sulthan
|
a736b4afa5 |
+20
-1
@@ -1,5 +1,24 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": []
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "CMD=$(python3 -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',d).get('command',''))\" 2>/dev/null || true); case \"$CMD\" in *grep*|*rg\\ *|*ripgrep*|*find\\ *|*fd\\ *|*ack\\ *|*ag\\ *) [ -f graphify-out/graph.json ] && echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"MANDATORY: graphify-out/graph.json exists. You MUST run `graphify query \\\"<question>\\\"` before grepping raw files. Only grep after graphify has oriented you, or to modify/debug specific lines.\"}}' || true ;; esac"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": "Read|Glob",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "HIT=$(python3 -c \"import json,sys;d=json.load(sys.stdin);t=d.get('tool_input',d);exts=('.py','.js','.ts','.tsx','.jsx','.astro','.vue','.svelte','.go','.rs','.java','.rb','.c','.h','.cpp','.hpp','.cc','.cs','.kt','.swift','.php','.scala','.lua','.sh','.md','.rst','.txt','.mdx');vals=[str(t.get('file_path') or ''),str(t.get('pattern') or ''),str(t.get('path') or '')];j=' '.join(vals).lower().replace(chr(92),'/');tails=[('.'+x.rsplit('.',1)[-1]) for v in vals if v for x in [v.lower().replace(chr(92),'/').rsplit('/',1)[-1]] if '.' in x];sys.stdout.write('1' if 'graphify-out/' not in j and any(tl in exts for tl in tails) else '')\" 2>/dev/null || true); if [ \"$HIT\" = 1 ] && [ -f graphify-out/graph.json ]; then echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"MANDATORY: graphify-out/graph.json exists. You MUST run graphify before reading source files. Use: `graphify query \\\"<question>\\\"` (scoped subgraph), `graphify explain \\\"<concept>\\\"`, or `graphify path \\\"<A>\\\" \\\"<B>\\\"`. Only read raw files after graphify has oriented you, or to modify/debug specific lines. This rule applies to subagents too \u2014 include it in every subagent prompt involving code exploration.\"}}'; fi || true"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
---
|
||||
name: gitea
|
||||
description: Use for every forge operation in this repo — read, create, comment on, label, close, or search an issue; create, review, merge, or check out a PR; and whenever `gh`, `issue://`, or `pr://` fails or a ticket number is ambiguous. This repo's forge is self-hosted Gitea driven by `tea`, not GitHub.
|
||||
---
|
||||
|
||||
# Gitea, not GitHub
|
||||
|
||||
`origin` is the self-hosted Gitea instance `gitea.violetcrown.my.id`, repo
|
||||
`sulthan/mangaBookmark`. Everything past plain git goes through
|
||||
[`tea`](https://gitea.com/gitea/tea) (0.14.2 on this machine).
|
||||
|
||||
**`gh` is not installed**, so the harness's `issue://<n>` and `pr://<n>` URIs
|
||||
error out (`GitHub CLI (gh) is not installed`, measured 2026-08-17). There is no
|
||||
fallback to add — read tickets with `tea`.
|
||||
|
||||
`tea` infers the repo from `origin`; auth lives in `tea login`, never a
|
||||
`GH_TOKEN`. Your Gitea username comes from `tea login list` — `tea` has no `@me`.
|
||||
|
||||
Flags are the environment's job: run `tea <command> --help` rather than trusting
|
||||
a remembered flag. This file carries only what `--help` will not tell you.
|
||||
|
||||
## Commands
|
||||
|
||||
| Job | Command |
|
||||
|---|---|
|
||||
| Read | `tea issue <n> --comments` / `tea pr <n> --comments` — `--comments` is not optional |
|
||||
| List | `tea issue list --state open\|closed\|all -o json --fields index,title,body,labels,state,author` |
|
||||
| Search | `tea issue list -k "<keyword>" -L "<label>" -A "<author>"` (`-K all` also searches PRs) |
|
||||
| Create | `tea issue create -t "..." -d "..."` (`-L`, `-a` optional) |
|
||||
| Comment | `tea comment <n> "..."` |
|
||||
| Label | `tea issue edit <n> --add-labels "..."` / `--remove-labels "..."` |
|
||||
| Close | `tea issue close <n>` / `tea pr close <n>` |
|
||||
| PR | `tea pr create --head <branch> --base main -t "..." -d "..."`, `tea pr checkout <n>`, `tea pr review <n>`, `tea pr merge <n>` |
|
||||
|
||||
## Traps
|
||||
|
||||
- **A bare read is a truncated read.** Without `--comments`, `tea issue <n>` and
|
||||
`tea pr <n>` print the opening body and drop every comment silently — no
|
||||
prompt, no marker, no hint that more exists (measured 2026-08-17: issue #123
|
||||
prints 40 lines bare, 132 with `--comments`). The comments are where the
|
||||
decisions live and the body is usually the stalest part of the ticket, so
|
||||
**every read that exists to understand an issue or PR passes `--comments`**,
|
||||
and understanding means body plus all comments plus whatever ticket they point
|
||||
at. Comment count is `tea issue list --fields index,comments`, so a read that
|
||||
shows fewer than that is incomplete. A PR's review comments are a second
|
||||
stream: `tea pr review-comments <n>`.
|
||||
- **One index space for issues and PRs.** A bare `#42` may be either: try
|
||||
`tea pr 42`, fall back to `tea issue 42`. Say which one you found.
|
||||
- **Output is rendered boxes**, not plain text. Anything you parse needs
|
||||
`-o json`, plus `--fields` to keep the payload small. `tea pr create` prints
|
||||
the PR URL on its last line.
|
||||
- **`close` takes no `--comment`.** Comment with `tea comment <n>`, then close.
|
||||
- **Gitea will not auto-create a label.** `tea labels list` first; missing one
|
||||
gets `tea labels create --name "..." --color "#rrggbb"` before the `edit`.
|
||||
- **Multi-line bodies go through a heredoc**, never inline escapes:
|
||||
```bash
|
||||
tea issue create -t "Title" -d "$(cat <<'EOF'
|
||||
body line one
|
||||
|
||||
- acceptance criterion
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
- **No sub-issue and no dependency command.** Gitea's API has issue
|
||||
dependencies, `tea` does not expose them, so parentage and blocking live as
|
||||
body lines — the shapes are in `docs/agents/issue-tracker.md`.
|
||||
|
||||
## Conventions this repo layers on top
|
||||
|
||||
Ticket bodies, wayfinding issues, and the PR-as-request-surface flag:
|
||||
`docs/agents/issue-tracker.md`. Triage label strings: `docs/agents/triage-labels.md`.
|
||||
A label named there still has to exist in the tracker before `--add-labels`.
|
||||
|
||||
Finish a forge action by stating the number you touched and its state after —
|
||||
"commented and closed #71" — so the write is checkable without a second query.
|
||||
@@ -1,133 +0,0 @@
|
||||
---
|
||||
name: implement-tickets
|
||||
description: "Orchestrate a batch of tickets: plan the briefs, then hand each ticket to its own implementer subagent in its own worktree."
|
||||
disable-model-invocation: true
|
||||
---
|
||||
|
||||
# Implement tickets
|
||||
|
||||
You are the **orchestrator**. You write briefs, dispatch, land results, and talk
|
||||
to the tracker. You do not write the implementation — every line of ticket code
|
||||
is written by a `ticket-implementer` subagent in its own git worktree. Reach for
|
||||
the editor yourself only for a merge conflict resolution.
|
||||
|
||||
Ticket source and tracker conventions: `docs/agents/issue-tracker.md`. `tea` usage: skill `gitea`.
|
||||
|
||||
## 1. Collect the tickets
|
||||
|
||||
The user's argument is the selector: issue numbers, a label, a parent issue, or
|
||||
nothing. With nothing, take the open issues labelled `ready-for-agent`.
|
||||
|
||||
Fetch each with `tea issue <n> --comments`, and read the **whole** body —
|
||||
acceptance criteria and the `Blocked by` line are what the rest of this skill
|
||||
runs on. A ticket whose blockers are still open is out of this batch unless a
|
||||
blocker is also in it.
|
||||
|
||||
## 2. Plan the batch
|
||||
|
||||
Explore enough of the codebase to write briefs a fresh context can act on: the
|
||||
files each ticket lands in, the patterns it must follow, the `AGENTS.md`
|
||||
invariants it touches.
|
||||
|
||||
Then decide three things:
|
||||
|
||||
- **Waves.** Blocking edges set the order; tickets with no open blocker inside
|
||||
the batch share a wave. Cap each wave at **3** concurrent tickets unless the
|
||||
user set another width.
|
||||
- **Contracts.** Two tickets in one wave that meet at a function signature, a
|
||||
JSON shape, a table column, or a token name: you decide the shape now and
|
||||
write the identical wording into both briefs. A contract left for the
|
||||
subagents to negotiate is a merge conflict you scheduled.
|
||||
- **Splits.** A ticket too big for one fresh context window goes into the wave
|
||||
as two briefs, or back to the user.
|
||||
|
||||
## 3. Get the plan approved
|
||||
|
||||
Present, and stop:
|
||||
|
||||
- the wave list, and for each ticket: number, title, one-line brief summary,
|
||||
the files or areas it will touch, its verification commands
|
||||
- every cross-ticket contract, verbatim as it will appear in the briefs
|
||||
- anything you had to assume
|
||||
|
||||
Wait for approval. Apply the user's edits to the plan, do not relitigate them.
|
||||
|
||||
## 4. Run a wave
|
||||
|
||||
Per ticket, before dispatch:
|
||||
|
||||
```bash
|
||||
git worktree add ../ticket-<n> -b ticket/<n>-<slug> <base> # base = the branch you are on
|
||||
cp .env ../ticket-<n>/ 2>/dev/null # gitignored, worktrees do not get it
|
||||
tea issue edit <n> --add-assignees <your gitea username> # tea login list has it
|
||||
```
|
||||
|
||||
Write the brief to `.scratch/<batch-slug>/t<n>-brief.md` using the template
|
||||
below, in the ubiquitous language of `CONTEXT.md` — a brief that says "scrape"
|
||||
where the domain says Poll hands the subagent the wrong model of the system.
|
||||
Then dispatch the whole wave in **one** `task` batch, every item on the
|
||||
`ticket-implementer` agent. Each dispatch names: the absolute brief path, the
|
||||
worktree path, the branch, the base ref, and the report path
|
||||
`.scratch/<batch-slug>/t<n>-report.md`.
|
||||
|
||||
<brief-template>
|
||||
|
||||
# Ticket #<n> — <title>
|
||||
|
||||
**Read first.** `tea issue <n> --comments` for this ticket, then the issue it
|
||||
refers to — the parent or spec — the same way. The comments carry decisions the
|
||||
body never got updated with. This brief stays the requirements; those two reads
|
||||
are the intent behind them.
|
||||
|
||||
**Goal.** The end-to-end behaviour this ticket makes work, from the user's side.
|
||||
|
||||
**Acceptance criteria.** Verbatim from the ticket.
|
||||
|
||||
**Contract.** The exact shared signatures / shapes / names this ticket must
|
||||
implement or consume, and which sibling ticket is on the other end. Omit when
|
||||
the ticket touches nothing shared.
|
||||
|
||||
**Where it lands.** The files and packages, and the existing pattern to follow
|
||||
in each.
|
||||
|
||||
**Binding invariants.** The `AGENTS.md` rules this change can break — name them.
|
||||
|
||||
**TDD seams.** Where a test comes first — run the `tdd` skill at each one and
|
||||
follow its red → green loop. Or "none — verify after".
|
||||
|
||||
**Verify.** The exact commands, e.g. `cd backend && go test ./...`,
|
||||
`node --test userscript/test/logic.test.js`.
|
||||
|
||||
**Out of scope.** What not to touch, especially a sibling ticket's files.
|
||||
|
||||
</brief-template>
|
||||
|
||||
## 5. Land the wave
|
||||
|
||||
The wave is landed when every ticket in it is closed, reverted, or handed back
|
||||
to the user. Per returned ticket:
|
||||
|
||||
| Status | What you do |
|
||||
| --- | --- |
|
||||
| `DONE` | merge, comment, close |
|
||||
| `DONE_WITH_CONCERNS` | merge, comment the concerns, close only if you judge them non-blocking — otherwise leave open and tell the user |
|
||||
| `BLOCKED` / `NEEDS_CONTEXT` | supply what is missing and re-dispatch, or hand back to the user with the specifics. Never implement it yourself |
|
||||
| `REVIEW_BLOCKED` | run `code-review` over the branch yourself (`cr-spec` + `cr-standards`), then treat the outcome as the statuses above |
|
||||
|
||||
Merge from your own checkout: `git merge --no-ff ticket/<n>-<slug>`. A textual
|
||||
conflict is yours to resolve (`resolving-merge-conflicts`). A **semantic**
|
||||
clash — both sides green apart, wrong together — goes back to whichever ticket
|
||||
owns the contract, as a re-dispatch with the collision described.
|
||||
|
||||
Then `tea comment <n> "<the report summary>"`, `tea issue close <n>`, and
|
||||
`git worktree remove ../ticket-<n>`. Keep the report file.
|
||||
|
||||
Only once the whole wave is landed does the next wave start — its briefs may
|
||||
need what this one changed.
|
||||
|
||||
## 6. Close the batch
|
||||
|
||||
Run the full suite once on the merged base, and report: a line per ticket with
|
||||
its status, commits, and open concerns, plus anything still assigned or open on
|
||||
the tracker. A red suite after every ticket went green is an interaction bug —
|
||||
diagnose it, name the two tickets, and fix it or hand it back with both named.
|
||||
@@ -14,7 +14,7 @@ parsers, helpers. UI, network, and storage behaviour are verified on-device.
|
||||
|
||||
```bash
|
||||
node --check userscript/manga-bookmark.user.js # parse check, silent on success
|
||||
node --test userscript/test/logic.test.js # 35 tests as of 2026-08-10
|
||||
node --test userscript/test/logic.test.js # 14 tests as of 2026-07-28
|
||||
```
|
||||
|
||||
Run both before every commit that touches the userscript.
|
||||
@@ -31,7 +31,7 @@ The test file installs four globals **before** requiring the userscript:
|
||||
|---|---|---|
|
||||
| `localStorage` | `Map`-backed stub | `loadCache`, `loadQueue`, and the key-migration IIFE touch it at module scope |
|
||||
| `location` | `{href, hostname, pathname, origin}` | read during boot |
|
||||
| `document` | `querySelector` for `meta[property="…"]` only, plus a no-op `addEventListener` | adapters read `og:title` (covers are the backend's, never scraped) |
|
||||
| `document` | `querySelector` for `meta[property="…"]` only, plus a no-op `addEventListener` | adapters read `og:title`/`og:image` |
|
||||
| `document.body` | **left `undefined`** | this is the whole trick |
|
||||
|
||||
`document.body === undefined` sends the userscript's boot block down its `else`
|
||||
|
||||
+30
-84
@@ -1,108 +1,54 @@
|
||||
# Copy to .env and fill in. Never commit the real .env.
|
||||
|
||||
# Secret every Reader's userscript credential is derived from (issue #24):
|
||||
# the backend rebuilds install URLs from it, and only SHA-256 hashes of the
|
||||
# credentials ever touch the database. Generate one:
|
||||
# Long random secret shared with the userscript's API_TOKEN. Generate one:
|
||||
# openssl rand -hex 32
|
||||
TOKEN_KEY=changeme-generate-a-long-random-token
|
||||
|
||||
# The owner's Discord user ID — seeded at startup as the first Reader, the
|
||||
# administrator (the only one who can revoke another Reader's sessions), and
|
||||
# the owner of every bookmark that predates registration. Discord snowflake,
|
||||
# e.g. 1046923170000000000.
|
||||
OWNER_DISCORD_ID=changeme-your-discord-user-id
|
||||
API_TOKEN=changeme-generate-a-long-random-token
|
||||
|
||||
# Comma-separated origins allowed to call the API (CORS). Both Asura domains
|
||||
# plus Demonic, Comix, Kagane, and the two novel sites. Add/remove as the
|
||||
# sites' hostnames change.
|
||||
ALLOWED_ORIGINS=https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to,https://novelfull.com,https://lightnovelworld.net
|
||||
|
||||
# Password for the bundled Postgres container, and therefore half of the
|
||||
# DATABASE_URL compose builds for the backend. Generate one:
|
||||
# openssl rand -hex 24
|
||||
POSTGRES_PASSWORD=changeme-generate-a-long-random-password
|
||||
|
||||
# Override only to point the backend at a Postgres compose does not run.
|
||||
# DATABASE_URL=postgres://user:pass@host:5432/bookmarks?sslmode=require
|
||||
|
||||
# Directory inside bookmark-api for immutable, content-addressed Cover bytes.
|
||||
# Compose builds the image and mounts its named volume at this path.
|
||||
COVER_DIR=/covers
|
||||
|
||||
# Public origin this deployment answers on, no trailing slash. Required: Cover
|
||||
# URLs go out absolute, because the userscript renders them on a Site's own
|
||||
# origin where a relative path would resolve against the Site (ADR-0007).
|
||||
PUBLIC_BASE_URL=https://bookmark-api.example.com
|
||||
# plus Demonic. Add/remove as the sites' hostnames change.
|
||||
ALLOWED_ORIGINS=https://asuracomic.net,https://asurascans.com,https://demonicscans.org
|
||||
|
||||
# --- Prod override (Traefik) only ---
|
||||
# Subdomain Traefik routes to this service (required by the prod override).
|
||||
# BOOKMARK_API_HOST=bookmark-api.example.com
|
||||
# MANGA_API_HOST=manga-api.example.com
|
||||
# Traefik's docker network name, if not "proxy".
|
||||
# PROXY_NETWORK=proxy
|
||||
# Traefik HTTPS entrypoint + cert resolver names, if yours differ from these.
|
||||
# TRAEFIK_ENTRYPOINT=websecure
|
||||
# TRAEFIK_CERTRESOLVER=le
|
||||
|
||||
# --- Web UI (Discord OAuth) ---
|
||||
# Sign-in is a Discord authorization code grant (ADR-0002), and it is also
|
||||
# registration: any member of the configured guild becomes a Reader on their
|
||||
# first successful login, with their own empty library. Create the application
|
||||
# at https://discord.com/developers/applications and register the exact
|
||||
# callback URL ($BOOKMARK_WEB_HOST/auth/discord/callback) as an OAuth2
|
||||
# redirect.
|
||||
DISCORD_CLIENT_ID=
|
||||
DISCORD_CLIENT_SECRET=
|
||||
# The guild whose membership gates sign-in (Developer Mode -> right-click the
|
||||
# server -> Copy Server ID).
|
||||
DISCORD_GUILD_ID=
|
||||
# Exact callback URL, e.g. https://bookmark.example.com/auth/discord/callback.
|
||||
# Discord matches it verbatim, so it must equal the registered redirect.
|
||||
DISCORD_REDIRECT_URI=
|
||||
# Optional: a role snowflake members must hold on top of guild membership.
|
||||
# Empty (the default) means membership alone suffices.
|
||||
# DISCORD_REQUIRED_ROLE=
|
||||
# --- Web UI ---
|
||||
# Password for the browser UI at https://$MANGA_WEB_HOST. Leave unset to
|
||||
# disable the web UI entirely (the routes are not registered at all).
|
||||
# Generate one: openssl rand -base64 18
|
||||
WEB_PASSWORD=
|
||||
|
||||
# Subdomain Traefik routes to the browser UI (required by the prod override).
|
||||
# Left commented on purpose: an example value here would be a silent
|
||||
# wrong-hostname fallback, and Traefik would publish the UI router on a domain
|
||||
# you do not own. The same container also answers on BOOKMARK_API_HOST for the
|
||||
# userscript's API.
|
||||
# BOOKMARK_WEB_HOST=bookmark.example.com
|
||||
# Subdomain Traefik routes to the browser UI (required by the prod override,
|
||||
# whether or not WEB_PASSWORD is set). Left commented on purpose: an example
|
||||
# value here would be a silent wrong-hostname fallback, and Traefik would
|
||||
# publish the UI router on a domain you do not own. The same container also
|
||||
# answers on MANGA_API_HOST for the userscript's API.
|
||||
# MANGA_WEB_HOST=manga.example.com
|
||||
|
||||
# --- Latest-chapter poller ---
|
||||
# The backend re-checks each bookmarked series' newest published chapter on its
|
||||
# own schedule, so latest_chapter stays fresh even when you never open the manga
|
||||
# sites. This runs in parallel with the userscript's own in-browser check.
|
||||
# Set to 0 to turn it off entirely. Pace is per Site (one Poll Lane per Site,
|
||||
# issue #100) and lives in the backend registry, not here — there is nothing
|
||||
# else to configure.
|
||||
# Set to 0 to turn it off entirely.
|
||||
# LATEST_CHAPTER_POLL_ENABLED=1
|
||||
#
|
||||
# Every Site rests an hour between checks and gaps ten seconds between fetches;
|
||||
# a Site with many Series tightens its own gap. See backend/internal/latest/sites.go.
|
||||
|
||||
# CDP endpoint of the browser, used for the two sites behind a Cloudflare
|
||||
# JavaScript challenge (kagane, novelfull) and by the web UI's kagane cover
|
||||
# proxy. Unset disables browser polling and serves 404 for covers not already
|
||||
# stored; those sites then rely on the userscript alone. That is also exactly
|
||||
# how an unreachable browser degrades, so a home machine that is off costs
|
||||
# chapter freshness and nothing else.
|
||||
# Two independent clocks. COOLDOWN is how long one series rests between checks;
|
||||
# INTERVAL is how often the poller wakes up and looks for series past that
|
||||
# cooldown. Shortening INTERVAL cannot shorten a COOLDOWN.
|
||||
# LATEST_CHAPTER_POLL_COOLDOWN=1h # per series, floor 15m
|
||||
# LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
|
||||
# LATEST_CHAPTER_POLL_BATCH=14 # series per wake
|
||||
# LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
|
||||
#
|
||||
# The browser does NOT run in this stack. It is its own compose unit on the
|
||||
# home machine (chrome/docker-compose.yml, chrome/.env.example) and is reached
|
||||
# over the tailnet, so set this to that machine's tailnet address:
|
||||
# Uses a ticker, not an immediate first run: the first poll happens one
|
||||
# INTERVAL after startup, not at startup. A container restarting more often
|
||||
# than INTERVAL never polls.
|
||||
#
|
||||
# BROWSER_WS_URL=ws://100.x.y.z:9222
|
||||
#
|
||||
# It must be the tailnet **IP**, never a MagicDNS hostname and never the old
|
||||
# Docker service name: Chrome's DevTools HTTP handler 500s any /json/version
|
||||
# request whose Host header isn't an IP or "localhost", which silently breaks
|
||||
# every kagane poll. Left unset here on purpose — a wrong default would poll a
|
||||
# stranger's address, and "no browser" is a safe, self-announcing state.
|
||||
# BROWSER_WS_URL=ws://100.x.y.z:9222
|
||||
|
||||
# Zone the backend stamps its log lines in. Cosmetic only. Nothing else in
|
||||
# the service has a zone: bookmark timestamps are unix ms, and the two real
|
||||
# time columns are timestamptz. Defaults to Asia/Jakarta; set to UTC for the
|
||||
# conventional server default.
|
||||
# API_TZ=Asia/Jakarta
|
||||
# BATCH x (COOLDOWN / INTERVAL) series hold the cooldown cadence — 84 with these
|
||||
# defaults. Beyond that the cadence stretches uniformly rather than breaking;
|
||||
# raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL.
|
||||
|
||||
+1
-4
@@ -5,12 +5,9 @@
|
||||
backend/server
|
||||
backend/backend
|
||||
.playwright-mcp/
|
||||
graphify-out/
|
||||
plans/
|
||||
.scratch/
|
||||
docs/superpowers/
|
||||
.superpowers/
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# impeccable-ignore-start
|
||||
# Ephemeral output, runtime state, and per-dev overrides.
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
---
|
||||
name: ticket-implementer
|
||||
description: Implements one ticket end to end inside its own git worktree - reads a brief file, implements, tests, commits, runs the two-axis code review through cr-spec and cr-standards, fixes findings, writes a report file, returns a short status contract. Dispatched by the implement-tickets skill.
|
||||
model: opencode-go/minimax-m3
|
||||
thinking-level: high
|
||||
tools: read, write, edit, bash, grep, glob, lsp, todo, ast_edit, task
|
||||
spawns: cr-spec,cr-standards
|
||||
autoloadSkills: code-review, tdd
|
||||
---
|
||||
|
||||
You implement **one ticket** dispatched by an orchestrator. Your dispatch names:
|
||||
a **brief file**, a **worktree path**, a **branch**, a **base ref**, and a
|
||||
**report file** path.
|
||||
|
||||
## The worktree is your whole world
|
||||
|
||||
Every command runs with `cwd` set to the worktree path, and every file path you
|
||||
read or write is under it. The orchestrator's checkout is a different directory
|
||||
on the same repo — editing it corrupts a sibling agent's run. If a command must
|
||||
run elsewhere, say so in the report instead of doing it.
|
||||
|
||||
Your branch is already checked out there. Never `git checkout`, `git switch`,
|
||||
`git rebase`, or `git worktree` anything.
|
||||
|
||||
## Order of work
|
||||
|
||||
1. Read the brief file. It is the single source of requirements — use its exact
|
||||
values verbatim.
|
||||
2. Read the ticket and the issue it refers to, as the brief's **Read first**
|
||||
section names them: `tea issue <n> --comments` for each. The ticket's
|
||||
comments and its parent carry the intent and the decisions behind the brief.
|
||||
Read no other ticket and no other brief.
|
||||
3. Read `AGENTS.md` in the worktree, plus the nested `AGENTS.md` for the area
|
||||
you touch. Its invariants bind you: security rules, design system, comment
|
||||
policy.
|
||||
4. Ask before writing code if requirements, acceptance criteria, approach, or
|
||||
dependencies are unclear. Asking is free; guessing is not.
|
||||
5. Implement exactly what the brief specifies. At each TDD seam the brief names,
|
||||
run the `tdd` skill and follow its red → green loop.
|
||||
Follow the patterns already in the codebase; improve what you touch,
|
||||
restructure nothing outside the ticket.
|
||||
6. Verify. Focused tests while iterating, the brief's full verification commands
|
||||
once at the end. Test output must be pristine.
|
||||
7. Commit to your branch. Reference the ticket number in the subject.
|
||||
8. Review (below), fix, re-verify, commit the fixes.
|
||||
9. Write the report file, then return the status contract.
|
||||
|
||||
## Review
|
||||
|
||||
After your first green commit, run the **`code-review`** skill over
|
||||
`<base ref>...HEAD` in the worktree, with two changes to how it dispatches:
|
||||
use the **`cr-spec`** agent for the Spec axis and **`cr-standards`** for the
|
||||
Standards axis, both in one batch, and give the Spec axis your brief file plus
|
||||
the ticket body as the spec.
|
||||
|
||||
Fix every Critical and Important finding, then re-run the tests that cover the
|
||||
amended code. Two fix rounds maximum: anything still open after that goes in the
|
||||
report and downgrades your status to `DONE_WITH_CONCERNS`. Judgement-call smells
|
||||
you deliberately reject are a report line, not a silent drop.
|
||||
|
||||
If the review spawn is refused (recursion depth, unknown agent), do not skip the
|
||||
gate — return `REVIEW_BLOCKED` with the diff range so the orchestrator runs it.
|
||||
|
||||
## Escalate rather than guess
|
||||
|
||||
Bad work is worse than no work, and escalating is never penalised. Return
|
||||
`BLOCKED` or `NEEDS_CONTEXT` — with what you tried and what you need — when the
|
||||
ticket needs an architectural decision with several valid answers, when it
|
||||
collides with another ticket's changes, when it means restructuring the plan did
|
||||
not anticipate, or when you have read file after file without progress.
|
||||
|
||||
## Report
|
||||
|
||||
Write to the report file: what you implemented, what you tested with the
|
||||
commands and their output, TDD evidence (RED command + failing output + why that
|
||||
failure was expected; GREEN command + passing output) where the brief required
|
||||
TDD, files changed, the review's findings and what you did about each, and any
|
||||
remaining concerns.
|
||||
|
||||
Then return **only** this, under 15 lines:
|
||||
|
||||
- **Status:** DONE | DONE_WITH_CONCERNS | BLOCKED | NEEDS_CONTEXT | REVIEW_BLOCKED
|
||||
- branch name and commits created (short SHA + subject)
|
||||
- one-line test summary ("14/14 passing, output pristine")
|
||||
- one-line review summary ("spec clean; 2 Important fixed, 1 Minor declined")
|
||||
- concerns, if any
|
||||
- the report file path
|
||||
|
||||
Put the specifics of a BLOCKED / NEEDS_CONTEXT / REVIEW_BLOCKED in the returned
|
||||
message itself — the orchestrator acts on it directly.
|
||||
@@ -1,47 +0,0 @@
|
||||
---
|
||||
description: Code-writer subagent for subagent-driven development. Fast model (ocg/deepseek-v4-flash) for mechanical, well-specified implementation tasks. Escalates complicated tasks so the controller can re-dispatch on minimax-m3.
|
||||
mode: subagent
|
||||
model: 9router/ocg/deepseek-v4-flash
|
||||
---
|
||||
|
||||
You are the implementer subagent for Subagent-Driven Development. You implement one task, exactly as specified, and report back with evidence.
|
||||
|
||||
## Before You Begin
|
||||
|
||||
If you have questions about requirements, acceptance criteria, approach, dependencies, or anything unclear in the task description — ask now. Raise concerns before starting work. Don't guess or make assumptions.
|
||||
|
||||
## Your Job
|
||||
|
||||
1. Implement exactly what the task specifies
|
||||
2. Write tests (follow TDD when the task says to)
|
||||
3. Verify the implementation works (run the focused test while iterating; run the full suite once before committing)
|
||||
4. Commit your work
|
||||
5. Self-review (below)
|
||||
6. Report back
|
||||
|
||||
Follow existing patterns in the codebase. Don't restructure code outside your task. Don't overbuild — only what was requested (YAGNI).
|
||||
|
||||
## When You're in Over Your Head
|
||||
|
||||
It is always OK to stop and say "this is too hard for me." Bad work is worse than no work. STOP and escalate when the task requires architectural judgment, multi-file integration you can't see clearly through, or you're reading file after file without progress.
|
||||
|
||||
**Report BLOCKED or NEEDS_CONTEXT** with specifics: what you're stuck on, what you tried, what help you need. If the task turns out more complicated than mechanical (design judgment, broad codebase understanding), escalate so the controller can re-dispatch you on the more capable minimax-m3 agent.
|
||||
|
||||
## Self-Review Before Reporting
|
||||
|
||||
- **Completeness:** everything in the spec implemented? edge cases handled?
|
||||
- **Quality:** names accurate? code clean and maintainable?
|
||||
- **Discipline:** avoided overbuilding? only what was requested?
|
||||
- **Testing:** do tests verify real behavior? output pristine (no stray warnings)?
|
||||
Fix what you find before reporting.
|
||||
|
||||
## Report Format
|
||||
|
||||
Report back with ONLY (under 15 lines):
|
||||
- **Status:** DONE | DONE_WITH_CONCERNS | BLOCKED | NEEDS_CONTEXT
|
||||
- Commits created (short SHA + subject)
|
||||
- One-line test summary (e.g. "14/14 passing, output pristine")
|
||||
- Concerns, if any
|
||||
- Report file path (if the controller gave you one)
|
||||
|
||||
If BLOCKED or NEEDS_CONTEXT, put the specifics in the final message itself — the controller acts on it directly. Use DONE_WITH_CONCERNS if you completed the work but have doubts. Never silently produce work you're unsure about.
|
||||
@@ -1,69 +0,0 @@
|
||||
---
|
||||
description: Reviewer subagent for subagent-driven development. Capable model (ocg/minimax-m3) for task-scoped and whole-branch code review; also the re-dispatch target when implementation tasks are complicated.
|
||||
mode: subagent
|
||||
model: 9router/ocg/minimax-m3
|
||||
---
|
||||
|
||||
You are the reviewer subagent for Subagent-Driven Development. You verify one task's implementation matches its requirements (spec compliance) and is well-built (code quality). You may also be dispatched for whole-branch review.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Task brief file (requirements — use exact values verbatim)
|
||||
- Implementer's report file
|
||||
- Diff file (commit list, stat summary, full diff with context)
|
||||
|
||||
## Method
|
||||
|
||||
Read the diff file once — it is your view of the change. The context lines ARE the changed files: do not read a changed file separately unless a hunk you must judge is cut off mid-function (say so in your report). Do not re-run git commands. Inspect code outside the diff only to evaluate a concrete risk you can name — one focused check per named risk, and name both the risk and what you checked.
|
||||
|
||||
Your review is read-only. Do not mutate the working tree, index, HEAD, or branch state.
|
||||
|
||||
## Do Not Trust the Report
|
||||
|
||||
Treat the implementer's report as unverified claims. It may be incomplete, inaccurate, or optimistic. Verify against the diff. Design rationales in the report ("kept it per YAGNI") are the implementer grading their own work — a stated rationale never downgrades a finding's severity.
|
||||
|
||||
## Tests
|
||||
|
||||
The implementer already ran the tests and reported results. Do not re-run the suite to confirm. Run a test only when reading the code raises a specific doubt no existing run answers — a focused test, never a package-wide suite. If heavy validation seems warranted, recommend it in your report instead. Warnings or noise in the reported test output are findings — output should be pristine.
|
||||
|
||||
## Part 1: Spec Compliance
|
||||
|
||||
Compare the diff against the brief:
|
||||
- **Missing:** requirements skipped, missed, or claimed without implementing
|
||||
- **Extra:** features not requested, over-engineering, nice-to-haves
|
||||
- **Misunderstood:** right feature built the wrong way, wrong problem solved
|
||||
|
||||
If a requirement can't be verified from this diff alone (lives in unchanged code or spans tasks), report it as a ⚠️ item instead of broadening your search.
|
||||
|
||||
## Part 2: Code Quality
|
||||
|
||||
- Clean separation of concerns? proper error handling? DRY without premature abstraction? edge cases?
|
||||
- Do new/changed tests verify real behavior, not mocks? edge cases covered?
|
||||
- Does each file have one clear responsibility? units independently testable? did this change create/significantly grow large files?
|
||||
|
||||
Point at evidence: file:line references for every finding. A tight report that cites lines gives the controller everything it needs.
|
||||
|
||||
## Calibration
|
||||
|
||||
Not everything is Critical. Important = this task can't be trusted until fixed: incorrect or fragile behavior, a missed requirement, maintainability damage you'd block a merge over (verbatim duplication of a logic block, swallowed errors, tests that assert nothing). "Coverage could be broader" and polish suggestions are Minor. If the plan explicitly mandates something this rubric calls a defect, that IS a finding — report Important, labeled plan-mandated. Acknowledge what was done well before listing issues.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Spec Compliance
|
||||
- ✅ Spec compliant | ❌ Issues found: [what's missing/extra/misunderstood, with file:line]
|
||||
- ⚠️ Cannot verify from diff: [requirements you couldn't verify, what the controller should check]
|
||||
|
||||
### Strengths
|
||||
[What's well done? Be specific.]
|
||||
|
||||
### Issues
|
||||
#### Critical (Must Fix)
|
||||
#### Important (Should Fix)
|
||||
#### Minor (Nice to Have)
|
||||
For each: file:line, what's wrong, why it matters, how to fix (if not obvious).
|
||||
|
||||
### Assessment
|
||||
**Task quality:** [Approved | Needs fixes]
|
||||
**Reasoning:** [1-2 sentence technical assessment]
|
||||
|
||||
Your final message is the report itself: begin directly with the spec-compliance verdict. Every line is a verdict, a finding with file:line, or a check you ran — no preamble, no process narration, no closing summary.
|
||||
@@ -1,193 +1,160 @@
|
||||
# AGENTS.md
|
||||
|
||||
Repo-wide guidance for coding agents.
|
||||
Guidance for OpenCode (and Claude Code) working in this repo.
|
||||
|
||||
## Status
|
||||
|
||||
Active. Backend (`backend/`) and userscript (`userscript/manga-bookmark.user.js`) built. Plan `plans/mangaBookmark.md` = original spec, may drift; trust code + design docs in `docs/superpowers/specs/` over plan.
|
||||
|
||||
## What this is
|
||||
|
||||
Read-progress tracker for two libraries — manga and novels — behind one self-hosted Go backend. Two separate Violentmonkey userscripts inject on-page UI (floating button + slide-in panel) and sync progress, so bookmarks unify across sites and devices:
|
||||
Manga read-progress tracker for user reading on **asurascans.com** (current domain; asuracomic.net 301s here) and **demonicscans.org** from **Bromite** (mobile Chromium). Userscript injects on-page UI (floating button + slide-in panel), syncs progress to self-hosted Go backend so bookmarks unify across both sites and devices.
|
||||
|
||||
- `manga-bookmark.user.js` — **asurascans.com** (asuracomic.net is dropped: its deep links 301 to the asurascans.com root, discarding the path), **demonicscans.org**, **comix.to**, **kagane.to**.
|
||||
- `novel-bookmark.user.js` — **novelfull.com**, **lightnovelworld.net**.
|
||||
## Hard constraints (drive design — do not violate)
|
||||
|
||||
One backend, one `bookmarks` table: a `kind` column (`manga`|`novel`) splits the libraries and the web UI switches between them. Rows are keyed `<site>:<series_id>`.
|
||||
|
||||
## Hard constraints (drive design — don't violate)
|
||||
|
||||
Nothing below is derivable from reading the code — it is why the code looks the
|
||||
way it does, plus dated measurements against services we don't control.
|
||||
|
||||
Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free where plain web APIs suffice — keeps portability across engines:
|
||||
- **Avoid `GM_*` unless needed.** Prefer page `localStorage` over `GM_setValue`/`GM_getValue`, on-page UI over `GM_registerMenuCommand`, plain `fetch()` over `GM_xmlhttpRequest` for cross-origin.
|
||||
- Cross-origin `fetch()` work **only** against CORS-enabled backend. Manga sites `https://`, so backend **must be HTTPS** (else mixed-content block).
|
||||
- Every site is its **own origin with its own `localStorage`** — a shared remote store is the only way to unify bookmarks. Cloud sync required, not optional.
|
||||
- Userscript run in **isolated world**, so embedded API token safe from site's JS.
|
||||
- Cloudflare's block on manga sites is **per-zone configuration plus request fingerprint, not IP reputation — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — a Site can turn its protection on overnight, which is exactly what comix.to did on 2026-08-12. An earlier version of this line blamed "Cloudflare's bot scoring"; that was wrong. The 1-99 bot score is Enterprise Bot Management only and does not exist for a free-plan zone, and no per-IP request rate is documented as an input to challenge issuance. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test.
|
||||
- **kagane.to, comix.to and novelfull.com are the exception to the above** — all three sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`). When that's unset, kagane and comix are skipped entirely (a plain fetch would only retrieve a challenge page) while novelfull pages are still attempted over plain TLS — its challenge is a live time-varying fact and its cover bytes never need the browser. comix turned hostile on 2026-08-12: its cover host `static.comix.to` is gated too, so its cover bytes go through the browser as well, and its page is read as an in-tab `fetch()` of the series URL rather than a rendered DOM — comix is an SPA, and rendering costs ~65 requests for the same server-rendered HTML one fetch returns. The three other sites poll fine over plain TLS.
|
||||
- **The CDP browser must look like a real browser, and stock headless images don't.** Measured 2026-08-08 against kagane.to, all from the same IP: `chromedp/headless-shell:stable` never cleared the challenge in 90s (`navigator.webdriver` true, empty plugin list, Chromium-branded client hints — suppressing `webdriver` alone changed nothing); `zenika/alpine-chrome` ships Chrome 124, refused outright; real Chrome with the default `--headless=new` UA never cleared, because the UA says `HeadlessChrome`; real Chrome with a stock UA **and** a non-UTC clock zone cleared in ~4s. Hence `chrome/` — a Debian image with `google-chrome-stable`, a version-derived UA, and `TZ`/`BROWSER_TZ`. Chrome reads the zone *name* through ICU from `/etc/localtime`'s symlink target, ignoring the file's contents, so mounting the host's `/etc/localtime` does **not** work; `/etc/timezone` is mounted instead.
|
||||
- **The browser is not in the API stack and must not be put back.** It's its own compose unit (`chrome/docker-compose.yml`) on a second machine, reached over the tailnet — it held 471 MiB on a 1974 MiB swapless VPS, and a residential egress avoids the cloud-hosting-IP signature Bot Fight Mode documentedly challenges (not a better "score" — free-plan zones have no score). Consequences that constrain code: `BROWSER_WS_URL` must be a tailnet **IP** (a MagicDNS name 500s at `/json/version`, same trap as the old Docker service name); the CDP port binds to the tailnet address only, since CDP authenticates nothing and that host has a real LAN; and the browser is on-demand, so an unreachable or asleep one must degrade exactly as an unset `BROWSER_WS_URL` — plain-TLS libraries unaffected, kagane/comix logged and skipped, stored covers still served. Never add `chromedp.NoModifyURL`: discovery per fetch is what makes a restarted Chrome invisible.
|
||||
- **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, and the challenge refuses it; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. A second earlier claim, that Cloudflare "scores" a UTC clock, was also wrong: the measurement is real but the mechanism is not documented anywhere — Cloudflare publishes no timezone signal, and free-plan zones carry no score at all. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one.
|
||||
- **A challenged page needs the tab kept open.** The interstitial takes seconds to solve and only then writes clearance into the browser's shared cookie jar. Navigate-read-close never clears anything; `BrowserFetcher.run` holds one tab and re-reads until the payload arrives.
|
||||
Bromite uses Chromium's **native** userscript engine, not Tampermonkey:
|
||||
- **No `GM_*` APIs anywhere.** No `GM_setValue`/`GM_getValue` (use page `localStorage`), no `GM_registerMenuCommand` (inject on-page UI), no `GM_xmlhttpRequest` for cross-origin (use plain `fetch()`). GM-free script also runs in desktop Tampermonkey/Violentmonkey for faster iteration.
|
||||
- Cross-origin `fetch()` works **only** against CORS-enabled backend. Manga sites `https://`, so backend **must be HTTPS** (else mixed-content block).
|
||||
- Asura and Demonic = **separate origins, separate `localStorage`** — shared remote store only way to unify bookmarks. Cloud sync required, not optional.
|
||||
- Userscript runs in **isolated world**, so embedded API token safe from site's JS.
|
||||
- Cloudflare's block on manga sites is **IP-reputation-based, not universal — not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s w/ real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier, untested assumption CGNAT dev IP would be blocked; wasn't, at least this date. Treat "does curl work now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare bot scoring can flip clean IP without notice. Any backend fetcher still needs graceful-degrade path for when challenged; adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Two Violentmonkey userscripts (isolated world, per-site adapters, localStorage cache)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> Postgres (volume)
|
||||
|
|
||||
| CDP over tailnet
|
||||
v
|
||||
on-demand Chrome, separate machine (chrome/)
|
||||
Bromite userscript (isolated world, per-site adapters, localStorage cache)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> SQLite (volume)
|
||||
```
|
||||
|
||||
Two deployable units on two machines: the API stack (`docker-compose.yml` + `docker-compose.prod.yml`, on the VPS) and the browser (`chrome/docker-compose.yml`, on the home machine). They share nothing but `BROWSER_WS_URL` and update independently. Backend-specific detail lives in `backend/AGENTS.md`, userscript-specific detail in `userscript/AGENTS.md`.
|
||||
- **Backend** (`backend/`): stdlib `net/http` (handful of routes, no framework) + `modernc.org/sqlite` (pure Go, `CGO_ENABLED=0` -> static binary -> distroless/scratch image). Reverse proxy terminates TLS; Go service listens plain `:8080`.
|
||||
- **Single-user store.** One `bookmarks` table keyed `<site>:<series_id>` (`asura`|`demonic`). Sync **last-write-wins**. Schema + endpoint list in plan.
|
||||
- **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth).
|
||||
- **Web UI:** same binary serves password-gated browser UI on second
|
||||
hostname — `GET /` (list, or login page when no session),
|
||||
`POST /login`, `POST /logout`, `GET /static/*`, htmx fragment endpoints
|
||||
under `/ui/*`. Templates/assets `go:embed`-ed, so `backend/Dockerfile`
|
||||
must copy `templates/` and `static/` plus `*.go`. Sessions = stateless
|
||||
HMAC cookies keyed off `API_TOKEN`; `WEB_PASSWORD` gates them, when empty
|
||||
web routes not registered at all. UI mutations read-modify-write
|
||||
through `Store.Get` + `Store.Upsert` so `updated_at` rule stays one
|
||||
place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`.
|
||||
- **Latest-chapter poller:** ticker goroutine in same binary re-checks
|
||||
each bookmarked series' newest published chapter from backend's own
|
||||
network access, so `latest_chapter` stays fresh when user not
|
||||
browsing. Second, parallel signal — userscript keeps own
|
||||
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged.
|
||||
Two independent clocks: per-bookmark cooldown (`latest_checked_at` column,
|
||||
enforced by `Store.DueForLatestCheck`'s WHERE clause) and wake interval.
|
||||
Row stamped *before* fetch so broken series waits full
|
||||
cooldown instead of retrying every tick; writes go through
|
||||
`Store.Get` + `Store.Upsert` so new chapter never reorders list.
|
||||
Fetches use `bogdanfinn/tls-client` w/ Chrome profile as defence in depth
|
||||
against fingerprint-based blocking; any failure logs and skips. See
|
||||
`docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`.
|
||||
Poller's `Store.Get` + `Store.Upsert` not wrapped in transaction, so
|
||||
userscript `PUT` committing between the two can be overwritten by
|
||||
poller's stale re-read — reverting read progress and, since stored
|
||||
value now differs, moving `updated_at` and reordering list. Known,
|
||||
accepted limitation for single-user deployment, not bug to fix.
|
||||
- **`updated_at` drives list order, moves only on real reading progress:** server applies timestamp when row new or `last_chapter_num` changes, else keeps stored value — favouriting series or recording newly published chapter must not reorder list. `PUT` therefore returns row **as stored**; clients must adopt that response over own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4.
|
||||
- **Lifecycle buckets:** `status` on each bookmark is `reading` | `archived` |
|
||||
`finished`, orthogonal to `favorite`. Archived and finished appear only in
|
||||
own tab — not All, Updated, Favourites, or recent strip. Poller keeps
|
||||
checking archived series, skips finished ones. `finished` settable only
|
||||
from web UI; `PUT /bookmarks/{key}` rejects it w/ 400.
|
||||
**Empty incoming status means "keep stored one"** — resolved on
|
||||
`VALUES` side of `Store.Upsert`, not conflict clause, since
|
||||
`excluded.*` = post-evaluation row and default applied there'd
|
||||
wipe bucket on every PUT from client predating column. See
|
||||
`docs/superpowers/specs/2026-07-27-status-buckets-design.md`.
|
||||
- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list), `DB_PATH`
|
||||
(default `/data/bookmarks.db`), `PORT` (default `8080`), `WEB_PASSWORD`
|
||||
(gates browser UI; unset disables it),
|
||||
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER`
|
||||
(background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`).
|
||||
`USERSCRIPT_PATH` (file served at `/u/{token}/manga-bookmark.user.js`,
|
||||
default `/userscript/manga-bookmark.user.js`, supplied by a bindmount).
|
||||
|
||||
### Userscript structure (single IIFE, `manga-bookmark.user.js`)
|
||||
|
||||
1. **Site adapters** — one per host, `detect(location, document)` returns page `type` + IDs. ID type/IDs from **URL regex** (most stable); pull `title`/`cover` from **`og:title`/`og:image` meta tags**, not CSS classes.
|
||||
2. **API client** — `apiGet/apiPut/apiDelete` w/ bearer header; `localStorage` key `mangabm:cache` for instant render + offline fallback.
|
||||
3. **Progress logic** — auto-upsert `last_chapter` only when `chapterNum >= stored last_chapter_num` (re-reading old chapters must not regress progress; unparseable -> set current). Manual panel override forces any value.
|
||||
4. **Retry queue** — every write goes through `pushBookmark`/`pushDelete`, so
|
||||
failed mutation parked in `localStorage` (`mangabm:queue`) and replayed on
|
||||
next navigation, reconnect, or `refresh()`. Entries are markers
|
||||
(`{key, op, sendStatus, attempts}`), never payloads — body read from
|
||||
cache at send time, so one entry per key gives ordering + coalescing for
|
||||
free. `sendStatus` **sticky**: while archive pending, later writes to
|
||||
that key keep carrying bucket, stops successful
|
||||
in-between write from silently un-archiving series. `refresh()` drains
|
||||
before fetching, overlays anything still pending, so list never
|
||||
flaps. 400 drops entry, 401 aborts pass and keeps queue,
|
||||
transient failures retry to cap of 10. Latest-chapter writes deliberately
|
||||
stay out of queue. See
|
||||
`docs/superpowers/specs/2026-07-27-offline-retry-queue-design.md`.
|
||||
5. **UI** — rendered inside **Shadow DOM** root to isolate from site CSS
|
||||
(critical on mobile). Three tabs (All / Favourites / Archived) + row of
|
||||
link chips to web UI and both manga sites; `WEB_BASE` sits in CONFIG
|
||||
block next to `API_BASE`.
|
||||
6. **SPA navigation** — Asura is Astro, client-routed on comic/chapter pages: patch `history.pushState`/`replaceState` + listen `popstate`, re-run `detect()` on URL change so auto-update fires w/o reload. Demonic uses classic reloads (initial `document-idle` run suffices).
|
||||
|
||||
### Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trusting)
|
||||
|
||||
- **asurascans.com**: series `/comics/<slug>` (slug carries a trailing
|
||||
site-wide build-hash suffix, e.g. `-059befe1`, that **rotates on every
|
||||
redeploy**), chapter `/comics/<slug>/chapter/<n>`. `seriesId` must strip
|
||||
the hash (`/-[0-9a-f]{8}$/`, `stripBuildHash` in the userscript,
|
||||
`asuraBuildHash` in the backend); URLs keep the full slug — stale-hash
|
||||
URLs 302 to current ones. Astro-rendered; chapter links present in raw
|
||||
server HTML.
|
||||
- **demonicscans.org**: series `/manga/<slug>` (slug may URL-encode punctuation, e.g. `%2527` for `'`), chapter `/title/<slug>/chapter/<n>/<page>` (older `chaptered.php?manga=<id>&chapter=<n>` form still exists as redirect, what series-page chapter-list anchors link through).
|
||||
Encodings (incl. triple-encoded punctuation like `%25252D`) are identical
|
||||
on /manga/ and /title/ pages, so decode-once seriesIds match — verified
|
||||
2026-07-28.
|
||||
|
||||
## Commands
|
||||
|
||||
Backend (`cd backend`):
|
||||
- Test all: `go test ./...` — **needs Docker.** Each test package starts a throwaway `postgres:17-alpine` container (`internal/pgtest`).
|
||||
- Test all: `go test ./...`
|
||||
- Single test: `go test -run TestName ./...`
|
||||
- Build static binary: `CGO_ENABLED=0 go build`
|
||||
|
||||
Local stack: `docker compose up` (bookmark-api + postgres only; `postgres-data` named volume, `restart: unless-stopped`). No browser — without `BROWSER_WS_URL` the poller logs and skips kagane and comix. To run one: `cd chrome && BROWSER_BIND_ADDR=172.17.0.1 docker compose up -d --build`, then `BROWSER_WS_URL=ws://172.17.0.1:9222` in the root `.env` (bridge gateway, so the API container can name it by IP).
|
||||
Local stack: `docker compose up` (named volume mounted at `/data`, `restart: unless-stopped`).
|
||||
|
||||
Live CDP proof (needs that browser and network, skipped otherwise):
|
||||
`SMOKE_BROWSER_WS_URL=ws://<ip>:<port> go test -run 'TestSmokeKagane|TestSmokeComix' ./internal/latest`
|
||||
— fetches a real kagane and comix cover and chapter list. A red run means the challenge is
|
||||
not clearing from this IP, which is a live fact to re-check, not necessarily a defect.
|
||||
A red `TestSmokeComix` reporting `ERR_CERT_COMMON_NAME_INVALID` is not the
|
||||
challenge: it means the resolver the browser container uses hijacks `comix.to`.
|
||||
Observed 2026-08-16 on one Indonesian ISP, which CNAMEs it to a block page
|
||||
(`aduankonten.id`). Check with `docker exec <browser> getent hosts comix.to`,
|
||||
and if it is hijacked, run the container with
|
||||
`--add-host comix.to:<ip> --add-host static.comix.to:<ip>` from a DoH lookup
|
||||
(`curl -H 'accept: application/dns-json' 'https://1.1.1.1/dns-query?name=comix.to&type=A'`).
|
||||
Machine-local, so don't put those hosts in `chrome/docker-compose.yml`.
|
||||
|
||||
Smoke test: `curl` endpoints with `Authorization: Bearer <token>`; confirm `OPTIONS` preflight return CORS headers and `/healthz` return 200.
|
||||
Smoke test: `curl` endpoints w/ `Authorization: Bearer <token>`; confirm `OPTIONS` preflight returns CORS headers and `/healthz` returns 200.
|
||||
|
||||
## Forge: Gitea, not GitHub
|
||||
|
||||
`origin` is self-hosted Gitea (`gitea.violetcrown.my.id`, repo `sulthan/mangaBookmark`), so **`gh` don't work here and the `issue://`/`pr://` URIs error out — drive the forge with `tea`.** How to run it — commands, traps, JSON output: skill `gitea`. Tracker conventions (ticket bodies, wayfinding, PR-as-request-surface flag): `docs/agents/issue-tracker.md`. Triage label strings: `docs/agents/triage-labels.md`.
|
||||
`origin` = self-hosted Gitea instance (`gitea.violetcrown.my.id`), so **`gh` doesn't work here — use `tea` (Gitea CLI) for anything past plain git.** Common ones:
|
||||
|
||||
## Design system
|
||||
- Open PR: `tea pr create --head <branch> --base main --title "..." --description "..."`
|
||||
- List / view / check out: `tea pr list`, `tea pr <n>`, `tea pr checkout <n>`
|
||||
- Issues: `tea issue create`, `tea issue list`
|
||||
- Auth lives in `tea login`, not `GH_TOKEN` env var.
|
||||
|
||||
Web UI + userscript panel follow **Cinder**. Tokens are the `:root` block in
|
||||
`backend/internal/web/static/style.css`; that file, `backend/internal/web/templates/*`,
|
||||
and the userscript `TEMPLATE`/`CSS` are the only places it is expressed.
|
||||
Source of truth for the visual language is the Claude Design project
|
||||
`BookmarkManager Web UI` (`969ac210-fe02-4c01-ae1b-9a271dcc779a`).
|
||||
|
||||
Core law: **ember means new chapter only** — no other state (busy, error,
|
||||
destruction) may use `--ember`; destruction gets `--danger`. No
|
||||
cards/corners/shadows, one `--measure: 760px` column, tokens only (never
|
||||
hardcode hex outside `:root`), both colour branches touched together. Any move
|
||||
that pulls a series out of the list (archive/finish/remove) must be
|
||||
confirm-gated via its own `.confirm-row`; only restore fires instantly.
|
||||
`tea` prints output as rendered boxes not plain text; PR URL lands on last line.
|
||||
|
||||
## Security invariants
|
||||
|
||||
Existing guarantees — don't regress:
|
||||
- Auth on `/bookmarks*`: require `Authorization: Bearer <API_TOKEN>`, **constant-time compare**, 401 otherwise.
|
||||
- CORS: reflect `Origin` only when in `ALLOWED_ORIGINS`; allow `GET,PUT,DELETE,OPTIONS` + headers `Authorization,Content-Type`; answer preflight `OPTIONS` w/ `204`.
|
||||
|
||||
- Auth on `/bookmarks*`: require `Authorization: Bearer <credential>` — the acting Reader's credential, matched by SHA-256 against `readers.token_sha256` — **constant-time compare** (via the hash, never the secret itself), 401 otherwise.
|
||||
- CORS: reflect `Origin` only when in `ALLOWED_ORIGINS`; allow `GET,PUT,DELETE,OPTIONS` + headers `Authorization,Content-Type`; answer preflight `OPTIONS` with `204`.
|
||||
## Relevant skills
|
||||
|
||||
## Secure coding rules (code you write here)
|
||||
`multi-stage-dockerfile` and `docker-compose-orchestration` for container work (referenced in plan).
|
||||
|
||||
Anchored to OWASP Top 10 / ASVS. Every rule below already has a working example in-tree — match it, don't start a second convention. AI-written backends fail on exactly these: broken access control, injection, weak session/error handling, invented dependencies.
|
||||
## graphify
|
||||
|
||||
Go backend:
|
||||
Project has knowledge graph at graphify-out/ w/ god nodes, community structure, cross-file relationships.
|
||||
|
||||
- SQL always parameterized (`$N`). Only compile-time constants (`bookmarkColumns`) may be concatenated into query text — never a request value, not even a validated one.
|
||||
- `html/template` only for anything a browser parses, never `text/template`. Never wrap stored or fetched strings in `template.HTML`/`JS`/`URL`; that switches off the escaping every template depends on.
|
||||
- Any outbound fetch of a client-supplied URL passes `fetchableSeriesURL` (site + `https` + host check) first. `series_url` arrives in a PUT body, so without the gate the poller will probe arbitrary hosts from the server's own network position. New fetch path reuses the gate rather than re-deriving one.
|
||||
- Cap every remote body with `io.LimitReader` (`maxBodyBytes`). An unbounded read is an OOM handed to whatever is on the other end.
|
||||
- Compare secrets with `hmac.Equal` / `subtle.ConstantTimeCompare`, never `==`. A credential is matched by the SHA-256 the `readers` table holds, which is already a fixed-width equality — a new secret comparison must not regress to `==`.
|
||||
- Errors: generic text to the client (`http.Error(w, "internal error", 500)`), detail to `log.Printf`. Never log `TOKEN_KEY`, a Reader's credential, `DISCORD_CLIENT_SECRET`, a session id, or a whole `Authorization` header.
|
||||
- Proxy headers are trusted only where they already are: `X-Forwarded-Proto` for the Secure cookie flag, **rightmost** `X-Forwarded-For` for client IP (leftmost is attacker-supplied). Don't read either anywhere else.
|
||||
- Session cookies keep `HttpOnly`, `SameSite`, `Secure`-when-HTTPS; expiry is enforced by the `sessions` table lookup, not a signature.
|
||||
- Stdlib crypto only. No hand-rolled hashing, no MD5/SHA-1 anywhere security-bearing.
|
||||
- Validate at the handler boundary before storing: body capped by `http.MaxBytesReader` (64 KB), empty `key` and unknown `status`/`kind` rejected with `400`. A bad value that reaches the store becomes every later reader's problem.
|
||||
Rules:
|
||||
- For codebase questions, first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships, `graphify explain "<concept>"` for focused concepts. Return scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
|
||||
- If graphify-out/wiki/index.md exists, use for broad navigation instead of raw source browsing.
|
||||
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain don't surface enough context.
|
||||
- After modifying code, run `graphify update .` to keep graph current (AST-only, no API cost).
|
||||
|
||||
Userscript:
|
||||
## OpenCode-specific
|
||||
|
||||
- Site-derived and stored strings render via `el(..., {text})` / `textContent`. `{html}` and `innerHTML` are for author-written literal markup only (`TEMPLATE`, `CSS`) — never a title, chapter label, or API response field. The page DOM belongs to a third-party site; treat it as attacker-controlled.
|
||||
- Isolated world protects the credential from the site's JS. It does not protect anything from an `innerHTML` sink you add yourself.
|
||||
- The userscripts carry `__API_TOKEN__` placeholders, substituted at serve time with the requesting Reader's credential (`internal/userscript`). Never put a real credential in the repo, docs, commit messages, or issues. Rotation is a web-UI action (epoch bump, `internal/token`); `TOKEN_KEY` in backend env is what derives every credential — never log it.
|
||||
- `fetch()` targets `API_BASE` only — no dynamic origin, no site-supplied URL. `authHeaders()` goes nowhere but the backend.
|
||||
- `localStorage` is shared with the site's own JS: cache and queue live there, credentials never do.
|
||||
- Wrap every `localStorage` read/write and `JSON.parse` in try/catch (quota, private mode, corrupt entry), as the existing helpers do.
|
||||
|
||||
Dependencies: stdlib first; a new module needs a stated reason. Confirm a package actually exists before adding it — a plausible name may be fiction (~20% of LLM-proposed packages don't resolve, which is how slopsquatting lands). Pin exact versions.
|
||||
|
||||
Review gate: auth, CORS, session, crypto, and the fetch gate are security-critical. Editing one is not a drive-by change — say which invariant you preserved and run `go test ./...` before calling it done.
|
||||
|
||||
## Comments
|
||||
|
||||
Comment only if code alone can't carry info. Cost per read — must earn spot.
|
||||
Wrong comment worse than none: it misleads readers and measurably degrades
|
||||
LLM performance on the file. Missing comment costs little. Bias to fewer.
|
||||
|
||||
Docstring on public/exported surface — exception, near-always worth it.
|
||||
Contract only: what it takes, returns, throws, mutates; units; pre/post
|
||||
conditions. Not a restatement of the body. Skip on private/obvious.
|
||||
|
||||
Inline — write for:
|
||||
- Why not what. Tradeoffs, non-obvious decisions, rejected alternatives.
|
||||
- heavy detail looking incidental — say so if "simplify" breaks it.
|
||||
- Non-local consequence, invisible from function alone.
|
||||
- Wire format / encoding / ordering / invariant — save callers re-deriving.
|
||||
- Gotcha/workaround, with ref (issue, RFC, vendor bug) if exists.
|
||||
- Domain/business rule not derivable from code.
|
||||
|
||||
Skip:
|
||||
- Restating code (no `// increment i` above `i++`).
|
||||
- Trivial getter/setter/pass-through.
|
||||
- Banners, dividers, `// helpers`.
|
||||
- Change narration (`// fix bug`, `// as requested`, `// new impl`) — git's job.
|
||||
- Commented-out code — delete.
|
||||
- TODO without concrete action + owner.
|
||||
- Narrating the plan you just reasoned through. Plan in prose or in your head;
|
||||
ship the code, not the transcript.
|
||||
- Anything restating a name that could be fixed by renaming instead.
|
||||
|
||||
Staleness filter: if the comment describes something likely to change
|
||||
independently of this line, it will rot and start lying. Either anchor it to
|
||||
something stable, assert it in a test, or leave it out.
|
||||
|
||||
Style: one dense comment over a function beats one per line inside. Tight; no
|
||||
worked example unless the bug is subtle. On edit, update or delete stale
|
||||
comments in the code you touch — silence beats a lie.
|
||||
|
||||
Test: "competent reader get this from code in a few sec?" Yes → skip.
|
||||
Needs detour through another file/spec/git-blame/external doc → write it.
|
||||
|
||||
## Writing an AGENTS.md
|
||||
|
||||
`AGENTS.md` is the single source of truth for agent guidance; every `CLAUDE.md`
|
||||
in this repo is a symlink to the `AGENTS.md` beside it. Edit `AGENTS.md`.
|
||||
|
||||
**Cite code, never docs, issues, or plans.** A spec, ADR, plan file, or Gitea
|
||||
issue records what was true when it was written and then goes stale silently;
|
||||
an agent that follows the pointer reads a decision that may already have been
|
||||
reversed. Code is the only source true at read time — cite a package, file,
|
||||
symbol, env var, or route. The sole non-code exception is a sibling
|
||||
`AGENTS.md`. If a doc holds a fact an agent needs, restate the fact here rather
|
||||
than linking to it.
|
||||
|
||||
**State a fact in prose only if the code cannot answer it.** Split by
|
||||
derivability:
|
||||
|
||||
- *Structure* — packages, routes, env vars, columns, struct fields. Rots fast,
|
||||
cheap to re-read. **Name the symbol, write nothing else.**
|
||||
- *Mechanism* — what a function does, how a flow proceeds. **Name the symbol
|
||||
plus at most one line of orientation.**
|
||||
- *Rationale* — why it is this way, what a "simplify" would break, what was
|
||||
tried and rejected. Not in the code and cannot be re-derived. **Write it out.**
|
||||
- *Measurement* — an observation against something we don't control. **Write it
|
||||
out with the date**; a dated fact is honest, an undated one pretends to be
|
||||
permanent.
|
||||
|
||||
Restating mechanism in prose is how these files rot: the code changes, the
|
||||
paragraph doesn't, and the next agent trusts the paragraph. A pointer degrades
|
||||
more honestly — and every symbol you name must actually exist, since a dead
|
||||
pointer is a bug, not a stale sentence.
|
||||
- Caveman mode active by default (`/home/tan/.config/opencode/AGENTS.md`). Keep comms terse — drop articles, fluff, pleasantries. Code/commits/security written normal.
|
||||
- `.superpowers/` and `.agents/` dirs hold skill definitions. Gitea at `gitea.violetcrown.my.id`.
|
||||
@@ -0,0 +1,177 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Status
|
||||
|
||||
Greenfield. Only `plans/mangaBookmark.md` exists — no code yet. That plan is the spec; read it before building. Two deliverables: a Go sync backend and a single Bromite-compatible userscript.
|
||||
|
||||
## What this is
|
||||
|
||||
A manga read-progress tracker for a user reading on **asurascans.com** (the current domain; asuracomic.net 301s here) and **demonicscans.org** from **Bromite** (mobile Chromium). A userscript injects on-page UI (floating button + slide-in panel) and syncs progress to a self-hosted Go backend so bookmarks unify across both sites and across devices.
|
||||
|
||||
## Hard constraints (these drive the design — do not violate)
|
||||
|
||||
Bromite uses Chromium's **native** userscript engine, not Tampermonkey:
|
||||
- **No `GM_*` APIs anywhere.** No `GM_setValue`/`GM_getValue` (use page `localStorage`), no `GM_registerMenuCommand` (inject on-page UI), no `GM_xmlhttpRequest` for cross-origin (use plain `fetch()`). Keeping the script GM-free also lets it run in desktop Tampermonkey/Violentmonkey for faster iteration.
|
||||
- Cross-origin `fetch()` works **only** against a CORS-enabled backend. Manga sites are `https://`, so backend **must be HTTPS** (mixed-content block otherwise).
|
||||
- Asura and Demonic are **separate origins with separate `localStorage`** — a shared remote store is the only way to unify bookmarks. Cloud sync is required, not optional.
|
||||
- Userscript runs in an **isolated world**, so the embedded API token is safe from the site's JS.
|
||||
- Cloudflare's block on fetching the manga sites is **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both the CGNAT dev machine *and* the deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. This contradicts an earlier, untested assumption that the CGNAT dev IP would be blocked; it was not, at least on this date. Treat "does curl work right now" as a live, time-varying fact to re-check, not a fixed property of a given machine — Cloudflare's bot scoring can flip a previously-clean IP without notice. Any backend fetcher still needs a graceful-degrade path for when it does get challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, or a direct probe) before finalizing, not assumed from a single earlier test.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Bromite userscript (isolated world, per-site adapters, localStorage cache)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> SQLite (volume)
|
||||
```
|
||||
|
||||
- **Backend** (`backend/`): stdlib `net/http` (a handful of routes, no framework) + `modernc.org/sqlite` (pure Go, `CGO_ENABLED=0` -> static binary -> distroless/scratch image). The reverse proxy terminates TLS; the Go service listens plain `:8080`.
|
||||
- **Single-user store.** One `bookmarks` table keyed `<site>:<series_id>` (`asura`|`demonic`). Sync is **last-write-wins**. Schema and endpoint list are in the plan.
|
||||
- **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth).
|
||||
- **Web UI:** the same binary serves a password-gated browser UI on a second
|
||||
hostname — `GET /` (list, or login page when there is no session),
|
||||
`POST /login`, `POST /logout`, `GET /static/*`, and htmx fragment endpoints
|
||||
under `/ui/*`. Templates and assets are `go:embed`-ed, so `backend/Dockerfile`
|
||||
must copy `templates/` and `static/` as well as `*.go`. Sessions are stateless
|
||||
HMAC cookies keyed off `API_TOKEN`; `WEB_PASSWORD` gates them and, when empty,
|
||||
the web routes are not registered at all. UI mutations read-modify-write
|
||||
through `Store.Get` + `Store.Upsert` so the `updated_at` rule stays in one
|
||||
place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`.
|
||||
**Design-tool caveat:** the templates link `/static/style.css` root-absolutely
|
||||
(correct — they are served from `/`), but the impeccable detector resolves a
|
||||
stylesheet href with `path.resolve(fileDir, href)`, which drops the directory
|
||||
on a leading `/` and silently skips the file. A relative href does not help
|
||||
either: the template's directory is not its served path. So
|
||||
`detect.mjs backend/templates` reports a **false clean** — always pass
|
||||
`backend/static` too. Its one finding there, `overused-font` on "Instrument
|
||||
Serif", is a deliberate identity choice, not debt.
|
||||
- **Every action that moves a series out of the list is confirm-gated.**
|
||||
Archive, finish, and remove each open their own `.confirm-row` disclosure
|
||||
(`toggleConfirmRow(key, kind)` in `filter.js`, `kind` ∈
|
||||
`archive|finish|remove`); restore fires instantly because it is the reversal.
|
||||
Remove's row wears the ember wash, the two reversible ones wear `.calm` grey.
|
||||
`--ember` stays reserved for the new-chapter signal: busy bar and inline
|
||||
error use `--mute`.
|
||||
- **Latest-chapter poller:** a ticker goroutine in the same binary re-checks
|
||||
each bookmarked series' newest published chapter from the backend's own
|
||||
network access, so `latest_chapter` stays fresh when the user is not
|
||||
browsing. It is a *second, parallel* signal — the userscript keeps its own
|
||||
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged.
|
||||
Two independent clocks: a per-bookmark cooldown (`latest_checked_at` column,
|
||||
enforced by `Store.DueForLatestCheck`'s WHERE clause) and a wake interval.
|
||||
The row is stamped *before* the fetch so a broken series waits out a full
|
||||
cooldown instead of retrying every tick, and writes go through
|
||||
`Store.Get` + `Store.Upsert` so a new chapter never reorders the list.
|
||||
Fetches use `bogdanfinn/tls-client` with a Chrome profile as defence in depth
|
||||
against fingerprint-based blocking; any failure logs and skips. See
|
||||
`docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`.
|
||||
The poller's `Store.Get` + `Store.Upsert` is not wrapped in a transaction, so
|
||||
a userscript `PUT` that commits between the two can be overwritten by the
|
||||
poller's stale re-read — reverting that read progress and, since the stored
|
||||
value now differs, moving `updated_at` and reordering the list. This is a
|
||||
known, accepted limitation for a single-user deployment, not a bug to fix.
|
||||
- **`updated_at` drives list order, so it moves only on real reading progress:** the server applies its timestamp when the row is new or `last_chapter_num` changes, and otherwise keeps the stored value — favouriting a series or recording a newly published chapter must not reorder the list. `PUT` therefore returns the row **as stored**, and clients must adopt that response rather than their own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4.
|
||||
- **Lifecycle buckets:** `status` on each bookmark is `reading` | `archived` |
|
||||
`finished`, orthogonal to `favorite`. Archived and finished appear only in
|
||||
their own tab — not in All, Updated, Favourites, or the recent strip. The
|
||||
poller keeps checking archived series and skips finished ones. `finished` is
|
||||
settable only from the web UI; `PUT /bookmarks/{key}` rejects it with 400.
|
||||
**An empty incoming status means "keep the stored one"** — resolved on the
|
||||
`VALUES` side of `Store.Upsert`, not in the conflict clause, because
|
||||
`excluded.*` is the post-evaluation row and a default applied there would
|
||||
wipe the bucket on every PUT from a client that predates the column. See
|
||||
`docs/superpowers/specs/2026-07-27-status-buckets-design.md`.
|
||||
- **Config via env:** `API_TOKEN`, `ALLOWED_ORIGINS` (comma list), `DB_PATH`
|
||||
(default `/data/bookmarks.db`), `PORT` (default `8080`), `WEB_PASSWORD`
|
||||
(gates the browser UI; unset disables it),
|
||||
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER`
|
||||
(background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`).
|
||||
`USERSCRIPT_PATH` (file served at `/u/{token}/manga-bookmark.user.js`,
|
||||
default `/userscript/manga-bookmark.user.js`, supplied by a bindmount).
|
||||
|
||||
### Userscript structure (single IIFE, `manga-bookmark.user.js`)
|
||||
|
||||
1. **Site adapters** — one per host, `detect(location, document)` returns page `type` + IDs. Identify type/IDs from **URL regex** (most stable); pull `title`/`cover` from **`og:title`/`og:image` meta tags**, not CSS classes.
|
||||
2. **API client** — `apiGet/apiPut/apiDelete` with bearer header; `localStorage` key `mangabm:cache` for instant render + offline fallback.
|
||||
3. **Progress logic** — auto-upsert `last_chapter` only when `chapterNum >= stored last_chapter_num` (re-reading old chapters must not regress progress; unparseable -> set current). Manual panel override forces any value.
|
||||
4. **Retry queue** — every write goes through `pushBookmark`/`pushDelete`, so a
|
||||
failed mutation is parked in `localStorage` (`mangabm:queue`) and replayed on
|
||||
the next navigation, reconnect, or `refresh()`. Entries are markers
|
||||
(`{key, op, sendStatus, attempts}`), never payloads — the body is read from
|
||||
the cache at send time, so one entry per key gives ordering and coalescing for
|
||||
free. `sendStatus` is **sticky**: while an archive is pending, later writes to
|
||||
that key keep carrying the bucket, which is what stops a successful
|
||||
in-between write from silently un-archiving the series. `refresh()` drains
|
||||
before it fetches and overlays anything still pending, so the list never
|
||||
flaps. A 400 drops the entry, a 401 aborts the pass and keeps the queue, and
|
||||
transient failures retry to a cap of 10. Latest-chapter writes deliberately
|
||||
stay out of the queue. See
|
||||
`docs/superpowers/specs/2026-07-27-offline-retry-queue-design.md`.
|
||||
5. **UI** — rendered inside a **Shadow DOM** root to isolate from site CSS
|
||||
(critical on mobile). Three tabs (All / Favourites / Archived) and a row of
|
||||
link chips to the web UI and both manga sites; `WEB_BASE` sits in the CONFIG
|
||||
block next to `API_BASE`. The FAB is a `7 × 44` edge tab whose *hit* area is
|
||||
widened to `28 × 72` by an invisible `#hit` child; `#fab` must keep
|
||||
`touch-action: none` and must **not** regain `overflow: hidden`. Because
|
||||
`touch-action` is resolved at gesture start, the strip cannot be both
|
||||
browser-scrolled and script-dragged, so `makeDraggable` splits by intent: a
|
||||
swipe from `#hit` scrolls via `window.scrollBy`, a hold of `ARM_MS` arms a
|
||||
reposition drag, and the visible sliver drags with no hold. See
|
||||
`docs/superpowers/specs/2026-07-28-edge-tab-hitbox-design.md`.
|
||||
6. **SPA navigation** — Asura is Astro, client-routed on the comic/chapter pages: patch `history.pushState`/`replaceState` + listen `popstate`, re-run `detect()` on URL change so auto-update fires without reload. Demonic uses classic reloads (initial `document-idle` run suffices).
|
||||
|
||||
### Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trusting)
|
||||
|
||||
- **asurascans.com**: series `/comics/<slug>` (slug carries a trailing
|
||||
site-wide build-hash suffix, e.g. `-059befe1`, that **rotates on every
|
||||
redeploy**), chapter `/comics/<slug>/chapter/<n>`. `seriesId` must strip
|
||||
the hash (`/-[0-9a-f]{8}$/`, `stripBuildHash` in the userscript,
|
||||
`asuraBuildHash` in the backend); URLs keep the full slug — stale-hash
|
||||
URLs 302 to current ones. Astro-rendered; chapter links present in raw
|
||||
server HTML.
|
||||
- **demonicscans.org**: series `/manga/<slug>` (slug may URL-encode punctuation, e.g. `%2527` for `'`), chapter `/title/<slug>/chapter/<n>/<page>` (older `chaptered.php?manga=<id>&chapter=<n>` form still exists as redirect, what series-page chapter-list anchors link through).
|
||||
Encodings (incl. triple-encoded punctuation like `%25252D`) are identical
|
||||
on /manga/ and /title/ pages, so decode-once seriesIds match — verified
|
||||
2026-07-28.
|
||||
|
||||
## Commands (once code exists)
|
||||
|
||||
Backend (`cd backend`):
|
||||
- Test all: `go test ./...`
|
||||
- Single test: `go test -run TestName ./...`
|
||||
- Build static binary: `CGO_ENABLED=0 go build`
|
||||
|
||||
Local stack: `docker compose up` (named volume mounted at `/data`, `restart: unless-stopped`).
|
||||
|
||||
Smoke test: `curl` the endpoints with `Authorization: Bearer <token>`; confirm `OPTIONS` preflight returns CORS headers and `/healthz` returns 200.
|
||||
|
||||
## Forge: Gitea, not GitHub
|
||||
|
||||
`origin` is a self-hosted Gitea instance (`gitea.violetcrown.my.id`), so **`gh` does not work here — use `tea` (Gitea CLI) for anything past plain git.** Common ones:
|
||||
|
||||
- Open a PR: `tea pr create --head <branch> --base main --title "..." --description "..."`
|
||||
- List / view / check out: `tea pr list`, `tea pr <n>`, `tea pr checkout <n>`
|
||||
- Issues: `tea issue create`, `tea issue list`
|
||||
- Auth lives in `tea login`, not a `GH_TOKEN` env var.
|
||||
|
||||
`tea` prints its output as rendered boxes rather than plain text; the PR URL lands on the last line.
|
||||
|
||||
## Security invariants
|
||||
|
||||
- Auth on `/bookmarks*`: require `Authorization: Bearer <API_TOKEN>`, **constant-time compare**, 401 otherwise.
|
||||
- CORS: reflect `Origin` only when in `ALLOWED_ORIGINS`; allow `GET,PUT,DELETE,OPTIONS` + headers `Authorization,Content-Type`; answer preflight `OPTIONS` with `204`.
|
||||
|
||||
## Relevant skills
|
||||
|
||||
`multi-stage-dockerfile` and `docker-compose-orchestration` for the container work (referenced in the plan).
|
||||
|
||||
## graphify
|
||||
|
||||
This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships.
|
||||
|
||||
Rules:
|
||||
- For codebase questions, first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
|
||||
- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing.
|
||||
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context.
|
||||
- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost).
|
||||
-159
@@ -1,159 +0,0 @@
|
||||
# Bookmark Manager
|
||||
|
||||
Read-progress tracker for serialised fiction. A reader browses third-party manga and
|
||||
novel sites; userscripts capture where they got to and sync it to a self-hosted backend,
|
||||
so progress survives across sites and devices.
|
||||
|
||||
## Language
|
||||
|
||||
**Series**:
|
||||
One ongoing work — a manga or a novel — as published by a Site. Identified by the canonical
|
||||
slug the Site itself publishes for it, never by its title and never by a Chapter Slug. A
|
||||
Series exists once and is shared by every Reader who bookmarks it; it owns the facts that
|
||||
are true regardless of who is reading — title, cover, Latest Chapter. A Reader cannot
|
||||
change them; they describe the Series, not anyone's relationship to it.
|
||||
_Avoid_: manga, title, book, comic
|
||||
|
||||
**Site**:
|
||||
One third-party source a Series is published on. A Series on two Sites is two Series.
|
||||
_Avoid_: source, host, provider, domain
|
||||
|
||||
**Chapter Slug**:
|
||||
A slug a Site builds its chapter addresses from. Not an identity: one Series may have
|
||||
several, any of them may differ from the slug that identifies the Series, and none is
|
||||
computable from another. Only the Site's own links say which ones a Series uses, so a
|
||||
Chapter Slug is always discovered, never derived.
|
||||
_Avoid_: series slug, url slug, permalink, chapter path
|
||||
|
||||
**Cover**:
|
||||
The image that stands for a Series wherever it is listed. A fact about the Series like
|
||||
its title — one Cover per Series, shared by every Reader, never per-Reader. Defined by
|
||||
what a Reader's browser can display, not by where the Site keeps the picture: an address
|
||||
no client can load is not a Cover, it is a missing one.
|
||||
_Avoid_: thumbnail, poster, image URL, artwork
|
||||
|
||||
**Reader**:
|
||||
A person with their own Progress. Exactly one per set of credentials, so there is no
|
||||
separate "account" concept to model — the credential belongs to the Reader.
|
||||
_Avoid_: user, account, member, subscriber
|
||||
|
||||
**Bookmark**:
|
||||
One Reader's tracked relationship with one Series, holding only what differs between
|
||||
Readers: Progress, Favourite, Lifecycle bucket. Facts about the Series itself belong
|
||||
to the Series, not here.
|
||||
_Avoid_: entry, item, record, subscription
|
||||
|
||||
**Orphan Series**:
|
||||
A Series no Reader bookmarks. Removing a Bookmark never removes the Series, so the row
|
||||
outlives every relationship to it: nothing reads it, no Poll visits it, and it still owns
|
||||
a Cover. A state of the Series, not a Lifecycle bucket — it says how many Readers hold it,
|
||||
never anything about a Reader.
|
||||
_Avoid_: dangling, unused, dead series, stale
|
||||
|
||||
**Library**:
|
||||
One of the two halves of the collection — manga or novel — selected by a Bookmark's
|
||||
`kind`. The web UI and the userscripts each address exactly one Library at a time.
|
||||
Not a per-person concept: "everything one person has bookmarked" is a different idea
|
||||
and must not be called a Library.
|
||||
_Avoid_: section, tab, category
|
||||
|
||||
**Progress**:
|
||||
The furthest chapter a reader has actually read in a Series. Only a change in Progress
|
||||
is real activity, so only Progress reorders the list.
|
||||
_Avoid_: position, bookmark (the noun is taken), last read
|
||||
|
||||
**Latest Chapter**:
|
||||
The highest-numbered chapter a Site has published for a Series. The number is what
|
||||
ranks it, never a date and never the Site's own "newest chapter" banner — where a Site
|
||||
disagrees with itself, its list of chapters is the record and its summary of that list
|
||||
is not. Established by a Poll and, between Polls, by a Sighting. Distinct from Progress
|
||||
in every way that matters: it is a fact about the Site, not about the reader, and it
|
||||
must never reorder the list.
|
||||
_Avoid_: newest, current chapter, update
|
||||
|
||||
**Poll**:
|
||||
The backend's own check of a Site for a Series's Latest Chapter, made without the
|
||||
Reader present. Performed once per Series no matter how many Readers bookmarked it —
|
||||
a Poll is work done on behalf of the Series, never on behalf of a Reader.
|
||||
_Avoid_: scrape, refresh, check, sync
|
||||
|
||||
**Poll Lane**:
|
||||
One Site's own stream of Polls, carrying the pace at which that Site is willing to be
|
||||
asked. Every Site has exactly one and no Lane can slow, block or borrow from another's;
|
||||
a Reader never has one and never influences one.
|
||||
_Avoid_: worker, queue, scheduler, batch, wave
|
||||
|
||||
**Lane Pass**:
|
||||
One sweep of a Poll Lane over the Series due on its Site: what it found waiting, how many it
|
||||
read, and whether it declined to work at all. A fact about the Lane rather than about any
|
||||
Series — a pass that read nothing is still a pass, and one that declined carries the reason it
|
||||
declined, since a Lane resting and a Lane stuck look identical from a count alone. Its record
|
||||
outlives the process that made it: "the poller has done nothing for six hours" is only
|
||||
answerable by something written down.
|
||||
_Avoid_: run, cycle, tick, batch, poll history
|
||||
|
||||
**Forced Poll**:
|
||||
A Poll the owner asks for by hand instead of waiting for the Series's turn. It jumps its
|
||||
Lane's queue and ignores every waiting rule — the rest between Polls, a Sighting standing
|
||||
in for a check, a Series only finished Readers hold — but never overrules a Site that is
|
||||
refusing us, the Lane's spacing between fetches, or a Series with no page to fetch. Asked
|
||||
for by marking the Series, never by commanding the poller, so it happens on the Lane's
|
||||
next pass rather than at the moment of asking.
|
||||
It also takes whatever Cover the Site publishes today: asking for one is asking to accept the
|
||||
page as it now stands, so it is the only read after Acquisition that can replace a Cover.
|
||||
_Avoid_: manual poll, refresh, retry, force refresh
|
||||
|
||||
**Paused Lane**:
|
||||
A Poll Lane the owner has stopped for a bounded time. It makes no Polls until the pause
|
||||
expires, so its Series stay due and unstamped exactly as they do when a Site cannot be
|
||||
reached. Every pause carries an expiry — a Lane cannot be stopped indefinitely — and it
|
||||
outlives a restart, being a fact about the Site rather than about the running process.
|
||||
_Avoid_: disabled, off, stopped, suspended, kill switch (that is the deploy-time switch)
|
||||
|
||||
**Stall**:
|
||||
A Poll Lane that owed Polls, made none, and has nothing to say for it. Distinct from the
|
||||
two conditions it resembles: a Site that refuses is exercising the pace it is entitled to,
|
||||
and a Lane the owner paused was told to stop — a Stall is neither asked for nor explained.
|
||||
It is the one fault no Reader surface can show: every Bookmark still opens, Progress still
|
||||
syncs, and Latest Chapter is quietly wrong for as long as it lasts.
|
||||
_Avoid_: outage, downtime, failure, backlog, lag
|
||||
|
||||
**Sighting**:
|
||||
What a Reader's browser happened to see of a Series's Latest Chapter while that Reader
|
||||
was on the page. It reports the same fact as a Poll but carries none of its authority:
|
||||
a Poll always overrules it, and only a Sighting on a Series no Reader else holds may
|
||||
defer one. A Sighting a later Poll contradicts downwards is a false Sighting, and
|
||||
enough of those cost the Reader the right to defer at all.
|
||||
_Avoid_: client report, user poll, observation, claim
|
||||
|
||||
**Acquisition**:
|
||||
The single read of a Series page made the moment the Series first exists, giving it
|
||||
both its Latest Chapter and its Cover without waiting for the Lane's pace. Distinct
|
||||
from a Poll in the two ways that matter: a Reader is present — it is triggered by
|
||||
their first Bookmark of that Series — and it establishes a Cover rather than refreshing
|
||||
facts, which no Poll does unless the owner forces one. It happens once in a Series's
|
||||
life; every later read of the same page is a Poll.
|
||||
_Avoid_: initial poll, first fetch, prefetch, warm-up
|
||||
|
||||
**Correction**:
|
||||
A Latest Chapter the owner sets by hand, on a Series no Poll can read. It reports the
|
||||
same fact as a Poll and carries even less authority than a Sighting: the next Poll
|
||||
overwrites it, so does any Reader's Sighting, and it is never a floor or a pin. It
|
||||
exists only because the Site page is unreadable — where a Poll can read the page, the
|
||||
Poll is the answer and a Correction is not wanted.
|
||||
_Avoid_: override, pin, manual value, fix
|
||||
|
||||
**New Chapter**:
|
||||
The state where Latest Chapter is ahead of Progress. The single condition the ember
|
||||
accent is permitted to signal.
|
||||
_Avoid_: unread, update available
|
||||
|
||||
**Lifecycle bucket**:
|
||||
Which of three mutually exclusive states a Bookmark sits in — reading, archived, or
|
||||
finished. A Bookmark is in exactly one. Orthogonal to being a favourite.
|
||||
_Avoid_: state, status (as a domain word), list
|
||||
|
||||
**Favourite**:
|
||||
A reader's manual pin on a Bookmark. Orthogonal to the Lifecycle bucket, and never a
|
||||
reason to reorder the list.
|
||||
_Avoid_: starred, pinned, priority
|
||||
-299
@@ -1,299 +0,0 @@
|
||||
# SQLite → Postgres cutover runbook
|
||||
|
||||
One-way, one-time. Moves the owner's reading history out of the retired SQLite
|
||||
volume (`<compose project>_bookmarks-data`, holding `/data/bookmarks.db`) and into
|
||||
the Postgres schema the migration runner builds. There is no dual-write period:
|
||||
the old database is read once, at cutover, from a **fresh export** — anything
|
||||
written to SQLite after the export is lost, so the old API must already be down.
|
||||
|
||||
Routine deploys are `REDEPLOY.md`; first-time setup is `DEPLOY.md`. This file is
|
||||
run once and then only ever read for reference.
|
||||
|
||||
Proven end to end on 2026-08-08 against a copy of `bookmarks-20260807-213515.db`
|
||||
into a scratch Postgres: 29 Bookmarks (18 reading, 11 archived, 7 favourites),
|
||||
29 Series, all owned by the seeded Reader, and every field of every row matching
|
||||
the source exactly. Production was not touched.
|
||||
|
||||
---
|
||||
|
||||
## 0. The generator is throwaway
|
||||
|
||||
It is written at cutover, run once, and deleted. It is deliberately **not** in
|
||||
this repository and never will be:
|
||||
|
||||
- Its output is the owner's personal reading history. That does not enter
|
||||
version control.
|
||||
- It reads SQLite. The backend module dropped `modernc.org/sqlite` (ADR-0001);
|
||||
a committed generator would drag the dependency back in through the side door.
|
||||
|
||||
So §3 specifies the transformation rather than shipping a script. It is a
|
||||
twenty-line program against a sixteen-column table (fifteen after `key`, which
|
||||
is dropped) — writing it from the spec below costs less than maintaining it
|
||||
would.
|
||||
|
||||
Beyond `DEPLOY.md`'s prerequisites (Docker and Compose), this runbook needs
|
||||
`python3`: its stdlib `sqlite3` module is the whole SQLite dependency, and §5's
|
||||
read-path check uses it in place of `jq`, which the server does not have. It
|
||||
does not have to run on the server — §3 only reads the snapshot copy, so it can
|
||||
run on a laptop and the resulting `import.sql` be copied over.
|
||||
|
||||
---
|
||||
|
||||
## 1. Stop the old API and take a fresh export
|
||||
|
||||
**Order matters.** Export after the API stops, or you migrate a snapshot that is
|
||||
already stale.
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark # wherever the checkout lives
|
||||
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
|
||||
BACKUP_DIR="$(cd .. && pwd)/$(basename "$PWD")-backups"; mkdir -p "$BACKUP_DIR"
|
||||
STAMP=$(date -u +%Y%m%d-%H%M%S)
|
||||
|
||||
# The volume is <compose project>_bookmarks-data, and the project name defaults
|
||||
# to the lowercased *directory* name, not the repo name — on this host the
|
||||
# checkout is ~/mangaBookmark, so the volume is mangabookmark_bookmarks-data.
|
||||
# Derive it exactly rather than with a `--filter name=` substring match, which
|
||||
# would return every volume whose name merely contains the string.
|
||||
VOL="$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_bookmarks-data"
|
||||
docker volume inspect "$VOL" >/dev/null && echo "$VOL"
|
||||
|
||||
$COMPOSE stop bookmark-api
|
||||
|
||||
# A clean SIGTERM closes the store, which checkpoints and unlinks the -wal, so
|
||||
# bookmarks.db alone is then the whole database. But `compose stop` SIGKILLs
|
||||
# after 10s, and a surviving -wal holds writes the main file does not — assert
|
||||
# it is gone rather than assuming the shutdown was clean.
|
||||
docker run --rm -v "$VOL":/d:ro alpine ls -l /d # -> bookmarks.db, alone
|
||||
|
||||
docker run --rm -v "$VOL":/from:ro -v "$BACKUP_DIR":/to \
|
||||
alpine cp /from/bookmarks.db "/to/bookmarks-$STAMP.db"
|
||||
|
||||
ls -lh "$BACKUP_DIR/bookmarks-$STAMP.db"
|
||||
```
|
||||
|
||||
If `-wal` and `-shm` are still there, the container was killed mid-write. Copy
|
||||
all three under the same basename and let SQLite replay the log when §3 opens
|
||||
it — copying only `bookmarks.db` silently drops whatever the log still holds.
|
||||
|
||||
Work on a **copy** of that file for the rest of this runbook. The export is the
|
||||
last line of retreat; nothing below should be able to write to it.
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/cutover && cp "$BACKUP_DIR/bookmarks-$STAMP.db" /tmp/cutover/snapshot.db
|
||||
chmod 444 /tmp/cutover/snapshot.db
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Bring up Postgres with the schema and the owner Reader
|
||||
|
||||
The new stack builds its own schema and seeds exactly one Reader from
|
||||
`OWNER_DISCORD_ID` — do not hand-write either. Pull the Postgres-era commit
|
||||
first: on a server that has only ever run the SQLite build, `--build` without a
|
||||
pull silently rebuilds the old image and the checks below fail with
|
||||
"relation readers does not exist".
|
||||
|
||||
```bash
|
||||
git pull --ff-only
|
||||
git log --oneline -1
|
||||
|
||||
# .env needs the new required vars (DATABASE_URL is built from
|
||||
# POSTGRES_PASSWORD; TOKEN_KEY, OWNER_DISCORD_ID and the DISCORD_* set are
|
||||
# required). Compose fails at start for a missing one.
|
||||
git diff HEAD@{1} HEAD -- .env.example docker-compose.yml docker-compose.prod.yml
|
||||
|
||||
$COMPOSE up -d --build
|
||||
docker logs bookmark-api --tail 20 # -> "listening on :8080"
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c '\dt'
|
||||
# -> bookmarks, readers, schema_migrations, series, sessions
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks \
|
||||
-c 'select id, discord_id from readers'
|
||||
# -> exactly one row, and discord_id is the owner's
|
||||
```
|
||||
|
||||
Two rows in `readers`, or zero, means `OWNER_DISCORD_ID` is wrong or the seed
|
||||
failed. Stop here — the import attaches history to "the oldest reader row", and
|
||||
that is only unambiguous while there is one.
|
||||
|
||||
`bookmarks` and `series` are empty at this point. That is what makes the import
|
||||
a plain sequence of `INSERT`s with no conflict handling.
|
||||
|
||||
---
|
||||
|
||||
## 3. Generate the import SQL
|
||||
|
||||
Read `/tmp/cutover/snapshot.db` and emit plain SQL on stdout. The old table is
|
||||
flat and its columns map one-for-one onto the split schema — no transformation
|
||||
beyond the split itself:
|
||||
|
||||
| SQLite `bookmarks` column | lands in | notes |
|
||||
|---|---|---|
|
||||
| `site`, `series_id` | both tables | the Series key; the wire `key` column is dropped, it is re-derived as `site:series_id` on read |
|
||||
| `title`, `series_url`, `cover`, `kind` | `series` | shared facts (ADR-0003) |
|
||||
| `latest_chapter`, `latest_chapter_num`, `latest_checked_at` | `series` | `latest_chapter_num` is nullable on **both** sides and `NULL` is meaningful — never coerce it to `0` |
|
||||
| `last_chapter`, `last_chapter_num`, `last_chapter_url` | `bookmarks` | Progress |
|
||||
| `favorite`, `status`, `updated_at` | `bookmarks` | `favorite` is `0`/`1` in SQLite and a real `boolean` in Postgres — emit `true`/`false` |
|
||||
| — | `bookmarks.reader_id` | the seeded owner |
|
||||
|
||||
**`latest_chapter_num` is the only column where `NULL` survives.** The SQLite
|
||||
table declares `title`, `series_url`, `cover`, `last_chapter`,
|
||||
`last_chapter_url` as bare `TEXT` and `last_chapter_num` as bare `REAL` — all
|
||||
six nullable — while their Postgres targets are `NOT NULL DEFAULT ''` /
|
||||
`NOT NULL DEFAULT 0`. One `NULL` in any of them aborts the whole import on a
|
||||
not-null violation. Coalesce them in the `SELECT` (`ifnull(title,'')`,
|
||||
`ifnull(last_chapter_num,0)`, …) rather than discovering it at §5. The
|
||||
2026-08-07 export happened to have none; a fresh export is not promised the
|
||||
same.
|
||||
|
||||
Rules the generator must follow:
|
||||
|
||||
- **Series first, Bookmarks second.** `bookmarks` has a foreign key onto
|
||||
`series (site, series_id)`; the reverse order fails on the first row.
|
||||
- **`SELECT DISTINCT` the Series.** The old key's uniqueness already makes
|
||||
`(site, series_id)` unique, so this is belt and braces — but if it ever
|
||||
collapses two rows, the count check in §5 catches it.
|
||||
- **Never hardcode the reader id.** Emit
|
||||
`INSERT INTO bookmarks (reader_id, …) SELECT id, … FROM owner`, where `owner`
|
||||
is a temp table built once at the top:
|
||||
`CREATE TEMP TABLE owner ON COMMIT DROP AS SELECT id FROM readers ORDER BY id LIMIT 1;`
|
||||
A literal id is a number nobody verifies; this one cannot be wrong.
|
||||
- **Wrap the whole file in `BEGIN; … COMMIT;`, temp table included.** Postgres
|
||||
has transactional DDL and DML: a failure half way leaves an empty database
|
||||
rather than half a library. The ordering is load-bearing —
|
||||
`ON COMMIT DROP` outside the transaction means the temp table drops itself
|
||||
the instant it is created (psql autocommits) and every
|
||||
`SELECT … FROM owner` then fails.
|
||||
- **Quote strings by doubling `'`.** Titles contain apostrophes and the URLs
|
||||
contain `%5C%27` escapes. Emit standard SQL literals only — no `E''` strings,
|
||||
no backslash escaping (`standard_conforming_strings` is on, so a backslash is
|
||||
a literal backslash and the URLs survive verbatim).
|
||||
|
||||
```bash
|
||||
python3 gen_import.py /tmp/cutover/snapshot.db > /tmp/cutover/import.sql
|
||||
wc -l /tmp/cutover/import.sql # -> 2 header + 29 series + 29 bookmarks + framing
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Review it by eye
|
||||
|
||||
29 rows is small enough to actually read, and this is the last point at which a
|
||||
mistake is free:
|
||||
|
||||
```bash
|
||||
less /tmp/cutover/import.sql
|
||||
grep -c '^INSERT INTO series' /tmp/cutover/import.sql # -> 29
|
||||
grep -c '^INSERT INTO bookmarks' /tmp/cutover/import.sql # -> 29
|
||||
```
|
||||
|
||||
Look for: a title whose apostrophe is not doubled, a `favorite` that is still
|
||||
`0`/`1`, a `latest_chapter_num` that turned into `0`, and any `reader_id`
|
||||
written as a bare number.
|
||||
|
||||
---
|
||||
|
||||
## 5. Apply it
|
||||
|
||||
```bash
|
||||
docker cp /tmp/cutover/import.sql "$($COMPOSE ps -q postgres)":/tmp/import.sql
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -v ON_ERROR_STOP=1 \
|
||||
-f /tmp/import.sql
|
||||
```
|
||||
|
||||
`ON_ERROR_STOP=1` is not optional: without it `psql` reports the error, keeps
|
||||
going, and exits `0` on a half-imported database.
|
||||
|
||||
Then the checklist. Every number here is asserted, not eyeballed:
|
||||
|
||||
```bash
|
||||
OWNER=$(grep -E '^OWNER_DISCORD_ID=' .env | cut -d= -f2)
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -x -c "
|
||||
SELECT (SELECT count(*) FROM bookmarks) AS bookmarks_total,
|
||||
(SELECT count(*) FROM bookmarks WHERE status='reading') AS reading,
|
||||
(SELECT count(*) FROM bookmarks WHERE status='archived')AS archived,
|
||||
(SELECT count(*) FROM series) AS series_total,
|
||||
(SELECT count(*) FROM readers) AS readers_total,
|
||||
(SELECT count(*) FROM bookmarks
|
||||
WHERE reader_id <> (SELECT id FROM readers WHERE discord_id='$OWNER'))
|
||||
AS not_owned_by_owner;"
|
||||
```
|
||||
|
||||
`not_owned_by_owner` resolves the Reader by **Discord id**, not by
|
||||
`ORDER BY id LIMIT 1`. The second form is the expression §3 tells the generator
|
||||
to import with, so comparing against it is true by construction and could never
|
||||
fail; resolving by Discord id is an independent check that the rows landed on
|
||||
the identity the owner will actually log in as. If that subquery returns NULL
|
||||
the whole count comes back `0` for the wrong reason — hence `readers_total`
|
||||
beside it.
|
||||
|
||||
Expected, for the 2026-08-07 export: `29`, `18`, `11`, `29`, `1`, `0`. Against a
|
||||
different export, the invariants rather than the literals are what hold:
|
||||
|
||||
- `bookmarks_total` equals the SQLite row count.
|
||||
- `reading + archived` equals `bookmarks_total` (nothing was `finished`).
|
||||
- `series_total` equals `SELECT count(*) FROM (SELECT DISTINCT site, series_id FROM bookmarks)`
|
||||
in the source.
|
||||
- `readers_total` is `1` and `not_owned_by_owner` is `0`.
|
||||
|
||||
Then spot-check the values themselves against the source — read position,
|
||||
favourite flag and latest chapter. Take the sample from each bucket explicitly:
|
||||
`ORDER BY updated_at DESC LIMIT 5` alone returns the most recently *progressed*
|
||||
rows, which are the ones least likely to be archived.
|
||||
|
||||
```bash
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
|
||||
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
|
||||
s.latest_chapter, b.status
|
||||
FROM bookmarks b JOIN series s USING (site, series_id)
|
||||
WHERE b.status='reading' ORDER BY b.updated_at DESC LIMIT 3;"
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
|
||||
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
|
||||
s.latest_chapter, b.status
|
||||
FROM bookmarks b JOIN series s USING (site, series_id)
|
||||
WHERE b.status='archived' ORDER BY b.updated_at DESC LIMIT 2;"
|
||||
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c "
|
||||
SELECT s.title, b.last_chapter, b.last_chapter_num, b.favorite,
|
||||
s.latest_chapter, b.status
|
||||
FROM bookmarks b JOIN series s USING (site, series_id)
|
||||
WHERE b.favorite ORDER BY b.updated_at DESC LIMIT 2;"
|
||||
```
|
||||
|
||||
Compare each against the same row in the snapshot — the generator's own source
|
||||
is the reference, so read it back with the same `python3` you used in §3.
|
||||
|
||||
Finally, prove the **read path**, not just the tables — this is the check that
|
||||
would catch a correct import behind a broken join:
|
||||
|
||||
```bash
|
||||
API=https://bookmark-api.violetcrown.my.id
|
||||
# Your own Reader credential: sign in to the web UI and take it from the
|
||||
# Userscripts panel's install link, or read the API_TOKEN constant out of an
|
||||
# already-installed script. There is no credential in .env to grep.
|
||||
TOKEN=<your Reader credential>
|
||||
curl -s -H "Authorization: Bearer $TOKEN" $API/bookmarks |
|
||||
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' # -> 29
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Afterwards
|
||||
|
||||
- **Keep the old SQLite volume for a month.** It is already undeclared in
|
||||
compose, so `docker compose down -v` cannot take it. Remove it by hand once
|
||||
the Postgres data has been trusted for a while. That happens in a shell where
|
||||
`$VOL` from §1 is long gone, so re-derive it:
|
||||
`docker volume rm "$(basename ~/mangaBookmark | tr '[:upper:]' '[:lower:]')_bookmarks-data"`
|
||||
(see `REDEPLOY.md` §1).
|
||||
- **Delete the generator and the working copies:** `rm -rf /tmp/cutover`. The
|
||||
timestamped export in `$BACKUP_DIR` is the copy that is kept.
|
||||
- **Take the first Postgres dump immediately** — `REDEPLOY.md` §1. Until that
|
||||
exists, the only backup of the migrated data is the SQLite file it came from.
|
||||
|
||||
If the import is wrong, there is nothing to unpick: drop the rows and start
|
||||
again from §3 — `TRUNCATE bookmarks, series;` leaves the seeded Reader and the
|
||||
schema in place.
|
||||
@@ -10,8 +10,8 @@ ACME/cert resolver, and control a domain.
|
||||
|
||||
- Docker + Docker Compose on the server.
|
||||
- A Traefik instance watching a Docker network (default name assumed: `proxy`).
|
||||
- DNS: an `A`/`AAAA` record for `bookmark-api.<yourdomain>` pointing at the server.
|
||||
- The repo copied to the server, e.g. `~/mangaBookmark/` (needs `backend/`,
|
||||
- DNS: an `A`/`AAAA` record for `manga-api.<yourdomain>` pointing at the server.
|
||||
- The repo copied to the server, e.g. `/opt/mangabm/` (needs `backend/`,
|
||||
`docker-compose.yml`, `docker-compose.prod.yml`, `.env.example`).
|
||||
|
||||
Confirm the Traefik network exists (create if not):
|
||||
@@ -25,48 +25,24 @@ docker network ls | grep proxy || docker network create proxy
|
||||
## 1. Configure `.env`
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark
|
||||
cd /opt/mangabm
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env`:
|
||||
|
||||
```ini
|
||||
# Required — secret every Reader's userscript credential is derived from.
|
||||
# Only SHA-256 hashes of credentials are stored.
|
||||
TOKEN_KEY=<paste output of: openssl rand -hex 32>
|
||||
|
||||
# Required — the owner's Discord user ID. Seeds the first Reader: the
|
||||
# administrator, and the owner of every bookmark that predates registration.
|
||||
# The value is the snowflake in your Discord profile (Settings →
|
||||
# Advanced → Developer Mode → right-click your name → Copy User ID).
|
||||
OWNER_DISCORD_ID=<discord user id>
|
||||
# Required — long random secret, also goes in the userscript.
|
||||
API_TOKEN=<paste output of: openssl rand -hex 32>
|
||||
|
||||
# CORS allowlist — leave as-is unless a site changes hostname.
|
||||
ALLOWED_ORIGINS=https://asurascans.com,https://demonicscans.org,https://comix.to,https://kagane.to,https://novelfull.com,https://lightnovelworld.net
|
||||
|
||||
# Required — password for the bundled Postgres container. Compose builds the
|
||||
# backend's DATABASE_URL out of it and has no fallback for either.
|
||||
POSTGRES_PASSWORD=<paste output of: openssl rand -hex 24>
|
||||
|
||||
# Leave unset. Only set this to point the backend at a Postgres compose does
|
||||
# not run; it then replaces the URL built from POSTGRES_PASSWORD above.
|
||||
# DATABASE_URL=postgres://user:pass@host:5432/bookmarks?sslmode=require
|
||||
|
||||
# Required path inside bookmark-api. Compose builds the image and mounts the
|
||||
# named cover-data volume at this path.
|
||||
COVER_DIR=/covers
|
||||
|
||||
# Required — the origin this deployment answers on, no trailing slash. Cover
|
||||
# URLs on the wire are absolute, because the userscript renders them on a
|
||||
# Site's own origin (ADR-0007). Same host as BOOKMARK_API_HOST below.
|
||||
PUBLIC_BASE_URL=https://bookmark-api.violetcrown.my.id
|
||||
ALLOWED_ORIGINS=https://asuracomic.net,https://asurascans.com,https://demonicscans.org
|
||||
|
||||
# Required for the Traefik override. Both have no fallback — compose refuses
|
||||
# to start without them. BOOKMARK_WEB_HOST is required even if the web UI
|
||||
# were unused; see 1b.
|
||||
BOOKMARK_API_HOST=bookmark-api.violetcrown.my.id
|
||||
BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
|
||||
# to start without them. MANGA_WEB_HOST is required even if you never set
|
||||
# WEB_PASSWORD; see 1b.
|
||||
MANGA_API_HOST=manga-api.violetcrown.my.id
|
||||
MANGA_WEB_HOST=manga.violetcrown.my.id
|
||||
|
||||
# Only if your Traefik setup differs from these defaults:
|
||||
# PROXY_NETWORK=proxy
|
||||
@@ -74,33 +50,13 @@ BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
|
||||
# TRAEFIK_CERTRESOLVER=le
|
||||
```
|
||||
|
||||
Generate + insert the two secrets in three lines:
|
||||
Generate + insert the token in one line:
|
||||
|
||||
```bash
|
||||
sed -i "s|^TOKEN_KEY=.*|TOKEN_KEY=$(openssl rand -hex 32)|" .env
|
||||
sed -i "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$(openssl rand -hex 24)|" .env
|
||||
grep -E '^TOKEN_KEY=' .env
|
||||
sed -i "s|^API_TOKEN=.*|API_TOKEN=$(openssl rand -hex 32)|" .env
|
||||
grep -E '^API_TOKEN=' .env # copy this — the userscript needs the same value
|
||||
```
|
||||
|
||||
`TOKEN_KEY` derives every Reader's userscript credential (issue #24); only
|
||||
SHA-256 hashes of the credentials are stored, so this secret is what a
|
||||
database leak alone cannot recover. Changing it invalidates every installed
|
||||
script at once.
|
||||
|
||||
`POSTGRES_PASSWORD` is read **only while the `postgres-data` volume is empty**,
|
||||
which in practice means at first boot. Changing it afterwards changes the URL
|
||||
the backend dials but not the password the database expects, and `bookmark-api`
|
||||
crash-loops on `password authentication failed`. Set it before §2 and leave it
|
||||
alone.
|
||||
|
||||
An `.env` written before issue #100 carries the old poll-pace names
|
||||
(`LATEST_CHAPTER_POLL_COOLDOWN`, `_BROWSER_COOLDOWN`, `_INTERVAL`, `_BATCH`,
|
||||
`_STAGGER`). All five are dead configuration now — the pace lives in the Site
|
||||
registry (`backend/internal/latest/sites.go`), so **delete those lines** and
|
||||
keep only the kill switch `LATEST_CHAPTER_POLL_ENABLED`. Leaving them behind
|
||||
is harmless (nothing reads them) but silently misleads the next person who
|
||||
edits the file.
|
||||
|
||||
> Match `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER` to your Traefik's actual
|
||||
> names (check your Traefik static config — common alternatives: `https`,
|
||||
> `myresolver`, `cloudflare`). Wrong names = no certificate issued.
|
||||
@@ -109,58 +65,44 @@ edits the file.
|
||||
|
||||
## 1b. Web UI
|
||||
|
||||
The browser UI is served by the same container on a second hostname. Sign-in
|
||||
is a Discord authorization code grant (ADR-0002): the owner's Discord account,
|
||||
gated by membership in one configured guild.
|
||||
The browser UI is served by the same container on a second hostname.
|
||||
|
||||
1. Add a DNS `A`/`AAAA` record for `bookmark.<yourdomain>` pointing at the
|
||||
server — the same address as `bookmark-api.<yourdomain>`.
|
||||
1. Add a DNS `A`/`AAAA` record for `manga.<yourdomain>` pointing at the server —
|
||||
the same address as `manga-api.<yourdomain>`.
|
||||
|
||||
2. Create the Discord application at <https://discord.com/developers/applications>:
|
||||
- **OAuth2 → Redirects:** add the exact callback URL
|
||||
`https://bookmark.violetcrown.my.id/auth/discord/callback`. Discord
|
||||
matches it verbatim — a trailing slash or different hostname breaks
|
||||
sign-in.
|
||||
- **OAuth2 → General:** note the Client ID, and generate a Client Secret.
|
||||
- No scopes or bot setup are needed in the dashboard; the service requests
|
||||
`identify` and `guilds.members.read` itself, and checks the *user's*
|
||||
membership of the guild, not the application's.
|
||||
|
||||
3. Set the variables in `.env`:
|
||||
2. Set both variables in `.env`:
|
||||
|
||||
```ini
|
||||
BOOKMARK_WEB_HOST=bookmark.violetcrown.my.id
|
||||
DISCORD_CLIENT_ID=<client id>
|
||||
DISCORD_CLIENT_SECRET=<client secret>
|
||||
DISCORD_GUILD_ID=<guild snowflake>
|
||||
DISCORD_REDIRECT_URI=https://bookmark.violetcrown.my.id/auth/discord/callback
|
||||
# Optional: only members holding this role may sign in.
|
||||
# DISCORD_REQUIRED_ROLE=<role snowflake>
|
||||
MANGA_WEB_HOST=manga.violetcrown.my.id
|
||||
WEB_PASSWORD=<paste output of: openssl rand -base64 18>
|
||||
```
|
||||
|
||||
The guild id is in Discord's client with Developer Mode on: right-click the
|
||||
server name → Copy Server ID. The four uncommented variables are required —
|
||||
the backend refuses to start without them. Guild membership *is*
|
||||
registration: any member of `DISCORD_GUILD_ID` becomes a Reader with their
|
||||
own library on their first sign-in. `OWNER_DISCORD_ID` from §1 is only the
|
||||
administrator — the Reader who can revoke another Reader's sessions.
|
||||
Generate and insert in one line:
|
||||
|
||||
4. Redeploy and check:
|
||||
```bash
|
||||
sed -i "s|^WEB_PASSWORD=.*|WEB_PASSWORD=$(openssl rand -base64 18)|" .env
|
||||
grep -E '^WEB_PASSWORD=' .env # this is what you type into the site
|
||||
```
|
||||
|
||||
3. Redeploy and check:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build
|
||||
curl -s -o /dev/null -w '%{http_code}\n' https://bookmark.violetcrown.my.id/
|
||||
curl -s -o /dev/null -w '%{http_code}\n' https://manga.violetcrown.my.id/
|
||||
```
|
||||
|
||||
Expected `200`, serving the login page with the Discord button. Signing in
|
||||
lands on the library; an account outside the guild is refused with a message
|
||||
that names neither the guild nor its id.
|
||||
Expected `200`, serving the login page.
|
||||
|
||||
Sessions are rows in the database: the cookie carries only an opaque id, and
|
||||
every request looks the row up and checks its expiry. Deleting a session row —
|
||||
or the whole `sessions` table — logs the browser out immediately; nothing is
|
||||
signed, so rotating a credential does not affect browser sessions. Sessions
|
||||
last 60 days.
|
||||
Leaving `WEB_PASSWORD` unset is safe: the web routes are not registered and `/`
|
||||
returns 404. The userscript's API on `MANGA_API_HOST` is unaffected either way.
|
||||
|
||||
`MANGA_WEB_HOST` itself is required by the prod override regardless — like
|
||||
`MANGA_API_HOST`, its Traefik label has no fallback, so `docker compose up`
|
||||
refuses to start without it even if `WEB_PASSWORD` is unset and the web UI is
|
||||
otherwise dormant.
|
||||
|
||||
Sessions are signed with a key derived from `API_TOKEN` and `WEB_PASSWORD`, so
|
||||
rotating either one logs every browser out. The session cookie lasts 60 days.
|
||||
|
||||
---
|
||||
|
||||
@@ -174,24 +116,11 @@ This merges the base file (build/image/env/volume) with the prod override
|
||||
(no host port, Traefik network + router labels). Always pass **both** `-f`
|
||||
flags — the prod file is not standalone.
|
||||
|
||||
Two services come up: `bookmark-api` (the backend) and `postgres` (its
|
||||
database, `postgres:17-alpine`). Postgres publishes no port — it sits alone
|
||||
with `bookmark-api` on an `internal: true` network — and stops everything if it
|
||||
is missing: `bookmark-api` waits for `pg_isready` to pass, then applies its
|
||||
embedded migrations, and only then listens. The schema is created that way;
|
||||
there is nothing to import by hand.
|
||||
|
||||
There is deliberately no browser here. Kagane and novelfull need one, and it
|
||||
runs on a **separate machine** over the tailnet — §7. Until you do that step,
|
||||
`BROWSER_WS_URL` is unset, the poller logs and skips those two sites, and
|
||||
everything else works normally.
|
||||
|
||||
Check it's up and healthy:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml ps
|
||||
# bookmark-api Up; postgres Up (healthy)
|
||||
docker logs bookmark-api --tail 20 # expect: "listening on :8080 ..."
|
||||
docker logs manga-api --tail 20 # expect: "listening on :8080 ..."
|
||||
```
|
||||
|
||||
---
|
||||
@@ -202,24 +131,21 @@ Give Traefik a few seconds to issue the cert, then:
|
||||
|
||||
```bash
|
||||
# Health (no auth) — must be valid TLS, no cert warning.
|
||||
curl -s https://bookmark-api.violetcrown.my.id/healthz # -> ok
|
||||
curl -s https://manga-api.violetcrown.my.id/healthz # -> ok
|
||||
|
||||
# Auth enforced.
|
||||
curl -s -o /dev/null -w '%{http_code}\n' \
|
||||
https://bookmark-api.violetcrown.my.id/bookmarks # -> 401
|
||||
https://manga-api.violetcrown.my.id/bookmarks # -> 401
|
||||
|
||||
# A Reader's own credential. It is derived, never stored in .env — take it from
|
||||
# the Userscripts panel's install link after signing in, or from an installed
|
||||
# script's API_TOKEN constant.
|
||||
TOKEN=<your Reader credential>
|
||||
TOKEN=$(grep -E '^API_TOKEN=' .env | cut -d= -f2)
|
||||
curl -s -H "Authorization: Bearer $TOKEN" \
|
||||
https://bookmark-api.violetcrown.my.id/bookmarks # -> []
|
||||
https://manga-api.violetcrown.my.id/bookmarks # -> []
|
||||
|
||||
# CORS preflight from a real site origin.
|
||||
curl -s -i -X OPTIONS \
|
||||
-H 'Origin: https://asurascans.com' \
|
||||
-H 'Access-Control-Request-Method: PUT' \
|
||||
https://bookmark-api.violetcrown.my.id/bookmarks/x | grep -i access-control
|
||||
https://manga-api.violetcrown.my.id/bookmarks/x | grep -i access-control
|
||||
# -> Access-Control-Allow-Origin: https://asurascans.com (+ Methods/Headers)
|
||||
```
|
||||
|
||||
@@ -230,30 +156,29 @@ a bad cert makes the browser block the userscript's `fetch()` (mixed content).
|
||||
|
||||
## 4. Configure the userscript
|
||||
|
||||
The bindmounted `userscript/*.user.js` files carry `__API_TOKEN__` placeholders
|
||||
and the deployment's `@downloadURL`/`@updateURL` lines. Check the metadata
|
||||
block — it ships hardcoded to this deployment's domain, so a deployer who
|
||||
copies the repo to another domain must edit the two lines or the script
|
||||
auto-updates from someone else's backend:
|
||||
Edit the config block at the top of `userscript/manga-bookmark.user.js`:
|
||||
|
||||
```js
|
||||
// @downloadURL https://bookmark-api.yourdomain.com/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
// @updateURL https://bookmark-api.yourdomain.com/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
const API_BASE = "https://manga-api.yourdomain.com"; // no trailing slash
|
||||
const API_TOKEN = "<same token as .env>";
|
||||
```
|
||||
|
||||
The backend substitutes `__API_TOKEN__` with the requesting Reader's derived
|
||||
credential at serve time (issue #24), so no real credential ever sits in the
|
||||
file. Only the `API_BASE` constant and the metadata hostname are deployer
|
||||
edits; do not put a credential in this file.
|
||||
The token sits in the userscript's isolated world — the manga sites' JS can't
|
||||
read it.
|
||||
|
||||
Also edit the `@downloadURL`/`@updateURL` metadata lines near the top of the
|
||||
file — they ship hardcoded to this deployment's domain and token, so a
|
||||
deployer who skips them ends up auto-updating from someone else's backend.
|
||||
See "Installing / updating the userscript" below for how those two lines are
|
||||
used.
|
||||
|
||||
---
|
||||
|
||||
## 5. Install on Bromite
|
||||
|
||||
1. Bromite → **Settings → User scripts** → enable (accept the permission prompt).
|
||||
2. Sign in to the web UI, open the **Userscripts** panel, and open the install
|
||||
link — Bromite detects `.user.js` and offers to install. The script already
|
||||
carries your credential; you never see or type one.
|
||||
2. Put the edited `manga-bookmark.user.js` on the device (save the file, or open
|
||||
its raw URL). Bromite detects `.user.js` and offers to install.
|
||||
3. Confirm install — the `@match` list covers both sites.
|
||||
4. Open a series on asurascans.com or demonicscans.org → a 📑 button appears
|
||||
bottom-right → tap → **+ Bookmark this**.
|
||||
@@ -261,15 +186,12 @@ edits; do not put a credential in this file.
|
||||
Optional desktop test: the script is `GM_*`-free, so the same file installs in
|
||||
Tampermonkey/Violentmonkey for quick checks before going mobile.
|
||||
|
||||
Rotating the credential in the same web-UI panel invalidates every installed
|
||||
copy immediately — reinstall on all devices, or they silently stop syncing.
|
||||
|
||||
---
|
||||
|
||||
## 6. Smoke-test the full loop
|
||||
|
||||
1. Bookmark a series on Asura.
|
||||
2. `curl -s -H "Authorization: Bearer $TOKEN" https://bookmark-api.yourdomain.com/bookmarks`
|
||||
2. `curl -s -H "Authorization: Bearer $TOKEN" https://manga-api.yourdomain.com/bookmarks`
|
||||
on the server — the series should appear.
|
||||
3. Open a chapter of that series — reopen the panel; last-read updates to that
|
||||
chapter (auto, never regresses on older chapters).
|
||||
@@ -278,206 +200,6 @@ copy immediately — reinstall on all devices, or they silently stop syncing.
|
||||
|
||||
---
|
||||
|
||||
## 7. The browser, on the home machine
|
||||
|
||||
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
|
||||
fingerprint clears, so the poller reaches them through a real Chrome over CDP.
|
||||
That browser does **not** run on the VPS: it held 471 MiB of a 1974 MiB box
|
||||
with no swap, and a residential IP avoids the cloud-hosting-IP signature Bot
|
||||
Fight Mode challenges anyway (ADR-0006). It
|
||||
is its own compose unit, deployed and updated independently of everything
|
||||
above.
|
||||
|
||||
Do this after §2, on the second machine. Both machines must already be on the
|
||||
same tailnet.
|
||||
|
||||
First, on the VPS, record what you are reclaiming — this is the whole point of
|
||||
the move and there is no way to measure it afterwards:
|
||||
|
||||
```bash
|
||||
free -m | awk '/^Mem:/ {print "available before:", $NF, "MiB"}'
|
||||
```
|
||||
|
||||
Take it again after §7 is finished and the old sidecar is gone. Expect roughly
|
||||
the sidecar's former footprint back (measured at 471 MiB working set, 595 MiB
|
||||
cgroup).
|
||||
|
||||
**On the home machine:**
|
||||
|
||||
```bash
|
||||
git clone <this repo> ~/mangaBookmark && cd ~/mangaBookmark/chrome
|
||||
|
||||
tailscale ip -4 # -> 100.x.y.z, this machine's tailnet IP
|
||||
cp .env.example .env
|
||||
echo "BROWSER_BIND_ADDR=$(tailscale ip -4)" >> .env
|
||||
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
The clone is only for `chrome/`; nothing else on this machine reads the rest of
|
||||
the repo. The unit is its own compose project (`bookmark-browser`), so it shares
|
||||
no volume, network or lifecycle with an API stack that happens to sit beside it.
|
||||
|
||||
`BROWSER_BIND_ADDR` has no default on purpose. CDP authenticates nothing —
|
||||
whatever reaches port 9222 drives the browser and, through it, this host — so
|
||||
the bind address *is* the access control, backed by Tailscale device identity.
|
||||
On the VPS that job was done by Docker network membership; this machine has a
|
||||
real LAN, so `0.0.0.0` would be a hole punched into your home network. Compose
|
||||
refuses to start rather than guess.
|
||||
|
||||
**Narrow it to the one device that needs it.** The bind address keeps CDP off
|
||||
your LAN; it still leaves port 9222 open to every device on the tailnet, and
|
||||
CDP has no login — a compromised phone is enough to drive this host. A new
|
||||
tailnet's policy is allow-all, so this is the step that makes "Tailscale
|
||||
identity is the access control" true rather than aspirational.
|
||||
|
||||
Tailscale has no `deny`, so a restriction is expressed by removing the blanket
|
||||
grant and enumerating what is left. That only works if the browser machine can
|
||||
be *excluded* from a selector that still covers your own devices — which is
|
||||
what tagging buys: a tagged device has no user, so `autogroup:member` and
|
||||
`autogroup:self` stop matching it. Tagging is the mechanism, not decoration.
|
||||
|
||||
In the admin console, under **Access controls**, the shipped policy grants
|
||||
`{"src": ["*"], "dst": ["*"], "ip": ["*"]}`. Replace it:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"tagOwners": {
|
||||
// Empty list: implicitly owned by the tailnet Owner/Admins, which is you.
|
||||
"tag:bookmark-api": [],
|
||||
"tag:bookmark-browser": [],
|
||||
},
|
||||
|
||||
"grants": [
|
||||
// The only thing on the tailnet that may drive the browser.
|
||||
{
|
||||
"src": ["tag:bookmark-api"],
|
||||
"dst": ["tag:bookmark-browser"],
|
||||
"ip": ["tcp:9222"],
|
||||
},
|
||||
// Your own devices reach your own devices, and the VPS, in full.
|
||||
{
|
||||
"src": ["autogroup:member"],
|
||||
"dst": ["autogroup:self", "tag:bookmark-api"],
|
||||
"ip": ["*"],
|
||||
},
|
||||
// On the browser machine you get SSH and nothing else. Widen this to `*`
|
||||
// and the restriction above is void; delete it and you are locked out.
|
||||
{
|
||||
"src": ["autogroup:member"],
|
||||
"dst": ["tag:bookmark-browser"],
|
||||
"ip": ["tcp:22"],
|
||||
},
|
||||
// Uncomment if you route traffic through an exit node — dropping the
|
||||
// blanket grant takes exit-node access with it.
|
||||
// {"src": ["autogroup:member"], "dst": ["autogroup:internet"], "ip": ["*"]},
|
||||
],
|
||||
|
||||
// Tagged devices left `autogroup:self`, so Tailscale SSH needs them named.
|
||||
// Irrelevant if you reach these boxes with ordinary sshd over the tailnet —
|
||||
// that is the `tcp:22` grant above.
|
||||
"ssh": [
|
||||
{
|
||||
"action": "check",
|
||||
"src": ["autogroup:member"],
|
||||
"dst": ["autogroup:self", "tag:bookmark-api", "tag:bookmark-browser"],
|
||||
"users": ["autogroup:nonroot", "root"],
|
||||
},
|
||||
],
|
||||
|
||||
// Run on every save, so a later edit that reopens 9222 is rejected outright.
|
||||
"tests": [
|
||||
{ "src": "tag:bookmark-api", "accept": ["tag:bookmark-browser:9222"] },
|
||||
{
|
||||
"src": "you@example.com",
|
||||
"accept": ["tag:bookmark-browser:22"],
|
||||
"deny": ["tag:bookmark-browser:9222"],
|
||||
},
|
||||
],
|
||||
}
|
||||
```
|
||||
|
||||
Then apply the tags — on the VPS and the home machine respectively:
|
||||
|
||||
```bash
|
||||
sudo tailscale up --advertise-tags=tag:bookmark-api
|
||||
sudo tailscale up --advertise-tags=tag:bookmark-browser
|
||||
```
|
||||
|
||||
Each re-authenticates in a browser and issues a new node key; the tailnet IP is
|
||||
unchanged, so `BROWSER_WS_URL` and `BROWSER_BIND_ADDR` still hold. Key expiry is
|
||||
disabled once a device is tagged, which is what you want for a server — an
|
||||
expired key would otherwise take the poller down every few months.
|
||||
|
||||
**Tagging replaces the device's user identity**, so do this only to machines
|
||||
that exist to run these services. If your "home machine" is also your daily
|
||||
driver, tag it anyway and reach it through the `:22` rule above, or skip the
|
||||
tag and accept that any device of yours can reach CDP.
|
||||
|
||||
Enforcement is by the destination's packet filter, so the check below is real,
|
||||
not advisory.
|
||||
|
||||
Prove the bind is tight, from the home machine itself:
|
||||
|
||||
```bash
|
||||
curl -s -m 3 http://$(tailscale ip -4):9222/json/version # -> JSON
|
||||
curl -s -m 3 http://<this machine's LAN IP>:9222/json/version
|
||||
# -> curl: (7) Failed to connect ... Connection refused
|
||||
```
|
||||
|
||||
The first call is also what wakes Chrome: it is not running until something
|
||||
connects, and it is reaped again after five idle minutes. A cold first response
|
||||
takes a few seconds; that is the browser starting, not a fault.
|
||||
|
||||
That check proves the *bind*, not the ACL — traffic that starts on the node is
|
||||
not filtered. Prove the ACL from somewhere else: on your laptop or phone the
|
||||
same URL must now time out, and from the VPS it must answer.
|
||||
|
||||
```bash
|
||||
# on any other device of yours -> hangs until timeout
|
||||
curl -s -m 5 http://<home machine tailnet IP>:9222/json/version
|
||||
# on the VPS -> JSON
|
||||
curl -s -m 20 http://<home machine tailnet IP>:9222/json/version
|
||||
```
|
||||
|
||||
**On the VPS:**
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark
|
||||
echo 'BROWSER_WS_URL=ws://100.x.y.z:9222' >> .env # the home machine's tailnet IP
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
It must be the tailnet **IP**. A MagicDNS hostname fails: Chrome's DevTools HTTP
|
||||
handler answers `/json/version` with a 500 for any `Host` header that is not an
|
||||
IP or `localhost`, and the failure looks like a broken site rather than a broken
|
||||
hostname.
|
||||
|
||||
**Prove it end to end.** This is the only check that says the challenge actually
|
||||
clears from that machine's egress — it fetches a real kagane cover and a real
|
||||
chapter list:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
SMOKE_BROWSER_WS_URL=ws://100.x.y.z:9222 go test -run TestSmokeKagane ./internal/latest
|
||||
```
|
||||
|
||||
A red run means "not clearing from this address right now", which is a live
|
||||
fact to re-check before it is a defect — a Site's Cloudflare settings, and the
|
||||
fingerprint this Chrome presents after an update, both move. Then, from
|
||||
the web UI, open a bookmarked kagane series and confirm the cover renders. Once
|
||||
a cover is stored it is served from Postgres forever after, so the browser being
|
||||
asleep, unreachable, or mid-power-outage costs chapter freshness and nothing
|
||||
visible.
|
||||
|
||||
Finally, take the VPS `free -m` reading again and compare it against the one
|
||||
from the top of this section.
|
||||
|
||||
**Updating the browser** is independent of the API stack and has its own
|
||||
runbook — `REDEPLOY.md` §8.
|
||||
|
||||
---
|
||||
|
||||
## Updating
|
||||
|
||||
Pull new code, then rebuild:
|
||||
@@ -486,14 +208,7 @@ Pull new code, then rebuild:
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build
|
||||
```
|
||||
|
||||
Data persists in the named volume `postgres-data` across rebuilds. (If this
|
||||
server predates the Postgres migration, the old SQLite volume `bookmarks-data`
|
||||
is still on disk and deliberately undeclared in compose so `down -v` cannot take
|
||||
it; see `REDEPLOY.md` §1 for when to remove it.)
|
||||
|
||||
The browser is a separate unit on a separate machine with its own update
|
||||
command — §7. Nothing above touches it, and it needs no coordination: the API
|
||||
picks up a restarted Chrome's new debugger UUID by itself.
|
||||
SQLite data persists in the named volume `bookmarks-data` across rebuilds.
|
||||
|
||||
---
|
||||
|
||||
@@ -502,20 +217,11 @@ picks up a restarted Chrome's new debugger UUID by itself.
|
||||
| Symptom | Likely cause / fix |
|
||||
|---------|--------------------|
|
||||
| No cert / TLS error at the domain | `TRAEFIK_ENTRYPOINT` or `TRAEFIK_CERTRESOLVER` name wrong; or DNS not resolving yet. Check `docker logs <traefik>`. |
|
||||
| 404 from Traefik | Service not on the `proxy` network, or `BOOKMARK_API_HOST` mismatch. Confirm `docker network inspect proxy` lists `bookmark-api`. |
|
||||
| 404 from Traefik | Service not on the `proxy` network, or `MANGA_API_HOST` mismatch. Confirm `docker network inspect proxy` lists `manga-api`. |
|
||||
| `fetch` fails in the userscript, `curl` works | Origin missing from `ALLOWED_ORIGINS`, or mixed content (backend not HTTPS). |
|
||||
| 401 with the right credential | The script's credential no longer matches the stored hash — most likely a rotation happened and the device was not reinstalled. Reinstall from the web UI. |
|
||||
| 401 after rotation, even right after reinstalling | `TOKEN_KEY` changed between the rotation and the reinstall; credentials are derived from it, so changing it invalidates every credential. Keep it stable. |
|
||||
| 401 with the right token | Trailing space/newline in `API_TOKEN`; regenerate and restart. |
|
||||
| Panel button absent | URL didn't match an adapter, or user scripts disabled in Bromite. |
|
||||
| `compose ... config` errors about `TOKEN_KEY`, `OWNER_DISCORD_ID` or `POSTGRES_PASSWORD` | Run compose from the dir with `.env`, or export the vars. All three are required and none has a fallback. |
|
||||
| `bookmark-api` restarts in a loop, `password authentication failed for user "bookmarks"` | `POSTGRES_PASSWORD` was changed after first boot; Postgres only applies it to an empty `postgres-data`. Restore the old value, or reset the role (`REDEPLOY.md` troubleshooting). |
|
||||
| `bookmark-api` never logs `listening on :8080` | It is blocked on `postgres` passing `pg_isready`, or a migration failed. `docker compose -f docker-compose.yml -f docker-compose.prod.yml logs postgres`. |
|
||||
| kagane rows never get a `latest_chapter`; log says `browser fetcher disabled` or nothing at all | `BROWSER_WS_URL` unset. Expected before §7 is done. |
|
||||
| kagane polls all fail; log shows a 500 from `/json/version` | `BROWSER_WS_URL` names a MagicDNS hostname (or any name). Chrome's DevTools handler only accepts an IP or `localhost` — use the tailnet IP. |
|
||||
| kagane polls fail with a connection error | Home machine off, off the tailnet, or the unit is down. `tailscale ping <machine>`, then `docker compose ps` in its `chrome/`. Costs freshness only; stored covers keep serving. |
|
||||
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series. |
|
||||
| `compose` in `chrome/` errors `set BROWSER_BIND_ADDR to this machine's tailnet IP` | No `chrome/.env`, or the variable is empty. Deliberate — it has no default so an unset value cannot publish CDP to the LAN. |
|
||||
| browser container restarts, or is OOM-killed | `docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'`. The 512 MiB cap is sized against a measured 645 MiB untuned peak; a real breach is a Chrome regression worth reading `docker logs` for, not a number to raise reflexively. |
|
||||
| `compose ... config` errors about `API_TOKEN` | Run compose from the dir with `.env`, or export the vars. |
|
||||
|
||||
Backend config reference and endpoint list: see `README.md`.
|
||||
|
||||
@@ -524,25 +230,20 @@ Backend config reference and endpoint list: see `README.md`.
|
||||
## Installing / updating the userscript
|
||||
|
||||
The backend serves the script itself, so Violentmonkey can auto-update it.
|
||||
Complements §4 above — the `@downloadURL`/`@updateURL` lines point at the
|
||||
credential-bearing path, so auto-updates come from the same place as the
|
||||
install.
|
||||
Complements §4 above — that step points `API_BASE`/`API_TOKEN` at your
|
||||
backend; this one points `@downloadURL`/`@updateURL` at the same place so
|
||||
auto-updates come from it too.
|
||||
|
||||
Install once, on the phone (Cromite + Violentmonkey): sign in to the web UI,
|
||||
open the **Userscripts** panel, and open the install link for the library —
|
||||
the script is served with your credential already inside it. Its
|
||||
`@downloadURL`/`@updateURL` point at the same credential-bearing path for
|
||||
updates:
|
||||
Install once, on the phone (Cromite + Violentmonkey):
|
||||
|
||||
```
|
||||
https://bookmark-api.<your-domain>/u/<your credential>/manga-bookmark.user.js
|
||||
https://manga-api.<your-domain>/u/<API_TOKEN>/manga-bookmark.user.js
|
||||
```
|
||||
|
||||
Violentmonkey offers to install it. The credential is in the path because
|
||||
Violentmonkey's update poll sends no `Authorization` header, and the script
|
||||
embeds the credential in plain text — an open URL would leak it. A wrong
|
||||
credential answers 404. The credential is derived from `TOKEN_KEY` and never
|
||||
appears anywhere but this URL and the rendered script.
|
||||
Open that URL in Cromite; Violentmonkey offers to install it. The token is in
|
||||
the path because Violentmonkey's update poll sends no `Authorization` header,
|
||||
and the script embeds `API_TOKEN` in plain text — an open URL would leak it. A
|
||||
wrong token answers 404.
|
||||
|
||||
Updating, without a redeploy:
|
||||
|
||||
|
||||
+15
-21
@@ -8,53 +8,47 @@ web
|
||||
|
||||
## Users
|
||||
|
||||
Members of one private Discord guild, each with their own library. Accounts exist and are created by signing in — there is no signup form, no invite code and no approval step: any member of the configured guild becomes a Reader on their first Discord login. The person running the deployment is the owner, seeded at startup, and the only Reader with an administrative capability (revoking another Reader's sessions).
|
||||
|
||||
Reading happens on **asurascans.com**, **demonicscans.org**, **comix.to** and **kagane.to** for manga and **novelfull.com** and **lightnovelworld.net** for novels, primarily via Bromite on mobile, with checks and corrections from a desktop browser. The web UI is the cross-device view into progress the userscripts capture while reading.
|
||||
Single user (self-hosted, no accounts, no multi-user planned). Reads manga on **asurascans.com** and **demonicscans.org** primarily via Bromite on mobile, also checks/updates from a desktop browser. The web UI is the cross-device view into progress captured by the userscript while reading.
|
||||
|
||||
## Product Purpose
|
||||
|
||||
Tracks read-progress ("last chapter read") per series across sites that each have their own separate `localStorage`. A Go backend unifies bookmarks into one store; the web UI is a Discord-gated browser view of one Reader's own bookmarks, for reviewing, favouriting, correcting, shelving or removing them, and jumping back into a series to continue reading. A background poller refreshes each series' latest-published-chapter so the list can flag "NEW" without the Reader visiting the site.
|
||||
Tracks read-progress ("last chapter read") per manga series across two otherwise-unrelated manga sites that each have their own separate `localStorage`. A Go backend unifies bookmarks into one store; the web UI is a password-gated browser view of that store for reviewing, favouriting, correcting, or removing bookmarks, and jumping back into a series to continue reading. A background poller also refreshes each series' latest-published-chapter so the list can flag "NEW" without the user visiting the site.
|
||||
|
||||
## Positioning
|
||||
|
||||
Not a public reading tracker or social app — a private, self-hosted sync layer for one Discord community, purpose-built for a fixed set of scraped sites. Multi-Reader, not multi-tenant: libraries are isolated, but the deployment belongs to one group and its membership is the whole access model.
|
||||
Not a public reading tracker or social app — a private, self-hosted sync layer purpose-built for two specific scraped sites, with no server-side account system (single bearer token + one password-gated session).
|
||||
|
||||
## Operating Context
|
||||
|
||||
- Primary reading device: Bromite (mobile Chromium), where a userscript captures progress automatically. Each Reader installs their own copy, rendered with their own credential.
|
||||
- Primary reading device: Bromite (mobile Chromium), where a userscript captures progress automatically.
|
||||
- Web UI is a secondary surface: checking list state, correcting a wrong chapter number, removing dead bookmarks, jumping to "continue reading."
|
||||
- Cover art and titles come from the source sites' `og:image`/`og:title` — real content, not placeholders. They are facts about the series, so they are shared between Readers who track it; progress is not.
|
||||
- List order is driven by `updated_at`, which moves only on real reading progress (not favouriting, not a newly detected chapter) — a UI constraint the design must not break.
|
||||
- Manga cover art and titles come from the source sites' `og:image`/`og:title` — real content, not placeholders.
|
||||
- List order is driven by `updated_at`, which moves only on real reading progress (not favouriting, not a newly detected chapter) — a UI constraint the redesign must not break.
|
||||
|
||||
## Capabilities and Constraints
|
||||
|
||||
- Two libraries (manga, novels) with lifecycle tabs: All / Updated / Favourites / Archived / Finished. Search-filter by title (client-side, `filter.js`).
|
||||
- Card actions: continue (opens source site), toggle favourite, manual chapter override, archive, finish, remove — each move out of the list confirm-gated.
|
||||
- "Continue reading" horizontal strip for series with an unread chapter.
|
||||
- A Reader with no bookmarks at all sees a deliberate empty library offering both userscript install links, not an error and not a blank page.
|
||||
- Isolation is the load-bearing invariant: two Readers cannot see or change each other's bookmarks. A series both track is one shared row polled once, with independent progress on each side.
|
||||
- The owner can revoke a specific Reader's sessions; nothing else in the UI differs by Reader.
|
||||
- htmx-driven partial updates, no client-side framework or build step — templates are Go `html/template`, `go:embed`-ed.
|
||||
- Two tabs: All / Favourites. Search-filter by title (client-side, `filter.js`).
|
||||
- Card actions: continue (opens source site), toggle favourite, manual chapter override, delete (with confirm).
|
||||
- "Continue reading" horizontal strip for recently-progressed series.
|
||||
- htmx-driven partial updates (card re-render on favourite/chapter/delete), no client-side framework/build step — templates are Go `html/template`, `go:embed`-ed.
|
||||
- Mobile-first is a hard functional constraint (primary device is a phone), not just a starting breakpoint.
|
||||
|
||||
## Brand Commitments
|
||||
|
||||
- Name: **BookmarkManager**.
|
||||
- **Dark-first is binding**: dark-by-default / light-follows-system-preference must be preserved as a design constraint, not just a starting default, because reading happens at night.
|
||||
- Name: **mangaBookmark**.
|
||||
- **Dark-first is binding**: current dark-by-default / light-follows-system-preference behavior must be preserved as a design constraint, not just a starting default, because reading happens at night.
|
||||
|
||||
## Evidence on Hand
|
||||
|
||||
- Live templates/CSS at `backend/internal/web/templates/*.html`, `backend/internal/web/static/style.css`, governed by the Cinder design system (`docs/design-system.md`).
|
||||
- No logo beyond the wordmark, no screenshots, no marketing copy; none should be fabricated.
|
||||
- Live templates/CSS at `backend/templates/*.html`, `backend/static/style.css` — current implemented UI, functional but not yet treated as an intentional design system.
|
||||
- No logo, screenshots, or marketing copy exist; none should be fabricated.
|
||||
|
||||
## Product Principles
|
||||
|
||||
- Dark-first, night-reading-optimized — never regress to a light-default or high-glare surface.
|
||||
- Mobile is the primary target; desktop is an enhancement, not the design center.
|
||||
- Progress data integrity over visual flourish: `updated_at`/list-ordering behavior is a correctness constraint the UI must respect, not decorate over.
|
||||
- A leak between Readers fails silently and looks like working software — isolation is asserted from both directions, never inferred from counting one Reader's rows.
|
||||
- No roles, no org chrome: the owner's Readers panel is one list with one button (revoke someone's sessions), not an admin console, and otherwise every Reader's view is the same.
|
||||
- No accounts, no multi-tenant chrome — the whole product is for one reader.
|
||||
- Prefer native platform affordances (system dark/light, native touch targets) over custom widgetry — this is a lean self-hosted tool, not a product to demo.
|
||||
|
||||
## Accessibility & Inclusion
|
||||
|
||||
@@ -1,35 +1,21 @@
|
||||
# Manga Bookmark
|
||||
|
||||
Track manga read-progress on **asurascans.com**,
|
||||
**demonicscans.org**, **comix.to**, and **kagane.to** from a phone (Bromite /
|
||||
mobile Chromium), synced to a self-hosted Go backend so bookmarks unify across
|
||||
all four sites and all devices.
|
||||
Track manga read-progress on **asurascans.com** (a.k.a. asuracomic.net) and
|
||||
**demonicscans.org** from a phone (Bromite / mobile Chromium), synced to a
|
||||
self-hosted Go backend so bookmarks unify across both sites and all devices.
|
||||
|
||||
Two parts:
|
||||
|
||||
- **`backend/`** — tiny Go (`net/http` + Postgres via pure-Go `pgx`) sync service. 4 routes,
|
||||
- **`backend/`** — tiny Go (`net/http` + pure-Go SQLite) sync service. 4 routes,
|
||||
static binary, distroless container.
|
||||
- **`userscript/manga-bookmark.user.js`** — single Bromite-compatible userscript
|
||||
(no `GM_*` APIs) that injects an on-page bookmark UI and syncs via `fetch()`.
|
||||
|
||||
```
|
||||
Bromite userscript (isolated world, Shadow DOM UI, localStorage cache)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> Postgres (volume)
|
||||
|
|
||||
| CDP over the tailnet
|
||||
v
|
||||
headless Chrome, on-demand,
|
||||
on a separate machine
|
||||
(chrome/, ADR-0006)
|
||||
-- fetch() HTTPS --> reverse proxy (TLS + CORS) --> Go net/http --> SQLite (volume)
|
||||
```
|
||||
|
||||
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
|
||||
fingerprint clears, so the poller reaches those two through a real Chrome over
|
||||
CDP. That browser is **not** part of the API stack: it is its own compose unit
|
||||
on a second machine, spawned on the first connection and reaped when idle. The
|
||||
API needs it only to discover new chapters and to fetch a kagane cover once —
|
||||
covers are stored, so the library renders in full with the browser switched off.
|
||||
|
||||
---
|
||||
|
||||
## 1. Backend
|
||||
@@ -38,46 +24,23 @@ covers are stored, so the library renders in full with the browser switched off.
|
||||
|
||||
| Var | Default | Notes |
|
||||
|-----|---------|-------|
|
||||
| `TOKEN_KEY` | *(required)* | Secret every Reader's userscript credential is derived from (issue #24); only SHA-256 hashes of credentials are stored. |
|
||||
| `OWNER_DISCORD_ID` | *(required)* | Discord user ID of the owner: seeded as the first Reader, owns every pre-registration bookmark, and is the only Reader who can revoke another's sessions. |
|
||||
| `ALLOWED_ORIGINS` | Asura + Demonic + Comix + Kagane origins | Comma-separated CORS allowlist. |
|
||||
| `DATABASE_URL` | *(required)* | Postgres connection URL, e.g. `postgres://bookmarks:…@postgres:5432/bookmarks?sslmode=disable`. Compose builds it from `POSTGRES_PASSWORD`. |
|
||||
| `COVER_DIR` | *(required)* | Filesystem volume for immutable, content-addressed Cover bytes. Compose builds the image and mounts `cover-data` at this path; standalone runs may choose another writable durable path. |
|
||||
| `API_TOKEN` | *(required)* | Bearer token shared with the userscript. |
|
||||
| `ALLOWED_ORIGINS` | Asura + Demonic origins | Comma-separated CORS allowlist. |
|
||||
| `DB_PATH` | `/data/bookmarks.db` | SQLite file location. |
|
||||
| `PORT` | `8080` | Plain HTTP; TLS terminated by the proxy. |
|
||||
| `BROWSER_WS_URL` | empty | CDP endpoint of the remote browser (`ws://<tailnet IP>:9222`), used to poll Kagane/Novelfull past their JS challenge and to fetch uncached Kagane covers. Must be an IP or `localhost` — Chrome's DevTools handler 500s any other Host header, MagicDNS names included. Unset disables both; stored covers still serve. |
|
||||
| `DISCORD_CLIENT_ID` | *(required)* | Discord application credentials for the browser sign-in (ADR-0002). |
|
||||
| `DISCORD_CLIENT_SECRET` | *(required)* | As above. Never logged, never echoed in an error. |
|
||||
| `DISCORD_GUILD_ID` | *(required)* | The one guild whose membership gates sign-in, checked at login only. Membership *is* registration: any member becomes a Reader on first login. |
|
||||
| `DISCORD_REDIRECT_URI` | *(required)* | Exact callback URL; Discord matches it verbatim against the registered redirect. |
|
||||
| `DISCORD_REQUIRED_ROLE` | empty | Role snowflake a member must additionally hold. Empty means guild membership alone suffices. |
|
||||
| `DISCORD_API_BASE` | `https://discord.com/api/v10` | Test seam — tests point it at a local stub so the real token exchange runs. |
|
||||
| `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. |
|
||||
| `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. |
|
||||
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. Pace is per Site in the registry — one Poll Lane per Site, each with its own rest and gap (issue #100) — so no other knobs exist. |
|
||||
|
||||
Compose reads a few more from the same `.env` that the backend never sees:
|
||||
`POSTGRES_PASSWORD` (required — `DATABASE_URL` is built from it, and Postgres
|
||||
only applies it while `postgres-data` is empty), `BOOKMARK_API_HOST` and
|
||||
`BOOKMARK_WEB_HOST` (required by the prod override), and the optional
|
||||
`PROXY_NETWORK` / `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER`. The browser
|
||||
unit has its own `chrome/.env` on its own machine — `BROWSER_BIND_ADDR`
|
||||
(required, the tailnet IP the CDP port is published on) and the optional
|
||||
`BROWSER_TZ`. Full commentary is in `.env.example` and `chrome/.env.example`;
|
||||
deployment order is `DEPLOY.md`.
|
||||
|
||||
### Endpoints
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/bookmarks` | Bearer | All bookmarks of the acting Reader. |
|
||||
| `GET` | `/bookmarks` | Bearer | All bookmarks (single-user). |
|
||||
| `PUT` | `/bookmarks/{key}` | Bearer | Upsert one series; returns the row as stored. |
|
||||
| `DELETE` | `/bookmarks/{key}` | Bearer | Remove one. |
|
||||
| `GET` | `/healthz` | none | `200 ok`. |
|
||||
| `GET` | `/u/{token}/manga-bookmark.user.js` | credential in path | Serves the userscript with the requesting Reader's credential substituted in and an mtime-derived `@version`. |
|
||||
| `GET` | `/u/{token}/manga-bookmark.user.js` | token in path | Serves the userscript with an mtime-derived `@version`. |
|
||||
|
||||
`key` is `<site>:<series_id>` — e.g. `asura:trash-of-the-counts-family-f886a8af`,
|
||||
`demonic:Infinite-Level-Up-in-Murim`, `comix:12345`, or
|
||||
`kagane:3fa85f64-5717-4562-b3fc-2c963f66afa6`. Sync is last-write-wins.
|
||||
`key` is `<site>:<series_id>` — e.g. `asura:trash-of-the-counts-family-f886a8af`
|
||||
or `demonic:Infinite-Level-Up-in-Murim`. Sync is last-write-wins.
|
||||
|
||||
`updated_at` orders the bookmark list, so it moves only on real reading
|
||||
progress: the server applies its timestamp when the row is new or
|
||||
@@ -94,45 +57,19 @@ go test ./... # unit + handler tests
|
||||
CGO_ENABLED=0 go build # static binary
|
||||
```
|
||||
|
||||
**`go test ./...` requires Docker.** The store talks to a real Postgres, so
|
||||
each test package starts a throwaway `postgres:17-alpine` container and gives
|
||||
every test its own database inside it (`internal/pgtest`). Nothing is stubbed
|
||||
and nothing reaches the network beyond the local Docker daemon.
|
||||
|
||||
### Run the stack
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# edit .env: set TOKEN_KEY (openssl rand -hex 32) and
|
||||
# POSTGRES_PASSWORD (openssl rand -hex 24)
|
||||
# edit .env: set API_TOKEN (openssl rand -hex 32)
|
||||
|
||||
docker compose up -d --build # binds 127.0.0.1:8080
|
||||
```
|
||||
|
||||
That brings up two services — the API and Postgres. The browser is deliberately
|
||||
not one of them; without `BROWSER_WS_URL` the poller logs and skips kagane and
|
||||
novelfull, and everything else works. To run one locally, publish it on the
|
||||
Docker bridge gateway so the API container can name it by IP:
|
||||
|
||||
```bash
|
||||
cd chrome
|
||||
echo 'BROWSER_BIND_ADDR=172.17.0.1' > .env
|
||||
docker compose up -d --build
|
||||
# then in the repo's own .env: BROWSER_WS_URL=ws://172.17.0.1:9222
|
||||
```
|
||||
|
||||
Bind it to `127.0.0.1` instead if you only want to drive it from the host, e.g.
|
||||
`SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeKagane ./internal/latest`.
|
||||
In production that address is the home machine's tailnet IP and nothing else —
|
||||
see `DEPLOY.md` §7 and ADR-0006.
|
||||
|
||||
Smoke test:
|
||||
|
||||
```bash
|
||||
# The credential is per Reader and derived, so there is no token in .env to
|
||||
# grep. Take yours from the Userscripts panel's install link after signing in,
|
||||
# or read it out of an installed script's API_TOKEN constant.
|
||||
TOKEN=<your Reader credential>
|
||||
TOKEN=$(grep '^API_TOKEN=' .env | cut -d= -f2)
|
||||
curl -s localhost:8080/healthz # ok
|
||||
curl -s localhost:8080/bookmarks # 401
|
||||
curl -s -H "Authorization: Bearer $TOKEN" localhost:8080/bookmarks # []
|
||||
@@ -146,7 +83,7 @@ curl -s -i -X OPTIONS -H 'Origin: https://asurascans.com' \
|
||||
|
||||
### Deploy behind your reverse proxy
|
||||
|
||||
Route `https://bookmark-api.<domain>` → the service on `:8080` (TLS at the proxy).
|
||||
Route `https://manga-api.<domain>` → the service on `:8080` (TLS at the proxy).
|
||||
|
||||
- **Host proxy** (nginx/Caddy on the host): the base compose already binds
|
||||
`127.0.0.1:8080`; point the proxy `proxy_pass http://127.0.0.1:8080;`.
|
||||
@@ -159,7 +96,7 @@ Route `https://bookmark-api.<domain>` → the service on `:8080` (TLS at the pro
|
||||
```
|
||||
Set `PROXY_NETWORK` in `.env` if your network isn't named `proxy`.
|
||||
|
||||
Verify: `https://bookmark-api.<domain>/healthz` returns `ok` over valid TLS (no
|
||||
Verify: `https://manga-api.<domain>/healthz` returns `ok` over valid TLS (no
|
||||
mixed-content), and an `OPTIONS` preflight from a real site origin returns the
|
||||
CORS headers.
|
||||
|
||||
@@ -167,18 +104,17 @@ CORS headers.
|
||||
|
||||
## 2. Userscript
|
||||
|
||||
### Install
|
||||
### Configure
|
||||
|
||||
Sign in to the web UI and open the **Userscripts** panel: it offers one
|
||||
install link per library. Each link serves a script rendered with your own
|
||||
credential already inside it — you never see, type or copy a credential. The
|
||||
served script carries `@downloadURL`/`@updateURL` pointing at its
|
||||
credential-bearing path, so Violentmonkey keeps auto-updating it.
|
||||
Edit the config block at the top of `userscript/manga-bookmark.user.js`:
|
||||
|
||||
The bindmounted files carry `__API_TOKEN__` placeholders; the backend
|
||||
substitutes the requesting Reader's credential at serve time, so no real
|
||||
credential is ever committed. Rotating the credential (same panel) invalidates
|
||||
every installed copy immediately — reinstall on all devices.
|
||||
```js
|
||||
const API_BASE = "https://manga-api.<domain>"; // no trailing slash
|
||||
const API_TOKEN = "<same token as backend>";
|
||||
```
|
||||
|
||||
The token lives in the userscript's **isolated world** — the manga sites' own
|
||||
JS cannot read it.
|
||||
|
||||
### Install on Bromite (mobile)
|
||||
|
||||
@@ -186,8 +122,8 @@ Bromite runs Chromium's native userscript engine (no Tampermonkey needed):
|
||||
|
||||
1. Bromite → **Settings → User scripts** → enable user scripts (allow the
|
||||
permission prompt).
|
||||
2. Open the install link from the web UI — Bromite detects the `.user.js` and
|
||||
offers to install it.
|
||||
2. Save the configured `manga-bookmark.user.js` to the device (or open its raw
|
||||
URL). Bromite detects the `.user.js` and offers to install it.
|
||||
3. Confirm the install; the `@match` list covers both sites.
|
||||
4. Open a series on either site — a 📑 button appears bottom-right.
|
||||
|
||||
@@ -245,7 +181,7 @@ userscript does the looking, from your own browser session:
|
||||
(`LATEST_CHECK_BATCH` / `LATEST_CHECK_THROTTLE_MS`). Failures are silent and
|
||||
simply retried after the window.
|
||||
|
||||
Freshness is tracked per device in `localStorage` under `bmgr:manga:lastchecked`
|
||||
Freshness is tracked per device in `localStorage` under `mangabm:lastchecked`
|
||||
and is deliberately not synced, since each device checks on its own.
|
||||
|
||||
This means a bookmark is as current as its last check — not the moment a
|
||||
@@ -256,25 +192,20 @@ an API.
|
||||
|
||||
## Adapter reference (verified live 2026-07-24)
|
||||
|
||||
The site adapters key everything off URL regex, with `title` from `og:title`
|
||||
(or the page's own heading where a site ships none). No adapter reads a cover:
|
||||
the backend acquires, stores and serves every Cover from its own origin
|
||||
(ADR-0007). Confirmed against live pages via Playwright:
|
||||
The site adapters key everything off URL regex, with `title`/`cover` from
|
||||
`og:title` / `og:image`. Confirmed against live pages via Playwright:
|
||||
|
||||
| Site | Series URL | Chapter URL | `series_id` |
|
||||
|------|-----------|-------------|-------------|
|
||||
| **Asura** (`asurascans.com`) | `/comics/<slug-hash>` | `/comics/<slug-hash>/chapter/<n>` | `<slug-hash>` |
|
||||
| **Demonic** (`demonicscans.org`) | `/manga/<slug>` | `/title/<slug>/chapter/<n>/<page>` (`chaptered.php?manga=<id>&chapter=<n>` 301s here) | `<slug>` |
|
||||
| **Comix** (`comix.to`) | `/title/<id>-<slug>` | `/title/<id>-<slug>/<uploadId>-chapter-<n>` | `<id>` |
|
||||
| **Kagane** (`kagane.to`) | `/series/<uuid>` | `/series/<uuid>/reader/<bookUuid>` | `<uuid>` |
|
||||
|
||||
Notes:
|
||||
- **`asuracomic.net` is no longer matched (deep links dead, re-checked
|
||||
2026-07-25).** They 301 to the `asurascans.com` **root**, discarding the path,
|
||||
at the edge — before the userscript gets a document — so nothing client-side
|
||||
can rescue them. The backend rejects stored addresses on that host too, since
|
||||
the poller pins each Site to one hostname. Reach series through
|
||||
`asurascans.com`.
|
||||
- **`asuracomic.net` deep links are dead (re-checked 2026-07-25).** They 301 to
|
||||
the `asurascans.com` **root**, discarding the path, at the edge — before the
|
||||
userscript gets a document — so nothing client-side can rescue them. Reach
|
||||
series through `asurascans.com`. The host stays matched in case the redirect
|
||||
starts preserving paths again.
|
||||
- Asura `og:title` carries a `Chapter N - Read Online \| Asura Scans` suffix that
|
||||
the adapter strips; Demonic chapter `og:title` is `<Title> Chapter N`.
|
||||
- Demonic's `<slug>` is identical on `/manga/…` and the canonical `/title/…`
|
||||
|
||||
+102
-228
@@ -9,16 +9,16 @@ Whole thing is ~5 minutes, most of it waiting on `docker build`. Order matters:
|
||||
**back up before you pull.** A backup taken after a bad migration is a backup of
|
||||
the damage.
|
||||
|
||||
Paths below assume the checkout is at `~/mangaBookmark`, which is where it lives
|
||||
on this deployment; substitute your own. The one absolute rule about paths:
|
||||
**backups live in a `-backups` sibling of the checkout**, never inside it. It
|
||||
Paths below assume the checkout is at `/opt/mangabm`; substitute your own. The
|
||||
one absolute rule about paths: **backups live in `../mangabm-backups/`**, a
|
||||
sibling of the project directory (`/opt/mangabm-backups`), never inside it. It
|
||||
sits outside the repo so `git pull`, `git clean -fd` and a bad `rm -rf` inside
|
||||
the checkout cannot take the backups with them.
|
||||
|
||||
```
|
||||
~/
|
||||
├── mangaBookmark/ <- the checkout (this repo)
|
||||
└── mangaBookmark-backups/ <- bookmarks-YYYYmmdd-HHMMSS.dump
|
||||
/opt/
|
||||
├── mangabm/ <- the checkout (this repo)
|
||||
└── mangabm-backups/ <- bookmarks-YYYYmmdd-HHMMSS.db
|
||||
```
|
||||
|
||||
---
|
||||
@@ -26,12 +26,12 @@ the checkout cannot take the backups with them.
|
||||
## 0. Preflight
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark
|
||||
cd /opt/mangabm
|
||||
|
||||
# Both -f flags, every time. The prod override is not standalone.
|
||||
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
|
||||
|
||||
$COMPOSE ps # bookmark-api should be Up
|
||||
$COMPOSE ps # manga-api should be Up
|
||||
git status --short # expect empty
|
||||
git log --oneline -1 # note this hash — it is your rollback target
|
||||
df -h /var/lib/docker | tail -1 # a build needs room
|
||||
@@ -44,117 +44,87 @@ dirty tree fails halfway and leaves you in a worse spot than either.
|
||||
Create the backup directory once, and make sure it is a sibling, not a child:
|
||||
|
||||
```bash
|
||||
BACKUP_DIR="$(cd .. && pwd)/$(basename "$PWD")-backups" # absolute — Docker needs it
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
echo "$BACKUP_DIR" # -> /home/sulthan/mangaBookmark-backups
|
||||
mkdir -p ../mangabm-backups
|
||||
BACKUP_DIR="$(cd .. && pwd)/mangabm-backups" # absolute — Docker needs it
|
||||
echo "$BACKUP_DIR" # -> /opt/mangabm-backups
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Back up the database
|
||||
|
||||
The database is Postgres, running as the `postgres` service on the named volume
|
||||
`postgres-data`. It has **no published port** — nothing outside the internal `db`
|
||||
network can reach it — so every command below goes in through the container:
|
||||
The database is a single SQLite file in the named Docker volume, at
|
||||
`/data/bookmarks.db` inside the container. Find the volume's real name — Compose
|
||||
prefixes it with the project directory:
|
||||
|
||||
```bash
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c '\dt'
|
||||
# -> bookmarks, covers, readers, schema_migrations, series, sessions
|
||||
docker volume ls --filter name=bookmarks-data
|
||||
# -> local mangabm_bookmarks-data
|
||||
VOL=$(docker volume ls --filter name=bookmarks-data -q | head -1)
|
||||
```
|
||||
|
||||
The `covers` table is metadata only after the filesystem cutover: bytes live in
|
||||
the separate `cover-data` volume. Back that volume up with the database dump;
|
||||
restoring only Postgres leaves stored Cover addresses without files.
|
||||
### Preferred: hot backup, no downtime
|
||||
|
||||
Inside the container that connects over the local socket as the `bookmarks`
|
||||
superuser, so no password is needed anywhere in this section. `-T` is not
|
||||
optional: without it Compose allocates a TTY, which rewrites `\n` to `\r\n` and
|
||||
silently corrupts any binary stream flowing back out — see the dump below.
|
||||
|
||||
### Preferred: hot dump, no downtime
|
||||
|
||||
`pg_dump` runs in a single repeatable-read transaction, so it writes one
|
||||
point-in-time-consistent snapshot while the API keeps serving. No stopping, no
|
||||
WAL to worry about — that is the server's problem, not yours.
|
||||
The store runs in **WAL mode**, so recent writes may still be sitting in
|
||||
`bookmarks.db-wal`. Copying `bookmarks.db` alone while the container runs can
|
||||
therefore silently drop the newest bookmarks. `VACUUM INTO` folds the WAL in and
|
||||
writes one consistent file, safe to run against a live database:
|
||||
|
||||
```bash
|
||||
STAMP=$(date -u +%Y%m%d-%H%M%S) # UTC, sorts chronologically as text
|
||||
$COMPOSE exec -T postgres pg_dump -U bookmarks -d bookmarks -Fc \
|
||||
> "$BACKUP_DIR/bookmarks-$STAMP.dump"
|
||||
docker run --rm \
|
||||
-v "$VOL":/data \
|
||||
-v "$BACKUP_DIR":/backup \
|
||||
alpine sh -c "apk add -q sqlite &&
|
||||
sqlite3 /data/bookmarks.db \"VACUUM INTO '/backup/bookmarks-$STAMP.db'\""
|
||||
|
||||
ls -lh "$BACKUP_DIR"/bookmarks-$STAMP.dump
|
||||
ls -lh "$BACKUP_DIR"/bookmarks-$STAMP.db
|
||||
```
|
||||
|
||||
`-Fc` is the custom archive format rather than plain SQL: it is compressed, and
|
||||
`pg_restore` can inspect and replay it selectively — list its table of contents,
|
||||
restore one table, restore schema without data, reorder. A plain `.sql` dump can
|
||||
only be piped into `psql` whole, and gives you no way to check what is in it
|
||||
short of reading it.
|
||||
`$STAMP` is the "time in the name" — `bookmarks-20260730-014233.db`. UTC, so the
|
||||
files sort in real order and never collide across a DST shift.
|
||||
|
||||
`$STAMP` is the "time in the name" — `bookmarks-20260730-014233.dump`. UTC, so
|
||||
the files sort in real order and never collide across a DST shift.
|
||||
Note the source volume is mounted **read-write**, which looks wrong for a backup
|
||||
and is not. Opening a WAL database requires creating the `-shm` shared-memory
|
||||
file; with `:ro` the command fails with `unable to open database file` and no
|
||||
backup is produced. `VACUUM INTO` never writes to the source itself.
|
||||
|
||||
Verify it before you trust it. An unreadable backup is worse than none, because
|
||||
you will act as though you have one:
|
||||
|
||||
```bash
|
||||
# 1. The dump parses and contains the tables. Uses the same image compose
|
||||
# already pulls, so nothing new to install.
|
||||
docker run --rm -v "$BACKUP_DIR":/backup postgres:17-alpine \
|
||||
pg_restore --list "/backup/bookmarks-$STAMP.dump" | grep 'TABLE DATA'
|
||||
# -> 1234; 0 0 TABLE DATA public bookmarks bookmarks
|
||||
# -> 1235; 0 0 TABLE DATA public covers bookmarks
|
||||
# -> 1236; 0 0 TABLE DATA public readers bookmarks
|
||||
# -> 1237; 0 0 TABLE DATA public schema_migrations bookmarks
|
||||
# -> 1238; 0 0 TABLE DATA public series bookmarks
|
||||
# -> 1239; 0 0 TABLE DATA public sessions bookmarks
|
||||
|
||||
# 2. Sanity-check the live row count you just captured.
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks \
|
||||
-c 'select count(*) from bookmarks'
|
||||
docker run --rm -v "$BACKUP_DIR":/backup alpine sh -c "apk add -q sqlite &&
|
||||
sqlite3 /backup/bookmarks-$STAMP.db 'PRAGMA integrity_check;' &&
|
||||
sqlite3 /backup/bookmarks-$STAMP.db 'SELECT count(*) FROM bookmarks;'"
|
||||
# -> ok
|
||||
# -> 37
|
||||
```
|
||||
|
||||
A custom-format archive stores row counts nowhere, so step 1 proves the file is
|
||||
a readable archive with the right tables in it, not that the rows are there;
|
||||
step 2 is the number those rows should be. It should match what the web UI
|
||||
shows. Zero on a server you know has bookmarks means the API and your `psql`
|
||||
are looking at different databases — check `DATABASE_URL`.
|
||||
The count should match what the web UI shows. Zero rows on a server you know has
|
||||
bookmarks means you backed up the wrong volume.
|
||||
|
||||
### Fallback: cold volume archive
|
||||
### Fallback: cold copy (no network for `apk add sqlite`)
|
||||
|
||||
Use this when you want the whole data directory rather than a logical dump — a
|
||||
like-for-like restore of the same Postgres major version onto the same host.
|
||||
|
||||
**The stack must be stopped first.** A running Postgres has dirty pages in
|
||||
shared buffers and WAL that has not been replayed into the data files, and `tar`
|
||||
walks the directory over several seconds while the server keeps writing to it.
|
||||
The archive you get is torn: files from different instants, possibly a
|
||||
half-written page. It may restore, start, and be quietly wrong. Online
|
||||
filesystem-level backup is `pg_basebackup`'s job, not `tar`'s; with the
|
||||
container stopped the shutdown checkpoint has already flushed everything and a
|
||||
plain archive of the volume is consistent.
|
||||
Stop the service first, then copy the database **and its sidecars** — the `-wal`
|
||||
is not optional, it is where the newest writes are:
|
||||
|
||||
```bash
|
||||
# Derived exactly, not with a `--filter name=` substring match plus `head -1`:
|
||||
# that quietly picks the first of however many volumes happen to contain the
|
||||
# string, and archiving the wrong data directory is not a visible failure.
|
||||
VOL="$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_postgres-data"
|
||||
docker volume inspect "$VOL" >/dev/null && echo "$VOL" # -> mangabookmark_postgres-data
|
||||
|
||||
$COMPOSE stop
|
||||
docker run --rm -v "$VOL":/from:ro -v "$BACKUP_DIR":/to alpine \
|
||||
tar czf "/to/postgres-data-$STAMP.tgz" -C /from .
|
||||
docker run --rm -v "$VOL":/data:ro -v "$BACKUP_DIR":/backup alpine sh -c "
|
||||
cp /data/bookmarks.db /backup/bookmarks-$STAMP.db
|
||||
[ -f /data/bookmarks.db-wal ] && cp /data/bookmarks.db-wal /backup/bookmarks-$STAMP.db-wal
|
||||
[ -f /data/bookmarks.db-shm ] && cp /data/bookmarks.db-shm /backup/bookmarks-$STAMP.db-shm
|
||||
ls -1 /backup"
|
||||
$COMPOSE start
|
||||
|
||||
ls -lh "$BACKUP_DIR"/postgres-data-$STAMP.tgz
|
||||
```
|
||||
|
||||
Costs ~15 seconds of downtime. Read-only on the source is safe here precisely
|
||||
because nothing is running against it. Restoring this variant means untarring it
|
||||
back into an *empty* `postgres-data` volume with the stack down — it is a whole
|
||||
data directory, not a file you can drop next to the live one, and it will only
|
||||
start under `postgres:17`.
|
||||
Costs ~10 seconds of downtime. A clean shutdown usually checkpoints the WAL away,
|
||||
so seeing only the `.db` file is normal and fine — the `[ -f ]` guards exist for
|
||||
the case where it did not. Restoring this variant means putting whichever files
|
||||
you got back together, under their original names.
|
||||
|
||||
Read-only is safe here precisely because nothing opens the database: it is a file
|
||||
copy, not a SQLite connection.
|
||||
|
||||
### Retention
|
||||
|
||||
@@ -162,21 +132,7 @@ Keep a month, drop the rest — a bookmark database this small compresses the
|
||||
decision to "disk is free, but not infinite":
|
||||
|
||||
```bash
|
||||
ls -1t "$BACKUP_DIR"/bookmarks-*.dump | tail -n +31 | xargs -r rm -v
|
||||
```
|
||||
|
||||
### A note on the old `bookmarks-data` volume
|
||||
|
||||
`bookmarks-data` is the **pre-migration SQLite volume**. It is deliberately not
|
||||
declared in `docker-compose.yml` any more, which is what keeps `docker compose
|
||||
down -v` from taking it with the rest of the stack. It is not the live database
|
||||
and nothing reads it — the one-way move out of it is `CUTOVER.md`. Once the
|
||||
Postgres data has been trusted for a while, remove it by hand — nothing else will.
|
||||
Its full name is `<compose project>_bookmarks-data`, and the project name is the
|
||||
lowercased directory name of the checkout:
|
||||
|
||||
```bash
|
||||
docker volume rm "$(basename "$PWD" | tr '[:upper:]' '[:lower:]')_bookmarks-data"
|
||||
ls -1t "$BACKUP_DIR"/bookmarks-*.db | tail -n +31 | xargs -r rm -v
|
||||
```
|
||||
|
||||
---
|
||||
@@ -215,16 +171,12 @@ rebuilt. The one exception is `userscript/manga-bookmark.user.js`, which is
|
||||
bindmounted read-only and read fresh per request.
|
||||
|
||||
```bash
|
||||
$COMPOSE ps # bookmark-api Up; postgres Up (healthy)
|
||||
docker logs bookmark-api --tail 20 # -> "listening on :8080 ..."
|
||||
$COMPOSE ps # Up, and recently (re)created
|
||||
docker logs manga-api --tail 20 # -> "listening on :8080 ..."
|
||||
```
|
||||
|
||||
Nothing in the log about the database, the migrations or the poller failing.
|
||||
`bookmark-api` waits on `postgres` reporting healthy before it starts and the
|
||||
binary applies any pending migration before it listens, so an API that never
|
||||
says "listening" is usually the database, not the code — `$COMPOSE logs
|
||||
postgres` first. The image is tagged
|
||||
`bookmarkmanager-backend:latest`, so the previous image is still on disk untagged —
|
||||
Nothing in the log about the database or the poller failing. The image is tagged
|
||||
`mangabm-backend:latest`, so the previous image is still on disk untagged —
|
||||
that is what makes the rollback in §6 quick.
|
||||
|
||||
---
|
||||
@@ -234,12 +186,9 @@ that is what makes the rollback in §6 quick.
|
||||
Same four API checks as `DEPLOY.md` §3, plus the web UI. Set the host names once:
|
||||
|
||||
```bash
|
||||
API=https://bookmark-api.violetcrown.my.id
|
||||
WEB=https://bookmark.violetcrown.my.id
|
||||
# Your own Reader credential - derived, never stored in .env. Take it from the
|
||||
# Userscripts panel's install link after signing in, or from an installed
|
||||
# script's API_TOKEN constant.
|
||||
TOKEN=<your Reader credential>
|
||||
API=https://manga-api.violetcrown.my.id
|
||||
WEB=https://manga.violetcrown.my.id
|
||||
TOKEN=$(grep -E '^API_TOKEN=' .env | cut -d= -f2)
|
||||
|
||||
curl -s $API/healthz # -> ok
|
||||
curl -s -o /dev/null -w '%{http_code}\n' $API/bookmarks # -> 401
|
||||
@@ -250,16 +199,9 @@ curl -s -i -X OPTIONS -H 'Origin: https://asurascans.com' \
|
||||
$API/bookmarks/x | grep -i access-control # -> allow-origin echoed
|
||||
```
|
||||
|
||||
`[]` from the third call is the alarm that matters: you are talking to an empty
|
||||
database, which means the API found a *different* Postgres than the one holding
|
||||
your data — a renamed project directory, a fresh `postgres-data`, or a
|
||||
`DATABASE_URL` override in `.env` pointing elsewhere. Stop and check, before
|
||||
touching anything else:
|
||||
|
||||
```bash
|
||||
$COMPOSE config --volumes # -> postgres-data
|
||||
$COMPOSE exec -T postgres psql -U bookmarks -d bookmarks -c 'select count(*) from bookmarks'
|
||||
```
|
||||
`[]` from the third call is the alarm that matters: the volume is not attached
|
||||
and you are looking at an empty database. Stop and check `$COMPOSE config
|
||||
--volumes` before touching anything else.
|
||||
|
||||
Web UI and its assets:
|
||||
|
||||
@@ -325,41 +267,33 @@ git checkout <previous-hash>
|
||||
$COMPOSE up -d --build
|
||||
```
|
||||
|
||||
**Database damaged** — restore the dump from §1. Stop **only the API**, not the
|
||||
whole stack: `pg_restore` needs the server up to restore into, and it needs
|
||||
`bookmark-api`'s connection pool gone, because `--clean` cannot drop a table
|
||||
other sessions are holding open.
|
||||
**Database damaged** — restore the backup from §1. Stop first: the running
|
||||
process holds the WAL, and dropping a file under a live SQLite connection
|
||||
corrupts what you were trying to save.
|
||||
|
||||
```bash
|
||||
$COMPOSE stop bookmark-api
|
||||
$COMPOSE stop
|
||||
|
||||
$COMPOSE exec -T postgres pg_restore -U bookmarks -d bookmarks --clean --if-exists \
|
||||
< "$BACKUP_DIR/bookmarks-<STAMP>.dump"
|
||||
docker run --rm -v "$VOL":/data -v "$BACKUP_DIR":/backup alpine sh -c '
|
||||
rm -f /data/bookmarks.db /data/bookmarks.db-wal /data/bookmarks.db-shm &&
|
||||
cp /backup/bookmarks-<STAMP>.db /data/bookmarks.db &&
|
||||
chown 65532:65532 /data/bookmarks.db &&
|
||||
ls -l /data'
|
||||
|
||||
$COMPOSE start bookmark-api
|
||||
docker logs bookmark-api --tail 20
|
||||
$COMPOSE start
|
||||
docker logs manga-api --tail 20
|
||||
curl -s -H "Authorization: Bearer $TOKEN" $API/bookmarks | head -c 200
|
||||
```
|
||||
|
||||
Three things here are easy to skip and all three bite:
|
||||
Two steps here are easy to skip and both bite:
|
||||
|
||||
- **`--clean --if-exists`.** Without `--clean` the dump's rows land *on top of*
|
||||
what is already there and you get primary-key collisions half way through, a
|
||||
partially restored database, and a non-zero exit you may not notice.
|
||||
`--if-exists` only suppresses the "does not exist" noise when the target is
|
||||
already empty; it is not the part doing the work.
|
||||
- **`-T` again.** Feeding a custom-format archive into a TTY-allocated `exec`
|
||||
corrupts it in flight and `pg_restore` fails with a garbled-header error on a
|
||||
file that is perfectly fine on disk.
|
||||
- **Stop the API, not Postgres.** `$COMPOSE stop` (everything) leaves you with
|
||||
nothing to restore into; leaving `bookmark-api` running leaves connections
|
||||
that block the drops *and* lets the poller write into a half-restored table.
|
||||
|
||||
No ownership fixing is needed any more — the Postgres image owns `postgres-data`
|
||||
itself and `pg_restore` writes through the server, not the filesystem.
|
||||
`schema_migrations` is inside the dump, so the database comes back at whatever
|
||||
schema version the backup was taken at; the migration runner applies anything
|
||||
newer the next time `bookmark-api` starts.
|
||||
- **Delete the stale `-wal` and `-shm`.** Leaving them beside a restored database
|
||||
mixes two different histories; SQLite will either refuse to open it or quietly
|
||||
reapply writes you meant to discard.
|
||||
- **`chown 65532:65532`.** The image is `distroless/static:nonroot` and runs as
|
||||
that uid, while the helper container above writes as root. A root-owned
|
||||
database opens read-only-ish: reads work, so `/bookmarks` looks fine, and then
|
||||
every write fails. That is the worst possible failure mode — it looks restored.
|
||||
|
||||
---
|
||||
|
||||
@@ -368,76 +302,24 @@ newer the next time `bookmark-api` starts.
|
||||
For a routine redeploy where nothing needs deciding:
|
||||
|
||||
```bash
|
||||
cd /opt/bookmarkmanager
|
||||
cd /opt/mangabm
|
||||
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
|
||||
BACKUP_DIR="$(cd .. && pwd)/bookmarkmanager-backups"; mkdir -p "$BACKUP_DIR"
|
||||
BACKUP_DIR="$(cd .. && pwd)/mangabm-backups"; mkdir -p "$BACKUP_DIR"
|
||||
VOL=$(docker volume ls --filter name=bookmarks-data -q | head -1)
|
||||
STAMP=$(date -u +%Y%m%d-%H%M%S)
|
||||
|
||||
$COMPOSE exec -T postgres pg_dump -U bookmarks -d bookmarks -Fc \
|
||||
> "$BACKUP_DIR/bookmarks-$STAMP.dump" &&
|
||||
docker run --rm -v "$BACKUP_DIR":/backup postgres:17-alpine \
|
||||
pg_restore --list "/backup/bookmarks-$STAMP.dump" > /dev/null &&
|
||||
docker run --rm -v "$VOL":/data -v "$BACKUP_DIR":/backup alpine sh -c \
|
||||
"apk add -q sqlite && sqlite3 /data/bookmarks.db \"VACUUM INTO '/backup/bookmarks-$STAMP.db'\" &&
|
||||
sqlite3 /backup/bookmarks-$STAMP.db 'PRAGMA integrity_check;'" &&
|
||||
git pull --ff-only &&
|
||||
$COMPOSE up -d --build &&
|
||||
sleep 5 &&
|
||||
curl -sf https://bookmark-api.violetcrown.my.id/healthz && echo " deploy ok"
|
||||
curl -sf https://manga-api.violetcrown.my.id/healthz && echo " deploy ok"
|
||||
```
|
||||
|
||||
The `&&` chain is deliberate: if the dump or its `pg_restore --list` check
|
||||
fails, nothing is pulled and nothing is rebuilt. A failed dump still leaves a
|
||||
short or empty `.dump` behind — the shell creates the file before `pg_dump`
|
||||
runs — so delete it rather than letting it sit in the backup directory looking
|
||||
like a backup. Then still do §5 by hand — no shell command can tell you the
|
||||
panel works on the phone.
|
||||
|
||||
---
|
||||
|
||||
## 8. The browser unit (separate machine, separate cadence)
|
||||
|
||||
Everything above is the API stack on the VPS. The headless browser is its own
|
||||
compose unit on the home machine (ADR-0006, `DEPLOY.md` §7) and is redeployed
|
||||
on its own schedule — it holds no data you can lose, so there is nothing to
|
||||
back up and no ordering constraint against the API.
|
||||
|
||||
```bash
|
||||
cd ~/mangaBookmark/chrome
|
||||
git pull --ff-only
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Then confirm it answers, and that a stopped-and-restarted Chrome is invisible
|
||||
to the API:
|
||||
|
||||
```bash
|
||||
curl -s -m 15 http://$(tailscale ip -4):9222/json/version | head -c 120
|
||||
# -> {"Browser":"Chrome/1xx...","webSocketDebuggerUrl":"ws://...<new uuid>"}
|
||||
```
|
||||
|
||||
The first call takes a few seconds: Chrome is not running until something
|
||||
connects, and it is reaped again after five idle minutes. The debugger UUID
|
||||
changes on every start and the API does not care — chromedp re-runs
|
||||
`/json/version` discovery per fetch, which is exactly why `chromedp.NoModifyURL`
|
||||
must never be added to `browser.go`.
|
||||
|
||||
**Rebuild is the Chrome upgrade path.** The image installs
|
||||
`google-chrome-stable` unpinned on purpose: a stale browser is what Cloudflare
|
||||
turns away, and the pinned Chrome 124 in `zenika/alpine-chrome` is the worked
|
||||
example. The `chrome-profile` volume survives `--build`, so clearance cookies
|
||||
are reused rather than re-solved.
|
||||
|
||||
Two things worth a glance after several days, both from the acceptance criteria
|
||||
of the move:
|
||||
|
||||
```bash
|
||||
docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'
|
||||
# -> 0 false
|
||||
free -m # the Gitea runner should still have its headroom
|
||||
```
|
||||
|
||||
Nothing here needs doing during an API redeploy. The API stack does not
|
||||
`depends_on` the browser, and an unreachable one degrades exactly as an unset
|
||||
`BROWSER_WS_URL`: plain-TLS libraries unaffected, kagane and comix logged
|
||||
and skipped, novelfull attempted over plain TLS, stored covers still served.
|
||||
The `&&` chain is deliberate: if the backup or its integrity check fails,
|
||||
nothing is pulled and nothing is rebuilt. Then still do §5 by hand — no shell
|
||||
command can tell you the panel works on the phone.
|
||||
|
||||
---
|
||||
|
||||
@@ -445,24 +327,16 @@ and skipped, novelfull attempted over plain TLS, stored covers still served.
|
||||
|
||||
| Symptom | Cause / fix |
|
||||
|---|---|
|
||||
| `/bookmarks` returns `[]` after redeploy | You are on an empty Postgres. Check `$COMPOSE config --volumes` lists `postgres-data`, that you passed both `-f` files, and that `.env` has no stray `DATABASE_URL` override. Do **not** re-bookmark; the data is still in the volume. |
|
||||
| `/bookmarks` returns `[]` after redeploy | Volume not attached — check `$COMPOSE config --volumes` and that you passed both `-f` files. Do **not** re-bookmark; the data is still in the volume. |
|
||||
| UI looks like plain Georgia / system sans | `static/fonts/` missing from the image, or the browser cached an old `style.css`. `/static/*` is served `max-age=3600`, so hard-reload or wait an hour. |
|
||||
| CSS or template change did not appear | You restarted without `--build`. Assets are `//go:embed`ed. |
|
||||
| Font answers `application/octet-stream` | Old binary — the `.woff2` MIME registration is in `web.go`. Rebuild. |
|
||||
| Everyone logged out of the web UI | The `sessions` table was wiped; sessions are database rows, not signed cookies. Expected after a deliberate revoke. |
|
||||
| `compose` errors about `BOOKMARK_WEB_HOST` | Run from the directory holding `.env`. Both host vars are required even when the web UI is unused. |
|
||||
| Everyone logged out of the web UI | `API_TOKEN` or `WEB_PASSWORD` changed; sessions are derived from both. Expected, just log in again. |
|
||||
| `compose` errors about `MANGA_WEB_HOST` | Run from the directory holding `.env`. Both host vars are required even when the web UI is unused. |
|
||||
| Userscript did not update on the phone | Violentmonkey polls on its own schedule; force a check. `@version` comes from the file's mtime, so confirm the pull actually touched it. |
|
||||
| `bookmark-api` crash-loops, log says `password authentication failed for user "bookmarks"` | `POSTGRES_PASSWORD` in `.env` no longer matches the one burned into `postgres-data` at first init — Postgres reads that variable only when initialising an empty volume. Put the old value back, or reset the role: `$COMPOSE exec postgres psql -U bookmarks -d bookmarks -c '\password bookmarks'` (prompts, so nothing lands in shell history) and then match `.env` to it. |
|
||||
| `compose` errors `set POSTGRES_PASSWORD in .env` | Unset. Compose builds the backend's `DATABASE_URL` out of it, so it is required even though you never write that URL yourself. Run from the directory holding `.env`. |
|
||||
| `postgres` never leaves `starting`; `bookmark-api` never starts either | The healthcheck (`pg_isready`) is failing and `bookmark-api` waits on it. `$COMPOSE logs postgres` — usually `postgres-data` was initialised by a different major version ("database files are incompatible with server"), or the disk is full. |
|
||||
| `pg_restore`: `cannot drop … other objects depend on it` / `being accessed by other users` | Live connections block `--clean`. `$COMPOSE stop bookmark-api` first (§6). If they persist: `$COMPOSE exec -T postgres psql -U bookmarks -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='bookmarks' and pid <> pg_backend_pid()"`. |
|
||||
| Dump is 0 bytes, or `pg_restore`: `did not find magic string in file header` | You ran `exec` without `-T`. The allocated TTY rewrites newlines in the binary stream and corrupts the archive in flight (§1). |
|
||||
| `git pull`: `could not read Username for 'https://…'` | The checkout's remote is the HTTPS clone URL and the server has no credential helper, so the pull prompts into a closed stdin. Switch it to SSH once — `git remote set-url origin ssh://git@gitea.violetcrown.my.id:2222/sulthan/mangaBookmark.git`. Gitea's SSH listens on **2222**, not 22; port 22 is the host's own sshd and answers `Permission denied (publickey)` no matter which key is registered. |
|
||||
| kagane rows stopped updating after a redeploy | Check `BROWSER_WS_URL` survived the `.env` edit and still names the home machine's tailnet **IP**. A hostname 500s at `/json/version`; an empty value disables the browser silently. Plain-TLS sites keep working either way, which is why this is easy to miss. |
|
||||
| kagane covers went blank in the web UI | Covers use the `cover-data` volume now. Restore/check that volume alongside Postgres; rows in `covers` are metadata only. If the database has rows but files are missing, the next browser-backed request refetches them; without a browser it remains a 404. |
|
||||
| Browser unit will not start: `set BROWSER_BIND_ADDR to this machine's tailnet IP` | `chrome/.env` is missing or the variable is empty. It has no default on purpose — an unset value must fail the deploy rather than publish an unauthenticated CDP port to the LAN. |
|
||||
| `bookmark-browser` shows `OOMKilled true` | The cap did its job. Read `docker logs bookmark-browser` before raising it — the sizing and what the cap protects are in ADR-0006. |
|
||||
| `apk add sqlite` fails (no network) | Use the cold-copy fallback in §1 — and copy `bookmarks.db-wal` too. |
|
||||
| Reads work but every write fails after a restore | Restored file is root-owned; the container is uid 65532. `chown 65532:65532` it (§6). |
|
||||
| Backup command: `unable to open database file` | Source volume mounted `:ro`. WAL needs to create `-shm`; mount it read-write (§1). |
|
||||
|
||||
Full first-time setup: `DEPLOY.md`. The one-off SQLite→Postgres move:
|
||||
`CUTOVER.md`. Config reference and endpoints: `README.md`.
|
||||
Full first-time setup: `DEPLOY.md`. Config reference and endpoints: `README.md`.
|
||||
UI conventions: `docs/design-system.md`.
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
# Only go source + module files, plus internal/ (which carries the
|
||||
# go:embed'd templates/static directories), are needed in the build context.
|
||||
# Only go source + module files, plus the go:embed'd templates/static
|
||||
# directories, are needed in the build context.
|
||||
*
|
||||
!go.mod
|
||||
!go.sum
|
||||
!*.go
|
||||
!internal/
|
||||
!internal/**
|
||||
!templates/
|
||||
!templates/**
|
||||
!static/
|
||||
!static/**
|
||||
|
||||
@@ -1,269 +0,0 @@
|
||||
Scope: `backend/`.
|
||||
|
||||
Each entry names the code that holds the truth — read that for *what it does*.
|
||||
The prose here is only what code cannot tell you: rationale, rejected
|
||||
alternatives, dated measurements, and invariants a plausible refactor would
|
||||
silently break.
|
||||
|
||||
### Layout
|
||||
|
||||
`backend/main.go` → `newRouter` is the composition root, the only place
|
||||
packages are wired. Packages under `backend/internal/`: `store`, `latest`,
|
||||
`session`, `httpmw`, `api`, `userscript`, `web`, `token`, `pgtest`. Root-level
|
||||
`*_test.go` exercise the full router; unit tests live beside their package.
|
||||
|
||||
Not visible from any single file: stdlib `net/http` with no framework,
|
||||
Postgres over `jackc/pgx/v5`, `CGO_ENABLED=0` static binary into a distroless
|
||||
image, TLS terminated by the reverse proxy so the service listens plain `:8080`.
|
||||
|
||||
### Schema — `internal/store/migrations/*.sql`, run by `store.migrate`
|
||||
|
||||
- Migration files are **append-only**. Editing an applied one changes nothing
|
||||
on a database that already recorded its version in `schema_migrations`, so
|
||||
the fix silently applies to new deployments only.
|
||||
- No column probing, no data-fixup migrations. Both were SQLite-era machinery
|
||||
and were removed deliberately — don't reintroduce either.
|
||||
|
||||
### Tests need Docker — `internal/pgtest`
|
||||
|
||||
`pgtest.Main` from `TestMain` starts one `postgres:17-alpine` per test binary;
|
||||
`pgtest.URL` hands each test its own database. A package whose tests touch the
|
||||
store must have that `TestMain` or it has no database at all.
|
||||
|
||||
### Reader-owned store — `internal/store`, `internal/token`
|
||||
- The Reader-owned tables are `readers`, `bookmarks`, `series`, and `sessions`; auxiliary `covers`, `poll_lanes`, and `poll_passes` are also defined in the migrations.
|
||||
|
||||
- **Credentials are derived, never stored.** `token.Token(TOKEN_KEY, discord_id, epoch)`
|
||||
is an HMAC; only its SHA-256 reaches `readers.token_sha256`. So install URLs
|
||||
can be rebuilt after any restart, and a database leak yields nothing usable.
|
||||
- **The owner's epoch-0 hash is refreshed at startup only while the row has
|
||||
never been rotated.** Drop that condition and a restart resurrects a
|
||||
rotated-away credential.
|
||||
- `Store.EnsureReader` never rewrites an existing row's hash — a returning
|
||||
Reader's login must not invalidate their installed scripts.
|
||||
- **Every read and write is scoped to the acting Reader**, resolved from the
|
||||
presented credential by `httpmw.Auth` and carried in the request context.
|
||||
There is no unauthenticated-by-Reader route and no global token.
|
||||
- **`series` holds what readers share, `bookmarks` only what differs.** A
|
||||
bookmark key is `(reader_id, site, series_id)` with no surrogate id; the wire
|
||||
`key` is derived as `site:series_id` on read.
|
||||
- `Store.Upsert` splits one flat body across both tables and enforces the
|
||||
ownership rule: client `title`/`series_url`/`cover` are written **only when
|
||||
the series row is new**, so one reader cannot retitle a shared series.
|
||||
- Sync is last-write-wins and the wire format stays flat — clients depend on
|
||||
both; neither is an implementation detail to tidy up.
|
||||
|
||||
### Web UI — `internal/web`
|
||||
|
||||
Routes, templates and assets are all in that package; `AdminPatterns()` and
|
||||
`adminRoutes()` enumerate the privileged ones.
|
||||
|
||||
- **`backend/Dockerfile` must copy the whole `internal/` tree**, not just
|
||||
`*.go`: templates and static assets are `go:embed`-ed from
|
||||
`internal/web/`.
|
||||
- **Guild membership *is* registration.** `discordCallback` gates on membership
|
||||
(plus `DISCORD_REQUIRED_ROLE` when set) and only then calls
|
||||
`Store.EnsureReader`, so a refusal creates nothing.
|
||||
- Sessions are rows, not signatures: the cookie carries an opaque id and
|
||||
expiry is checked on lookup, which is what makes deleting the row an instant
|
||||
revocation.
|
||||
- UI mutations go through `Store.Get` + `Store.Upsert` so the `updated_at` rule
|
||||
below stays in exactly one place.
|
||||
- `listView.Fresh` exists because a Reader with no bookmarks at all needs
|
||||
install links, not an empty-filter message.
|
||||
- **Design-tool caveat:** `detect.mjs backend/internal/web/templates` reports a
|
||||
**false clean**. Templates link `/static/style.css` root-absolutely (correct —
|
||||
it is served from `/`), but the detector resolves hrefs with
|
||||
`path.resolve(fileDir, href)`, which drops the directory on a leading `/` and
|
||||
skips the file silently; a relative href doesn't help either, since a
|
||||
template's directory isn't its served path. Always pass
|
||||
`backend/internal/web/static` too. The one finding there, `overused-font` on
|
||||
"Instrument Serif", is a deliberate identity choice, not debt.
|
||||
|
||||
### Confirm gating — `internal/web/static/filter.js`, `toggleConfirmRow(key, kind)`
|
||||
|
||||
Every action that pulls a series out of the list (`archive|finish|remove`) opens
|
||||
its own `.confirm-row`; restore fires instantly because it is the reversal.
|
||||
Remove wears the ember wash, the two reversible ones wear `.calm` grey.
|
||||
**`--ember` is reserved for the new-chapter signal** — the busy bar and inline
|
||||
errors must use `--mute`, or the one colour that means "something to read"
|
||||
stops meaning it.
|
||||
|
||||
### Latest-chapter poller — `internal/latest`, Site registry in `sites.go`
|
||||
|
||||
One goroutine per Site (a Poll Lane) re-checks that Site's bookmarked series
|
||||
from the backend's own network position, so `latest_chapter` stays fresh while
|
||||
nobody is browsing. The userscript's `reportLatestChapter` is a second,
|
||||
parallel signal — it PUTs every read, unchanged numbers included, because an
|
||||
unchanged read is exactly the Sighting worth deferring a Poll on.
|
||||
|
||||
- **Pace lives in the Site registry, not config.** Two clocks: per-series rest
|
||||
(`series.latest_checked_at`, enforced in `Store.DueForLatestCheck`'s WHERE)
|
||||
and per-Lane gap (`effectiveGap`). The five env knobs that used to size one
|
||||
shared pace are gone; don't add them back.
|
||||
- **The poller walks Series, not Bookmarks** — a series several readers hold is
|
||||
fetched once per cycle, and the due queue orders `reader_count DESC,
|
||||
latest_checked_at ASC` so the widely-read ones win contention.
|
||||
- **The series row is stamped *before* the fetch**, so a permanently broken
|
||||
series waits out its rest instead of being retried every tick.
|
||||
- `Store.SetLatestChapter` is a single-column UPDATE, deliberately not a
|
||||
read-modify-write of the bookmark: it therefore cannot revert read progress
|
||||
or move `updated_at`. The old stale-re-read race died with the Get+Upsert
|
||||
flow — don't restore one here.
|
||||
|
||||
**Sightings** (`Store.RecordSighting`, the due query's HAVING clause,
|
||||
`latest.checkOne`) let a Reader's own page read defer a Poll.
|
||||
|
||||
- Recorded by the PUT handler **before** the Upsert, because the raise test
|
||||
needs the row as it stands.
|
||||
- A Series is deferred only while it has exactly one Bookmark, was sighted
|
||||
within one Rest, and is under `sightingCeilingRests` since its last Poll — so
|
||||
a shared Series is never deferred and nothing goes six hours unpolled
|
||||
whatever arrives.
|
||||
- A *higher* report clears the attribution rather than crediting it: the value
|
||||
the Poll then stores is its own, so a later retraction isn't the Reader's
|
||||
fault.
|
||||
- `store.SightingDisagreementLimit` contradictions stop a Reader deferring —
|
||||
their reports still write the Latest Chapter — and
|
||||
`store.SightingAgreementsToClear` agreements forgive them, as does the
|
||||
owner's clear-marks control.
|
||||
- Deferral is recomputed from live facts each round, so nothing needs
|
||||
invalidating when a Series gains a second Bookmark. The one input read
|
||||
earlier is the Reader's marks, so crossing or clearing a threshold takes
|
||||
effect from their next Sighting and the standing already bought lasts out its
|
||||
rest.
|
||||
|
||||
**Refusals and browser loss are Lane-local.** Two `errChallengeHeld` in a pass
|
||||
stop that Site for `RefuseBackoff` while other Lanes continue. An
|
||||
`errBrowserInterrupted` (remote Chrome restarted) sets a shared Poller flag so
|
||||
the *other* browser Lanes skip their passes for the same window — otherwise a
|
||||
restarting Chrome stamps one Series per Lane per pass, burning rests on
|
||||
failures. The flag decays and they probe again.
|
||||
|
||||
- **`isInterstitial` matches the orchestration path
|
||||
`/cdn-cgi/challenge-platform/h/`, never the bare prefix.** Cloudflare injects
|
||||
`/cdn-cgi/challenge-platform/scripts/jsd/main.js` into ordinary 200 pages
|
||||
once a zone turns JS detections on, which demonic did on 2026-08-16: the
|
||||
prefix match read every real demonic page as a refusal and parked the Lane in
|
||||
backoff while plain TLS was returning full series pages.
|
||||
- Fetches use `bogdanfinn/tls-client` with a Chrome profile as defence in depth
|
||||
against fingerprint blocking; any failure logs and skips.
|
||||
- kagane, comix and novelfull sit behind Cloudflare JS challenges the TLS
|
||||
client can't clear, so they go over CDP (`BROWSER_WS_URL`). kagane and comix
|
||||
are simply not polled when it's unset — a plain fetch would only retrieve a
|
||||
challenge page — while novelfull still attempts plain TLS, because its
|
||||
challenge is a live time-varying fact and its cover bytes never need a browser.
|
||||
- **comix's browser read is an in-tab `fetch()` of the Series URL, not a DOM
|
||||
render.** It is an SPA: rendering cost ~65 requests for the same
|
||||
server-rendered HTML one fetch returns (measured 2026-08-12).
|
||||
- Browser Lanes wake Chrome only when 5+ Series are due or one has waited 15m,
|
||||
and cover work runs in the background so a slow CDN can't eat a Lane's gap.
|
||||
|
||||
### Covers — `Store.OnSeriesCreated`, `latest.Acquirer`, `latest.CoverBytesFetcher`, `Store.SetSeriesCover`
|
||||
|
||||
Acquired once when the first Bookmark of a Series is created, then served from
|
||||
our own origin by the public `GET /covers/{addr}`.
|
||||
|
||||
- Acquisition runs in a goroutine: the Reader's PUT must neither block on a
|
||||
Site nor fail with one. Every failure is logged and dropped, leaving the
|
||||
Bookmark intact.
|
||||
- The wire `cover` is the absolute `PUBLIC_BASE_URL + /covers/{sha256}` once
|
||||
bytes exist and `""` before — **never an address that 404s**. Absolute
|
||||
because the userscript renders it on a Site's origin.
|
||||
- `GET /covers/{addr}` is public and uncredentialed by design: no cookie or
|
||||
token of ours may travel to a Site's origin.
|
||||
- A client-sent `cover` is decoded and discarded, permanently — wire
|
||||
compatibility, not an oversight.
|
||||
- **One route serves all six Sites.** No proxy, no per-Site rewrite, no second
|
||||
place that decides a renderable address: the wire `cover` is it. Templates
|
||||
render `.Cover` and nothing else. The old kagane-only serving path
|
||||
(`/img/kagane/{id}` plus a template rewrite) is gone; don't reintroduce a
|
||||
per-Site route because one Site's CDN misbehaves.
|
||||
- The only Site names left in cover code are in `browserOnlyCoverURL`
|
||||
(`internal/latest`): kagane answers a plain fetch with a challenge *and*
|
||||
`cross-origin-resource-policy: same-origin`, and `static.comix.to` answers
|
||||
with the same challenge its pages serve. Every other Site's CDN answers plain
|
||||
TLS.
|
||||
- **comix cover bytes must arrive by direct navigation, not an in-page fetch:**
|
||||
its Series page sets `cross-origin-embedder-policy: require-corp`, which
|
||||
fails a page-context fetch of `static.comix.to`.
|
||||
- With no browser configured, kagane and comix Covers are simply absent;
|
||||
novelfull still gets one whenever its page answers a plain request.
|
||||
|
||||
### `updated_at` drives list order — `Store.Upsert`
|
||||
|
||||
The server applies its own timestamp only when the row is new or
|
||||
`last_chapter_num` changes, else it keeps the stored value. **Favouriting a
|
||||
series, or a newly published chapter arriving, must not reorder the list** —
|
||||
only real reading progress moves a row. Consequently `PUT` returns the row **as
|
||||
stored** and clients must adopt that response rather than their own payload.
|
||||
|
||||
### Lifecycle buckets — `status` on each bookmark
|
||||
|
||||
`reading` | `archived` | `finished`, orthogonal to `favorite`. Archived and
|
||||
finished appear only in their own tab, never in All, Updated, Favourites or the
|
||||
recent strip. The poller keeps checking archived series and skips finished ones.
|
||||
|
||||
- `finished` is settable only from the web UI; `PUT /bookmarks/{key}` rejects
|
||||
it with 400.
|
||||
- **An empty incoming status means "keep the stored one"**, and it is resolved
|
||||
on the `VALUES` side of `Store.Upsert`, not in the conflict clause:
|
||||
`excluded.*` is the post-evaluation row, so a default applied there would
|
||||
wipe the bucket on every PUT from a client predating the column.
|
||||
|
||||
### Config — `Config` / `loadConfig` / `loadLatestPoll` in `backend/main.go`
|
||||
|
||||
That function is the complete list of env vars, their defaults, and which are
|
||||
required. What it can't tell you:
|
||||
|
||||
- `PUBLIC_BASE_URL` must be an absolute origin because every Cover URL on the
|
||||
wire is built from it and the userscript renders on a Site's origin.
|
||||
- `BROWSER_WS_URL` **must be a tailnet IP, never a hostname** — Chrome's
|
||||
DevTools handler 500s `/json/version` for any Host that isn't an IP or
|
||||
`localhost`. Unset (the default) disables browser polling.
|
||||
- `USERSCRIPT_PATH` / `NOVEL_USERSCRIPT_PATH` are bindmounted files; the
|
||||
`__API_TOKEN__` placeholder inside them is substituted with the requesting
|
||||
Reader's credential at serve time.
|
||||
- Pace is per Site in the registry, not env. The
|
||||
`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER` knobs are
|
||||
gone on purpose.
|
||||
- The 1h rest for browser Sites is safe on documented grounds: a challenged
|
||||
page costs seconds of a serialized single-tab browser, free-plan zones carry
|
||||
no bot score and no published per-IP rate input, and `cf_clearance` expires
|
||||
in 30 minutes, so every cadence at or above 1h re-solves anyway.
|
||||
|
||||
### Userscript install & rotation — `internal/userscript`, `internal/token`
|
||||
|
||||
Session-gated `GET /install/{manga,novel}-bookmark.user.js` renders the
|
||||
bindmounted script with the acting Reader's derived credential substituted in,
|
||||
so the credential never appears in page markup, the address bar, or a redirect.
|
||||
`?download=1` adds `Content-Disposition: attachment` for mobile Violentmonkey,
|
||||
which ignores a `.user.js` navigation. `POST /rotate-token` is an atomic epoch
|
||||
bump plus hash rewrite and invalidates every installed copy — the panel must
|
||||
keep warning to reinstall on all devices.
|
||||
|
||||
### Owner-only admin — `internal/web/admin.go`
|
||||
|
||||
- **Every route reaching past the acting Reader is listed in `adminRoutes()`
|
||||
and wrapped in `requireOwner` at registration** — add it there, not as a
|
||||
check inside a handler; `web.AdminPatterns()` is what the gate test walks. A
|
||||
non-owner gets 404, never 403.
|
||||
- The one owner comparison left outside the gate is in `index`
|
||||
(`view.Owner = readerID == h.store.OwnerID()`): it gates a link, not an
|
||||
endpoint, so it is a rendering decision a registration-time wrapper cannot
|
||||
express. Do not "unify" it into the gate.
|
||||
- The Lanes page reads the pass log, never a running poller: `lanesView()` in
|
||||
`admin_lanes.go` projects `store.LatestLanePasses()` and
|
||||
`store.LanePassOutcomes()` (ADR-0012), so a restart answers the instant the
|
||||
database is up. Browser configuration is a config fact and reachability is
|
||||
derived from recent browser-Site passes inside `latest.RefuseBackoff` — no
|
||||
reporter interface exists to fake.
|
||||
- A pass that returns before computing figures (refusal backoff, sidecar down)
|
||||
carries the previous pass's numbers forward rather than recording zeroes.
|
||||
- **`Checked` next to `Due` is what separates a stopped Lane from a quiet one**,
|
||||
so neither may be dropped from the row.
|
||||
- Due-without-Checked is **not** by itself a stall: a browser Lane under both
|
||||
wake thresholds records its pass with the `SkipAsleep` skip and renders
|
||||
"browser asleep", and that never counts toward `Attention`. It is the
|
||||
commonest healthy state for kagane, comix and novelfull, so spending the
|
||||
stall mark on it would train the owner to ignore the mark that matters.
|
||||
@@ -1 +0,0 @@
|
||||
AGENTS.md
|
||||
+13
-13
@@ -1,35 +1,35 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# --- build stage: compile a static, CGO-free binary ---
|
||||
FROM golang:1.26-alpine AS build
|
||||
ARG COVER_DIR=/covers
|
||||
FROM golang:1.24-alpine AS build
|
||||
WORKDIR /src
|
||||
|
||||
# Dependencies first for layer caching (changes rarely).
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Then source (changes often). internal/web carries the go:embed'd
|
||||
# templates/static assets — missing them turns the embed directive into a
|
||||
# build error, so the whole tree must land before `go build`.
|
||||
# Then source (changes often).
|
||||
# Source plus the go:embed'd assets. Missing either directory turns the embed
|
||||
# directive into a build error, so both must be copied before `go build`.
|
||||
COPY *.go ./
|
||||
COPY internal/ ./internal/
|
||||
COPY templates/ ./templates/
|
||||
COPY static/ ./static/
|
||||
|
||||
# Static binary: the Postgres driver (jackc/pgx) is pure Go, so CGO_ENABLED=0
|
||||
# leaves no libc dependency.
|
||||
# Static binary: pure-Go sqlite means CGO_ENABLED=0 -> no libc dependency.
|
||||
# -trimpath + -ldflags strip paths and debug info for a smaller image.
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/server .
|
||||
|
||||
# Create the source directory; runtime COPY sets ownership for the named volume.
|
||||
RUN mkdir -p "$COVER_DIR"
|
||||
# Data dir with the runtime user's ownership so the mounted volume inherits it.
|
||||
RUN mkdir -p /out/data
|
||||
|
||||
# --- runtime stage: distroless static, non-root ---
|
||||
FROM gcr.io/distroless/static:nonroot
|
||||
ARG COVER_DIR=/covers
|
||||
WORKDIR /
|
||||
COPY --from=build --chown=65532:65532 ${COVER_DIR} ${COVER_DIR}
|
||||
COPY --from=build /out/server /server
|
||||
COPY --from=build --chown=65532:65532 /out/data /data
|
||||
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 8080
|
||||
USER nonroot:nonroot
|
||||
ENV PORT=8080
|
||||
ENV DB_PATH=/data/bookmarks.db PORT=8080
|
||||
ENTRYPOINT ["/server"]
|
||||
|
||||
@@ -1,720 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/pgtest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
// testTokenKey derives every test Reader's credential; it must match the key
|
||||
// newTestStoreURL seeds the owner with, or derived credentials authenticate
|
||||
// nothing.
|
||||
const testTokenKey = "test-token-key"
|
||||
|
||||
// testDiscordID is the owner row's discord_id (newTestStoreURL); the derived
|
||||
// credential is a function of it.
|
||||
const testDiscordID = "test-owner"
|
||||
|
||||
// testCoverBaseURL is the public origin cover URLs are built from, standing in
|
||||
// for PUBLIC_BASE_URL.
|
||||
const testCoverBaseURL = "https://bookmarks.test"
|
||||
|
||||
func testConfig() Config {
|
||||
return Config{
|
||||
TokenKey: testTokenKey,
|
||||
AllowedOrigins: []string{"https://asurascans.com", "https://demonicscans.org"},
|
||||
Port: "8080",
|
||||
}
|
||||
}
|
||||
|
||||
// ownerCredential is the owner's epoch-0 derived credential: the string the
|
||||
// install links carry and the userscript routes authenticate.
|
||||
func ownerCredential() string {
|
||||
return token.Token([]byte(testTokenKey), testDiscordID, 0)
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
|
||||
func newTestServer(t *testing.T) http.Handler {
|
||||
t.Helper()
|
||||
return newRouter(newTestStore(t), testConfig())
|
||||
}
|
||||
|
||||
func newTestStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
s, _ := newTestStoreURL(t)
|
||||
return s
|
||||
}
|
||||
|
||||
// newTestStoreURL is newTestStore plus the database URL, for tests that need
|
||||
// to reach the same database directly.
|
||||
func newTestStoreURL(t *testing.T) (*store.Store, string) {
|
||||
t.Helper()
|
||||
url := pgtest.URL(t)
|
||||
s, err := store.Open(url, store.Owner{
|
||||
DiscordID: testDiscordID, TokenHash: token.Hash(ownerCredential()),
|
||||
}, t.TempDir(), testCoverBaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("store.Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s, url
|
||||
}
|
||||
|
||||
// auth authenticates a request as the owner Reader, whose derived credential
|
||||
// is the only thing the API accepts.
|
||||
func auth(req *http.Request) *http.Request {
|
||||
req.Header.Set("Authorization", "Bearer "+ownerCredential())
|
||||
return req
|
||||
}
|
||||
|
||||
func floatPtr(f float64) *float64 { return &f }
|
||||
|
||||
// seedForCheck inserts a bookmark (and with it its series) and forces the
|
||||
// series' latest_checked_at.
|
||||
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: key,
|
||||
Site: site,
|
||||
SeriesID: seriesID,
|
||||
SeriesURL: seriesURL,
|
||||
UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %q: %v", key, err)
|
||||
}
|
||||
if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
|
||||
t.Fatalf("seed mark %q: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
ts, err := s.LatestCheckedAt(site, seriesID)
|
||||
if err != nil {
|
||||
t.Fatalf("LatestCheckedAt %q: %v", key, err)
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
func TestHealthzNoAuth(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("healthz status = %d, want 200", rr.Code)
|
||||
}
|
||||
if rr.Body.String() != "ok" {
|
||||
t.Fatalf("healthz body = %q, want ok", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRequired(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
header string
|
||||
}{
|
||||
{"no header", ""},
|
||||
{"bad token", "Bearer wrong"},
|
||||
{"not bearer", "Basic " + ownerCredential()},
|
||||
{"empty bearer", "Bearer "},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/bookmarks", nil)
|
||||
if tc.header != "" {
|
||||
req.Header.Set("Authorization", tc.header)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", rr.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthAccepted(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodGet, "/bookmarks", nil)))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); got != "[]\n" {
|
||||
t.Fatalf("empty list body = %q, want []", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORSPreflight(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
req := httptest.NewRequest(http.MethodOptions, "/bookmarks/asura:foo-1", nil)
|
||||
req.Header.Set("Origin", "https://asurascans.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "PUT")
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("preflight status = %d, want 204", rr.Code)
|
||||
}
|
||||
if got := rr.Header().Get("Access-Control-Allow-Origin"); got != "https://asurascans.com" {
|
||||
t.Fatalf("Allow-Origin = %q, want reflected origin", got)
|
||||
}
|
||||
if got := rr.Header().Get("Access-Control-Allow-Methods"); got == "" {
|
||||
t.Fatal("Allow-Methods missing")
|
||||
}
|
||||
if got := rr.Header().Get("Access-Control-Allow-Headers"); got == "" {
|
||||
t.Fatal("Allow-Headers missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORSDisallowedOrigin(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
req := httptest.NewRequest(http.MethodOptions, "/bookmarks", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if got := rr.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("Allow-Origin = %q, want empty for disallowed origin", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBookmarkRoundTrip(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
key := "asura:solo-leveling-123"
|
||||
in := store.Bookmark{
|
||||
Title: "Solo Leveling",
|
||||
SeriesURL: "https://asurascans.com/series/solo-leveling-123",
|
||||
Cover: "https://asurascans.com/cover.jpg",
|
||||
LastChapter: "Chapter 10",
|
||||
LastChapterNum: 10,
|
||||
LastChapterURL: "https://asurascans.com/series/solo-leveling-123/chapter/10",
|
||||
}
|
||||
body, _ := json.Marshal(in)
|
||||
|
||||
// PUT
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, "/bookmarks/"+key, bytes.NewReader(body))))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status = %d, want 200", rr.Code)
|
||||
}
|
||||
var stored store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &stored); err != nil {
|
||||
t.Fatalf("decode PUT response: %v", err)
|
||||
}
|
||||
if stored.Key != key || stored.Site != "asura" || stored.SeriesID != "solo-leveling-123" {
|
||||
t.Fatalf("derived fields wrong: %+v", stored)
|
||||
}
|
||||
if stored.UpdatedAt == 0 {
|
||||
t.Fatal("server did not set updated_at")
|
||||
}
|
||||
|
||||
// GET
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodGet, "/bookmarks", nil)))
|
||||
var list []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &list); err != nil {
|
||||
t.Fatalf("decode list: %v", err)
|
||||
}
|
||||
if len(list) != 1 || list[0].Key != key || list[0].LastChapterNum != 10 {
|
||||
t.Fatalf("GET list wrong: %+v", list)
|
||||
}
|
||||
|
||||
// PUT again (upsert, progress advance)
|
||||
in.LastChapter, in.LastChapterNum = "Chapter 11", 11
|
||||
body, _ = json.Marshal(in)
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, "/bookmarks/"+key, bytes.NewReader(body))))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("second PUT status = %d", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodGet, "/bookmarks", nil)))
|
||||
json.Unmarshal(rr.Body.Bytes(), &list)
|
||||
if len(list) != 1 || list[0].LastChapterNum != 11 {
|
||||
t.Fatalf("upsert did not update in place: %+v", list)
|
||||
}
|
||||
|
||||
// DELETE
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodDelete, "/bookmarks/"+key, nil)))
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("DELETE status = %d, want 204", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodGet, "/bookmarks", nil)))
|
||||
json.Unmarshal(rr.Body.Bytes(), &list)
|
||||
if len(list) != 0 {
|
||||
t.Fatalf("after delete list = %+v, want empty", list)
|
||||
}
|
||||
}
|
||||
|
||||
// The wire contract (ADR-0004): GET and PUT speak exactly the flat field set
|
||||
// they always did, with the series-owned fields as siblings of the bookmark
|
||||
// fields, not nested. Asserted as a key set, not by inspection.
|
||||
func TestFlatWireFieldSet(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
key := "comix:some-title"
|
||||
in := store.Bookmark{
|
||||
Key: key,
|
||||
Site: "comix",
|
||||
SeriesID: "some-title",
|
||||
Title: "Some Title",
|
||||
SeriesURL: "https://comix.to/title/some-title",
|
||||
Cover: "https://comix.to/covers/some-title.jpg",
|
||||
LastChapter: "Chapter 7",
|
||||
LastChapterNum: 7,
|
||||
LastChapterURL: "https://comix.to/title/some-title/ch/7",
|
||||
Favorite: true,
|
||||
LatestChapter: "Chapter 8",
|
||||
LatestChapterNum: floatPtr(8),
|
||||
Status: store.StatusArchived,
|
||||
Kind: store.KindManga,
|
||||
}
|
||||
body, _ := json.Marshal(in)
|
||||
|
||||
wantKeys := map[string]bool{
|
||||
"key": true, "site": true, "series_id": true, "title": true,
|
||||
"series_url": true, "cover": true, "last_chapter": true,
|
||||
"last_chapter_num": true, "last_chapter_url": true, "favorite": true,
|
||||
"latest_chapter": true, "latest_chapter_num": true, "updated_at": true,
|
||||
"status": true, "kind": true,
|
||||
}
|
||||
checkFlat := func(t *testing.T, payload []byte) map[string]json.RawMessage {
|
||||
t.Helper()
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(payload, &obj); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(obj) != len(wantKeys) {
|
||||
t.Fatalf("field count = %d, want %d (%s)", len(obj), len(wantKeys), payload)
|
||||
}
|
||||
for k := range obj {
|
||||
if !wantKeys[k] {
|
||||
t.Fatalf("unexpected field %q", k)
|
||||
}
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
// PUT
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, "/bookmarks/"+key, bytes.NewReader(body))))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status = %d, want 200", rr.Code)
|
||||
}
|
||||
checkFlat(t, rr.Body.Bytes())
|
||||
|
||||
// Every field round-trips with its value, and updated_at is server-stamped.
|
||||
var stored store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &stored); err != nil {
|
||||
t.Fatalf("decode PUT response: %v", err)
|
||||
}
|
||||
latestNum := floatPtr(8)
|
||||
want := store.Bookmark{
|
||||
Key: key, Site: "comix", SeriesID: "some-title",
|
||||
Title: in.Title, SeriesURL: in.SeriesURL,
|
||||
LastChapter: in.LastChapter, LastChapterNum: in.LastChapterNum,
|
||||
LastChapterURL: in.LastChapterURL, Favorite: true,
|
||||
LatestChapter: in.LatestChapter, LatestChapterNum: latestNum,
|
||||
Status: store.StatusArchived, Kind: store.KindManga,
|
||||
}
|
||||
// Cover is deliberately absent above: the client's cover is discarded, and
|
||||
// this wiring acquires none, so the field is present and empty (ADR-0007).
|
||||
if stored.Title != want.Title || stored.SeriesURL != want.SeriesURL || stored.Cover != want.Cover ||
|
||||
stored.LastChapter != want.LastChapter || stored.LastChapterNum != want.LastChapterNum ||
|
||||
stored.LastChapterURL != want.LastChapterURL || stored.Favorite != want.Favorite ||
|
||||
stored.LatestChapter != want.LatestChapter ||
|
||||
stored.LatestChapterNum == nil || *stored.LatestChapterNum != *want.LatestChapterNum ||
|
||||
stored.Status != want.Status || stored.Kind != want.Kind {
|
||||
t.Fatalf("PUT response = %+v, want %+v", stored, want)
|
||||
}
|
||||
if stored.UpdatedAt == 0 {
|
||||
t.Fatal("updated_at not server-stamped")
|
||||
}
|
||||
|
||||
// GET reports the same flat shape.
|
||||
list := getBookmarks(t, srv)
|
||||
if len(list) != 1 {
|
||||
t.Fatalf("list = %d items, want 1", len(list))
|
||||
}
|
||||
body2, _ := json.Marshal(list[0])
|
||||
checkFlat(t, body2)
|
||||
}
|
||||
|
||||
// A PUT naming an existing series must ignore client-supplied title, cover and
|
||||
// URL — the security boundary from ADR-0003, where a hostile site's scraped
|
||||
// values could otherwise land on a shared row — while progress still lands.
|
||||
func TestPutExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
key := "asura:solo"
|
||||
|
||||
first := putBookmark(t, srv, key, store.Bookmark{
|
||||
Title: "Solo Leveling",
|
||||
SeriesURL: "https://asurascans.com/comics/solo",
|
||||
Cover: "https://asurascans.com/covers/solo.jpg",
|
||||
LastChapterNum: 10,
|
||||
})
|
||||
|
||||
second := putBookmark(t, srv, key, store.Bookmark{
|
||||
Title: "Scraped Rename",
|
||||
SeriesURL: "https://evil.example/solo",
|
||||
Cover: "https://evil.example/solo.jpg",
|
||||
LastChapterNum: 11,
|
||||
})
|
||||
if second.Title != first.Title || second.SeriesURL != first.SeriesURL || second.Cover != first.Cover {
|
||||
t.Fatalf("stored = %+v, want original title/url/cover kept", second)
|
||||
}
|
||||
if second.LastChapterNum != 11 {
|
||||
t.Fatalf("LastChapterNum = %v, want 11 — progress must still land", second.LastChapterNum)
|
||||
}
|
||||
}
|
||||
|
||||
// putBookmark PUTs b at key and returns the bookmark the server echoes back,
|
||||
// which is the row as actually stored (not the request payload).
|
||||
func putBookmark(t *testing.T, srv http.Handler, key string, b store.Bookmark) store.Bookmark {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(b)
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, "/bookmarks/"+key, bytes.NewReader(body))))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("PUT %s status = %d, body = %s", key, rr.Code, rr.Body.String())
|
||||
}
|
||||
var out store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode PUT response: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func getBookmarks(t *testing.T, srv http.Handler) []store.Bookmark {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodGet, "/bookmarks", nil)))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("GET status = %d", rr.Code)
|
||||
}
|
||||
var list []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &list); err != nil {
|
||||
t.Fatalf("decode list: %v", err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// updated_at drives list ordering, so it must move only on a real progress
|
||||
// advance — never on a favorite toggle or a latest-chapter capture.
|
||||
func TestUpsertConditionalUpdatedAt(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(store.Bookmark) store.Bookmark
|
||||
wantBumped bool
|
||||
}{
|
||||
{
|
||||
name: "unchanged progress",
|
||||
mutate: func(b store.Bookmark) store.Bookmark { return b },
|
||||
wantBumped: false,
|
||||
},
|
||||
{
|
||||
name: "changed progress",
|
||||
mutate: func(b store.Bookmark) store.Bookmark {
|
||||
b.LastChapter, b.LastChapterNum = "Chapter 11", 11
|
||||
return b
|
||||
},
|
||||
wantBumped: true,
|
||||
},
|
||||
{
|
||||
name: "favorite only",
|
||||
mutate: func(b store.Bookmark) store.Bookmark {
|
||||
b.Favorite = true
|
||||
return b
|
||||
},
|
||||
wantBumped: false,
|
||||
},
|
||||
{
|
||||
name: "latest chapter only",
|
||||
mutate: func(b store.Bookmark) store.Bookmark {
|
||||
b.LatestChapter, b.LatestChapterNum = "Chapter 15", floatPtr(15)
|
||||
return b
|
||||
},
|
||||
wantBumped: false,
|
||||
},
|
||||
{
|
||||
name: "unrelated metadata only",
|
||||
mutate: func(b store.Bookmark) store.Bookmark {
|
||||
b.Title, b.Cover = "Renamed", "https://example.test/new.jpg"
|
||||
return b
|
||||
},
|
||||
wantBumped: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
key := fmt.Sprintf("asura:cond-%d", i)
|
||||
|
||||
first := putBookmark(t, srv, key, store.Bookmark{
|
||||
Title: "Test",
|
||||
LastChapter: "Chapter 10",
|
||||
LastChapterNum: 10,
|
||||
})
|
||||
if first.UpdatedAt == 0 {
|
||||
t.Fatal("new bookmark did not get updated_at set")
|
||||
}
|
||||
|
||||
// Guarantee a later wall-clock ms so a real bump is observable.
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
|
||||
second := putBookmark(t, srv, key, tc.mutate(first))
|
||||
if tc.wantBumped && second.UpdatedAt <= first.UpdatedAt {
|
||||
t.Fatalf("updated_at = %d, want > %d", second.UpdatedAt, first.UpdatedAt)
|
||||
}
|
||||
if !tc.wantBumped && second.UpdatedAt != first.UpdatedAt {
|
||||
t.Fatalf("updated_at = %d, want preserved %d", second.UpdatedAt, first.UpdatedAt)
|
||||
}
|
||||
|
||||
// The PUT response must match what a subsequent GET reports.
|
||||
list := getBookmarks(t, srv)
|
||||
if len(list) != 1 {
|
||||
t.Fatalf("list = %+v, want 1 item", list)
|
||||
}
|
||||
if list[0].UpdatedAt != second.UpdatedAt {
|
||||
t.Fatalf("GET updated_at = %d, PUT echoed %d", list[0].UpdatedAt, second.UpdatedAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFavoriteRoundTrip(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
key := "demonic:some-series"
|
||||
|
||||
stored := putBookmark(t, srv, key, store.Bookmark{Title: "Fav", Favorite: true})
|
||||
if !stored.Favorite {
|
||||
t.Fatalf("PUT response favorite = false, want true")
|
||||
}
|
||||
|
||||
list := getBookmarks(t, srv)
|
||||
if len(list) != 1 || !list[0].Favorite {
|
||||
t.Fatalf("favorite did not round-trip: %+v", list)
|
||||
}
|
||||
|
||||
// Unfavoriting must persist too (guards against a write that only ever ORs in true).
|
||||
stored = putBookmark(t, srv, key, store.Bookmark{Title: "Fav", Favorite: false})
|
||||
if stored.Favorite {
|
||||
t.Fatal("PUT response favorite = true after unfavorite")
|
||||
}
|
||||
list = getBookmarks(t, srv)
|
||||
if len(list) != 1 || list[0].Favorite {
|
||||
t.Fatalf("unfavorite did not round-trip: %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestChapterNullable(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
key := "asura:latest-test"
|
||||
|
||||
// Never captured: latest_chapter_num must serialize as JSON null.
|
||||
body, _ := json.Marshal(store.Bookmark{Title: "No latest yet"})
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, "/bookmarks/"+key, bytes.NewReader(body))))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status = %d", rr.Code)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), `"latest_chapter_num":null`) {
|
||||
t.Fatalf("want latest_chapter_num null in response, got %s", rr.Body.String())
|
||||
}
|
||||
|
||||
list := getBookmarks(t, srv)
|
||||
if len(list) != 1 || list[0].LatestChapterNum != nil {
|
||||
t.Fatalf("latest_chapter_num = %v, want nil", list[0].LatestChapterNum)
|
||||
}
|
||||
|
||||
// Once captured it round-trips as a value.
|
||||
stored := putBookmark(t, srv, key, store.Bookmark{
|
||||
Title: "No latest yet",
|
||||
LatestChapter: "Chapter 162",
|
||||
LatestChapterNum: floatPtr(162),
|
||||
})
|
||||
if stored.LatestChapterNum == nil || *stored.LatestChapterNum != 162 {
|
||||
t.Fatalf("PUT response latest_chapter_num = %v, want 162", stored.LatestChapterNum)
|
||||
}
|
||||
list = getBookmarks(t, srv)
|
||||
if len(list) != 1 || list[0].LatestChapterNum == nil || *list[0].LatestChapterNum != 162 {
|
||||
t.Fatalf("latest chapter did not round-trip: %+v", list)
|
||||
}
|
||||
if list[0].LatestChapter != "Chapter 162" {
|
||||
t.Fatalf("latest_chapter = %q, want %q", list[0].LatestChapter, "Chapter 162")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDiscord(t *testing.T) {
|
||||
t.Setenv("DISCORD_CLIENT_ID", "client-1")
|
||||
t.Setenv("DISCORD_CLIENT_SECRET", "client-secret-1")
|
||||
t.Setenv("DISCORD_GUILD_ID", "guild-1")
|
||||
t.Setenv("DISCORD_REQUIRED_ROLE", "role-9")
|
||||
t.Setenv("DISCORD_REDIRECT_URI", "https://bm.example.com/auth/discord/callback")
|
||||
t.Setenv("DISCORD_API_BASE", "https://stub.example/api")
|
||||
if got := loadConfig().Discord; got.ClientID != "client-1" || got.ClientSecret != "client-secret-1" ||
|
||||
got.GuildID != "guild-1" || got.RequiredRole != "role-9" ||
|
||||
got.RedirectURI != "https://bm.example.com/auth/discord/callback" ||
|
||||
got.APIBase != "https://stub.example/api" {
|
||||
t.Fatalf("Discord config = %+v, want every field set", got)
|
||||
}
|
||||
|
||||
// API base falls back to the Discord default; the role is optional.
|
||||
t.Setenv("DISCORD_REQUIRED_ROLE", "")
|
||||
t.Setenv("DISCORD_API_BASE", "")
|
||||
got := loadConfig().Discord
|
||||
if got.RequiredRole != "" {
|
||||
t.Fatalf("RequiredRole = %q, want empty by default", got.RequiredRole)
|
||||
}
|
||||
if got.APIBase != "https://discord.com/api/v10" {
|
||||
t.Fatalf("APIBase = %q, want the Discord default", got.APIBase)
|
||||
}
|
||||
}
|
||||
|
||||
// A userscript PUT body has no latest_checked_at field. If the column is ever
|
||||
// moved into bookmarkColumns, this test catches it: the PUT would reset the
|
||||
// cooldown and the poller would re-fetch that series on every single tick.
|
||||
func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
srv := newRouter(s, testConfig())
|
||||
|
||||
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 777)
|
||||
|
||||
// Exactly what the userscript sends: no latest_checked_at key at all.
|
||||
body := `{"key":"asura:x","site":"asura","series_id":"x",
|
||||
"series_url":"https://asurascans.com/comics/x",
|
||||
"last_chapter":"Chapter 5","last_chapter_num":5}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body))
|
||||
req.Header.Set("Authorization", "Bearer "+ownerCredential())
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := readLatestCheckedAt(t, s, "asura:x"); got != 777 {
|
||||
t.Fatalf("latest_checked_at = %d after client PUT, want 777 preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Sighting deferral (issue #103) only reaches production through the PUT
|
||||
// handler: the store and poller can be right and the feature still dead if the
|
||||
// handler never records the report. Asserted where a client can see it - the
|
||||
// series stops being due the moment the PUT lands.
|
||||
func TestPutRecordsASighting(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
srv := newRouter(s, testConfig())
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
hour := time.Hour.Milliseconds()
|
||||
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", now-2*hour)
|
||||
due, err := s.DueForLatestCheck("asura", now-hour, now-6*hour)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("due before the PUT = %d series, want 1", len(due))
|
||||
}
|
||||
|
||||
body := `{"key":"asura:x","site":"asura","series_id":"x",
|
||||
"series_url":"https://asurascans.com/comics/x",
|
||||
"last_chapter":"Chapter 5","last_chapter_num":5,
|
||||
"latest_chapter":"Chapter 9","latest_chapter_num":9}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, auth(req))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
due, err = s.DueForLatestCheck("asura", now-hour, now-6*hour)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 0 {
|
||||
t.Fatalf("due after the PUT = %d series, want 0: the handler recorded no Sighting", len(due))
|
||||
}
|
||||
}
|
||||
|
||||
// The userscript route is registered outside the web UI's Discord auth, so it
|
||||
// must keep working whatever the web config — see internal/userscript for the
|
||||
// handler's own behaviour. The credential in the path is the owner's derived
|
||||
// one, and the served script carries it substituted in.
|
||||
func TestUserscriptServedWithWebUIDisabled(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "manga-bookmark.user.js")
|
||||
if err := os.WriteFile(path, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
s := newTestStore(t)
|
||||
cfg := testConfig() // no Discord config needed for the userscript route
|
||||
cfg.UserscriptPath = path
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/u/"+ownerCredential()+"/manga-bookmark.user.js", nil)
|
||||
newRouter(s, cfg).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+ownerCredential()+`"`) {
|
||||
t.Fatalf("served script does not carry the requesting Reader's credential:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Both scripts are served from the same handler, outside the web UI's auth —
|
||||
// a wrong credential is a 404, never a 401.
|
||||
func TestNovelUserscriptServed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
novelPath := filepath.Join(dir, "novel-bookmark.user.js")
|
||||
if err := os.WriteFile(novelPath, []byte("// novel\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
s := newTestStore(t)
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.NovelUserscriptPath = novelPath
|
||||
srv := newRouter(s, cfg)
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet,
|
||||
"/u/"+ownerCredential()+"/novel-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/javascript") {
|
||||
t.Fatalf("Content-Type = %q, want text/javascript", ct)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet,
|
||||
"/u/wrong-token/novel-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("wrong token status = %d, want 404", rr.Code)
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
func getCover(t *testing.T, srv http.Handler, path string, cookie *http.Cookie) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
if cookie != nil {
|
||||
req.AddCookie(cookie)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
// The acquired Cover is served from this deployment's own origin, to any
|
||||
// browser rendering a third-party page — no session, no credential (ADR-0007).
|
||||
func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
|
||||
const sourceURL = "https://cdn.asurascans.com/covers/solo.webp"
|
||||
srv, st := newWebTestServer(t, testConfig())
|
||||
if err := st.PutCover(sourceURL, []byte("\x00webp-bytes"), "image/webp"); err != nil {
|
||||
t.Fatalf("PutCover: %v", err)
|
||||
}
|
||||
|
||||
// The wire URL is what a client actually requests, so the path under test
|
||||
// is taken from it rather than rebuilt by hand.
|
||||
wire := st.CoverWireURL(store.CoverAddress(sourceURL))
|
||||
path, ok := strings.CutPrefix(wire, testCoverBaseURL)
|
||||
if !ok {
|
||||
t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL)
|
||||
}
|
||||
rr := getCover(t, srv, path, nil)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 without any credential", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); got != "\x00webp-bytes" {
|
||||
t.Fatalf("body = %q, want the stored bytes", got)
|
||||
}
|
||||
if got := rr.Header().Get("Content-Type"); got != "image/webp" {
|
||||
t.Fatalf("Content-Type = %q, want the stored one", got)
|
||||
}
|
||||
// Content-addressed bytes never change, so a client that has them must
|
||||
// never need to ask again.
|
||||
if got := rr.Header().Get("Cache-Control"); !strings.Contains(got, "immutable") {
|
||||
t.Fatalf("Cache-Control = %q, want an immutable cache directive", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicCoverRejectsUnknownAddress(t *testing.T) {
|
||||
srv, _ := newWebTestServer(t, testConfig())
|
||||
cases := map[string]string{
|
||||
"unknown": "/covers/" + store.CoverAddress("https://cdn.example/never-stored.jpg"),
|
||||
"malformed": "/covers/not-an-address",
|
||||
"traversal": "/covers/../../etc/passwd",
|
||||
"empty": "/covers/",
|
||||
}
|
||||
for name, path := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if rr := getCover(t, srv, path, nil); rr.Code == http.StatusOK {
|
||||
t.Fatalf("%s: status = 200, want anything but a served body", path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A content type outside the image set is never echoed back. The old kagane
|
||||
// proxy could fetch text/html from a challenged fetch and had to refuse it;
|
||||
// the general route's only input is the store, and the store refuses to
|
||||
// record anything that is not an image — but the guarantee is pinned at the
|
||||
// serving boundary, not the write gate, so a poisoned row (migrated data, a
|
||||
// writer that skips the gate) is also never served.
|
||||
func TestPublicCoverNeverEchoesNonImage(t *testing.T) {
|
||||
const sourceURL = "https://cdn.example/cover"
|
||||
st, dsn := newTestStoreURL(t)
|
||||
// The write gate refuses non-image content types outright.
|
||||
if err := st.PutCover(sourceURL, []byte("<script>"), "text/html"); err == nil {
|
||||
t.Fatal("PutCover accepted a non-image content type")
|
||||
}
|
||||
// A legitimate row, then the content type flipped behind the store's back:
|
||||
// the bytes exist at the address, so only the type is hostile.
|
||||
address := store.CoverAddress(sourceURL)
|
||||
if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil {
|
||||
t.Fatalf("seed row: %v", err)
|
||||
}
|
||||
db, err := sql.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", dsn, err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`UPDATE covers SET content_type = 'text/html' WHERE address = $1`, address); err != nil {
|
||||
t.Fatalf("poison row: %v", err)
|
||||
}
|
||||
rr := getCover(t, newRouter(st, testConfig()), "/covers/"+address, nil)
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatalf("status = 200, want a refusal for a non-image row (body %q)", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point of acquiring bytes is that the UI shows them: the card's
|
||||
// <img> must carry the public address, not a third-party URL and not a
|
||||
// placeholder.
|
||||
func TestListRendersAcquiredCover(t *testing.T) {
|
||||
const sourceURL = "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg"
|
||||
srv, st := newWebTestServer(t, testConfig())
|
||||
if _, err := st.Upsert(st.OwnerID(), store.Bookmark{
|
||||
Key: "comix:n8we", Site: "comix", SeriesID: "n8we", Title: "Dungeons and Crayons",
|
||||
SeriesURL: "https://comix.to/title/n8we", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := st.SetSeriesCover("comix", "n8we", sourceURL, []byte("\xff\xd8jpeg"), "image/jpeg"); err != nil {
|
||||
t.Fatalf("SetSeriesCover: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/ui/list", nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddress(sourceURL) + `"`
|
||||
if !strings.Contains(rr.Body.String(), want) {
|
||||
t.Fatalf("rendered list does not contain %s", want)
|
||||
}
|
||||
}
|
||||
+16
-15
@@ -1,13 +1,11 @@
|
||||
module bookmarkmanager/backend
|
||||
module mangabm/backend
|
||||
|
||||
go 1.26
|
||||
go 1.24.1
|
||||
|
||||
require (
|
||||
github.com/bogdanfinn/fhttp v0.6.8
|
||||
github.com/bogdanfinn/tls-client v1.15.1
|
||||
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f
|
||||
github.com/chromedp/chromedp v0.16.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
modernc.org/sqlite v1.34.4
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -17,20 +15,23 @@ require (
|
||||
github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect
|
||||
github.com/bogdanfinn/utls v1.7.7-barnius // indirect
|
||||
github.com/bogdanfinn/websocket v1.5.5-barnius // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.8.0 // indirect
|
||||
modernc.org/strutil v1.2.0 // indirect
|
||||
modernc.org/token v1.1.0 // indirect
|
||||
)
|
||||
|
||||
+46
-34
@@ -14,44 +14,28 @@ github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5Zl
|
||||
github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg=
|
||||
github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI=
|
||||
github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI=
|
||||
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk=
|
||||
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0=
|
||||
github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
|
||||
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg=
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
|
||||
@@ -62,6 +46,8 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
|
||||
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.0.0-20211104170005-ce137452f963/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
@@ -70,14 +56,40 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
|
||||
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
|
||||
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
|
||||
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
|
||||
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
|
||||
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
|
||||
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
|
||||
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
|
||||
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
|
||||
modernc.org/sqlite v1.34.4 h1:sjdARozcL5KJBvYQvLlZEmctRgW9xqIZc2ncN7PU0P8=
|
||||
modernc.org/sqlite v1.34.4/go.mod h1:3QQFCG2SEMtc2nv+Wq4cQCH7Hjcg+p/RMlS1XK+zwbk=
|
||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type bookmarkHandler struct {
|
||||
store *Store
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if v != nil {
|
||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||
log.Printf("encode response: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// list returns all bookmarks. GET /bookmarks
|
||||
func (h *bookmarkHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := h.store.List()
|
||||
if err != nil {
|
||||
log.Printf("list: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, items)
|
||||
}
|
||||
|
||||
// put upserts one bookmark. PUT /bookmarks/{key}
|
||||
func (h *bookmarkHandler) put(w http.ResponseWriter, r *http.Request) {
|
||||
key := r.PathValue("key")
|
||||
if key == "" {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var b Bookmark
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&b); err != nil {
|
||||
http.Error(w, "invalid JSON body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Path key is authoritative; derive site/series_id from it when the body
|
||||
// omits them so the stored row is always self-consistent.
|
||||
b.Key = key
|
||||
if b.Site == "" || b.SeriesID == "" {
|
||||
if site, series, ok := strings.Cut(key, ":"); ok {
|
||||
if b.Site == "" {
|
||||
b.Site = site
|
||||
}
|
||||
if b.SeriesID == "" {
|
||||
b.SeriesID = series
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An empty status is "no opinion" and Upsert keeps the stored bucket.
|
||||
// Finishing a series is a web-UI decision, so the JSON API refuses it
|
||||
// rather than trusting every client to leave it alone.
|
||||
switch b.Status {
|
||||
case "", statusReading, statusArchived:
|
||||
case statusFinished:
|
||||
http.Error(w, "status "+statusFinished+" can only be set from the web UI",
|
||||
http.StatusBadRequest)
|
||||
return
|
||||
default:
|
||||
http.Error(w, "invalid status", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Candidate timestamp, not a decision: Upsert keeps the stored one unless
|
||||
// reading progress actually moved. Any client value is ignored.
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
|
||||
stored, err := h.store.Upsert(b)
|
||||
if err != nil {
|
||||
log.Printf("upsert: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Echo the stored row: clients adopt this as their cached copy, so it must
|
||||
// carry the authoritative updated_at rather than the candidate above.
|
||||
writeJSON(w, http.StatusOK, stored)
|
||||
}
|
||||
|
||||
// delete removes one bookmark. DELETE /bookmarks/{key}
|
||||
func (h *bookmarkHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||
key := r.PathValue("key")
|
||||
if key == "" {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.Delete(key); err != nil {
|
||||
log.Printf("delete: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func healthz(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/httpmw"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// Handler serves the userscript-facing JSON bookmark API.
|
||||
type Handler struct {
|
||||
Store *store.Store
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if v != nil {
|
||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||
log.Printf("encode response: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// List returns all bookmarks of the acting Reader. GET /bookmarks
|
||||
func (h *Handler) List(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := h.Store.List(httpmw.ReaderID(r))
|
||||
if err != nil {
|
||||
log.Printf("list: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, items)
|
||||
}
|
||||
|
||||
// Put upserts one bookmark. PUT /bookmarks/{key}
|
||||
func (h *Handler) Put(w http.ResponseWriter, r *http.Request) {
|
||||
key := r.PathValue("key")
|
||||
if key == "" {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var b store.Bookmark
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&b); err != nil {
|
||||
http.Error(w, "invalid JSON body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// A body may carry a cover, and it is discarded here rather than
|
||||
// rejected: an older installed userscript may still send one, and
|
||||
// ADR-0004's compatibility argument depends on those scripts continuing
|
||||
// to work. The Cover is acquired server-side (ADR-0007), so the field is
|
||||
// permanently inert - not pending removal, and not a value any later code
|
||||
// should start reading.
|
||||
b.Cover = ""
|
||||
|
||||
// Path key is authoritative; derive site/series_id from it when the body
|
||||
// omits them so the stored row is always self-consistent.
|
||||
b.Key = key
|
||||
if b.Site == "" || b.SeriesID == "" {
|
||||
if site, series, ok := strings.Cut(key, ":"); ok {
|
||||
if b.Site == "" {
|
||||
b.Site = site
|
||||
}
|
||||
if b.SeriesID == "" {
|
||||
b.SeriesID = series
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An empty status is "no opinion" and Upsert keeps the stored bucket.
|
||||
// Finishing a series is a web-UI decision, so the JSON API refuses it
|
||||
// rather than trusting every client to leave it alone.
|
||||
switch b.Status {
|
||||
case "", store.StatusReading, store.StatusArchived:
|
||||
case store.StatusFinished:
|
||||
http.Error(w, "status "+store.StatusFinished+" can only be set from the web UI",
|
||||
http.StatusBadRequest)
|
||||
return
|
||||
default:
|
||||
http.Error(w, "invalid status", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Same rule as status: empty means "keep the stored value". An unknown
|
||||
// value is a client bug, not something to silently coerce to manga.
|
||||
switch b.Kind {
|
||||
case "", store.KindManga, store.KindNovel:
|
||||
default:
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid kind"})
|
||||
return
|
||||
}
|
||||
|
||||
// Candidate timestamp, not a decision: Upsert keeps the stored one unless
|
||||
// reading progress actually moved. Any client value is ignored.
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
|
||||
// A userscript PUT is a Sighting: the Reader's browser was on the Series
|
||||
// page and read its Latest Chapter (issue #103). Recorded before the
|
||||
// Upsert, which is what makes the raise comparison possible, and never
|
||||
// from the web UI's own read-modify-write — a Reader toggling a favourite
|
||||
// has not looked at the Site and must not postpone a Poll. A failure here
|
||||
// costs a deferral, not the write, so it is logged and dropped.
|
||||
readerID := httpmw.ReaderID(r)
|
||||
if err := h.Store.RecordSighting(readerID, b.Site, b.SeriesID, b.LatestChapterNum, b.UpdatedAt); err != nil {
|
||||
log.Printf("record sighting: %v", err)
|
||||
}
|
||||
|
||||
stored, err := h.Store.Upsert(readerID, b)
|
||||
if err != nil {
|
||||
log.Printf("upsert: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Echo the stored row: clients adopt this as their cached copy, so it must
|
||||
// carry the authoritative updated_at rather than the candidate above.
|
||||
writeJSON(w, http.StatusOK, stored)
|
||||
}
|
||||
|
||||
// Delete removes one bookmark. DELETE /bookmarks/{key}
|
||||
func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
|
||||
key := r.PathValue("key")
|
||||
if key == "" {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.Store.Delete(httpmw.ReaderID(r), key); err != nil {
|
||||
log.Printf("delete: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// Healthz answers the unauthenticated liveness check. GET /healthz
|
||||
func Healthz(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
// Cover serves stored cover bytes. GET /covers/{address}
|
||||
//
|
||||
// Public on purpose: the userscript renders these on Sites the deployment
|
||||
// does not control, where no credential of ours may be sent, and the address
|
||||
// is the SHA-256 of a URL the Site already publishes (ADR-0007). An unknown
|
||||
// address is a 404 rather than an error - "no Cover yet" is a normal state,
|
||||
// and the clients fall back to their placeholder.
|
||||
func (h *Handler) Cover(w http.ResponseWriter, r *http.Request) {
|
||||
body, contentType, ok, err := h.Store.CoverByAddress(r.PathValue("address"))
|
||||
if err != nil {
|
||||
log.Printf("cover: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
// Refuse anything the write gate would not have recorded: a poisoned row
|
||||
// (migrated data, a writer that skips the gate) must never be echoed back
|
||||
// as bytes of a type no Cover may have.
|
||||
if _, ok := store.CoverContentType(contentType); !ok {
|
||||
log.Printf("cover %s: refusing non-image content type %q", r.PathValue("address"), contentType)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
// Content-addressed, so the bytes at this URL can never change. Public
|
||||
// rather than private: no credential gates the route.
|
||||
w.Header().Set("Cache-Control", "public, max-age=604800, immutable")
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// acquireTimeout bounds one creation-time acquisition end to end: the series
|
||||
// page plus the cover bytes. Nothing is waiting on it — the Reader's write has
|
||||
// already returned — so this only stops a stalled Site from holding a
|
||||
// goroutine and a connection open forever.
|
||||
const acquireTimeout = 45 * time.Second
|
||||
|
||||
// Acquirer gives a Series its Latest Chapter and its Cover the moment the
|
||||
// first Bookmark creates it, instead of leaving the Reader to wait out the
|
||||
// poll queue — which is ordered by Reader count, so a Series with one Reader
|
||||
// sits behind every popular one (ADR-0007).
|
||||
//
|
||||
// Both facts come from a single series-page fetch, which is also why no
|
||||
// client-supplied cover hint is worth accepting: the page has to be fetched
|
||||
// for the chapter signal regardless, so a hint would save no request while
|
||||
// adding a client-controlled input to a server-side fetch.
|
||||
//
|
||||
// Every failure path is "log and move on". The Bookmark, its progress and its
|
||||
// Latest Chapter are already committed; a Site that is down or a Cover that
|
||||
// cannot be produced must not disturb any of them, and the Series is simply
|
||||
// left blank until the poll's own cover pass (#61) fills it.
|
||||
type Acquirer struct {
|
||||
Store *store.Store
|
||||
// Fetch retrieves the series page over plain TLS. Nil with a nil
|
||||
// BrowserFetch disables acquisition entirely.
|
||||
Fetch Fetcher
|
||||
// BrowserFetch retrieves kagane and novelfull pages through the browser
|
||||
// sidecar, the only thing that clears their Cloudflare challenge. The
|
||||
// per-site fallback policy lives in fetcherFor. Nil leaves those Sites
|
||||
// unacquired when no fallback applies.
|
||||
BrowserFetch Fetcher
|
||||
// Covers retrieves the cover bytes. Nil leaves the Cover blank and the
|
||||
// chapter half working.
|
||||
Covers CoverBytesFetcher
|
||||
// BrowserCoverFetch retrieves browser-claimed cover bytes through the
|
||||
// sidecar. Nil leaves those Covers blank; nothing falls back to a plain
|
||||
// fetch, which would only ever retrieve a challenge page.
|
||||
BrowserCoverFetch BrowserCoverFetcher
|
||||
// Ctx cancels in-flight acquisitions at shutdown. A hook signature has
|
||||
// nowhere to pass one, so it lives here; nil means context.Background.
|
||||
Ctx context.Context
|
||||
|
||||
inflight sync.WaitGroup
|
||||
}
|
||||
|
||||
// acquireSlots caps how many creation-time fetches run at once. A Reader whose
|
||||
// userscript bulk-syncs creates many Series at once, and a burst of
|
||||
// simultaneous requests from one server IP is the traffic shape most likely to
|
||||
// move that IP's bot score — the same reason the poller staggers its batch.
|
||||
var acquireSlots = make(chan struct{}, 2)
|
||||
|
||||
// Acquire starts one acquisition and returns immediately: a Reader's bookmark
|
||||
// action may not block on a third-party Site's latency, nor fail with it. It
|
||||
// is the store's OnSeriesCreated hook, so it only ever runs for a Series no
|
||||
// Reader had bookmarked before.
|
||||
func (a *Acquirer) Acquire(sr store.Series) {
|
||||
a.inflight.Add(1)
|
||||
go func() {
|
||||
defer a.inflight.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("acquire %q: recovered from panic: %v", sr.Key(), r)
|
||||
}
|
||||
}()
|
||||
parent := a.Ctx
|
||||
if parent == nil {
|
||||
parent = context.Background()
|
||||
}
|
||||
select {
|
||||
case acquireSlots <- struct{}{}:
|
||||
defer func() { <-acquireSlots }()
|
||||
case <-parent.Done():
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, acquireTimeout)
|
||||
defer cancel()
|
||||
a.acquire(ctx, sr)
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait blocks until every started acquisition has finished. It exists for
|
||||
// tests: an asynchronous side effect is otherwise unobservable without
|
||||
// polling for it.
|
||||
func (a *Acquirer) Wait() { a.inflight.Wait() }
|
||||
|
||||
func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
|
||||
if a.Fetch == nil && a.BrowserFetch == nil {
|
||||
return
|
||||
}
|
||||
facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, a.BrowserFetch, a.Fetch)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, errNotFetchable):
|
||||
// series_url arrives in a client-supplied PUT body, so without the
|
||||
// gate a token-holder chooses what the server fetches from its own
|
||||
// network position.
|
||||
log.Printf("acquire %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
||||
case errors.Is(err, errNoFetcher):
|
||||
log.Printf("acquire %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||
default:
|
||||
log.Printf("acquire %q: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// This page just served the same purpose a poll tick would have; without
|
||||
// the stamp the row stays due and the poller refetches it immediately.
|
||||
//
|
||||
// Stamped after success — the reverse of the poller, which stamps before
|
||||
// the fetch: the Reader is here, watching the Series they just created, so
|
||||
// a failed acquisition must leave the row due for a fast retry rather than
|
||||
// consuming the rest. The stamp happens even when the page read
|
||||
// succeeded but produced no facts to persist.
|
||||
if err := a.Store.MarkLatestChecked(sr.Site, sr.SeriesID, time.Now().UnixMilli()); err != nil {
|
||||
log.Printf("acquire %q: mark checked: %v", sr.Key(), err)
|
||||
}
|
||||
|
||||
if facts.HasLatest {
|
||||
if err := a.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil {
|
||||
log.Printf("acquire %q: set latest chapter: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
if !facts.HasCover {
|
||||
return
|
||||
}
|
||||
bytes, contentType, err := fetchCoverBytes(ctx, facts.Cover, a.BrowserCoverFetch, a.Covers)
|
||||
if err != nil {
|
||||
log.Printf("acquire %q: fetch cover %s: %v", sr.Key(), facts.Cover, err)
|
||||
return
|
||||
}
|
||||
if err := a.Store.SetSeriesCover(sr.Site, sr.SeriesID, facts.Cover, bytes, contentType); err != nil {
|
||||
log.Printf("acquire %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -1,457 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// The series page carries both facts, which is the whole argument for taking
|
||||
// them from one fetch.
|
||||
const asuraSeriesAndCoverFixture = asuraSeriesFixture + asuraCoverFixture
|
||||
|
||||
const (
|
||||
acquireKey = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
acquireSeriesID = "chronicles-of-the-demon-faction-f886a8af"
|
||||
acquireSeriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
acquireCoverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
|
||||
)
|
||||
|
||||
const (
|
||||
kaganeKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
kaganeSeriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
kaganeSeriesURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
kaganeImageID = "019fe11a-84c3-7fc3-a84b-88787374b617"
|
||||
kaganeCoverSrc = "https://kagane.to/api/v2/image/" + kaganeImageID + "/compressed"
|
||||
)
|
||||
|
||||
// kagane's browser-fetched body is one JSON object carrying both the chapter
|
||||
// list (series_books) and the cover image ids (series_covers), so the single
|
||||
// acquisition fetch yields both facts.
|
||||
const kaganeSeriesAndCoverFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",` +
|
||||
`"series_books":[{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41}],` +
|
||||
`"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en",` +
|
||||
`"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}]}`
|
||||
|
||||
const (
|
||||
novelfullKey = "novelfull:reverend-insanity"
|
||||
novelfullSeriesID = "reverend-insanity"
|
||||
novelfullSeriesURI = "https://novelfull.com/reverend-insanity.html"
|
||||
novelfullCoverURL = "https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp"
|
||||
)
|
||||
|
||||
// newAcquirer wires an acquirer onto the store's creation hook, which is how
|
||||
// main wires it: the write path is what starts an acquisition.
|
||||
func newAcquirer(s *store.Store, page *fakeFetcher, covers *fakeBytesCoverFetcher) *Acquirer {
|
||||
a := &Acquirer{Store: s, Fetch: page, Covers: covers}
|
||||
s.OnSeriesCreated = a.Acquire
|
||||
return a
|
||||
}
|
||||
|
||||
func bookmarkNewSeries(t *testing.T, s *store.Store, seriesURL string) store.Bookmark {
|
||||
t.Helper()
|
||||
stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: acquireKey, Site: "asura", SeriesID: acquireSeriesID,
|
||||
Title: "Chronicles of the Demon Faction", SeriesURL: seriesURL,
|
||||
Cover: "https://evil.example/client-supplied.jpg", UpdatedAt: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
return stored
|
||||
}
|
||||
|
||||
func readBookmark(t *testing.T, s *store.Store, key string) store.Bookmark {
|
||||
t.Helper()
|
||||
b, ok, err := s.Get(s.OwnerID(), key)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get %q = %v, %v", key, ok, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func bookmarkNewKaganeSeries(t *testing.T, s *store.Store) store.Bookmark {
|
||||
t.Helper()
|
||||
stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: kaganeKey, Site: "kagane", SeriesID: kaganeSeriesID,
|
||||
Title: "Infinite Decryption", SeriesURL: kaganeSeriesURL, UpdatedAt: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
return stored
|
||||
}
|
||||
|
||||
func bookmarkNewNovelfullSeries(t *testing.T, s *store.Store) store.Bookmark {
|
||||
t.Helper()
|
||||
stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: novelfullKey, Site: "novelfull", SeriesID: novelfullSeriesID,
|
||||
Title: "Reverend Insanity", SeriesURL: novelfullSeriesURI, UpdatedAt: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
return stored
|
||||
}
|
||||
|
||||
// The reported bug: a Reader bookmarks a Series nobody holds and expects the
|
||||
// Cover, not a broken image. Both facts come from the one series-page fetch.
|
||||
func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
|
||||
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
||||
acq := newAcquirer(s, page, covers)
|
||||
|
||||
// The write itself must not carry the acquisition: it returns before the
|
||||
// Cover exists, and the field is empty until the bytes land.
|
||||
stored := bookmarkNewSeries(t, s, acquireSeriesURL)
|
||||
if stored.Cover != "" {
|
||||
t.Fatalf("Cover on the creating write = %q, want empty", stored.Cover)
|
||||
}
|
||||
acq.Wait()
|
||||
|
||||
if got := page.callCount(); got != 1 {
|
||||
t.Fatalf("series page fetches = %d, want exactly 1", got)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, acquireKey)
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want)
|
||||
}
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(acquireCoverURL))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("stored cover = (%q, %q), want the fetched bytes", body, contentType)
|
||||
}
|
||||
}
|
||||
|
||||
// A pause governs the Lane only: a Reader's first bookmark of a Series on a
|
||||
// paused Site still reads the page, because acquisition is the creation-time
|
||||
// fetch, not the poll queue (issue #147).
|
||||
func TestAcquireIgnoresLanePause(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
if err := s.PauseLane("asura", time.Now().Add(6*time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
|
||||
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
||||
acq := newAcquirer(s, page, covers)
|
||||
|
||||
bookmarkNewSeries(t, s, acquireSeriesURL)
|
||||
acq.Wait()
|
||||
|
||||
if got := page.callCount(); got != 1 {
|
||||
t.Fatalf("series page fetches on a paused Site = %d, want 1", got)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, acquireKey)
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
||||
}
|
||||
}
|
||||
|
||||
// A Series that already exists is not re-acquired: no fetch, and the Cover it
|
||||
// already has is left alone.
|
||||
func TestAcquireSkipsAnExistingSeries(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
|
||||
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
||||
acq := newAcquirer(s, page, covers)
|
||||
|
||||
bookmarkNewSeries(t, s, acquireSeriesURL)
|
||||
acq.Wait()
|
||||
bookmarkNewSeries(t, s, acquireSeriesURL)
|
||||
acq.Wait()
|
||||
|
||||
if got := page.callCount(); got != 1 {
|
||||
t.Fatalf("series page fetches = %d, want 1 — an existing series is not re-acquired", got)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, acquireKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A Site that is down costs the Cover and nothing else.
|
||||
func TestAcquireFailureLeavesTheBookmarkIntact(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
page *fakeFetcher
|
||||
covers *fakeBytesCoverFetcher
|
||||
// wantLatest is the chapter that still lands; 0 means none did.
|
||||
wantLatest float64
|
||||
}{
|
||||
{
|
||||
"the series page is unreachable",
|
||||
&fakeFetcher{err: errors.New("connection reset")},
|
||||
&fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
||||
0,
|
||||
},
|
||||
{
|
||||
"the series page answers with a challenge",
|
||||
&fakeFetcher{body: challengeFixture, status: 200},
|
||||
&fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
||||
0,
|
||||
},
|
||||
{
|
||||
"only the cover bytes fail",
|
||||
&fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200},
|
||||
&fakeBytesCoverFetcher{err: errors.New("403")},
|
||||
181,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
acq := newAcquirer(s, tc.page, tc.covers)
|
||||
|
||||
stored := bookmarkNewSeries(t, s, acquireSeriesURL)
|
||||
acq.Wait()
|
||||
|
||||
got := readBookmark(t, s, acquireKey)
|
||||
if got.Cover != "" {
|
||||
t.Fatalf("Cover = %q, want empty rather than an address that 404s", got.Cover)
|
||||
}
|
||||
if got.Title != stored.Title || got.UpdatedAt != stored.UpdatedAt {
|
||||
t.Fatalf("bookmark = %+v, want it untouched by the failed acquisition", got)
|
||||
}
|
||||
if tc.wantLatest == 0 {
|
||||
if got.LatestChapterNum != nil {
|
||||
t.Fatalf("LatestChapterNum = %v, want none captured", *got.LatestChapterNum)
|
||||
}
|
||||
return
|
||||
}
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != tc.wantLatest {
|
||||
t.Fatalf("LatestChapterNum = %v, want %v", got.LatestChapterNum, tc.wantLatest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// series_url arrives in a client-supplied body, so the acquisition reuses the
|
||||
// poller's gate rather than deriving a second one: a non-https scheme, a
|
||||
// site the parsers do not know, or a host pinned to another site is refused
|
||||
// before the server spends a request from its own network position.
|
||||
func TestAcquireRefusesAnUnfetchableSeriesURL(t *testing.T) {
|
||||
for _, seriesURL := range []string{
|
||||
"http://asurascans.com/comics/x",
|
||||
"file:///etc/passwd",
|
||||
"",
|
||||
} {
|
||||
t.Run(seriesURL, func(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
|
||||
acq := newAcquirer(s, page, &fakeBytesCoverFetcher{})
|
||||
|
||||
bookmarkNewSeries(t, s, seriesURL)
|
||||
acq.Wait()
|
||||
|
||||
if got := page.callCount(); got != 0 {
|
||||
t.Fatalf("fetches for %q = %d, want 0", seriesURL, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// blockingFetcher stands in for a Site that never answers, so a synchronous
|
||||
// acquisition would be visible as a stalled write rather than a slow one.
|
||||
type blockingFetcher struct {
|
||||
release <-chan struct{}
|
||||
body string
|
||||
}
|
||||
|
||||
func (f *blockingFetcher) Get(ctx context.Context, _ string) (string, int, error) {
|
||||
select {
|
||||
case <-f.release:
|
||||
return f.body, 200, nil
|
||||
case <-ctx.Done():
|
||||
return "", 0, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// The Reader's write may not wait on a third-party Site: with the acquisition
|
||||
// wedged on an unanswering page, the PUT still returns.
|
||||
func TestAcquireDoesNotBlockTheWrite(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
release := make(chan struct{})
|
||||
acq := &Acquirer{Store: s, Fetch: &blockingFetcher{release: release, body: asuraSeriesAndCoverFixture}}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
upserted := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: acquireKey, Site: "asura", SeriesID: acquireSeriesID,
|
||||
Title: "Chronicles of the Demon Faction", SeriesURL: acquireSeriesURL, UpdatedAt: 1000,
|
||||
})
|
||||
upserted <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-upserted:
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the creating write blocked on the acquisition")
|
||||
}
|
||||
close(release)
|
||||
acq.Wait()
|
||||
}
|
||||
|
||||
// The second symptom of #47: a kagane Series bookmarked from a chapter page
|
||||
// gets its Cover at creation, with the bytes fetched through the browser
|
||||
// sidecar — the only path that clears the challenge — into the
|
||||
// content-addressed store.
|
||||
func TestAcquireKaganeCoverThroughBrowser(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
tlsPage := &fakeFetcher{body: "", status: 403}
|
||||
browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
|
||||
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
||||
acq := &Acquirer{
|
||||
Store: s, Fetch: tlsPage, BrowserFetch: browserPage,
|
||||
BrowserCoverFetch: covers, Covers: &fakeBytesCoverFetcher{},
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
bookmarkNewKaganeSeries(t, s)
|
||||
acq.Wait()
|
||||
|
||||
if got := tlsPage.callCount(); got != 0 {
|
||||
t.Fatalf("plain-TLS page fetches = %d, want 0 — kagane pages are browser-only", got)
|
||||
}
|
||||
if got := browserPage.callCount(); got != 1 {
|
||||
t.Fatalf("browser page fetches = %d, want 1", got)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("browser cover fetches = %d, want 1", got)
|
||||
}
|
||||
if got := covers.calls[0]; got != kaganeCoverSrc {
|
||||
t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc)
|
||||
}
|
||||
got := readBookmark(t, s, kaganeKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(kaganeCoverSrc); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
|
||||
}
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(kaganeCoverSrc))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/webp" {
|
||||
t.Fatalf("stored cover = (%q, %q), want the browser-fetched bytes", body, contentType)
|
||||
}
|
||||
}
|
||||
|
||||
// novelfull needs the browser only for its HTML: the cover URL comes out of
|
||||
// the browser-fetched page, but the bytes go over plain TLS through the
|
||||
// ordinary gated fetcher, never through the browser (issue #62).
|
||||
func TestAcquireNovelfullCoverOverPlainTLS(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
browserPage := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
|
||||
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
||||
acq := &Acquirer{
|
||||
Store: s, Fetch: &fakeFetcher{body: "", status: 403},
|
||||
BrowserFetch: browserPage, Covers: covers,
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
bookmarkNewNovelfullSeries(t, s)
|
||||
acq.Wait()
|
||||
|
||||
if got := browserPage.callCount(); got != 1 {
|
||||
t.Fatalf("browser page fetches = %d, want 1", got)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetches = %d, want 1 — novelfull bytes never touch the browser", got)
|
||||
}
|
||||
if got := covers.calls[0]; got != novelfullCoverURL {
|
||||
t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
|
||||
}
|
||||
got := readBookmark(t, s, novelfullKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
|
||||
// With no browser sidecar configured, kagane is simply not acquired: no
|
||||
// request is spent on a page that could only ever answer with a challenge,
|
||||
// and nothing falls back to a plain fetch.
|
||||
func TestAcquireKaganeSkippedWithoutBrowser(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
tlsPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
|
||||
acq := &Acquirer{
|
||||
Store: s, Fetch: tlsPage,
|
||||
Covers: &fakeBytesCoverFetcher{body: []byte("x"), contentType: "image/webp"},
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
bookmarkNewKaganeSeries(t, s)
|
||||
acq.Wait()
|
||||
|
||||
if got := tlsPage.callCount(); got != 0 {
|
||||
t.Fatalf("plain-TLS fetches for kagane = %d, want 0", got)
|
||||
}
|
||||
if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
|
||||
t.Fatalf("Cover = %q, want empty without a browser", got.Cover)
|
||||
}
|
||||
}
|
||||
|
||||
// The byte half of "nothing falls back to a plain fetch": with a browser for
|
||||
// the page but none for the bytes, a kagane Cover stays absent and the TLS
|
||||
// cover fetcher is never consulted.
|
||||
func TestAcquireKaganeBytesNeverFallBackToPlainTLS(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
|
||||
tlsCovers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
||||
acq := &Acquirer{
|
||||
Store: s, Fetch: &fakeFetcher{body: "", status: 403},
|
||||
BrowserFetch: browserPage, Covers: tlsCovers,
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
bookmarkNewKaganeSeries(t, s)
|
||||
acq.Wait()
|
||||
|
||||
if got := tlsCovers.callCount(); got != 0 {
|
||||
t.Fatalf("plain-TLS cover fetches = %d, want 0 — kagane bytes are browser-only", got)
|
||||
}
|
||||
if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
|
||||
t.Fatalf("Cover = %q, want empty without a browser cover fetcher", got.Cover)
|
||||
}
|
||||
}
|
||||
|
||||
// novelfull's no-browser degradation differs from kagane's: only its HTML
|
||||
// needs the sidecar, so when the page body is available — the challenge is a
|
||||
// live time-varying fact that sometimes answers a plain request — the Cover
|
||||
// still lands, bytes over plain TLS.
|
||||
func TestAcquireNovelfullCoverWithoutBrowser(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
tlsPage := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
|
||||
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
||||
acq := &Acquirer{Store: s, Fetch: tlsPage, Covers: covers}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
bookmarkNewNovelfullSeries(t, s)
|
||||
acq.Wait()
|
||||
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, novelfullKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/runtime"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
|
||||
// challenge clears in a few seconds when it clears at all; anything longer is a
|
||||
// challenge that is not going to pass, and the caller's rest was already
|
||||
// stamped before this ran.
|
||||
const challengeTimeout = 45 * time.Second
|
||||
|
||||
var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
||||
|
||||
// comixSeriesPathRe matches the one path shape comixRead will open: a Series
|
||||
// page, "/title/<id>-<slug>". Verified live 2026-08-12.
|
||||
var comixSeriesPathRe = regexp.MustCompile(`^/title/[^/?#]+/?$`)
|
||||
|
||||
// BrowserFetcher retrieves pages through a remote headless Chrome over the
|
||||
// DevTools Protocol.
|
||||
//
|
||||
// It exists for one reason: kagane.to, novelfull.com and comix.to sit behind a
|
||||
// Cloudflare JavaScript challenge. Verified 2026-08-03 (kagane), 2026-08-05
|
||||
// (novelfull) and 2026-08-12 (comix), plain HTTP and bogdanfinn/tls-client
|
||||
// with a Chrome_133 profile both get 403 with cf-mitigated: challenge on every
|
||||
// path, including the API, robots.txt and images. Clearing it requires
|
||||
// executing the challenge script, which only a real browser does.
|
||||
//
|
||||
// The request is made *inside* the page rather than by extracting cf_clearance
|
||||
// and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent
|
||||
// and often the TLS fingerprint, so replaying it means keeping three things in
|
||||
// sync that break silently and separately. The browser's own cookie jar
|
||||
// persists across polls, so the challenge is solved once every few hours.
|
||||
//
|
||||
// The three sites differ in what a cleared tab is asked for: kagane fetches a
|
||||
// JSON API from inside the page (the list exists nowhere else), comix fetches
|
||||
// its own Series URL from inside the page (the served HTML carries the facts,
|
||||
// and rendering the SPA costs ~65 requests instead of one), and novelfull
|
||||
// renders its list into the HTML so the cleared DOM is the payload.
|
||||
type BrowserFetcher struct {
|
||||
allocCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
// One page at a time: caps the browser's memory — it runs under a hard
|
||||
// cgroup cap on a shared machine — and keeps series from sharing page state.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
var _ Fetcher = (*BrowserFetcher)(nil)
|
||||
|
||||
// NewBrowserFetcher connects to a Chrome over CDP. The browser is not a
|
||||
// sidecar: it runs on a separate machine and is reached over the tailnet
|
||||
// (ADR-0006), so wsURL is that machine's tailnet address, e.g.
|
||||
// ws://100.64.0.5:9222.
|
||||
//
|
||||
// It must be an IP, never a hostname — not MagicDNS, not a Docker service
|
||||
// name. Chrome's DevTools HTTP handler 500s any /json/version request whose
|
||||
// Host header isn't an IP or "localhost" (confirmed 2026-08-03), so a name
|
||||
// fails at discovery and surfaces as a dead site rather than a bad URL.
|
||||
//
|
||||
// Do not add chromedp.NoModifyURL here: that option skips the /json/version
|
||||
// discovery request entirely and dials wsURL as if it were already the full
|
||||
// debugger endpoint, but Chrome only accepts connections at
|
||||
// /devtools/browser/<uuid>, a path chosen fresh at every Chrome start — dialing
|
||||
// the bare host:port 404s. The default (discovery) path works precisely
|
||||
// because Chrome's /json/version response echoes back the Host header of the
|
||||
// discovery request in webSocketDebuggerUrl, so as long as wsURL is an IP this
|
||||
// process can reach, the URL chromedp gets back already points at it. That is
|
||||
// also why a Chrome restarted behind a stable endpoint needs no reconnect
|
||||
// here: the fresh UUID arrives with the next discovery.
|
||||
func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
|
||||
if wsURL == "" {
|
||||
return nil, fmt.Errorf("empty browser websocket url")
|
||||
}
|
||||
ctx, cancel := chromedp.NewRemoteAllocator(context.Background(), wsURL)
|
||||
return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil
|
||||
}
|
||||
|
||||
func (f *BrowserFetcher) Close() {
|
||||
f.cancel()
|
||||
}
|
||||
|
||||
// Get navigates to seriesURL, lets any challenge resolve, then reads the
|
||||
// payload the Site's registry entry describes (the shapes are listed on
|
||||
// BrowserFetcher). The returned body is whatever the Site's chapter list lives
|
||||
// in, which is what the entry's LatestChapter parse expects.
|
||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||
var body string
|
||||
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
|
||||
// entries' Read funcs are expected to refuse any address owned by another
|
||||
// Site, and the loop must not depend on that staying true.
|
||||
for _, name := range browserBackedSites() {
|
||||
s := sites[name]
|
||||
read, ok := s.Browser.Read(seriesURL, &body)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := f.run(ctx, seriesURL, read,
|
||||
func() bool { return s.Browser.Done(body) }); err != nil {
|
||||
// Challenge never cleared, or the payload was refused.
|
||||
// Indistinguishable from here and handled identically by the caller.
|
||||
if errors.Is(err, errChallengeHeld) {
|
||||
return "", 403, nil
|
||||
}
|
||||
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
|
||||
}
|
||||
return body, 200, nil
|
||||
}
|
||||
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
|
||||
}
|
||||
|
||||
// kaganeRead builds the in-tab fetch of kagane's chapter-list API: the
|
||||
// request must be made from inside the page so it carries the clearance
|
||||
// cookie, and the API is the only place the list exists. Refusing any other
|
||||
// address is the per-Site half of the SSRF gate, kept deliberately behind
|
||||
// fetchableSeriesURL (see browserRead.Read).
|
||||
func kaganeRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
apiURL, ok := kaganeAPIURL(seriesURL)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return chromedp.Evaluate(
|
||||
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
|
||||
out, awaitPromise), true
|
||||
}
|
||||
|
||||
// novelfullRead reads the cleared DOM. novelfull renders its chapter list
|
||||
// into the served HTML, so there is no API to call from inside the page — the
|
||||
// challenge-cleared DOM is the payload.
|
||||
func novelfullRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
if !novelfullSeriesURL(seriesURL) {
|
||||
return nil, false
|
||||
}
|
||||
return chromedp.OuterHTML("html", out, chromedp.ByQuery), true
|
||||
}
|
||||
|
||||
// comixRead fetches the Series page from inside the cleared tab. comix is an
|
||||
// SPA: rendering the page costs ~65 requests, while one same-origin fetch of
|
||||
// the same address returns the server-rendered HTML — 24.5 KB, ~480 ms,
|
||||
// carrying both parser anchors (measured 2026-08-12, issue #98). So this is
|
||||
// kaganeRead's shape, not novelfullRead's, even though the payload is HTML.
|
||||
// Refusing any other address is the per-Site half of the SSRF gate.
|
||||
func comixRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
pageURL, ok := comixSeriesPageURL(seriesURL)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return chromedp.Evaluate(
|
||||
`fetch(`+jsString(pageURL)+`).then(r => r.ok ? r.text() : "")`,
|
||||
out, awaitPromise), true
|
||||
}
|
||||
|
||||
// Image retrieves one cover's bytes through the browser sidecar, and its
|
||||
// content type.
|
||||
//
|
||||
// It exists because kagane and comix serve covers behind the same challenge as
|
||||
// their pages — kagane additionally with
|
||||
// `cross-origin-resource-policy: same-origin` — so the bytes are only
|
||||
// reachable from inside a browser that already holds the clearance cookie
|
||||
// (verified 2026-08-08 for kagane, 2026-08-12 for comix). Acquisition through
|
||||
// the sidecar is the only route.
|
||||
//
|
||||
// The image URL is navigated to rather than fetched from another page of the
|
||||
// Site: the challenge only runs on a top-level navigation, and once it clears
|
||||
// the document *is* the image, so a same-origin fetch of location.href reads
|
||||
// it straight back out of the cache. For comix the navigation is also the only
|
||||
// route that works at all — its Series page sets
|
||||
// `cross-origin-embedder-policy: require-corp`, which fails a page-context
|
||||
// fetch of the cover host.
|
||||
//
|
||||
// The challenge is not solved by the first read: WaitReady("body") is satisfied
|
||||
// by the interstitial too. run holds the tab open until the in-page fetch
|
||||
// succeeds, which is what gives the challenge script the seconds it needs.
|
||||
func (f *BrowserFetcher) Image(ctx context.Context, imageURL string) ([]byte, string, error) {
|
||||
if !browserOnlyCoverURL(imageURL) {
|
||||
return nil, "", fmt.Errorf("not a browser-fetchable cover url: %q", imageURL)
|
||||
}
|
||||
var dataURL string
|
||||
err := f.run(ctx, imageURL,
|
||||
chromedp.Evaluate(`fetch(location.href).then(r => r.ok
|
||||
? r.blob().then(b => new Promise(res => {
|
||||
const fr = new FileReader();
|
||||
fr.onload = () => res(fr.result);
|
||||
fr.readAsDataURL(b);
|
||||
}))
|
||||
: "")`, &dataURL, awaitPromise),
|
||||
func() bool { return dataURL != "" })
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageURL, err)
|
||||
}
|
||||
// "data:image/webp;base64,<payload>".
|
||||
head, payload, ok := strings.Cut(dataURL, ";base64,")
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("browser image %s: not a data url", imageURL)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(payload)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageURL, err)
|
||||
}
|
||||
return raw, strings.TrimPrefix(head, "data:"), nil
|
||||
}
|
||||
|
||||
// errChallengeHeld reports that the budget ran out with the interstitial still
|
||||
// up. Distinct from a transport failure: it means "this site said no", which
|
||||
// the poller answers with a refusal backoff for that Site's Lane (issue #100).
|
||||
var errChallengeHeld = errors.New("challenge held")
|
||||
|
||||
// errBrowserInterrupted distinguishes a remote Chrome restart from the
|
||||
// caller's own deadline. chromedp reports both as context.Canceled.
|
||||
var errBrowserInterrupted = errors.New("browser interrupted")
|
||||
|
||||
func classifyBrowserError(ctx context.Context, browserLost bool, err error) error {
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
if !browserLost {
|
||||
return err
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", errBrowserInterrupted, err)
|
||||
}
|
||||
|
||||
func browserConnectionLost(ctx context.Context) bool {
|
||||
c := chromedp.FromContext(ctx)
|
||||
if c == nil || c.Browser == nil {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case <-c.Browser.LostConnection:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// challengePollInterval paces re-reads while a challenge solves itself.
|
||||
const challengePollInterval = 2 * time.Second
|
||||
|
||||
// isInterstitial reports whether html is Cloudflare's challenge page rather
|
||||
// than the site's own. Matched on the challenge orchestration path
|
||||
// (/cdn-cgi/challenge-platform/h/<b|g|x>/orchestrate/...), which is stable
|
||||
// across the interstitial's wording and locale — the visible "Just a
|
||||
// moment..." title is neither.
|
||||
//
|
||||
// The bare "/cdn-cgi/challenge-platform/" prefix is NOT enough: Cloudflare
|
||||
// injects /cdn-cgi/challenge-platform/scripts/jsd/main.js into ordinary 200
|
||||
// pages when JS detections are on, so matching the prefix declared every real
|
||||
// demonic page a refusal and parked that Lane in 15m backoff (observed
|
||||
// 2026-08-16, demonic turned detections on).
|
||||
func isInterstitial(html string) bool {
|
||||
return strings.Contains(html, "/cdn-cgi/challenge-platform/h/")
|
||||
}
|
||||
|
||||
// run navigates to target and re-reads until done reports an answer, bounded by
|
||||
// challengeTimeout and by the caller's own deadline, in a tab that is closed on
|
||||
// return so one wedged page cannot poison later calls.
|
||||
//
|
||||
// Holding the tab open across re-reads is the whole point. A Cloudflare
|
||||
// interstitial needs several seconds of a live page to solve itself and write
|
||||
// clearance into the browser's shared cookie jar; reading once and closing the
|
||||
// tab — which is what this did before 2026-08-08 — never gives it that window,
|
||||
// so every fetch lands on the interstitial and the clearance that would have
|
||||
// unblocked all the later ones is never obtained.
|
||||
func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.Action, done func() bool) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
callerCtx := ctx
|
||||
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
|
||||
defer cancel()
|
||||
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
|
||||
defer cancelTab()
|
||||
// Bind the caller's deadline to the tab.
|
||||
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
|
||||
defer cancelDeadline()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cancelDeadline()
|
||||
}()
|
||||
|
||||
if err := chromedp.Run(tabCtx,
|
||||
chromedp.Navigate(target),
|
||||
chromedp.WaitReady("body", chromedp.ByQuery),
|
||||
); err != nil {
|
||||
return classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
|
||||
}
|
||||
var lastErr error
|
||||
for {
|
||||
// The challenge reloads the page when it passes, which tears down the
|
||||
// execution context mid-read. That is a retry, not a failure.
|
||||
if err := chromedp.Run(tabCtx, read); err != nil {
|
||||
err = classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
|
||||
if errors.Is(err, errBrowserInterrupted) {
|
||||
return err
|
||||
}
|
||||
lastErr = err
|
||||
} else if done() {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if err := callerCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if lastErr != nil {
|
||||
return fmt.Errorf("%w (last read: %v)", errChallengeHeld, lastErr)
|
||||
}
|
||||
return errChallengeHeld
|
||||
case <-time.After(challengePollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
|
||||
// chapter list. Returning false for anything else is a second line of defence
|
||||
// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
|
||||
// series_url is client-supplied, so the host is pinned here too.
|
||||
func kaganeAPIURL(seriesURL string) (string, bool) {
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil || u.Scheme != "https" || u.Hostname() != "kagane.to" {
|
||||
return "", false
|
||||
}
|
||||
m := kaganeSeriesRe.FindStringSubmatch(u.Path)
|
||||
if m == nil {
|
||||
return "", false
|
||||
}
|
||||
return "https://kagane.to/api/v2/series/" + m[1], true
|
||||
}
|
||||
|
||||
// novelfullSeriesURL reports whether seriesURL is a novelfull series page this
|
||||
// fetcher will open. novelfull's chapter list is in the served HTML, so unlike
|
||||
// kagane there is no API to call from inside the page — the challenge-cleared
|
||||
// DOM is the payload. The host is pinned here for the same reason kagane's is:
|
||||
// series_url is client-supplied and a headless browser is a strong SSRF
|
||||
// primitive.
|
||||
func novelfullSeriesURL(seriesURL string) bool {
|
||||
u, err := url.Parse(seriesURL)
|
||||
return err == nil && u.Scheme == "https" && u.Hostname() == "novelfull.com" &&
|
||||
strings.HasSuffix(u.Path, ".html")
|
||||
}
|
||||
|
||||
// comixSeriesPageURL returns the address comixRead fetches inside the tab: the
|
||||
// Series page itself, rebuilt from the pinned host and path so nothing else
|
||||
// travels. Host-pinned here for the same reason kagane's is — series_url is
|
||||
// client-supplied and a headless browser is a strong SSRF primitive.
|
||||
func comixSeriesPageURL(seriesURL string) (string, bool) {
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil || u.Scheme != "https" || u.Hostname() != "comix.to" ||
|
||||
!comixSeriesPathRe.MatchString(u.Path) {
|
||||
return "", false
|
||||
}
|
||||
return "https://comix.to" + u.Path, true
|
||||
}
|
||||
|
||||
// awaitPromise makes Evaluate resolve the promise rather than returning a
|
||||
// serialised Promise object.
|
||||
func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
|
||||
return p.WithAwaitPromise(true)
|
||||
}
|
||||
|
||||
// jsString renders s as a JavaScript string literal for embedding in an
|
||||
// Evaluate expression. The URL is host-pinned by kaganeAPIURL before it gets
|
||||
// here, but quoting it properly is what keeps that guarantee intact.
|
||||
func jsString(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKaganeAPIURL(t *testing.T) {
|
||||
const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
tests := []struct {
|
||||
name string
|
||||
seriesURL string
|
||||
want string
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "series page maps to its API endpoint",
|
||||
seriesURL: "https://kagane.to/series/" + uuid,
|
||||
want: "https://kagane.to/api/v2/series/" + uuid,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "trailing slash is tolerated",
|
||||
seriesURL: "https://kagane.to/series/" + uuid + "/",
|
||||
want: "https://kagane.to/api/v2/series/" + uuid,
|
||||
wantOK: true,
|
||||
},
|
||||
{"not a series path", "https://kagane.to/search", "", false},
|
||||
{"foreign host", "https://evil.example/series/" + uuid, "", false},
|
||||
{"garbage", "://", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := kaganeAPIURL(tt.seriesURL)
|
||||
if ok != tt.wantOK || got != tt.want {
|
||||
t.Errorf("kaganeAPIURL(%q) = %q, %v; want %q, %v",
|
||||
tt.seriesURL, got, ok, tt.want, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNovelfullSeriesURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
want bool
|
||||
}{
|
||||
{"series page", "https://novelfull.com/reverend-insanity.html", true},
|
||||
{"foreign host", "https://evil.example/reverend-insanity.html", false},
|
||||
{"not https", "http://novelfull.com/reverend-insanity.html", false},
|
||||
{"not a series page", "https://novelfull.com/genre/Fantasy", false},
|
||||
{"garbage", "://nope", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := novelfullSeriesURL(tc.url); got != tc.want {
|
||||
t.Fatalf("novelfullSeriesURL(%q) = %v, want %v", tc.url, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestComixSeriesPageURL(t *testing.T) {
|
||||
const series = "https://comix.to/title/n8we-dungeons-and-crayons"
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
want string
|
||||
}{
|
||||
{"series page", series, series},
|
||||
{"trailing slash kept", series + "/", series + "/"},
|
||||
// Query and fragment are dropped: only the pinned path travels.
|
||||
{"query dropped", series + "?tab=chapters", series},
|
||||
{"foreign host", "https://evil.example/title/x", ""},
|
||||
{"lookalike host", "https://comix.to.evil.example/title/x", ""},
|
||||
{"not https", "http://comix.to/title/x", ""},
|
||||
{"not a series path", "https://comix.to/search", ""},
|
||||
{"chapter page", series + "/11139891-chapter-80", ""},
|
||||
{"garbage", "://nope", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, ok := comixSeriesPageURL(tc.url)
|
||||
if ok != (tc.want != "") || got != tc.want {
|
||||
t.Fatalf("comixSeriesPageURL(%q) = %q, %v; want %q", tc.url, got, ok, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The browser is an SSRF primitive and a cover address can originate in a
|
||||
// client-supplied PUT body, so this gate decides what it may navigate to.
|
||||
func TestBrowserOnlyCoverURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
url string
|
||||
want bool
|
||||
}{
|
||||
{"https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg", true},
|
||||
{"https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed", true},
|
||||
// Every other Site's CDN answers plain TLS.
|
||||
{"https://gg.asuracomic.net/covers/x.webp", false},
|
||||
{"http://static.comix.to/039d/x.jpg", false},
|
||||
{"https://static.comix.to.evil.example/039d/x.jpg", false},
|
||||
{"https://evil.example/static.comix.to/x.jpg", false},
|
||||
{"https://static.comix.to/039d/x.jpg?next=http://169.254.169.254/", false},
|
||||
{"https://static.comix.to/039d/x.svg", false},
|
||||
{"https://static.comix.to/../etc/passwd.jpg", false},
|
||||
{"https://static.comix.to/", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.url, func(t *testing.T) {
|
||||
if got := browserOnlyCoverURL(tc.url); got != tc.want {
|
||||
t.Fatalf("browserOnlyCoverURL(%q) = %v, want %v", tc.url, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The jsd script is injected into ordinary 200 pages when a zone turns JS
|
||||
// detections on; only the orchestration path means the page itself is the
|
||||
// challenge. Conflating the two parked the demonic Lane in refusal backoff
|
||||
// while every fetch was in fact the real series page (observed 2026-08-16).
|
||||
func TestIsInterstitial(t *testing.T) {
|
||||
if !isInterstitial(challengeFixture) {
|
||||
t.Fatal("challenge page not detected as interstitial")
|
||||
}
|
||||
const jsdInjected = `<html><head><title>The Possessed Grappler</title>
|
||||
<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head>
|
||||
<body><a href="/chaptered.php?manga=13721&chapter=22">Chapter 22</a></body></html>`
|
||||
if isInterstitial(jsdInjected) {
|
||||
t.Fatal("real page carrying the injected jsd script misread as interstitial")
|
||||
}
|
||||
if got, ok := demonicLatestChapter("", jsdInjected); !ok || got.Label != "Chapter 22" {
|
||||
t.Fatalf("demonicLatestChapter = %+v, ok = %v, want Chapter 22", got, ok)
|
||||
}
|
||||
}
|
||||
func TestClassifyBrowserInterruption(t *testing.T) {
|
||||
if err := classifyBrowserError(context.Background(), true, context.Canceled); !errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("classifyBrowserError(context.Canceled) = %v, want browser interruption", err)
|
||||
}
|
||||
if err := classifyBrowserError(context.Background(), false, context.Canceled); errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("ordinary cancellation misclassified as browser interruption: %v", err)
|
||||
}
|
||||
|
||||
caller, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := classifyBrowserError(caller, true, context.Canceled); errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("caller cancellation misclassified as browser interruption: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// CoverBytesFetcher retrieves one cover from its source URL. The caller owns
|
||||
// persistence; this seam keeps network policy independent from the store.
|
||||
type CoverBytesFetcher interface {
|
||||
Fetch(ctx context.Context, sourceURL string) (body []byte, contentType string, err error)
|
||||
}
|
||||
|
||||
// fetchCoverBytes routes a cover's byte retrieval by URL shape, not by Site
|
||||
// name: the browser fetcher's module claims the addresses only it can fetch
|
||||
// (kagane's image route answers a plain fetch with a challenge and
|
||||
// `cross-origin-resource-policy: same-origin`, static.comix.to answers one with
|
||||
// the same challenge its pages serve), and everything else goes over
|
||||
// plain TLS. Missing fetchers degrade to an error the caller logs, never a
|
||||
// fallback onto a path that cannot succeed. One routing rule for the poll and
|
||||
// the acquirer, so the two cannot drift apart.
|
||||
func fetchCoverBytes(ctx context.Context, cover string, browser BrowserCoverFetcher, tls CoverBytesFetcher) ([]byte, string, error) {
|
||||
if browserOnlyCoverURL(cover) {
|
||||
if browser == nil {
|
||||
return nil, "", errors.New("no cover fetcher")
|
||||
}
|
||||
return browser.Image(ctx, cover)
|
||||
}
|
||||
if tls == nil {
|
||||
return nil, "", errors.New("no cover fetcher")
|
||||
}
|
||||
return tls.Fetch(ctx, cover)
|
||||
}
|
||||
|
||||
// CoverResolver resolves a host before any connection is attempted. Tests
|
||||
// inject it to exercise hostile DNS results without touching the live network.
|
||||
type CoverResolver func(context.Context, string) ([]netip.Addr, error)
|
||||
|
||||
// maxCoverBytes caps one cover, separately from the series-page maxBodyBytes:
|
||||
// a cover is a bounded binary asset, not a text page, and 4 MiB rejected 12%
|
||||
// of asurascans covers measured 2026-08-17 (p90 4.52 MB, max 8.57 MB — two of
|
||||
// the three over-cap files were JPEGs, not the animated GIF of issue #71).
|
||||
// 10 MiB is ~18% headroom over that worst case and matches the GitHub and
|
||||
// Discord image limits; see docs/research/gif-maximum-byte-size.md. GIF itself
|
||||
// has no maximum size, so this number is policy, not format.
|
||||
const maxCoverBytes = 10 << 20
|
||||
|
||||
// TLSCoverFetcher retrieves image bytes with the standard HTTPS client. Unlike
|
||||
// TLSFetcher, it does not need a browser fingerprint: cover hosts are public
|
||||
// CDNs and the response is accepted only after the destination gate passes.
|
||||
type TLSCoverFetcher struct {
|
||||
client *http.Client
|
||||
resolve CoverResolver
|
||||
}
|
||||
|
||||
var _ CoverBytesFetcher = (*TLSCoverFetcher)(nil)
|
||||
|
||||
const coverRequestTimeout = 30 * time.Second
|
||||
|
||||
var carrierGradeNAT = netip.MustParsePrefix("100.64.0.0/10")
|
||||
|
||||
// NewCoverFetcher builds the production cover client with the real resolver.
|
||||
func NewCoverFetcher() *TLSCoverFetcher {
|
||||
return NewCoverFetcherWithResolver(nil)
|
||||
}
|
||||
|
||||
// NewCoverFetcherWithResolver builds a cover client using resolve, or the real
|
||||
// system resolver when resolve is nil.
|
||||
func NewCoverFetcherWithResolver(resolve CoverResolver) *TLSCoverFetcher {
|
||||
if resolve == nil {
|
||||
resolve = defaultCoverResolver
|
||||
}
|
||||
return newCoverFetcher(newCoverHTTPClient(resolve), resolve)
|
||||
}
|
||||
|
||||
func newCoverFetcher(client *http.Client, resolve CoverResolver) *TLSCoverFetcher {
|
||||
f := &TLSCoverFetcher{client: client, resolve: resolve}
|
||||
client.CheckRedirect = func(req *http.Request, _ []*http.Request) error {
|
||||
if err := f.validateURL(req.Context(), req.URL); err != nil {
|
||||
return fmt.Errorf("redirect destination: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func defaultCoverResolver(ctx context.Context, host string) ([]netip.Addr, error) {
|
||||
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
||||
}
|
||||
|
||||
func newCoverHTTPClient(resolve CoverResolver) *http.Client {
|
||||
base, ok := http.DefaultTransport.(*http.Transport)
|
||||
if !ok {
|
||||
base = &http.Transport{}
|
||||
}
|
||||
transport := base.Clone()
|
||||
// A proxy would make the dial target the proxy rather than the cover host,
|
||||
// defeating destination classification. Cover fetching is direct by design.
|
||||
transport.Proxy = nil
|
||||
dialer := &net.Dialer{}
|
||||
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("split cover address %q: %w", address, err)
|
||||
}
|
||||
addrs, err := resolveCoverHost(ctx, host, resolve)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if !publicCoverAddress(addr) {
|
||||
return nil, fmt.Errorf("cover host resolves to refused address %s", addr)
|
||||
}
|
||||
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(addr.String(), port))
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("cover host %q has no reachable address", host)
|
||||
}
|
||||
return &http.Client{Transport: transport, Timeout: coverRequestTimeout}
|
||||
}
|
||||
|
||||
func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
|
||||
u, err := url.Parse(sourceURL)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("parse cover URL: %w", err)
|
||||
}
|
||||
if err := f.validateURL(ctx, u); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("build cover request: %w", err)
|
||||
}
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("fetch cover: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", fmt.Errorf("fetch cover: status %d", resp.StatusCode)
|
||||
}
|
||||
raw, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", resp.Header.Get("Content-Type"))
|
||||
}
|
||||
contentType, ok := store.CoverContentType(raw)
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", raw)
|
||||
}
|
||||
if resp.ContentLength > maxCoverBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxCoverBytes)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverBytes+1))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("read cover: %w", err)
|
||||
}
|
||||
if len(body) > maxCoverBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxCoverBytes)
|
||||
}
|
||||
return body, contentType, nil
|
||||
}
|
||||
|
||||
// This gate deliberately differs from fetchableSeriesURL: cover hosts are
|
||||
// site-independent CDNs, so a Site host allowlist would reject valid covers.
|
||||
func (f *TLSCoverFetcher) validateURL(ctx context.Context, u *url.URL) error {
|
||||
if u == nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
|
||||
return errors.New("cover URL must use HTTPS without credentials")
|
||||
}
|
||||
host := u.Hostname()
|
||||
if host == "" {
|
||||
return errors.New("cover URL has no host")
|
||||
}
|
||||
addrs, err := resolveCoverHost(ctx, host, f.resolve)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve cover host %q: %w", host, err)
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return fmt.Errorf("resolve cover host %q: no addresses", host)
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if !publicCoverAddress(addr) {
|
||||
return fmt.Errorf("cover host %q resolves to refused address %s", host, addr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveCoverHost(ctx context.Context, host string, resolve CoverResolver) ([]netip.Addr, error) {
|
||||
if literal, err := netip.ParseAddr(host); err == nil {
|
||||
return []netip.Addr{literal.Unmap()}, nil
|
||||
}
|
||||
return resolve(ctx, strings.TrimSuffix(host, "."))
|
||||
}
|
||||
|
||||
func publicCoverAddress(addr netip.Addr) bool {
|
||||
addr = addr.Unmap()
|
||||
return addr.IsValid() && addr.IsGlobalUnicast() &&
|
||||
!addr.IsLoopback() && !addr.IsPrivate() && !addr.IsLinkLocalUnicast() &&
|
||||
!carrierGradeNAT.Contains(addr)
|
||||
}
|
||||
@@ -1,249 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCoverFetcherFetchesPublicHTTPSImage(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.TLS == nil {
|
||||
t.Fatal("cover request was not made over TLS")
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
io.WriteString(w, "cover-bytes")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
transport := server.Client().Transport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // test server certificate
|
||||
transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String())
|
||||
}
|
||||
client := &http.Client{Transport: transport}
|
||||
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
body, contentType, err := fetcher.Fetch(context.Background(), "https://cdn.example/cover.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("Fetch = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCoverFetcherRechecksResolverBeforeConnection(t *testing.T) {
|
||||
var requests int
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
requests++
|
||||
}))
|
||||
defer server.Close()
|
||||
_, port, err := net.SplitHostPort(server.Listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("server address: %v", err)
|
||||
}
|
||||
|
||||
resolves := 0
|
||||
fetcher := NewCoverFetcherWithResolver(func(context.Context, string) ([]netip.Addr, error) {
|
||||
resolves++
|
||||
if resolves == 1 {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
}
|
||||
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
|
||||
})
|
||||
_, _, err = fetcher.Fetch(context.Background(), "https://cdn.example:"+port+"/cover.jpg")
|
||||
if err == nil {
|
||||
t.Fatal("Fetch accepted a destination that became private")
|
||||
}
|
||||
if resolves != 2 {
|
||||
t.Fatalf("resolver calls = %d, want preflight and dial checks", resolves)
|
||||
}
|
||||
if requests != 0 {
|
||||
t.Fatalf("requests = %d, want 0", requests)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func coverResponse(status int, contentType, location string, body []byte) *http.Response {
|
||||
header := make(http.Header)
|
||||
if contentType != "" {
|
||||
header.Set("Content-Type", contentType)
|
||||
}
|
||||
if location != "" {
|
||||
header.Set("Location", location)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Header: header,
|
||||
Body: io.NopCloser(bytes.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverFetcherRefusesUnsafeDestinationsBeforeRequest(t *testing.T) {
|
||||
var calls int
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return coverResponse(http.StatusOK, "image/jpeg", "", []byte("must not reach network")), nil
|
||||
})}
|
||||
resolve := func(_ context.Context, host string) ([]netip.Addr, error) {
|
||||
switch host {
|
||||
case "loopback.example":
|
||||
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
|
||||
case "private.example":
|
||||
return []netip.Addr{netip.MustParseAddr("10.0.0.1")}, nil
|
||||
case "linklocal.example":
|
||||
return []netip.Addr{netip.MustParseAddr("169.254.1.1")}, nil
|
||||
case "unique-local.example":
|
||||
return []netip.Addr{netip.MustParseAddr("fc00::1")}, nil
|
||||
case "cgnat.example":
|
||||
return []netip.Addr{netip.MustParseAddr("100.64.0.1")}, nil
|
||||
default:
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
}
|
||||
}
|
||||
fetcher := newCoverFetcher(client, resolve)
|
||||
|
||||
tests := []string{
|
||||
"http://public.example/cover.jpg",
|
||||
"https://127.0.0.1/cover.jpg",
|
||||
"https://10.0.0.1/cover.jpg",
|
||||
"https://169.254.1.1/cover.jpg",
|
||||
"https://[fc00::1]/cover.jpg",
|
||||
"https://100.64.0.1/cover.jpg",
|
||||
"https://loopback.example/cover.jpg",
|
||||
"https://private.example/cover.jpg",
|
||||
"https://linklocal.example/cover.jpg",
|
||||
"https://unique-local.example/cover.jpg",
|
||||
"https://cgnat.example/cover.jpg",
|
||||
}
|
||||
for _, sourceURL := range tests {
|
||||
t.Run(sourceURL, func(t *testing.T) {
|
||||
calls = 0
|
||||
if _, _, err := fetcher.Fetch(context.Background(), sourceURL); err == nil {
|
||||
t.Fatal("Fetch accepted refused destination")
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Fatalf("network calls = %d, want 0", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverFetcherStopsRedirectIntoPrivateAddress(t *testing.T) {
|
||||
var calls int
|
||||
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if req.URL.Hostname() != "cdn.example" {
|
||||
t.Fatalf("redirect reached %s", req.URL)
|
||||
}
|
||||
return coverResponse(http.StatusFound, "", "https://internal.example/cover.jpg", nil), nil
|
||||
})}
|
||||
fetcher := newCoverFetcher(client, func(_ context.Context, host string) ([]netip.Addr, error) {
|
||||
if host == "internal.example" {
|
||||
return []netip.Addr{netip.MustParseAddr("192.168.1.1")}, nil
|
||||
}
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/cover.jpg"); err == nil {
|
||||
t.Fatal("Fetch followed redirect into private address")
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("network calls = %d, want only public first hop", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverFetcherRejectsOversizedBody(t *testing.T) {
|
||||
var calls int
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxCoverBytes+1))
|
||||
response.ContentLength = -1
|
||||
return response, nil
|
||||
})}
|
||||
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/large.webp"); err == nil {
|
||||
t.Fatal("Fetch accepted oversized body")
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("network calls = %d, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Covers between the series-page cap and the cover cap must be accepted: the
|
||||
// 4 MiB page cap rejected 12% of asurascans covers (issue #71).
|
||||
func TestCoverFetcherAcceptsCoverOverPageCap(t *testing.T) {
|
||||
body := bytes.Repeat([]byte("x"), maxBodyBytes+1)
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return coverResponse(http.StatusOK, "image/gif", "", body), nil
|
||||
})}
|
||||
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
got, contentType, err := fetcher.Fetch(context.Background(), "https://cdn.example/big.gif")
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch rejected a %d-byte cover: %v", len(body), err)
|
||||
}
|
||||
if len(got) != len(body) {
|
||||
t.Fatalf("body = %d bytes, want %d", len(got), len(body))
|
||||
}
|
||||
if contentType != "image/gif" {
|
||||
t.Fatalf("content type = %q, want image/gif", contentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverFetcherRejectsNonImage(t *testing.T) {
|
||||
var calls int
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return coverResponse(http.StatusOK, "text/html", "", []byte("challenge")), nil
|
||||
})}
|
||||
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/challenge"); err == nil {
|
||||
t.Fatal("Fetch accepted non-image response")
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("network calls = %d, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// comix labels its covers "image/jpg", which is not a registered type but is
|
||||
// what the Site actually answers with; the bytes are stored under the real
|
||||
// name so one image cannot land under two spellings.
|
||||
func TestCoverFetcherCanonicalisesJpgAlias(t *testing.T) {
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return coverResponse(http.StatusOK, "image/jpg", "", []byte("cover-bytes")), nil
|
||||
})}
|
||||
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
body, contentType, err := fetcher.Fetch(context.Background(), "https://static.comix.to/cover.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("Fetch = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
||||
}
|
||||
}
|
||||
@@ -1,732 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// Fetcher retrieves a series page. It exists as an interface so tests can inject
|
||||
// a fake: nothing in the test suite may touch the network or the TLS client.
|
||||
type Fetcher interface {
|
||||
Get(ctx context.Context, url string) (body string, status int, err error)
|
||||
}
|
||||
|
||||
// BrowserCoverFetcher retrieves one cover's bytes through the browser-backed
|
||||
// path — the only route that clears the challenge kagane's and comix's image
|
||||
// URLs answer a plain fetch with. Satisfied by BrowserFetcher.
|
||||
type BrowserCoverFetcher interface {
|
||||
Image(ctx context.Context, imageURL string) (body []byte, contentType string, err error)
|
||||
}
|
||||
|
||||
// Poller re-checks each bookmarked series' newest published chapter on a
|
||||
// schedule, independent of the userscript's own in-browser checks. The two run
|
||||
// in parallel and report the same observable fact, so whichever writes last wins
|
||||
// and neither needs to know about the other.
|
||||
//
|
||||
// Every Site gets its own Poll Lane: one independent stream of Polls with its
|
||||
// own pace, running concurrently with every other Site's (issue #100). Rest
|
||||
// time and gap live in the Site registry, not here — see sites.go. Rest is
|
||||
// enforced by the WHERE clause in DueForLatestCheck rather than by any timer;
|
||||
// the gap is enforced by the Lane sleeping between fetches.
|
||||
type Poller struct {
|
||||
Store *store.Store
|
||||
Fetch Fetcher
|
||||
// BrowserFetch handles sites behind a JavaScript challenge that Fetch
|
||||
// cannot clear. Nil disables those sites entirely rather than falling back
|
||||
// to Fetch, which would only ever retrieve a challenge page.
|
||||
BrowserFetch Fetcher
|
||||
// CoverFetch is optional; failures are logged and never affect the chapter poll.
|
||||
CoverFetch BrowserCoverFetcher
|
||||
// CoverBytesFetch is optional; it handles plain-TLS sources through the
|
||||
// same failure-isolated prefetch path.
|
||||
CoverBytesFetch CoverBytesFetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
// eligibleCount reports how many of a Site's Series are eligible for
|
||||
// polling, defaulting to Store.EligibleSeriesCount. Injected so tests can
|
||||
// fail the count alone: the eligible query shares the due query's tables,
|
||||
// so no real store failure can reach this path without breaking the due
|
||||
// query first (issue #141).
|
||||
eligibleCount func(site string) (int, error)
|
||||
|
||||
// refuseUntil gates a Site's Lane after it refused twice in one run: no
|
||||
// Series of that Site is attempted again before this time (issue #100).
|
||||
// The stamp is durable — the pass gate reads it from the store, so a
|
||||
// restart does not forget the refusal; nothing of it lives in memory.
|
||||
// browserDownAt is when a browser Lane last lost the sidecar; the other
|
||||
// browser Lanes skip their passes for the next RefuseBackoff, so a
|
||||
// restarting Chrome does not stamp one Series per pass per Lane (story 20).
|
||||
mu sync.Mutex
|
||||
browserDownAt time.Time
|
||||
// coverWG tracks in-flight cover work. Covers heal in the background so a
|
||||
// slow cover host cannot delay the next Series-page Poll; tests join it
|
||||
// before asserting on cover fetches.
|
||||
coverWG sync.WaitGroup
|
||||
}
|
||||
|
||||
// fillBlankCover gives a Series its Cover when it has none. The blank state is
|
||||
// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows
|
||||
// created before acquisition existed, and rows whose creation-time fetch
|
||||
// failed, both heal here. A non-blank CoverAddress is left alone — refetching
|
||||
// would add a request per Series per cycle and change artwork under the Reader
|
||||
// for no visible reason. A row that already carries a source URL is owned by
|
||||
// prefetchCover instead; this path only records a Cover address already
|
||||
// extracted from the series page.
|
||||
//
|
||||
// Failures are logged against the Series and never returned: the chapter poll
|
||||
// must not notice. A failed fill is retried the next time this Series is due;
|
||||
// there is no separate retry queue.
|
||||
func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, cover string) {
|
||||
if sr.CoverAddress != "" || sr.Cover != "" {
|
||||
return
|
||||
}
|
||||
if cover == "" {
|
||||
return
|
||||
}
|
||||
// Like healCover, the fill runs in the background: a large import of
|
||||
// blanks would otherwise pay one og:image fetch per Series against the
|
||||
// Lane's gap (issue #100, story 12).
|
||||
p.coverWG.Add(1)
|
||||
go func() {
|
||||
defer p.coverWG.Done()
|
||||
p.storeCover(ctx, sr, cover)
|
||||
}()
|
||||
}
|
||||
|
||||
// prefetchCover heals Series that already carry a third-party source URL but
|
||||
// no stored address — the state left by client-supplied covers before
|
||||
// acquisition moved server-side. Every Site takes the same path; fetchCoverBytes
|
||||
// routes by URL shape, so browser-claimed URLs still need the sidecar. New
|
||||
// blanks have no source URL and go through fillBlankCover from the series page
|
||||
// instead.
|
||||
func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) {
|
||||
if sr.Cover == "" || sr.CoverAddress != "" {
|
||||
return
|
||||
}
|
||||
body, contentType, found, err := p.Store.GetCover(sr.Cover)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil {
|
||||
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
p.storeCover(ctx, sr, sr.Cover)
|
||||
}
|
||||
|
||||
// storeCover fetches bytes for sourceURL and points the Series at them. Every
|
||||
// failure is logged against the Series and swallowed so the chapter poll
|
||||
// cannot see it.
|
||||
func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL string) {
|
||||
bytes, contentType, err := fetchCoverBytes(ctx, sourceURL, p.CoverFetch, p.CoverBytesFetch)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch cover %s: %v", sr.Key(), sourceURL, err)
|
||||
return
|
||||
}
|
||||
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sourceURL, bytes, contentType); err != nil {
|
||||
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// fetcherFor returns the fetcher a site's page needs, or nil when the site
|
||||
// cannot be fetched at all right now. A Site whose registry entry carries a
|
||||
// Browser read — kagane, comix and novelfull, all behind a Cloudflare
|
||||
// JavaScript challenge no TLS fingerprint clears — prefers the browser; when it
|
||||
// is absent, the entry's Fallback decides whether plain TLS may take over. One
|
||||
// routing rule for the poll and the acquirer, so the two cannot drift apart.
|
||||
func fetcherFor(site string, browser, tls Fetcher) Fetcher {
|
||||
s, known := sites[site]
|
||||
if !known {
|
||||
// No registry entry means nothing to fetch or parse; fail closed even
|
||||
// though the only caller gates first, so a future caller that skips
|
||||
// the gate cannot hand an arbitrary https URL to the TLS fetcher.
|
||||
return nil
|
||||
}
|
||||
if s.Browser == nil {
|
||||
return tls
|
||||
}
|
||||
if browser != nil {
|
||||
return browser
|
||||
}
|
||||
if s.Browser.Fallback {
|
||||
return tls
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run polls until ctx is cancelled: one goroutine per Site Lane, each pacing
|
||||
// itself by the Site's effective gap. Lanes share nothing but the store and
|
||||
// the browser fetcher's single tab (BrowserFetcher serializes itself), so one
|
||||
// hostile Site burns only its own budget.
|
||||
// laneNames returns every registry Site in the deterministic order both Run
|
||||
// and runOnce iterate: sorted, so lane behaviour and its tests agree on who
|
||||
// runs first.
|
||||
func laneNames() []string { return SiteNames() }
|
||||
|
||||
func (p *Poller) Run(ctx context.Context) {
|
||||
names := laneNames()
|
||||
log.Printf("latest-chapter poller: %d lanes, rest=%s gap=%s", len(names), defaultRest, defaultGap)
|
||||
for _, name := range names {
|
||||
go p.lane(ctx, name)
|
||||
}
|
||||
<-ctx.Done()
|
||||
log.Println("latest-chapter poller: stopped")
|
||||
}
|
||||
|
||||
// lane is one Site's Poll Lane: one pass, then sleep the pace the pass
|
||||
// reported, then another pass, until ctx is cancelled. The sleep is the whole
|
||||
// pace discipline — a pass that fetched nothing still reports its gap so the
|
||||
// Lane wakes often enough to notice Series as they become due. The pass shares
|
||||
// the Poller's browser-down state, so a sidecar loss is noticed once and the
|
||||
// other browser Lanes skip passes until the backoff window decays.
|
||||
func (p *Poller) lane(ctx context.Context, name string) {
|
||||
for {
|
||||
pace := p.runLanePass(ctx, name, true)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(pace):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runOnce processes one round: one pass of every Lane, back to back, no real
|
||||
// time passing. This is the deterministic entry point the test suite drives a
|
||||
// round at a time. The production Run loop does the same work paced by its own
|
||||
// sleeps; pacing is the only difference.
|
||||
func (p *Poller) runOnce(ctx context.Context) {
|
||||
for _, name := range laneNames() {
|
||||
p.runLanePass(ctx, name, false)
|
||||
}
|
||||
}
|
||||
|
||||
// One skip value per way a Lane Pass can return before its loop (issue #141);
|
||||
// empty means the pass reached the loop. The values are wire strings — stored
|
||||
// in poll_passes and read by the Lanes page — so they are stable, not prose.
|
||||
const (
|
||||
// Exported so the web layer renders a skip's reason without retyping the
|
||||
// wire string (issue #145); the values are storage and page-stable.
|
||||
SkipPaused = "paused" // the pause row was read at the top
|
||||
SkipRefusing = "refusing" // refusal backoff
|
||||
SkipSidecarDown = "sidecar-down" // a sibling browser Lane lost Chrome
|
||||
SkipNoFetcher = "no-fetcher" // browser Site, no browser configured, no fallback
|
||||
SkipDueQuery = "due-query" // the due query failed
|
||||
SkipAsleep = "asleep" // under both browser wake thresholds
|
||||
SkipEligibleCount = "eligible-count" // the eligible count failed
|
||||
SkipNothingEligible = "nothing-eligible" // nothing eligible; sleeps a full rest
|
||||
)
|
||||
|
||||
// readOutcome classifies one Series read for the pass row's outcome counts
|
||||
// (issue #141). The classification the read already makes is counted, never a
|
||||
// second taxonomy: refused is the Site holding a challenge, unreachable the
|
||||
// browser interrupting, noChapter a 200 with real HTML but no chapter links,
|
||||
// unfetchable the host pin or a missing fetcher, and errors everything else.
|
||||
type readOutcome int
|
||||
|
||||
const (
|
||||
outcomeSuccess readOutcome = iota
|
||||
outcomeRefused
|
||||
outcomeUnreachable
|
||||
outcomeNoChapter
|
||||
outcomeUnfetchable
|
||||
outcomeError
|
||||
)
|
||||
|
||||
// outcomeCounts are the five named outcome counts of one pass. A success
|
||||
// count is derived, never stored: checked minus the four, with unreachable
|
||||
// excluded because the sidecar-loss path returns before the checked counter
|
||||
// increments (issue #141).
|
||||
type outcomeCounts struct {
|
||||
refused, unreachable, noChapter, unfetchable, errors int
|
||||
}
|
||||
|
||||
func (c *outcomeCounts) add(o readOutcome) {
|
||||
switch o {
|
||||
case outcomeRefused:
|
||||
c.refused++
|
||||
case outcomeUnreachable:
|
||||
c.unreachable++
|
||||
case outcomeNoChapter:
|
||||
c.noChapter++
|
||||
case outcomeUnfetchable:
|
||||
c.unfetchable++
|
||||
case outcomeError:
|
||||
c.errors++
|
||||
}
|
||||
}
|
||||
|
||||
// passRecord is what one pass's durable row will be: the skip value and
|
||||
// outcome counts filled in along the pass's return path. recordPass assembles
|
||||
// the row, so every exit records exactly once.
|
||||
type passRecord struct {
|
||||
site string
|
||||
ranAt int64
|
||||
skip string
|
||||
counts outcomeCounts
|
||||
}
|
||||
|
||||
// lanePassRetention is how far back a Lane's pass log is kept. It is not the
|
||||
// display window: retention is how far back a question can reach, and the
|
||||
// window is what the owner is shown (issue #139).
|
||||
const lanePassRetention = 14 * 24 * time.Hour
|
||||
|
||||
// runLanePass processes one pass of one Site's Lane: select the due Series,
|
||||
// pace through them, and report how long the Lane should wait before its next
|
||||
// pass. paced spaces consecutive fetches by the Site's effective gap — the
|
||||
// production Lane's rate limit; the deterministic test entry runs back to back.
|
||||
func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.Duration {
|
||||
now := p.Now()
|
||||
// Durable pass log (issue #141): one row per exit. The figures are filled
|
||||
// in as the pass measures them; a pass that returns before measuring
|
||||
// carries the previous pass's forward inside recordPass.
|
||||
fig := passFigures{}
|
||||
rec := passRecord{site: name, ranAt: now.UnixMilli()}
|
||||
defer func() { p.recordPass(rec, fig) }()
|
||||
|
||||
// One Lane row read at the top of a pass, serving two gates (issue #139).
|
||||
// Both stamps outlive our process, so the gates read the durable row
|
||||
// rather than memory: a refusal is the Site's mood and a pause the
|
||||
// owner's order, and neither is lost to a restart.
|
||||
pausedUntil, refuseUntil, err := p.Store.LaneGates(name)
|
||||
if err != nil {
|
||||
// Fail open: a store that cannot answer the gate cannot record the
|
||||
// pass either, and one Lane must not stall on its own gate read.
|
||||
log.Printf("latest poll %s: lane gates: %v", name, err)
|
||||
}
|
||||
if pausedUntil > now.UnixMilli() {
|
||||
// Paused ahead of the refusal check: no Series is touched, so the
|
||||
// queue stays intact for when the pause lifts (issue #141, #147).
|
||||
rec.skip = SkipPaused
|
||||
log.Printf("latest poll %s: paused until %s, skipping pass", name, time.UnixMilli(pausedUntil).Format(time.RFC3339))
|
||||
return time.Duration(pausedUntil-now.UnixMilli()) * time.Millisecond
|
||||
}
|
||||
if refuseUntil > now.UnixMilli() {
|
||||
// Cooling down after a refusal: do not attempt this Site at all.
|
||||
rec.skip = SkipRefusing
|
||||
return time.Duration(refuseUntil-now.UnixMilli()) * time.Millisecond
|
||||
}
|
||||
if isBrowserSite(name) {
|
||||
if downFor, down := p.browserDownFor(now); down && downFor < RefuseBackoff {
|
||||
// A sibling browser Lane lost the sidecar within the backoff
|
||||
// window: skip this pass, so a restarting Chrome does not stamp
|
||||
// this Site's Series one pass at a time. After RefuseBackoff the
|
||||
// flag decays and the Lane probes again (issue #100, story 20).
|
||||
rec.skip = SkipSidecarDown
|
||||
log.Printf("latest poll %s: browser lane skipping pass (sidecar down %s ago)", name, downFor)
|
||||
return RefuseBackoff - downFor
|
||||
}
|
||||
}
|
||||
s := sites[name]
|
||||
f := fetcherFor(name, p.BrowserFetch, p.Fetch)
|
||||
if f == nil {
|
||||
// No fetcher at all right now (browser absent, no fallback): every
|
||||
// Series stays unstamped and due, so a browser that appears after a
|
||||
// restart finds its full queue waiting (issue #100).
|
||||
rec.skip = SkipNoFetcher
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
|
||||
due, err := p.Store.DueForLatestCheck(name, now.Add(-s.Rest).UnixMilli(),
|
||||
now.Add(-sightingCeilingRests*s.Rest).UnixMilli())
|
||||
if err != nil {
|
||||
rec.skip = SkipDueQuery
|
||||
log.Printf("latest poll %s: due query: %v", name, err)
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
fig.Due = len(due)
|
||||
if s.Browser != nil && f == p.BrowserFetch && !browserWakeDue(due, now, s.Rest) && !anyForced(due) {
|
||||
// Below both thresholds Chrome stays asleep (ADR-0005 on-demand
|
||||
// browser): waking it for a single Poll would cost a challenge solve
|
||||
// per request. A forced Series is the one exception — a human asking
|
||||
// is not the machine waking itself (issue #146). The Lane still paces
|
||||
// at the default gap, which is what the owner's page must show rather
|
||||
// than a zero.
|
||||
rec.skip = SkipAsleep
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
if s.Browser != nil {
|
||||
// Browser Lanes share one tab, so their combined ceiling is about 360
|
||||
// Polls an hour. When they cannot keep up, the wait past the rest time
|
||||
// grows — log by how much, every pass, so the decision to give them
|
||||
// more pages is made from a measurement rather than a guess.
|
||||
if behind := maxSeriesWait(due, now, s.Rest) - s.Rest; behind > 0 {
|
||||
log.Printf("latest poll %s: browser lane behind by %s (browser Sites cannot keep up with the hour)", name, behind)
|
||||
}
|
||||
}
|
||||
|
||||
eligible, err := p.countEligible(name)
|
||||
if err != nil {
|
||||
rec.skip = SkipEligibleCount
|
||||
log.Printf("latest poll %s: eligible count: %v", name, err)
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
gap, clamped := effectiveGap(s, eligible)
|
||||
fig.Gap, fig.Clamped = gap, clamped
|
||||
if clamped {
|
||||
log.Printf("latest poll %s: gap clamped to %s floor (eligible series=%d)", name, minGap, eligible)
|
||||
}
|
||||
if eligible == 0 {
|
||||
// Nothing to poll for the foreseeable future; sleep a full rest instead
|
||||
// of re-querying every gap.
|
||||
rec.skip = SkipNothingEligible
|
||||
return s.Rest
|
||||
}
|
||||
|
||||
refusals := 0
|
||||
for i, sr := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if refusals >= 2 {
|
||||
// This Site refused twice in a row: the remaining Series are left
|
||||
// unstamped and due, and the Lane waits RefuseBackoff before
|
||||
// trying it again.
|
||||
break
|
||||
}
|
||||
if paced && i > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break
|
||||
case <-time.After(gap):
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
outcome := p.checkOne(ctx, sr)
|
||||
if outcome == outcomeUnreachable {
|
||||
// The mid-loop browser loss writes an empty skip on purpose: the
|
||||
// pass returns before the checked counter increments, so its row
|
||||
// is stall-shaped (due > 0, checked 0, skip ''), and a stall is
|
||||
// the exact signal this exit produces. A tenth skip value would
|
||||
// make it legible but is deliberately not invented here.
|
||||
rec.counts.add(outcome)
|
||||
p.setBrowserDown(now)
|
||||
log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, RefuseBackoff)
|
||||
return gap
|
||||
}
|
||||
if outcome == outcomeRefused {
|
||||
refusals++
|
||||
} else {
|
||||
refusals = 0
|
||||
}
|
||||
rec.counts.add(outcome)
|
||||
fig.Checked++
|
||||
}
|
||||
if fig.Checked > 0 {
|
||||
log.Printf("latest poll %s: due=%d checked=%d", name, len(due), fig.Checked)
|
||||
}
|
||||
if refusals >= 2 {
|
||||
// The refusal outlives the process: the durable stamp gates a restart,
|
||||
// so a Site that just told us to back off is not re-probed.
|
||||
if err := p.Store.SetLaneRefusal(name, now.Add(RefuseBackoff).UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %s: persist refusal: %v", name, err)
|
||||
}
|
||||
log.Printf("latest poll %s: refused twice this run, waiting %s", name, RefuseBackoff)
|
||||
return RefuseBackoff
|
||||
}
|
||||
return gap
|
||||
}
|
||||
|
||||
// passFigures are the numbers one pass measured for its durable row (issue
|
||||
// #141): due and checked as the pass saw them, the pace it chose, and whether
|
||||
// the gap sat on the floor. A pass that returned before measuring keeps the
|
||||
// previous pass's figures via carry-forward in recordPass; the in-memory
|
||||
// snapshot those once mirrored into is gone — the page reads the durable row
|
||||
// now (issue #145).
|
||||
type passFigures struct {
|
||||
Due, Checked int
|
||||
Gap time.Duration
|
||||
Clamped bool
|
||||
}
|
||||
|
||||
// recordPass writes the durable row for one pass (issue #141). Called deferred
|
||||
// from runLanePass so every return path records exactly one row. A pass that
|
||||
// never computed its own figures — its gap is zero — carries the previous
|
||||
// pass's due, gap, clamped and checked forward rather than stating zeroes it
|
||||
// did not measure; the skip column says why it declined, so the zeroes that
|
||||
// remain (due-query, no-fetcher) read as explanations rather than
|
||||
// measurements.
|
||||
func (p *Poller) recordPass(rec passRecord, fig passFigures) {
|
||||
row := store.LanePass{
|
||||
Site: rec.site,
|
||||
RanAt: rec.ranAt,
|
||||
Skip: rec.skip,
|
||||
Due: fig.Due,
|
||||
Checked: fig.Checked,
|
||||
GapMS: fig.Gap.Milliseconds(),
|
||||
Clamped: fig.Clamped,
|
||||
Refused: rec.counts.refused,
|
||||
Unreachable: rec.counts.unreachable,
|
||||
NoChapter: rec.counts.noChapter,
|
||||
Unfetchable: rec.counts.unfetchable,
|
||||
Errors: rec.counts.errors,
|
||||
}
|
||||
if row.GapMS == 0 {
|
||||
// The pass never computed a gap, so it has no figures of its own:
|
||||
// carry the previous pass's, in one latest-per-Site read — the
|
||||
// recorder needs one Site, not six (issue #139).
|
||||
if prev, ok, err := p.Store.LatestLanePass(rec.site); err != nil {
|
||||
log.Printf("latest poll %s: previous pass: %v", rec.site, err)
|
||||
} else if ok {
|
||||
row.Due, row.Checked = prev.Due, prev.Checked
|
||||
row.GapMS, row.Clamped = prev.GapMS, prev.Clamped
|
||||
}
|
||||
}
|
||||
if err := p.Store.RecordLanePass(row, rec.ranAt-lanePassRetention.Milliseconds()); err != nil {
|
||||
log.Printf("latest poll %s: record lane pass: %v", rec.site, err)
|
||||
}
|
||||
}
|
||||
|
||||
// countEligible routes the eligible count through the test seam when one is
|
||||
// set, else the store.
|
||||
func (p *Poller) countEligible(site string) (int, error) {
|
||||
if p.eligibleCount != nil {
|
||||
return p.eligibleCount(site)
|
||||
}
|
||||
return p.Store.EligibleSeriesCount(site)
|
||||
}
|
||||
|
||||
// setBrowserDown records when a browser Lane lost the sidecar. It is Poller
|
||||
// state rather than pass state so the other browser Lanes see it too.
|
||||
func (p *Poller) setBrowserDown(now time.Time) {
|
||||
p.mu.Lock()
|
||||
p.browserDownAt = now
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// browserDownFor reports how long the sidecar has been down and that it is
|
||||
// down at all — the zero time means never down, which must not read as a
|
||||
// zero-duration loss. The window decays: once RefuseBackoff passes without a
|
||||
// fresh loss, Lanes probe again.
|
||||
func (p *Poller) browserDownFor(now time.Time) (time.Duration, bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.browserDownAt.IsZero() {
|
||||
return 0, false
|
||||
}
|
||||
return now.Sub(p.browserDownAt), true
|
||||
}
|
||||
|
||||
// isBrowserSite reports whether the registry routes this Site's page through
|
||||
// the browser sidecar.
|
||||
func isBrowserSite(name string) bool {
|
||||
return sites[name].Browser != nil
|
||||
}
|
||||
|
||||
// browserWakeDue reports whether a browser Lane may start a run: five or more
|
||||
// of its Series are due, or any one of them has been due for browserWakeAge.
|
||||
// Below both thresholds the Lane leaves Chrome asleep — Series Polled together
|
||||
// become due together, so the group naturally stays clustered, and the age
|
||||
// rule exists to stop a Series that drifted out of the group from starving.
|
||||
func browserWakeDue(due []store.Series, now time.Time, rest time.Duration) bool {
|
||||
if len(due) >= browserWakeCount {
|
||||
return true
|
||||
}
|
||||
return maxSeriesWait(due, now, rest) >= browserWakeAge
|
||||
}
|
||||
|
||||
// anyForced reports whether the due list holds a forced Series: one whose
|
||||
// owner check-now request (issue #146) has not been answered yet. A human
|
||||
// asking wakes a sleeping Chrome even below the wake thresholds; the request
|
||||
// itself still ages visibly if the home machine is off.
|
||||
func anyForced(due []store.Series) bool {
|
||||
for _, sr := range due {
|
||||
if sr.Forced {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// maxSeriesWait returns how long the most-overdue of the due Series has been
|
||||
// waiting past its due moment (0 when due is empty).
|
||||
func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.Duration {
|
||||
var oldest time.Duration
|
||||
for _, sr := range due {
|
||||
if w := now.Sub(time.UnixMilli(sr.LatestCheckedAt).Add(rest)); w > oldest {
|
||||
oldest = w
|
||||
}
|
||||
}
|
||||
return oldest
|
||||
}
|
||||
|
||||
// checkOne re-checks one series. Every failure path here is "log and move on":
|
||||
// the poller is a best-effort enhancement, and no single bad series may stall a
|
||||
// Lane or take down the process. The returned outcome classifies the read for
|
||||
// the pass row (issue #141), so the Lane can count a refusal, a lost browser,
|
||||
// a chapter-less page, an unfetchable address or a transport error without
|
||||
// re-deriving the taxonomy.
|
||||
func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("latest poll %q: recovered from panic: %v", sr.Key(), r)
|
||||
outcome = outcomeError
|
||||
}
|
||||
}()
|
||||
|
||||
// Stamped before the fetch, not after, so an error, a timeout, or a shutdown
|
||||
// mid-request still consumes the rest. Otherwise a renamed or deleted
|
||||
// series would be retried on every single pass forever. The userscript
|
||||
// stamps in the same order and for the same reason (L471-473). A Series
|
||||
// never reaches checkOne without a fetcher — runLanePass skips those — so
|
||||
// the stamp means "attempted", and an untried Series stays due.
|
||||
if err := p.Store.MarkLatestChecked(sr.Site, sr.SeriesID, p.Now().UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %q: mark checked: %v", sr.Key(), err)
|
||||
return outcomeError
|
||||
}
|
||||
|
||||
facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, p.BrowserFetch, p.Fetch)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, errNotFetchable):
|
||||
// The rest above is already consumed, so a row that never
|
||||
// passes the gate is retried at rest pace rather than
|
||||
// hot-looping.
|
||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
||||
return outcomeUnfetchable
|
||||
case errors.Is(err, errNoFetcher):
|
||||
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||
return outcomeUnfetchable
|
||||
}
|
||||
// A legacy cover heals independently of the page read: its source may
|
||||
// answer — a CDN — while the origin does not, so a fetch failure does
|
||||
// not skip the heal, matching the order the shared read replaced.
|
||||
p.healCover(ctx, sr)
|
||||
log.Printf("latest poll %q: %v", sr.Key(), err)
|
||||
if errors.Is(err, errChallengeHeld) {
|
||||
return outcomeRefused
|
||||
}
|
||||
if errors.Is(err, errBrowserInterrupted) {
|
||||
return outcomeUnreachable
|
||||
}
|
||||
return outcomeError
|
||||
}
|
||||
// A legacy cover source is healed independently of the page read.
|
||||
p.healCover(ctx, sr)
|
||||
// Cover fill is independent of the chapter signal: a page that lost its
|
||||
// chapter list may keep its og:image, and a blank Series heals either way.
|
||||
p.fillBlankCover(ctx, sr, facts.Cover)
|
||||
if !facts.HasLatest {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a rest instead of hot-looping.
|
||||
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
|
||||
return outcomeNoChapter
|
||||
}
|
||||
|
||||
// The Poll is the oracle for whatever Sighting last raised this Series
|
||||
// (issue #103), and the judgement is free: the comparison below already
|
||||
// exists, and no extra request is made to reach it.
|
||||
p.judgeSighting(sr, facts.Latest.Num)
|
||||
|
||||
// Equality, not >, mirroring the userscript (L427): a site that retracts a
|
||||
// chapter should correct the stored number downward. The comparison is
|
||||
// against the due-query snapshot; a concurrent write in between only costs
|
||||
// one redundant UPDATE of the same absolute value, never a wrong one.
|
||||
if sr.LatestChapterNum != nil && *sr.LatestChapterNum == facts.Latest.Num {
|
||||
return outcomeSuccess
|
||||
}
|
||||
|
||||
// Series-level write: the row is shared, so one update refreshes every
|
||||
// bookmark joining to it, and the bookmark's updated_at is never touched —
|
||||
// a newly published chapter is not reading progress and must not reorder
|
||||
// the list.
|
||||
if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil {
|
||||
log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err)
|
||||
return outcomeError
|
||||
}
|
||||
log.Printf("latest poll %q: latest is now %s", sr.Key(), facts.Latest.Label)
|
||||
return outcomeSuccess
|
||||
}
|
||||
|
||||
// judgeSighting settles the Sighting the Series' stored Latest Chapter is owed
|
||||
// to, if any, against what the Site actually publishes. The asymmetry is the
|
||||
// whole of the detection rule and is what keeps it free of false alarms: a Poll
|
||||
// finding a *lower* number than stored means the Reader who raised it reported
|
||||
// a chapter that does not exist, while a Poll finding a higher one is only the
|
||||
// Site publishing since and means nothing about the report. Equality confirms
|
||||
// the report, which is how an honest Reader earns back a mark.
|
||||
//
|
||||
// A Series with no attribution — the stored value is a Poll's own, or a
|
||||
// previous Poll already judged the report — is nobody's to answer for.
|
||||
func (p *Poller) judgeSighting(sr store.Series, found float64) {
|
||||
if sr.LatestRaisedBy == nil || sr.LatestChapterNum == nil {
|
||||
return
|
||||
}
|
||||
stored := *sr.LatestChapterNum
|
||||
if found > stored {
|
||||
// The report is neither confirmed nor contradicted, but it is answered:
|
||||
// the value about to be stored is the Poll's own, so leaving the
|
||||
// attribution would credit this Reader with the next Poll's agreement
|
||||
// and blame them if the Site later retracts.
|
||||
if err := p.Store.ClearSightingAttribution(sr.Site, sr.SeriesID, *sr.LatestRaisedBy); err != nil {
|
||||
log.Printf("latest poll %q: clear sighting attribution: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if found < stored {
|
||||
// Logged with both numbers and the Reader, because that is what tells a
|
||||
// broken Site adapter (which marks every Reader of that Site at once)
|
||||
// from one Reader deliberately lying.
|
||||
log.Printf("latest poll %q: sighting contradicted: reader %d raised it to %v, site publishes %v",
|
||||
sr.Key(), *sr.LatestRaisedBy, stored, found)
|
||||
}
|
||||
if err := p.Store.RecordSightingOutcome(sr.Site, sr.SeriesID, *sr.LatestRaisedBy, found == stored); err != nil {
|
||||
log.Printf("latest poll %q: record sighting outcome: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// healCover runs prefetchCover in the background. Cover bytes come from a
|
||||
// different host — often a CDN — and heal once in a Series's life, so they
|
||||
// must not consume a Lane's gap: a large import with many blanks would
|
||||
// otherwise make every Latest Chapter go stale behind a slow image host
|
||||
// (issue #100).
|
||||
func (p *Poller) healCover(ctx context.Context, sr store.Series) {
|
||||
p.coverWG.Add(1)
|
||||
go func() {
|
||||
defer p.coverWG.Done()
|
||||
p.prefetchCover(ctx, sr)
|
||||
}()
|
||||
}
|
||||
|
||||
// waitCovers blocks until every in-flight cover heal finishes. Tests call it
|
||||
// after a round before asserting on cover fetches.
|
||||
func (p *Poller) waitCovers() {
|
||||
p.coverWG.Wait()
|
||||
}
|
||||
|
||||
// fetchableSeriesURL reports whether site is a Site the registry knows and
|
||||
// seriesURL is safe to hand to a fetcher: an https URL whose host matches the
|
||||
// Site's pinned hostname exactly. series_url comes from client-supplied PUT
|
||||
// bodies, so this is a defence against the poller being used to probe
|
||||
// arbitrary hosts from the server's own network position, not just a check
|
||||
// against wasted requests. The pin guards different things per Site — a
|
||||
// browser Site guards a control that executes JavaScript and carries cookies,
|
||||
// a parser Site guards a wasted request — but the rule is one rule, from the
|
||||
// registry.
|
||||
func fetchableSeriesURL(site, seriesURL string) bool {
|
||||
s, known := sites[site]
|
||||
if !known {
|
||||
return false
|
||||
}
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return u.Scheme == "https" && u.Hostname() == s.Host
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,73 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// seriesRead carries the two facts the poll and the acquirer both extract
|
||||
// from a series page. Persistence, stamps and scheduling stay with the
|
||||
// callers, so the policies that keep the two flows distinct (stamp order,
|
||||
// rests) are not swallowed by the module.
|
||||
type seriesRead struct {
|
||||
Latest latestChapter
|
||||
HasLatest bool
|
||||
Cover string
|
||||
HasCover bool
|
||||
// BodyLen is the fetched body's length, surfaced because the no-chapter
|
||||
// log uses it to tell a markup change from a body the size cap cut short.
|
||||
BodyLen int
|
||||
}
|
||||
|
||||
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
||||
// failures so each caller keeps its own distinct log line for all three.
|
||||
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
|
||||
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
|
||||
// is how a Lane tells a refusal from an ordinary failure (issue #100).
|
||||
var (
|
||||
errNotFetchable = errors.New("series url not fetchable")
|
||||
errNoFetcher = errors.New("no fetcher for site")
|
||||
)
|
||||
|
||||
// readSeriesPage performs the series-page read the poll and the acquirer have
|
||||
// in common: gate the address, choose the route, fetch the page, extract the
|
||||
// Latest Chapter and the Cover address. It persists nothing and stamps
|
||||
// nothing.
|
||||
//
|
||||
// series_url arrives in a client-supplied PUT body (PUT /bookmarks/{key}
|
||||
// accepts any string), so the gate is not an optimisation against burning a
|
||||
// request on an unknown site: without it, the server would issue a GET from
|
||||
// its own network position to whatever URL a token-holder writes, including
|
||||
// link-local/internal addresses or non-https schemes.
|
||||
func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fetcher) (seriesRead, error) {
|
||||
if !fetchableSeriesURL(site, seriesURL) {
|
||||
return seriesRead{}, fmt.Errorf("%w: site=%q url=%q", errNotFetchable, site, seriesURL)
|
||||
}
|
||||
f := fetcherFor(site, browser, tls)
|
||||
if f == nil {
|
||||
return seriesRead{}, fmt.Errorf("%w: site %q", errNoFetcher, site)
|
||||
}
|
||||
body, status, err := f.Get(ctx, seriesURL)
|
||||
if err != nil {
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: %w", seriesURL, err)
|
||||
}
|
||||
if status == 403 {
|
||||
// Cloudflare's challenge response for these Sites (cf-mitigated). The
|
||||
// browser fetcher returns exactly this on a held interstitial, and a
|
||||
// plain-TLS 403 means the same: the Site is refusing.
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
|
||||
}
|
||||
if status != 200 {
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
|
||||
}
|
||||
if isInterstitial(body) {
|
||||
// A 200 that is the challenge page, not the payload: the TLS route can
|
||||
// receive this where the browser would have kept re-reading. Same
|
||||
// refusal as the 403.
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: interstitial body", errChallengeHeld, seriesURL)
|
||||
}
|
||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||
}
|
||||
@@ -1,494 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// Sightings (issue #103) are specified at the Poller seam, with the store as
|
||||
// the way in: a Sighting is seeded the way handlers.Put performs one, a round
|
||||
// is run against the injected fetcher and a frozen clock, and the assertions
|
||||
// are the two observable facts — whether the Series was fetched, and what the
|
||||
// stored Latest Chapter is afterwards. Nothing here asserts counter arithmetic
|
||||
// through an internal call or reads how a deferral is represented in a row.
|
||||
|
||||
const (
|
||||
sightingSlug = "chronicles-of-the-demon-faction-f886a8af"
|
||||
sightingKey = "asura:" + sightingSlug
|
||||
sightingURL = "https://asurascans.com/comics/" + sightingSlug
|
||||
)
|
||||
|
||||
// sightingFixtureLatest is the newest chapter asuraSeriesFixture publishes.
|
||||
const sightingFixtureLatest = 181.0
|
||||
|
||||
// sight performs one Sighting exactly as the JSON API does (handlers.Put):
|
||||
// RecordSighting against the row as stored, then the Upsert that stores the
|
||||
// reported value. The order is load-bearing — the raise comparison has nothing
|
||||
// to compare against once the Upsert has landed — and the bookmark's own fields
|
||||
// are carried over untouched, which is what a userscript PUT does when it
|
||||
// echoes back the row it cached.
|
||||
func sight(t *testing.T, s *store.Store, readerID int64, key string, num float64, at time.Time) {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
b, found, err := s.Get(readerID, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("sight %q: get: %v found=%v", key, err, found)
|
||||
}
|
||||
if err := s.RecordSighting(readerID, site, seriesID, &num, at.UnixMilli()); err != nil {
|
||||
t.Fatalf("sight %q: %v", key, err)
|
||||
}
|
||||
b.LatestChapter = fmt.Sprintf("Chapter %v", num)
|
||||
b.LatestChapterNum = &num
|
||||
b.UpdatedAt = at.UnixMilli()
|
||||
if _, err := s.Upsert(readerID, b); err != nil {
|
||||
t.Fatalf("sight %q: upsert: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// secondReader is another Reader on the same database. The owner seed is the
|
||||
// only reader-creation path in this package, so a second Open as a different
|
||||
// owner is how a test gets one (as TestRunOnceFetchesSharedSeriesOnce does).
|
||||
func secondReader(t *testing.T, dbURL string) *store.Store {
|
||||
t.Helper()
|
||||
other, err := store.Open(dbURL,
|
||||
store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))},
|
||||
t.TempDir(), testCoverBaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Open second reader: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { other.Close() })
|
||||
return other
|
||||
}
|
||||
|
||||
func readLatestNum(t *testing.T, s *store.Store, readerID int64, key string) float64 {
|
||||
t.Helper()
|
||||
b, ok, err := s.Get(readerID, key)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get %q: %v ok=%v", key, err, ok)
|
||||
}
|
||||
if b.LatestChapterNum == nil {
|
||||
t.Fatalf("%q has no latest chapter", key)
|
||||
}
|
||||
return *b.LatestChapterNum
|
||||
}
|
||||
|
||||
// A Series only one Reader bookmarks is the case where being wrong can hurt
|
||||
// nobody but the Reader who reported it, so their Sighting stands in for the
|
||||
// Poll and the round leaves the Series alone.
|
||||
func TestSightingOnSolitarySeriesDefersPoll(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("fetched %d times after a Sighting on a solitary Series, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// On a shared Series the Sighting still writes the Latest Chapter for everyone,
|
||||
// but the Poll happens on schedule anyway — which is what corrects a wrong
|
||||
// value within the hour instead of letting it persist.
|
||||
func TestSightingOnSharedSeriesDoesNotDeferPoll(t *testing.T) {
|
||||
s, dbURL := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
other := secondReader(t, dbURL)
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
sight(t, s, s.OwnerID(), sightingKey, 200, now.Add(-10*time.Minute))
|
||||
|
||||
// The Sighting updated the shared row immediately, before any Poll.
|
||||
if got := readLatestNum(t, s, s.OwnerID(), sightingKey); got != 200 {
|
||||
t.Fatalf("latest after the Sighting = %v, want 200", got)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times after a Sighting on a shared Series, want 1", got)
|
||||
}
|
||||
if got := readLatestNum(t, s, s.OwnerID(), sightingKey); got != sightingFixtureLatest {
|
||||
t.Fatalf("latest after the Poll = %v, want the Site's own %v", got, sightingFixtureLatest)
|
||||
}
|
||||
}
|
||||
|
||||
// Reporting a chapter is not reading one: a Sighting may move the Latest
|
||||
// Chapter and nothing else. Both the solitary and the shared case, because the
|
||||
// deferral branch must not be where this guarantee lives.
|
||||
func TestSightingLeavesProgressAndOrderingUntouched(t *testing.T) {
|
||||
for _, shared := range []bool{false, true} {
|
||||
name := "solitary"
|
||||
if shared {
|
||||
name = "shared"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, dbURL := newTestStore(t)
|
||||
read := 5.0
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, SeriesURL: sightingURL,
|
||||
LastChapter: "Chapter 5", LastChapterNum: read,
|
||||
LastChapterURL: sightingURL + "/chapter/5", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if shared {
|
||||
other := secondReader(t, dbURL)
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
sight(t, s, s.OwnerID(), sightingKey, 200, time.UnixMilli(9_000_000))
|
||||
|
||||
b, ok, err := s.Get(s.OwnerID(), sightingKey)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get: %v ok=%v", err, ok)
|
||||
}
|
||||
if b.LatestChapterNum == nil || *b.LatestChapterNum != 200 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 200", b.LatestChapterNum)
|
||||
}
|
||||
if b.LastChapterNum != read {
|
||||
t.Fatalf("LastChapterNum = %v, want %v: a Sighting is not Progress", b.LastChapterNum, read)
|
||||
}
|
||||
if b.UpdatedAt != 1000 {
|
||||
t.Fatalf("updated_at moved to %d: a Sighting must not reorder the list", b.UpdatedAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The ceiling is what makes trusting a client report safe: however recently a
|
||||
// Series was sighted, one that has not been Polled in six hours is Polled.
|
||||
func TestSightingCeilingForcesPoll(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-7*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times past the %s ceiling, want 1", got, sightingCeilingRests*defaultRest)
|
||||
}
|
||||
}
|
||||
|
||||
// Deferral is decided from live facts every round, so a Series that gains a
|
||||
// second Bookmark stops deferring at once — and one that loses it defers again.
|
||||
func TestDeferralFollowsTheBookmarkCount(t *testing.T) {
|
||||
s, dbURL := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("solitary Series fetched %d times, want 0", got)
|
||||
}
|
||||
|
||||
other := secondReader(t, dbURL)
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("shared Series fetched %d times, want 1", got)
|
||||
}
|
||||
|
||||
// The Poll above consumed the rest, so move past it before asking again.
|
||||
if err := other.Delete(other.OwnerID(), sightingKey); err != nil {
|
||||
t.Fatalf("delete second bookmark: %v", err)
|
||||
}
|
||||
later := now.Add(2 * time.Hour)
|
||||
p.Now = func() time.Time { return later }
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, later.Add(-time.Minute))
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("Series fetched %d times after returning to one Bookmark, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A Series nobody reports any more returns to the normal schedule on its own:
|
||||
// the Sighting's standing lasts one rest, not forever.
|
||||
func TestDeferralExpiresWithoutFurtherSightings(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("fetched %d times while the Sighting stood, want 0", got)
|
||||
}
|
||||
|
||||
p.Now = func() time.Time { return now.Add(90 * time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times once the Sighting aged out, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// demonicFixture publishes one chapter in demonicscans' live page shape, so a
|
||||
// test can make a Site publish an arbitrary number rather than the one the
|
||||
// captured fixture froze.
|
||||
func demonicFixture(num float64) string {
|
||||
return fmt.Sprintf(
|
||||
`<a href="/chaptered.php?manga=11799&chapter=%v" class="chplinks" title="Catastrophic Necromancer %v">Chapter %v</a>`,
|
||||
num, num, num)
|
||||
}
|
||||
|
||||
const (
|
||||
demonicKey = "demonic:Catastrophic-Necromancer"
|
||||
demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
||||
)
|
||||
|
||||
// contradictOnce reports a chapter that does not exist and then runs the round
|
||||
// that catches it, returning when that round ran so a caller can chain the
|
||||
// next one. The wait is one rest and a minute: a Sighting stands in for exactly
|
||||
// one rest, so that is the first moment this solitary Series is Polled again.
|
||||
func contradictOnce(t *testing.T, s *store.Store, p *Poller, sightAt time.Time, real float64) time.Time {
|
||||
t.Helper()
|
||||
sight(t, s, s.OwnerID(), demonicKey, real+500, sightAt)
|
||||
at := sightAt.Add(defaultRest + time.Minute)
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != real {
|
||||
t.Fatalf("latest after the Poll = %v, want the Site's own %v", got, real)
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
func seedDemonic(t *testing.T, s *store.Store, checkedAt int64) {
|
||||
t.Helper()
|
||||
seedForCheck(t, s, demonicKey, demonicURL, checkedAt)
|
||||
}
|
||||
|
||||
// A Poll finding a lower number than stored means the Sighting that raised it
|
||||
// was false. The Reader is named — not the Series flagged — and both numbers are
|
||||
// logged, because that is what tells a broken adapter from a deliberate lie.
|
||||
func TestPollContradictingASightingNamesTheReaderAndBothNumbers(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
contradictOnce(t, s, p, now, 296)
|
||||
|
||||
got := logs.String()
|
||||
for _, want := range []string{
|
||||
fmt.Sprintf("reader %d", s.OwnerID()), "796", "296", demonicKey,
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("contradiction log = %q, want it to name %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Three contradictions cost the Reader the right to defer. Nothing here writes
|
||||
// a counter: the marks are earned through Polls, which is the only way
|
||||
// production produces them.
|
||||
func TestThreeContradictionsStopDeferral(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
start := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, start)
|
||||
at := start
|
||||
for range store.SightingDisagreementLimit {
|
||||
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
|
||||
}
|
||||
fetchesSoFar := f.callCount()
|
||||
|
||||
// The marked Reader sights the same solitary Series again. It still writes
|
||||
// the Latest Chapter — the penalty removes a privilege, it does not silence
|
||||
// anyone — but the Poll is no longer postponed: the round below runs while a
|
||||
// trusted Reader's Sighting would still be standing, and fetches anyway.
|
||||
sight(t, s, s.OwnerID(), demonicKey, 900, at.Add(31*time.Minute))
|
||||
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != 900 {
|
||||
t.Fatalf("latest after a marked Reader's Sighting = %v, want 900", got)
|
||||
}
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != fetchesSoFar+1 {
|
||||
t.Fatalf("marked Reader's Sighting still deferred the Poll (fetches %d, want %d)",
|
||||
got, fetchesSoFar+1)
|
||||
}
|
||||
}
|
||||
|
||||
// The owner's remedy for a mark a broken Site adapter produced restores the
|
||||
// privilege without a wait and without SQL.
|
||||
func TestClearingMarksRestoresDeferral(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
start := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, start)
|
||||
at := start
|
||||
for range store.SightingDisagreementLimit {
|
||||
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
|
||||
}
|
||||
if err := s.ClearReaderMarks(s.OwnerID()); err != nil {
|
||||
t.Fatalf("ClearReaderMarks: %v", err)
|
||||
}
|
||||
|
||||
fetchesSoFar := f.callCount()
|
||||
sight(t, s, s.OwnerID(), demonicKey, 900, at.Add(31*time.Minute))
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != fetchesSoFar {
|
||||
t.Fatalf("fetched %d times after the marks were cleared, want %d: deferral must resume",
|
||||
got, fetchesSoFar)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovery is automatic but expensive: twenty Polls that each confirm a
|
||||
// Sighting of this Reader's clear the marks. Each round needs a new chapter,
|
||||
// because only a report that raises the stored number is attributed and so only
|
||||
// that one can be confirmed.
|
||||
func TestTwentyAgreementsClearTheMarks(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
start := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, start)
|
||||
at := start
|
||||
for range store.SightingDisagreementLimit {
|
||||
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
|
||||
}
|
||||
|
||||
chapter := 296.0
|
||||
for range store.SightingAgreementsToClear {
|
||||
chapter++
|
||||
sight(t, s, s.OwnerID(), demonicKey, chapter, at.Add(time.Minute))
|
||||
f.body = demonicFixture(chapter) // the Site publishes what was reported
|
||||
at = at.Add(defaultRest + time.Minute)
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
}
|
||||
|
||||
fetchesSoFar := f.callCount()
|
||||
chapter++
|
||||
sight(t, s, s.OwnerID(), demonicKey, chapter, at.Add(31*time.Minute))
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != fetchesSoFar {
|
||||
t.Fatalf("fetched %d times after %d confirmations, want %d: the marks must be forgiven",
|
||||
got, store.SightingAgreementsToClear, fetchesSoFar)
|
||||
}
|
||||
}
|
||||
|
||||
// A Poll finding a higher number is the Site publishing since the Sighting and
|
||||
// means nothing about the Reader — no mark, and no credit either.
|
||||
func TestPollFindingHigherNumberIsNotAContradiction(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
// Reported truthfully, then the Site published one more.
|
||||
sight(t, s, s.OwnerID(), demonicKey, 295, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
// One rest on, the Sighting has lapsed and the Poll happens.
|
||||
p.Now = func() time.Time { return now.Add(7 * time.Hour) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times past the ceiling, want 1", got)
|
||||
}
|
||||
|
||||
// Unmarked, so a fresh Sighting still defers.
|
||||
at := now.Add(9 * time.Hour)
|
||||
sight(t, s, s.OwnerID(), demonicKey, 296, at.Add(-time.Minute))
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("a Reader whose report the Site overtook lost the right to defer (fetches %d, want 1)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A Poll that overtakes a Sighting takes ownership of the row: the value stored
|
||||
// afterwards is the Poll's own, so a later retraction is not the Reader's fault
|
||||
// and must not be charged to them.
|
||||
func TestAttributionDoesNotSurviveAPollThatOvertookIt(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), demonicKey, 295, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
at := now.Add(defaultRest + time.Minute)
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != 296 {
|
||||
t.Fatalf("latest after the Poll = %v, want the Site's own 296", got)
|
||||
}
|
||||
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
|
||||
f.body = demonicFixture(290) // the Site retracts what only the Poll wrote
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if strings.Contains(logs.String(), "sighting contradicted") {
|
||||
t.Fatalf("a retraction of the Poll's own value was charged to a Reader: %s", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A PUT with no Latest Chapter in it — a favourite toggle, progress written
|
||||
// from a chapter page — is nobody looking at the Series page, so it buys no
|
||||
// deferral. Otherwise a client could suppress a Series' Polls while reporting
|
||||
// nothing, and with nothing reported there would be nothing to judge.
|
||||
func TestPutWithoutALatestChapterDoesNotDefer(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
// The handler's own call, with the field the client omitted.
|
||||
if err := s.RecordSighting(s.OwnerID(), "demonic", "Catastrophic-Necromancer",
|
||||
nil, now.Add(-time.Minute).UnixMilli()); err != nil {
|
||||
t.Fatalf("RecordSighting: %v", err)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times after a PUT carrying no chapter, want 1", got)
|
||||
}
|
||||
}
|
||||
@@ -1,540 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"html"
|
||||
"log"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// latestChapter is the newest chapter a series page advertises.
|
||||
type latestChapter struct {
|
||||
Num float64
|
||||
Label string
|
||||
}
|
||||
|
||||
// site answers the fixed questions every series-page read asks of its Site
|
||||
// (ADR-0009): the host its addresses must carry, how to find the Latest
|
||||
// Chapter and the Cover address in a body, and — for a Site behind a
|
||||
// JavaScript challenge — how to read its payload from a cleared tab. One
|
||||
// entry describes everything about one Site, and nowhere else gets to compare
|
||||
// the site string.
|
||||
type site struct {
|
||||
// Host is the exact hostname a series_url for this Site must carry.
|
||||
Host string
|
||||
// LatestChapter finds the newest chapter in a fetched body.
|
||||
LatestChapter func(seriesURL, body string) (latestChapter, bool)
|
||||
// Cover finds the Cover address in a fetched body.
|
||||
Cover func(seriesURL, body string) (string, bool)
|
||||
// Rest is how long a Series of this Site rests between Polls.
|
||||
Rest time.Duration
|
||||
// Gap is the Lane's strictest pace: at least one second must pass between
|
||||
// two consecutive Series-page Polls of this Site (issue #100).
|
||||
Gap time.Duration
|
||||
// Browser reads this Site's payload from a cleared browser tab; nil
|
||||
// means the page is fetched over plain TLS.
|
||||
Browser *browserRead
|
||||
}
|
||||
|
||||
type browserRead struct {
|
||||
// Read builds the tab read for seriesURL, refusing (false) an address
|
||||
// this Site will not open in a browser — the per-Site half of the SSRF
|
||||
// gate, kept deliberately behind fetchableSeriesURL: a headless browser
|
||||
// executes JavaScript and carries cookies, and series_url is
|
||||
// client-supplied.
|
||||
Read func(seriesURL string, out *string) (chromedp.Action, bool)
|
||||
// Done reports whether the payload arrived.
|
||||
Done func(body string) bool
|
||||
// Fallback allows the plain-TLS fetcher when no browser is configured.
|
||||
// False skips the Site instead. kagane and comix are false — a plain fetch
|
||||
// would only ever retrieve a challenge page — and novelfull is true,
|
||||
// because its challenge is a live time-varying fact (AGENTS.md).
|
||||
Fallback bool
|
||||
}
|
||||
|
||||
// asuraSlugRe pulls the series slug out of a stored series_url.
|
||||
// Shape verified live 2026-07-26: https://asurascans.com/comics/<slug>, where
|
||||
// the slug carries a trailing build-hash suffix (e.g. "-f886a8af") that
|
||||
// rotates on every site redeploy — callers must strip it (asuraBuildHash)
|
||||
// before using the slug to scope anything.
|
||||
var asuraSlugRe = regexp.MustCompile(`/comics/([^/?#]+)`)
|
||||
|
||||
// asuraBuildHash matches the trailing "-xxxxxxxx" site-wide build ID Asura
|
||||
// appends to every series slug. It rotates on each site redeploy, so it is
|
||||
// never part of a stable series_id. Must stay in sync with stripBuildHash in
|
||||
// userscript/manga-bookmark.user.js.
|
||||
var asuraBuildHash = regexp.MustCompile(`-[0-9a-f]{8}$`)
|
||||
|
||||
// demonicChapterRe matches the pre-redirect anchors demonic series pages link
|
||||
// through. Both the raw "&" and the HTML-escaped "&" forms occur.
|
||||
var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?chapter=([0-9.]+)`)
|
||||
|
||||
// comixSlugRe pulls the "<id>-<slug>" segment out of a stored series_url.
|
||||
// Only the id prefix is stable; the slug tail follows the title.
|
||||
var comixSlugRe = regexp.MustCompile(`/title/([^/?#]+)`)
|
||||
|
||||
func comixSeriesID(seriesURL string) (string, bool) {
|
||||
m := comixSlugRe.FindStringSubmatch(seriesURL)
|
||||
if m == nil {
|
||||
return "", false
|
||||
}
|
||||
id := m[1]
|
||||
if i := strings.Index(id, "-"); i != -1 {
|
||||
id = id[:i]
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
// kaganeChapterRe matches the chapter numbers in a kagane API response. This
|
||||
// branch is fed by the browser fetcher, so the body is JSON rather than HTML —
|
||||
// there are no anchors to scan.
|
||||
var kaganeChapterRe = regexp.MustCompile(`"chapter_no":"([0-9.]+)"`)
|
||||
|
||||
// novelfullSlugRe pulls the series slug out of a stored series_url. novelfull
|
||||
// series pages are "/<slug>.html"; their chapter anchors are
|
||||
// "/<slug>/chapter-<n>[-<title-slug>].html". Verified live 2026-08-05.
|
||||
var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`)
|
||||
|
||||
// lnwChapterRe matches any chapter-shaped address on lightnovelworld. Unlike
|
||||
// asura, novelfull and comix — which scope to their stored series slug so a
|
||||
// foreign chapter link cannot contribute — this Site's chapter addresses carry
|
||||
// the Chapter Slug, which is not the Series identity: one Series may publish
|
||||
// under several Chapter Slugs (measured 2026-08-11: a sampled novel serves
|
||||
// 1-99 under one slug and 100-423 under another), so no stored-slug pattern can
|
||||
// cover a Series' whole list. An unscoped match is safe because
|
||||
// lnwLatestChapter truncates the body at the comment thread before scanning
|
||||
// (lnwCommentMarker); without that, a visitor's comment could set the Latest
|
||||
// Chapter on the shared Series row.
|
||||
var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-([0-9.]+)/`)
|
||||
|
||||
// lnwCommentMarker is the boundary of lightnovelworld's server-rendered
|
||||
// wpdiscuz comment thread. It occurs exactly once per page and follows every
|
||||
// chapter anchor (measured 2026-08-11,
|
||||
// docs/research/lightnovelworld-chapter-vs-series-slug.md §6), so cutting the
|
||||
// body at its first occurrence keeps the whole chapter list while excluding a
|
||||
// region any visitor can write to. Absent means the page shape changed: the
|
||||
// body is skipped, never scanned whole.
|
||||
const lnwCommentMarker = "wpd-threads"
|
||||
|
||||
// maxChapter returns the highest chapter number the regex finds in body. A
|
||||
// maximum rather than a first or last, ported from the userscript's
|
||||
// latestChapterFromAnchors (asura L123-133, demonic L183-193): neither site
|
||||
// lists chapters in a dependable order.
|
||||
//
|
||||
// The userscript's asura rule additionally requires the anchor text to match
|
||||
// /Chapter\s+[\d.]+/i. That check exists only to skip the "First Chapter"
|
||||
// shortcut, which points at chapter/1 and therefore can never win a maximum,
|
||||
// so it is redundant once a maximum is taken.
|
||||
func maxChapter(re *regexp.Regexp, body string) (latestChapter, bool) {
|
||||
var best latestChapter
|
||||
found := false
|
||||
for _, m := range re.FindAllStringSubmatch(body, -1) {
|
||||
// [0-9.]+ can swallow a trailing separator, e.g. "chapter/12." in a
|
||||
// sentence; ParseFloat would reject the whole match.
|
||||
raw := strings.Trim(m[1], ".")
|
||||
num, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !found || num > best.Num {
|
||||
best = latestChapter{Num: num, Label: "Chapter " + raw}
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
|
||||
// asuraLatestChapter scopes chapter links to this series' own slug, which
|
||||
// replaces the userscript's anchor-text check with a stronger guarantee: a
|
||||
// chapter link belonging to some other series cannot contribute even if the
|
||||
// page starts carrying them.
|
||||
func asuraLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
||||
m := asuraSlugRe.FindStringSubmatch(seriesURL)
|
||||
if m == nil {
|
||||
return latestChapter{}, false
|
||||
}
|
||||
// Stored URLs predating a redeploy may carry a stale build hash; chapter
|
||||
// hrefs in the fetched body carry the current one. Strip to the stable ID
|
||||
// and make the hash optional in the pattern, so scoping survives
|
||||
// rotations.
|
||||
slug := asuraBuildHash.ReplaceAllString(m[1], "")
|
||||
// Compiled per call rather than cached: this runs once per fetch, which is
|
||||
// at most a few times a minute, and the slug varies per series.
|
||||
re := regexp.MustCompile(`/comics/` + regexp.QuoteMeta(slug) + `(?:-[0-9a-f]{8})?/chapter/([0-9.]+)`)
|
||||
return maxChapter(re, body)
|
||||
}
|
||||
|
||||
// demonicLatestChapter is not scoped: demonicChapterRe matches any
|
||||
// chaptered.php?manga=<id> anchor, because the stored series_id is a slug,
|
||||
// not the numeric id the URL carries, so it cannot be scoped.
|
||||
func demonicLatestChapter(_, body string) (latestChapter, bool) {
|
||||
return maxChapter(demonicChapterRe, body)
|
||||
}
|
||||
|
||||
// comixLatestChapter reads comix's SPA: the served HTML carries a JSON state
|
||||
// blob instead of chapter anchors, and latestChapterUrl is the only place the
|
||||
// newest chapter appears. Scoping to this series' id prefix keeps a
|
||||
// "recommended" strip's entries from winning the maximum.
|
||||
func comixLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
||||
id, ok := comixSeriesID(seriesURL)
|
||||
if !ok {
|
||||
return latestChapter{}, false
|
||||
}
|
||||
re := regexp.MustCompile(`"latestChapterUrl":"/title/` + regexp.QuoteMeta(id) + `-[^"]*-chapter-([0-9.]+)"`)
|
||||
return maxChapter(re, body)
|
||||
}
|
||||
|
||||
// kaganeLatestChapter scans the kagane series API JSON that the browser read
|
||||
// fetched from inside the page; the match rides on the property name,
|
||||
// regardless of the surrounding JSON shape.
|
||||
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
|
||||
return maxChapter(kaganeChapterRe, body)
|
||||
}
|
||||
|
||||
// novelfullLatestChapter is scoped to this series' slug for the same reason
|
||||
// asura is: page 1 carries a "latest chapters" widget and a "you may also
|
||||
// like" strip, and neither may contribute to the maximum.
|
||||
func novelfullLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil {
|
||||
return latestChapter{}, false
|
||||
}
|
||||
m := novelfullSlugRe.FindStringSubmatch(u.Path)
|
||||
if m == nil {
|
||||
return latestChapter{}, false
|
||||
}
|
||||
re := regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`)
|
||||
return maxChapter(re, body)
|
||||
}
|
||||
|
||||
// lnwLatestChapter truncates the body at the comment thread before scanning:
|
||||
// it is the one region of the page any visitor can write to (see lnwChapterRe).
|
||||
// A body without the marker is skipped, never scanned whole — a redesign must
|
||||
// degrade into staleness, not into a wrong shared value; the logged body length
|
||||
// tells a markup change from a body the size cap cut short.
|
||||
func lnwLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
||||
i := strings.Index(body, lnwCommentMarker)
|
||||
if i < 0 {
|
||||
log.Printf("latest poll %q: no %s marker in %d bytes", seriesURL, lnwCommentMarker, len(body))
|
||||
return latestChapter{}, false
|
||||
}
|
||||
return maxChapter(lnwChapterRe, body[:i])
|
||||
}
|
||||
|
||||
// latestChapterFrom returns the highest chapter number body advertises for this
|
||||
// series, via the Site's registry entry. ok is false when the body yields
|
||||
// nothing usable — an unknown site, an empty body, a Cloudflare challenge page,
|
||||
// and a site redesign all land here, and the caller treats all four identically.
|
||||
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
|
||||
if fn := sites[site].LatestChapter; fn != nil {
|
||||
return fn(seriesURL, body)
|
||||
}
|
||||
return latestChapter{}, false
|
||||
}
|
||||
|
||||
var metaTagRe = regexp.MustCompile(`(?is)<meta\b[^>]*>`)
|
||||
var doubleQuotedMetaAttrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*"([^"]*)"`)
|
||||
var singleQuotedMetaAttrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*'([^']*)'`)
|
||||
|
||||
// comix's server-rendered page embeds query data in this JSON script; parsing
|
||||
// the target detail entry avoids matching posters from recommended results.
|
||||
var comixInitialDataRe = regexp.MustCompile(`(?is)<script\b[^>]*\bid\s*=\s*["']initial-data["'][^>]*>(.*?)</script>`)
|
||||
|
||||
// kaganeImageURLRe matches the canonical compressed image route kagane's API
|
||||
// publishes — the only cover URL form the extractor emits and the browser
|
||||
// fetcher accepts. The URL is matched in full (scheme, host, id shape) rather
|
||||
// than trusted: the value a fetcher is pointed at may have been client-
|
||||
// supplied, and a headless browser is a strong SSRF primitive.
|
||||
var kaganeImageURLRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`)
|
||||
|
||||
// comixImageURLRe matches comix's cover host and path shape. Pinned in full
|
||||
// (scheme, host, path characters, image extension) for the same reason
|
||||
// kaganeImageURLRe is: the address reaches a headless browser, and it can
|
||||
// originate in a client-supplied PUT body. No dot is allowed inside the path,
|
||||
// so no traversal or second extension can hide in it. Shape from a live page,
|
||||
// 2026-08-10: /039d/i/1/34/6a6742bf15736@280.jpg.
|
||||
var comixImageURLRe = regexp.MustCompile(`^https://static\.comix\.to/[A-Za-z0-9@/_-]+\.(?:jpg|jpeg|png|webp)$`)
|
||||
|
||||
// browserOnlyCoverURL reports whether the browser sidecar is the only fetcher
|
||||
// for cover bytes at imageURL. kagane's image route answers a plain fetch with
|
||||
// a challenge and `cross-origin-resource-policy: same-origin`, and
|
||||
// static.comix.to answers one with the same Cloudflare challenge its pages
|
||||
// serve (measured 2026-08-12, issue #98), so a TLS fetch would only ever
|
||||
// retrieve a challenge page and must not be attempted (ADR-0007). This is the
|
||||
// byte-fetch router's per-Site knowledge; it lives in the extraction module,
|
||||
// which owns those URL shapes.
|
||||
func browserOnlyCoverURL(imageURL string) bool {
|
||||
return kaganeImageURLRe.MatchString(imageURL) ||
|
||||
comixImageURLRe.MatchString(imageURL)
|
||||
}
|
||||
|
||||
// kagane's browser-fetched series response publishes cover image IDs under
|
||||
// series_covers. The API's canonical compressed image route is the only URL
|
||||
// form accepted by the store and browser fetcher; no rendition is guessed.
|
||||
func kaganeCoverURL(body string) string {
|
||||
var response struct {
|
||||
SeriesCovers []struct {
|
||||
ImageID string `json:"image_id"`
|
||||
} `json:"series_covers"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &response); err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, cover := range response.SeriesCovers {
|
||||
// Validate the assembled URL against the same regex the browser
|
||||
// fetcher enforces, so the extractor can never emit an address the
|
||||
// fetch would refuse.
|
||||
imageURL := "https://kagane.to/api/v2/image/" + cover.ImageID + "/compressed"
|
||||
if kaganeImageURLRe.MatchString(imageURL) {
|
||||
return imageURL
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func comixCoverURL(seriesURL, body string) string {
|
||||
id, ok := comixSeriesID(seriesURL)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
data := comixInitialDataRe.FindStringSubmatch(body)
|
||||
if data == nil {
|
||||
return ""
|
||||
}
|
||||
var state struct {
|
||||
Queries map[string]json.RawMessage `json:"queries"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(data[1]), &state); err != nil {
|
||||
return ""
|
||||
}
|
||||
raw := state.Queries[`["manga","detail","`+id+`"]`]
|
||||
if len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
var detail struct {
|
||||
Poster struct {
|
||||
Medium string `json:"medium"`
|
||||
} `json:"poster"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &detail); err != nil {
|
||||
return ""
|
||||
}
|
||||
return publishedCoverURL(detail.Poster.Medium)
|
||||
}
|
||||
|
||||
// ogImageCover reads the og:image metadata shared by asura, demonic and
|
||||
// lightnovelworld.
|
||||
func ogImageCover(_, body string) (string, bool) {
|
||||
cover := metaContent(body, "property", "og:image")
|
||||
return cover, cover != ""
|
||||
}
|
||||
|
||||
func novelfullCoverEntry(_, body string) (string, bool) {
|
||||
cover := metaContent(body, "name", "image")
|
||||
return cover, cover != ""
|
||||
}
|
||||
|
||||
func comixCoverEntry(seriesURL, body string) (string, bool) {
|
||||
cover := comixCoverURL(seriesURL, body)
|
||||
return cover, cover != ""
|
||||
}
|
||||
|
||||
func kaganeCoverEntry(_, body string) (string, bool) {
|
||||
cover := kaganeCoverURL(body)
|
||||
return cover, cover != ""
|
||||
}
|
||||
|
||||
// coverFrom reports false for unknown sites, challenge bodies, and pages with
|
||||
// no usable cover, via the Site's registry entry.
|
||||
func coverFrom(site, seriesURL, body string) (string, bool) {
|
||||
if fn := sites[site].Cover; fn != nil {
|
||||
return fn(seriesURL, body)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// metaContent returns the content of the first <meta> whose attrName is
|
||||
// attrValue. It keeps scanning after an empty match so a later published cover
|
||||
// is not hidden by an empty tag.
|
||||
func metaContent(body, attrName, attrValue string) string {
|
||||
for _, tag := range metaTagRe.FindAllString(body, -1) {
|
||||
attrs := make(map[string]string)
|
||||
for _, m := range doubleQuotedMetaAttrRe.FindAllStringSubmatch(tag, -1) {
|
||||
attrs[strings.ToLower(m[1])] = m[2]
|
||||
}
|
||||
for _, m := range singleQuotedMetaAttrRe.FindAllStringSubmatch(tag, -1) {
|
||||
attrs[strings.ToLower(m[1])] = m[2]
|
||||
}
|
||||
if strings.EqualFold(attrs[strings.ToLower(attrName)], attrValue) {
|
||||
if cover := publishedCoverURL(attrs["content"]); cover != "" {
|
||||
return cover
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func publishedCoverURL(value string) string {
|
||||
value = strings.TrimSpace(html.UnescapeString(value))
|
||||
return strings.ReplaceAll(value, " ", "%20")
|
||||
}
|
||||
|
||||
// Poll Lane constants (issue #100). The per-Site structure is deliberately
|
||||
// uniform at first — every Site rests an hour and gaps ten seconds — but it
|
||||
// exists so a single Site can be slowed if it turns hostile, and the numbers
|
||||
// stay in the registry so the structure has a place to differ.
|
||||
const (
|
||||
// defaultRest is how long every Series rests between Polls.
|
||||
defaultRest = time.Hour
|
||||
// defaultGap is the strictest pace of every Lane unless the eligible
|
||||
// Series count forces it tighter.
|
||||
defaultGap = 10 * time.Second
|
||||
// minGap floors the effective gap. One request per second is already an
|
||||
// order of magnitude past the strictest rate rule a free-plan Site can
|
||||
// express (docs/research/cloudflare-bot-scoring-and-poll-cadence.md);
|
||||
// below it the Lane is outrunning its own plan and says so loudly.
|
||||
minGap = time.Second
|
||||
// RefuseBackoff is how long a Lane waits after its Site refused twice in
|
||||
// one run before attempting it again. Exported so the web layer can derive
|
||||
// browser reachability from the pass log over the same window (issue #145).
|
||||
RefuseBackoff = 15 * time.Minute
|
||||
// browserWakeCount and browserWakeAge gate a browser Lane's run: five or
|
||||
// more due Series, or any one of them waiting this long, or Chrome stays
|
||||
// asleep (ADR-0005 on-demand browser).
|
||||
browserWakeCount = 5
|
||||
browserWakeAge = 15 * time.Minute
|
||||
// sightingCeilingRests caps Sighting deferral (issue #103): however many
|
||||
// Sightings arrive, a Series unpolled for this many of its Site's rests is
|
||||
// Polled. It is what makes a client report safe to trust — a wrong Latest
|
||||
// Chapter dies within the ceiling deterministically, rather than in
|
||||
// expectation the way a randomised audit would have it. Six, so a Series a
|
||||
// Reader visits constantly still gets one authoritative check per working
|
||||
// day-part.
|
||||
sightingCeilingRests = 6
|
||||
)
|
||||
|
||||
// effectiveGap is a Site's pace: the registry gap, or one rest divided by the
|
||||
// eligible Series count when that is smaller, never below one second. The
|
||||
// denominator follows defaultRest rather than a literal hour so a Site whose
|
||||
// rest is ever changed keeps its per-Series pace in step. The second return is
|
||||
// true when the one-second floor engaged (and the Lane logs a warning naming
|
||||
// the Site, every round it does).
|
||||
func effectiveGap(s site, eligible int) (time.Duration, bool) {
|
||||
gap := s.Gap
|
||||
if eligible > 0 {
|
||||
if perSeries := defaultRest / time.Duration(eligible); perSeries < gap {
|
||||
gap = perSeries
|
||||
}
|
||||
}
|
||||
if gap < minGap {
|
||||
return minGap, true
|
||||
}
|
||||
return gap, false
|
||||
}
|
||||
|
||||
// sites is the registry: one entry per Site, keyed by the stored site string.
|
||||
// Adding a Site means adding an entry here and nowhere else — the dispatch
|
||||
// functions above and the poller's route list are lookups into this map. An
|
||||
// unknown site string resolves to the zero entry, which fails the existing
|
||||
// not-fetchable and no-fetcher paths unchanged.
|
||||
var sites = map[string]site{
|
||||
"asura": {
|
||||
Host: "asurascans.com",
|
||||
LatestChapter: asuraLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
"demonic": {
|
||||
Host: "demonicscans.org",
|
||||
LatestChapter: demonicLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
"comix": {
|
||||
Host: "comix.to",
|
||||
LatestChapter: comixLatestChapter,
|
||||
Cover: comixCoverEntry,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
Read: comixRead,
|
||||
// The interstitial is served in place of the page, so "arrived"
|
||||
// has to exclude it explicitly, as novelfull's does.
|
||||
Done: func(body string) bool { return body != "" && !isInterstitial(body) },
|
||||
// Never falls back: a plain fetch of a comix page or cover
|
||||
// retrieves only a challenge page (measured 2026-08-12).
|
||||
Fallback: false,
|
||||
},
|
||||
},
|
||||
"kagane": {
|
||||
Host: "kagane.to",
|
||||
LatestChapter: kaganeLatestChapter,
|
||||
Cover: kaganeCoverEntry,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
Read: kaganeRead,
|
||||
Done: func(body string) bool { return body != "" },
|
||||
// Never falls back: a plain fetch of a kagane page or cover would
|
||||
// only ever retrieve a challenge page (verified 2026-08-03).
|
||||
Fallback: false,
|
||||
},
|
||||
},
|
||||
"novelfull": {
|
||||
Host: "novelfull.com",
|
||||
LatestChapter: novelfullLatestChapter,
|
||||
Cover: novelfullCoverEntry,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
Read: novelfullRead,
|
||||
// The interstitial has a DOM too, so "the payload arrived" has to
|
||||
// exclude it explicitly.
|
||||
Done: func(body string) bool { return body != "" && !isInterstitial(body) },
|
||||
Fallback: true,
|
||||
},
|
||||
},
|
||||
"lightnovelworld": {
|
||||
Host: "lightnovelworld.net",
|
||||
LatestChapter: lnwLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
}
|
||||
|
||||
// SiteNames returns every registry Site, sorted. The admin Series list's Site
|
||||
// select needs the full registry, not just the Sites that have rows, and
|
||||
// laneNames() is the poller's copy of the same list — both read this.
|
||||
func SiteNames() []string {
|
||||
names := make([]string, 0, len(sites))
|
||||
for name := range sites {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// browserBackedSites is derived from the registry: the Sites whose pages are
|
||||
// read through the browser sidecar. Sorted so callers that range it (the
|
||||
// browser fetcher's dispatch) see a stable order instead of map-iteration
|
||||
// noise.
|
||||
func browserBackedSites() []string {
|
||||
out := make([]string, 0, len(sites))
|
||||
for name, s := range sites {
|
||||
if s.Browser != nil {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,475 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
|
||||
// fetched 2026-07-26. The first anchor is the "First Chapter" shortcut: it is a
|
||||
// real chapter link with no "Chapter N" text, and it must not be mistaken for
|
||||
// the latest just because it parses.
|
||||
const asuraSeriesFixture = `
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/1" class="py-3 rounded-md bg-[#E8E8E8]"><svg class="w-4 h-4"></svg>First Chapter</a>
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/179" data-astro-prefetch="hover" class="group flex"><span class="font-medium">Chapter 179</span></a>
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/181" data-astro-prefetch="hover" class="group flex"><span class="font-medium">Chapter 181</span></a>
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/180" data-astro-prefetch="hover" class="group flex"><span class="font-medium">Chapter 180</span></a>
|
||||
`
|
||||
|
||||
// A chapter link belonging to a different series, of the kind a "you might also
|
||||
// like" strip would introduce. Slug scoping must exclude it.
|
||||
const asuraCrossSeriesFixture = asuraSeriesFixture + `
|
||||
<a href="/comics/some-other-series-aabbccdd/chapter/999" class="group flex"><span>Chapter 999</span></a>
|
||||
`
|
||||
|
||||
// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer fetched
|
||||
// 2026-07-26. Note the raw "&", the doubled space after <a, and the decimal
|
||||
// chapters, all as they appear live.
|
||||
const demonicSeriesFixture = `
|
||||
<a href="/chaptered.php?manga=11799&chapter=0.5" class="chplinks" title="Catastrophic Necromancer 0.5">Chapter 0.5</a>
|
||||
<a href="/chaptered.php?manga=11799&chapter=294" class="chplinks" title="Catastrophic Necromancer 294">Chapter 294</a>
|
||||
<a href="/chaptered.php?manga=11799&chapter=296" class="chplinks" title="Catastrophic Necromancer 296">Chapter 296</a>
|
||||
<a href="/chaptered.php?manga=11799&chapter=295" class="chplinks" title="Catastrophic Necromancer 295">Chapter 295</a>
|
||||
`
|
||||
|
||||
// What Cloudflare serves instead of the page when an IP's bot score flips.
|
||||
const challengeFixture = `<!DOCTYPE html><html><head><title>Just a moment...</title>
|
||||
<script src="/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page/v1"></script></head>
|
||||
<body><div id="challenge-running">Checking your browser</div></body></html>`
|
||||
|
||||
// Trimmed from the server-rendered HTML of
|
||||
// https://comix.to/title/n8we-dungeons-and-crayons fetched 2026-08-03. comix is
|
||||
// an SPA: the page ships a JSON state blob rather than a list of chapter
|
||||
// anchors, and latestChapterUrl is where the newest chapter actually lives.
|
||||
//
|
||||
// Still the right fixture after comix moved behind the challenge (#98): the
|
||||
// browser read is an in-tab fetch of the Series URL, so the body a poll parses
|
||||
// is this same server-rendered HTML, not a rendered DOM. Confirmed against a
|
||||
// live cleared tab 2026-08-16 (TestSmokeComix): the in-tab fetch returned
|
||||
// 24793 bytes of server-rendered HTML that these same parses read a chapter
|
||||
// and a cover out of.
|
||||
const comixSeriesFixture = `
|
||||
{"firstChapterUrl":"/title/n8we-dungeons-and-crayons/5038739-chapter-1","latestChapterUrl":"/title/n8we-dungeons-and-crayons/11139891-chapter-80"},
|
||||
{""manga","recommended","n8we",1]":{"items":[{"latestChapterUrl":"/title/qqwrm-full-time-awakening/99999999-chapter-999"}]}
|
||||
`
|
||||
|
||||
// The kagane branch is fed by the browser fetcher, so the body is API JSON, not
|
||||
// HTML. Trimmed from GET /api/v2/series/<uuid> on 2026-08-03.
|
||||
const kaganeAPIFixture = `
|
||||
{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption",
|
||||
"series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1},
|
||||
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
|
||||
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
|
||||
`
|
||||
|
||||
// Trimmed from https://novelfull.com/reverend-insanity.html fetched 2026-08-05.
|
||||
// The page carries a newest-first "latest chapters" widget above an
|
||||
// oldest-first paginated list, so the newest anchor is deliberately NOT last —
|
||||
// only a maximum finds it. The final anchor belongs to another series and must
|
||||
// be excluded by slug scoping.
|
||||
const novelfullSeriesFixture = `
|
||||
<div class="l-chapters">
|
||||
<a href="/reverend-insanity/chapter-2334-fang-yuan-and-giant-sun.html">Chapter 2334</a>
|
||||
<a href="/reverend-insanity/chapter-2333-three-venerables.html">Chapter 2333</a>
|
||||
</div>
|
||||
<ul class="list-chapter">
|
||||
<li><a href="/reverend-insanity/chapter-1.html">Chapter 1</a></li>
|
||||
<li><a href="/reverend-insanity/chapter-2.html">Chapter 2</a></li>
|
||||
</ul>
|
||||
<a href="/release-that-witch/chapter-9999.html">Chapter 9999</a>
|
||||
`
|
||||
|
||||
// Trimmed from
|
||||
// https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/
|
||||
// fetched 2026-08-11, whole document (~307 KB decoded; the wire body is ~32 KB
|
||||
// zstd-compressed). This
|
||||
// novel publishes its chapters under two Chapter Slugs: 1–99 at
|
||||
// …-not-chapter-<n>/ and 100–423 at …-not-them-all-chapter-<n>/, and the page
|
||||
// lists them newest-first, so the …-not-them-all anchors precede the …-not
|
||||
// anchors. Every anchor through the comment-thread marker is verbatim page
|
||||
// text (the site renders this novel's chapter titles as "[ ... words ]"). The
|
||||
// comment block after the marker is the real wpdiscuz comment #wpd-comm-358_0
|
||||
// from https://lightnovelworld.net/novel/the-sword-illuminates-the-great-wilderness/
|
||||
// — the pinned page serves zero comments — with its share/link/vote/reply
|
||||
// boilerplate trimmed. The comment's body carried no link, so the bare <a
|
||||
// href> to https://lightnovelworld.net/overgeared-chapter-2059/ inside
|
||||
// wpd-comment-text is the one composed element; that URL is a real chapter of
|
||||
// a real different novel (overgeared; fetched, HTTP 200).
|
||||
const lnwSeriesFixture = `
|
||||
<li data-ID="102741">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all-chapter-404/">
|
||||
<div class="epl-num">Vol. 1 Ch. 404</div>
|
||||
<div class="epl-title">[ ... words ]</div>
|
||||
<div class="epl-date">April 12, 2026</div>
|
||||
</a>
|
||||
</li>
|
||||
<li data-ID="102780">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all-chapter-423/">
|
||||
<div class="epl-num">Vol. 1 Ch. 423</div>
|
||||
<div class="epl-title">[ ... words ]</div>
|
||||
<div class="epl-date">April 7, 2026</div>
|
||||
</a>
|
||||
</li>
|
||||
<li data-ID="102527">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all-chapter-300/">
|
||||
<div class="epl-num">Vol. 1 Ch. 300</div>
|
||||
<div class="epl-title">[ ... words ]</div>
|
||||
<div class="epl-date">April 4, 2026</div>
|
||||
</a>
|
||||
</li>
|
||||
<li data-ID="102325">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all-chapter-200/">
|
||||
<div class="epl-num">Vol. 1 Ch. 200</div>
|
||||
<div class="epl-title">[ ... words ]</div>
|
||||
<div class="epl-date">March 29, 2026</div>
|
||||
</a>
|
||||
</li>
|
||||
<li data-ID="102121">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all-chapter-100/">
|
||||
<div class="epl-num">Vol. 1 Ch. 100</div>
|
||||
<div class="epl-title">[ ... words ]</div>
|
||||
<div class="epl-date">March 22, 2026</div>
|
||||
</a>
|
||||
</li>
|
||||
<li data-ID="26014">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-chapter-99/">
|
||||
<div class="epl-num">Vol. 1 Ch. 99</div>
|
||||
<div class="epl-title">Chapter 99</div>
|
||||
<div class="epl-date">November 5, 2025</div>
|
||||
</a>
|
||||
</li>
|
||||
<li data-ID="25916">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-chapter-50/">
|
||||
<div class="epl-num">Vol. 1 Ch. 50</div>
|
||||
<div class="epl-title">Chapter 50</div>
|
||||
<div class="epl-date">October 29, 2025</div>
|
||||
</a>
|
||||
</li>
|
||||
<li class='tseplsfrst' data-ID="25818">
|
||||
<a href="https://lightnovelworld.net/all-jobs-and-classes-i-just-wanted-one-skill-not-chapter-1/">
|
||||
<div class="epl-num">Vol. 1 Ch. 1</div>
|
||||
<div class="epl-title">Chapter 01</div>
|
||||
<div class="epl-date">October 11, 2025</div>
|
||||
</a>
|
||||
</li>
|
||||
<div id="wpd-threads" class="wpd-thread-wrapper">
|
||||
<div class="wpd-thread-list">
|
||||
<div id='wpd-comm-358_0' class='comment byuser comment-author-jimbear even thread-even depth-1 wpd-comment wpd_comment_level-1'><div class="wpd-comment-wrap wpd-blog-user wpd-blog-subscriber">
|
||||
<div class="wpd-comment-left ">
|
||||
<div class="wpd-avatar ">
|
||||
<img alt='hasbi asy' src='https://secure.gravatar.com/avatar/3c1792cb31cab842f90e7c463f0948e98536cf938aebbd2f678f937ca60fb799?s=64&d=mm&r=g' srcset='https://secure.gravatar.com/avatar/3c1792cb31cab842f90e7c463f0948e98536cf938aebbd2f678f937ca60fb799?s=128&d=mm&r=g 2x' class='avatar avatar-64 photo' height='64' width='64' decoding='async'/>
|
||||
</div>
|
||||
<div class="wpd-comment-label" wpd-tooltip="Member" wpd-tooltip-position="right">
|
||||
<span>Member</span>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div id="comment-358" class="wpd-comment-right">
|
||||
<div class="wpd-comment-header">
|
||||
<div class="wpd-comment-author ">
|
||||
hasbi asy
|
||||
</div>
|
||||
<div class="wpd-comment-date" title="July 9, 2026 1:44 am">
|
||||
<i class='far fa-clock' aria-hidden='true'></i>
|
||||
1 month ago
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="wpd-comment-text">
|
||||
<p>where’s everyone</p>
|
||||
|
||||
<a href="https://lightnovelworld.net/overgeared-chapter-2059/">https://lightnovelworld.net/overgeared-chapter-2059/</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id='wpdiscuz_form_anchor-358_0'></div>
|
||||
</div>
|
||||
</div>
|
||||
`
|
||||
|
||||
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
|
||||
// (redirected to ...-00dcbf97) on 2026-08-10.
|
||||
const asuraCoverFixture = `<meta property="og:image" content="https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp">`
|
||||
|
||||
// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer on 2026-08-10.
|
||||
// The source publishes the raw space in this URL.
|
||||
const demonicCoverFixture = `<meta property="og:image" content="https://readermc.org/images/thumbnails/Catastrophic Necromancer.webp">`
|
||||
|
||||
// Trimmed from https://comix.to/title/n8we-dungeons-and-crayons on 2026-08-10.
|
||||
// The state includes a recommended poster before the target detail object and
|
||||
// nested IDs inside that object; no og:image is present.
|
||||
const comixCoverFixture = `<script type="application/json" id="initial-data">{"queries":{"[\"manga\",\"recommended\",\"n8we\",1]":{"poster":{"medium":"https://static.comix.to/recommended@280.jpg","large":"https://static.comix.to/recommended.jpg"}},"[\"manga\",\"detail\",\"n8we\"]":{"chapters":[{"hid":"nested"}],"poster":{"medium":"https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg","large":"https://static.comix.to/039d/i/1/34/6a6742bf15736.jpg"}}}}</script>`
|
||||
|
||||
// Trimmed from GET https://kagane.to/api/v2/series/019fe11a-8670-7cf3-8343-0b02057d3787 on 2026-08-10.
|
||||
const kaganeCoverFixture = `{"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}]}`
|
||||
|
||||
// Trimmed from https://novelfull.com/reverend-insanity.html on 2026-08-10.
|
||||
const novelfullCoverFixture = `<meta name="image" content="https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp">`
|
||||
|
||||
// Trimmed from
|
||||
// https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/
|
||||
// on 2026-08-11.
|
||||
const lnwCoverFixture = `<meta property="og:image" content="https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg" />`
|
||||
|
||||
func TestCoverFrom(t *testing.T) {
|
||||
const comixURL = "https://comix.to/title/n8we-dungeons-and-crayons"
|
||||
tests := []struct {
|
||||
name string
|
||||
site string
|
||||
seriesURL string
|
||||
body string
|
||||
wantOK bool
|
||||
wantCover string
|
||||
}{
|
||||
{
|
||||
name: "asura uses published metadata URL",
|
||||
site: "asura", body: asuraCoverFixture, wantOK: true,
|
||||
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
||||
},
|
||||
{
|
||||
name: "demonic escapes raw spaces",
|
||||
site: "demonic", body: demonicCoverFixture, wantOK: true,
|
||||
wantCover: "https://readermc.org/images/thumbnails/Catastrophic%20Necromancer.webp",
|
||||
},
|
||||
{
|
||||
name: "comix takes target medium poster",
|
||||
site: "comix", seriesURL: comixURL, body: comixCoverFixture, wantOK: true,
|
||||
wantCover: "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg",
|
||||
},
|
||||
{
|
||||
name: "kagane reads API cover image ID",
|
||||
site: "kagane", body: kaganeCoverFixture, wantOK: true,
|
||||
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
||||
},
|
||||
{
|
||||
name: "novelfull reads image metadata",
|
||||
site: "novelfull", body: novelfullCoverFixture, wantOK: true,
|
||||
wantCover: "https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp",
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld reads og image",
|
||||
site: "lightnovelworld", body: lnwCoverFixture, wantOK: true,
|
||||
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
|
||||
},
|
||||
{
|
||||
name: "later metadata cover survives empty match",
|
||||
site: "asura",
|
||||
body: `<meta property="og:image" content="">` + asuraCoverFixture,
|
||||
wantOK: true,
|
||||
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
||||
},
|
||||
{
|
||||
name: "page without cover is empty",
|
||||
site: "asura", body: `<meta property="og:title" content="No Cover">`,
|
||||
},
|
||||
{
|
||||
name: "unknown site is empty",
|
||||
site: "unknown", body: asuraCoverFixture,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := coverFrom(tt.site, tt.seriesURL, tt.body)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("ok = %v, want %v (got %q)", ok, tt.wantOK, got)
|
||||
}
|
||||
if got != tt.wantCover {
|
||||
t.Errorf("cover = %q, want %q", got, tt.wantCover)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverFromChallenge(t *testing.T) {
|
||||
tests := []struct {
|
||||
site string
|
||||
seriesURL string
|
||||
}{
|
||||
{"asura", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"},
|
||||
{"demonic", "https://demonicscans.org/manga/Catastrophic-Necromancer"},
|
||||
{"comix", "https://comix.to/title/n8we-dungeons-and-crayons"},
|
||||
{"kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"},
|
||||
{"novelfull", "https://novelfull.com/reverend-insanity.html"},
|
||||
{"lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.site, func(t *testing.T) {
|
||||
if got, ok := coverFrom(tt.site, tt.seriesURL, challengeFixture); ok || got != "" {
|
||||
t.Fatalf("cover = %q, ok = %v, want empty", got, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestChapterFrom(t *testing.T) {
|
||||
const asuraURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
const demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
site string
|
||||
seriesURL string
|
||||
body string
|
||||
wantOK bool
|
||||
wantNum float64
|
||||
wantLabel string
|
||||
}{
|
||||
{
|
||||
name: "asura takes the max, not the last listed",
|
||||
site: "asura", seriesURL: asuraURL, body: asuraSeriesFixture,
|
||||
wantOK: true, wantNum: 181, wantLabel: "Chapter 181",
|
||||
},
|
||||
{
|
||||
name: "asura ignores another series' chapter links",
|
||||
site: "asura", seriesURL: asuraURL, body: asuraCrossSeriesFixture,
|
||||
wantOK: true, wantNum: 181, wantLabel: "Chapter 181",
|
||||
},
|
||||
{
|
||||
name: "asura scoping survives a build-hash rotation",
|
||||
site: "asura",
|
||||
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-059befe1",
|
||||
body: asuraCrossSeriesFixture,
|
||||
wantOK: true, wantNum: 181, wantLabel: "Chapter 181",
|
||||
},
|
||||
{
|
||||
name: "asura with an unparseable series url",
|
||||
site: "asura", seriesURL: "https://asurascans.com/", body: asuraSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "demonic takes the max across raw and escaped ampersands",
|
||||
site: "demonic", seriesURL: demonicURL, body: demonicSeriesFixture,
|
||||
wantOK: true, wantNum: 296, wantLabel: "Chapter 296",
|
||||
},
|
||||
{
|
||||
name: "demonic keeps decimal chapters parseable",
|
||||
site: "demonic", seriesURL: demonicURL,
|
||||
body: `<a href="/chaptered.php?manga=11799&chapter=0.5">Chapter 0.5</a>`,
|
||||
wantOK: true, wantNum: 0.5, wantLabel: "Chapter 0.5",
|
||||
},
|
||||
{
|
||||
name: "empty body",
|
||||
site: "asura", seriesURL: asuraURL, body: "",
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "cloudflare challenge page",
|
||||
site: "asura", seriesURL: asuraURL, body: challengeFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "demonic markup handed to the asura rule",
|
||||
site: "asura", seriesURL: asuraURL, body: demonicSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "unknown site",
|
||||
site: "mangadex", seriesURL: "https://example.com/x", body: asuraSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "comix reads latestChapterUrl, scoped to this series",
|
||||
site: "comix",
|
||||
seriesURL: "https://comix.to/title/n8we-dungeons-and-crayons",
|
||||
body: comixSeriesFixture,
|
||||
wantOK: true, wantNum: 80, wantLabel: "Chapter 80",
|
||||
},
|
||||
{
|
||||
name: "comix yields nothing on a challenge page",
|
||||
site: "comix",
|
||||
seriesURL: "https://comix.to/title/n8we-dungeons-and-crayons",
|
||||
body: challengeFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "kagane takes the max chapter_no from API json",
|
||||
site: "kagane",
|
||||
seriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
body: kaganeAPIFixture,
|
||||
wantOK: true, wantNum: 41, wantLabel: "Chapter 41",
|
||||
},
|
||||
{
|
||||
name: "kagane yields nothing on a challenge page",
|
||||
site: "kagane",
|
||||
seriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
||||
body: challengeFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "novelfull takes the max and ignores another series",
|
||||
site: "novelfull",
|
||||
seriesURL: "https://novelfull.com/reverend-insanity.html",
|
||||
body: novelfullSeriesFixture,
|
||||
wantOK: true, wantNum: 2334, wantLabel: "Chapter 2334",
|
||||
},
|
||||
{
|
||||
name: "novelfull yields nothing on a challenge page",
|
||||
site: "novelfull",
|
||||
seriesURL: "https://novelfull.com/reverend-insanity.html",
|
||||
body: challengeFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "novelfull with an unparseable series url",
|
||||
site: "novelfull",
|
||||
seriesURL: "https://novelfull.com/genre/Fantasy",
|
||||
body: novelfullSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
// Stored before the slug split, so the address carries the ...-not
|
||||
// Chapter Slug; the 100-423 block under the other slug must still win.
|
||||
// The body is the chapter-list portion of lnwSeriesFixture with the
|
||||
// comment block omitted; the marker is kept, because a body without it
|
||||
// is skipped, not scanned.
|
||||
{
|
||||
name: "lightnovelworld max spans both chapter slugs",
|
||||
site: "lightnovelworld",
|
||||
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not/",
|
||||
body: strings.SplitN(lnwSeriesFixture, lnwCommentMarker, 2)[0] + lnwCommentMarker,
|
||||
wantOK: true, wantNum: 423, wantLabel: "Chapter 423",
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld comment anchor cannot set the latest chapter",
|
||||
site: "lightnovelworld",
|
||||
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||
body: lnwSeriesFixture,
|
||||
wantOK: true, wantNum: 423, wantLabel: "Chapter 423",
|
||||
},
|
||||
// Marker removed from the fixture, comment block still present: a
|
||||
// redesign must degrade into a skip, never into the comment's number.
|
||||
{
|
||||
name: "lightnovelworld body without the comment marker is skipped",
|
||||
site: "lightnovelworld",
|
||||
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||
body: strings.ReplaceAll(lnwSeriesFixture, lnwCommentMarker, ""),
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld yields nothing on a challenge page",
|
||||
site: "lightnovelworld",
|
||||
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||
body: challengeFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := latestChapterFrom(tt.site, tt.seriesURL, tt.body)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("ok = %v, want %v (got %+v)", ok, tt.wantOK, got)
|
||||
}
|
||||
if !tt.wantOK {
|
||||
return
|
||||
}
|
||||
if got.Num != tt.wantNum {
|
||||
t.Errorf("Num = %v, want %v", got.Num, tt.wantNum)
|
||||
}
|
||||
if got.Label != tt.wantLabel {
|
||||
t.Errorf("Label = %q, want %q", got.Label, tt.wantLabel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// TestSmokeComix answers "is comix's challenge clearing from this browser right
|
||||
// now" — a live, time-varying fact, so a red run is something to re-check
|
||||
// before it is a defect. Needs the real browser unit with outbound network:
|
||||
//
|
||||
// cd chrome && BROWSER_BIND_ADDR=127.0.0.1 docker compose up -d --build
|
||||
// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeComix ./internal/latest
|
||||
//
|
||||
// It walks the whole read: the in-tab page fetch, both parses, and the Cover
|
||||
// bytes by direct navigation to static.comix.to. The Cover address comes out of
|
||||
// the page rather than being pinned in the test, because a stored one rots.
|
||||
func TestSmokeComix(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
const seriesURL = "https://comix.to/title/m12d-classmate"
|
||||
|
||||
f, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
body, status, err := f.Get(ctx, seriesURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
t.Logf("status=%d bytes=%d", status, len(body))
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status)
|
||||
}
|
||||
chapter, ok := latestChapterFrom("comix", seriesURL, body)
|
||||
if !ok {
|
||||
t.Fatalf("no latest chapter in %d bytes — page shape changed", len(body))
|
||||
}
|
||||
t.Logf("latest chapter: %v %q", chapter.Num, chapter.Label)
|
||||
|
||||
cover, ok := coverFrom("comix", seriesURL, body)
|
||||
if !ok {
|
||||
t.Fatalf("no cover address in %d bytes — page shape changed", len(body))
|
||||
}
|
||||
t.Logf("cover: %s", cover)
|
||||
if !browserOnlyCoverURL(cover) {
|
||||
t.Fatalf("cover %q is not claimed by the browser gate: the pin and the live URL shape disagree", cover)
|
||||
}
|
||||
|
||||
bytes, contentType, err := f.Image(ctx, cover)
|
||||
if err != nil {
|
||||
t.Fatalf("Image: %v", err)
|
||||
}
|
||||
if len(bytes) < 1000 {
|
||||
t.Fatalf("cover is %d bytes, want a real image", len(bytes))
|
||||
}
|
||||
t.Logf("fetched %d bytes of %s", len(bytes), contentType)
|
||||
if _, ok := store.CoverContentType(contentType); !ok {
|
||||
t.Fatalf("content type %q is not storable", contentType)
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// TestSmokeKaganeImage is the live proof that the acquisition path's browser
|
||||
// fetch actually clears Cloudflare and returns image bytes. It needs the real
|
||||
// browser unit with outbound network, so it runs only when SMOKE_BROWSER_WS_URL
|
||||
// is set:
|
||||
//
|
||||
// cd chrome && BROWSER_BIND_ADDR=127.0.0.1 docker compose up -d --build
|
||||
// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeKaganeImage ./internal/latest
|
||||
//
|
||||
// Not chromedp/headless-shell: its challenge never clears (see chrome/Dockerfile),
|
||||
// so a red run there proves nothing about kagane.
|
||||
func TestSmokeKaganeImage(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
const imageURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed" // SP Baby's cover
|
||||
|
||||
// The same URL through a plain client is what any other fetcher would get.
|
||||
// Asserting on it keeps the test honest about why the browser is needed.
|
||||
req, err := http.NewRequest(http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res, err := (&http.Client{Timeout: 15 * time.Second}).Do(req); err == nil {
|
||||
res.Body.Close()
|
||||
if res.StatusCode == http.StatusOK {
|
||||
t.Log("note: kagane answered a plain request 200 — the challenge is not up right now")
|
||||
}
|
||||
}
|
||||
|
||||
f, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
body, contentType, err := f.Image(ctx, imageURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Image: %v", err)
|
||||
}
|
||||
if len(body) < 1000 {
|
||||
t.Fatalf("body is %d bytes, want a real image", len(body))
|
||||
}
|
||||
if contentType != "image/webp" {
|
||||
t.Fatalf("content type = %q, want image/webp", contentType)
|
||||
}
|
||||
// WebP files start with "RIFF....WEBP".
|
||||
if string(body[:4]) != "RIFF" || string(body[8:12]) != "WEBP" {
|
||||
t.Fatalf("body is not a WebP: % x", body[:12])
|
||||
}
|
||||
t.Logf("fetched %d bytes of %s", len(body), contentType)
|
||||
|
||||
// The browser module claims only the cover URL shape it can clear a
|
||||
// challenge for; anything else must be refused before any navigation.
|
||||
if _, _, err := f.Image(ctx, "https://cdn.example/cover.jpg"); err == nil {
|
||||
t.Fatal("Image accepted a cover URL the browser module does not claim")
|
||||
}
|
||||
}
|
||||
|
||||
// Control for the test above: the poller's own kagane path, same sidecar. If
|
||||
// this fails too, the sidecar is not clearing the challenge at all and the
|
||||
// image result says nothing about Image itself.
|
||||
func TestSmokeKaganeGet(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
f, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
body, status, err := f.Get(ctx, "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
t.Logf("status=%d bytes=%d head=%.80q", status, len(body), body)
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSmokeAcquireKaganeCover proves the #62 acquisition path end to end
|
||||
// against the real browser: a kagane Series bookmarked at creation gets its
|
||||
// Cover, bytes fetched through the sidecar into the content-addressed store.
|
||||
// Same SMOKE_BROWSER_WS_URL gate as the tests above; a red run means the
|
||||
// challenge is not clearing from this IP (a live fact to re-check), not
|
||||
// necessarily a defect in the pipeline.
|
||||
func TestSmokeAcquireKaganeCover(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
const (
|
||||
seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
|
||||
coverURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed"
|
||||
)
|
||||
s, _ := newTestStore(t)
|
||||
bf, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer bf.Close()
|
||||
tlsF, err := NewTLSFetcher()
|
||||
if err != nil {
|
||||
t.Fatalf("NewTLSFetcher: %v", err)
|
||||
}
|
||||
acq := &Acquirer{
|
||||
Store: s, Fetch: tlsF, BrowserFetch: bf,
|
||||
BrowserCoverFetch: bf, Covers: NewCoverFetcher(),
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: "kagane:" + seriesID, Site: "kagane", SeriesID: seriesID,
|
||||
Title: "smoke", SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
acq.Wait()
|
||||
|
||||
got, found, err := s.Get(s.OwnerID(), "kagane:"+seriesID)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: %v found=%v", err, found)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q — the acquire path did not store the browser-fetched bytes", got.Cover, want)
|
||||
}
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
||||
}
|
||||
if len(body) < 1000 {
|
||||
t.Fatalf("stored cover is %d bytes, want a real image", len(body))
|
||||
}
|
||||
if contentType != "image/webp" {
|
||||
t.Fatalf("content type = %q, want image/webp", contentType)
|
||||
}
|
||||
t.Logf("stored %d bytes of %s", len(body), contentType)
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// lnwSeriesPageFloor is the smallest body that can still be a whole
|
||||
// lightnovelworld Series page. Whole pages measured 685 KB..1.18 MB on
|
||||
// 2026-08-11 and carry the marker at ~94% of the document, so a body under
|
||||
// 100 KB is a challenge, a notice, or a truncated read — asserting on it
|
||||
// would report the marker missing when it was never fetched.
|
||||
const lnwSeriesPageFloor = 100 << 10
|
||||
|
||||
// TestSmokeLnwCommentBoundary is the live proof that the comment-thread marker
|
||||
// the lightnovelworld chapter scan truncates at (lnwCommentMarker,
|
||||
// "wpd-threads") still holds on the Site. The scan depends on it: when the
|
||||
// marker vanishes every Series is skipped and logged — correct, but silent
|
||||
// until a Reader notices their Latest Chapter has stopped moving. It runs only
|
||||
// when SMOKE_LNW_SERIES_URL is set — the URL of the live Series page to check.
|
||||
// The immortality-simulator page measured 2026-08-11
|
||||
// (docs/research/lightnovelworld-chapter-vs-series-slug.md) is the default to
|
||||
// point it at:
|
||||
//
|
||||
// SMOKE_LNW_SERIES_URL=https://lightnovelworld.net/novel/immortality-simulator/ go test -v -run TestSmokeLnwCommentBoundary ./internal/latest
|
||||
//
|
||||
// A red run means the Site's markup has moved — the marker is gone, occurs
|
||||
// more than once, or no longer follows the last chapter anchor — and the scan
|
||||
// in sites.go is now skipping this Site. Revisit sites.go before anything
|
||||
// else; the test is not flaky. A Cloudflare challenge or a non-200 is
|
||||
// distinguished from a marker failure by the "not a marker failure" messages
|
||||
// below, which carry the observed status and body length.
|
||||
func TestSmokeLnwCommentBoundary(t *testing.T) {
|
||||
seriesURL := os.Getenv("SMOKE_LNW_SERIES_URL")
|
||||
if seriesURL == "" {
|
||||
t.Skip("SMOKE_LNW_SERIES_URL unset")
|
||||
}
|
||||
if !fetchableSeriesURL("lightnovelworld", seriesURL) {
|
||||
t.Fatalf("%q is not a fetchable lightnovelworld series URL", seriesURL)
|
||||
}
|
||||
|
||||
f, err := NewTLSFetcher()
|
||||
if err != nil {
|
||||
t.Fatalf("NewTLSFetcher: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
body, status, err := f.Get(ctx, seriesURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("status = %d, body %d bytes — not a marker failure; the Site did not answer this IP with a Series page", status, len(body))
|
||||
}
|
||||
if len(body) < lnwSeriesPageFloor {
|
||||
t.Fatalf("body %d bytes — not a whole Series page (measured 685 KB..1.18 MB); not a marker failure, likely a Cloudflare challenge or a non-Series response", len(body))
|
||||
}
|
||||
if !lnwChapterRe.MatchString(body) {
|
||||
t.Fatalf("no chapter anchor in %d bytes — not a lightnovelworld Series page; not a marker failure, likely a Cloudflare challenge or a non-Series response", len(body))
|
||||
}
|
||||
markerIdx := strings.Index(body, lnwCommentMarker)
|
||||
if failures := checkLnwCommentBoundary(body); len(failures) > 0 {
|
||||
t.Fatalf("%s (body %d bytes)", strings.Join(failures, "; "), len(body))
|
||||
}
|
||||
t.Logf("ok: %q once at byte %d, body %d bytes", lnwCommentMarker, markerIdx, len(body))
|
||||
}
|
||||
|
||||
// checkLnwCommentBoundary verifies the three marker assertions against a
|
||||
// fetched Series body: the marker occurs exactly once, every chapter anchor
|
||||
// precedes it, and at least one anchor precedes it at all. It returns one
|
||||
// human-readable failure per broken assertion — with observed offsets and body
|
||||
// length — and empty when the page is healthy.
|
||||
func checkLnwCommentBoundary(body string) []string {
|
||||
markerIdx := strings.Index(body, lnwCommentMarker)
|
||||
switch n := strings.Count(body, lnwCommentMarker); {
|
||||
case n == 0:
|
||||
return []string{fmt.Sprintf("%q occurs 0 times in %d bytes, want exactly 1", lnwCommentMarker, len(body))}
|
||||
case n != 1:
|
||||
return []string{fmt.Sprintf("%q occurs %d times in %d bytes (first at byte %d), want exactly 1", lnwCommentMarker, n, len(body), markerIdx)}
|
||||
}
|
||||
lastAnchor, anchorsBefore := -1, 0
|
||||
for _, m := range lnwChapterRe.FindAllStringIndex(body, -1) {
|
||||
if m[0] < markerIdx {
|
||||
anchorsBefore++
|
||||
}
|
||||
lastAnchor = m[0]
|
||||
}
|
||||
var failures []string
|
||||
if lastAnchor >= markerIdx {
|
||||
failures = append(failures, fmt.Sprintf("last chapter anchor at byte %d does not precede the marker at byte %d", lastAnchor, markerIdx))
|
||||
}
|
||||
if anchorsBefore == 0 {
|
||||
failures = append(failures, fmt.Sprintf("no chapter anchor before the marker at byte %d — the truncated prefix the scan sees yields nothing", markerIdx))
|
||||
}
|
||||
return failures
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
// Package pgtest runs the Postgres the test suite needs: one throwaway
|
||||
// container per test binary, one fresh database per test. Docker is therefore
|
||||
// a hard prerequisite for `go test ./...`.
|
||||
//
|
||||
// Rolled by hand rather than pulled in as a dependency — it is one `docker
|
||||
// run`, one `docker port` and a ping loop, against a module list that is
|
||||
// otherwise stdlib plus what the poller genuinely needs.
|
||||
package pgtest
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
const (
|
||||
image = "postgres:17-alpine"
|
||||
readyLimit = 60 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
adminURL string
|
||||
dbSeq atomic.Int64
|
||||
)
|
||||
|
||||
// Main starts the container, runs the package's tests and tears the container
|
||||
// down. Every test package that touches the store calls it from TestMain:
|
||||
//
|
||||
// func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
func Main(m *testing.M) int {
|
||||
id, url, err := start()
|
||||
if err != nil {
|
||||
fmt.Println("pgtest:", err)
|
||||
return 1
|
||||
}
|
||||
defer exec.Command("docker", "rm", "-f", id).Run()
|
||||
|
||||
adminURL = url
|
||||
return m.Run()
|
||||
}
|
||||
|
||||
// URL creates a database of its own for t and returns a connection URL for it.
|
||||
// Nothing drops it again: the container goes away wholesale when Main returns.
|
||||
func URL(t testing.TB) string {
|
||||
t.Helper()
|
||||
if adminURL == "" {
|
||||
t.Fatal("pgtest: no container; this package needs TestMain to call pgtest.Main")
|
||||
}
|
||||
// Generated, never derived from the test name, so it needs no quoting and
|
||||
// cannot collide when tests run in parallel.
|
||||
name := "test_" + strconv.FormatInt(dbSeq.Add(1), 10)
|
||||
|
||||
admin, err := sql.Open("pgx", adminURL)
|
||||
if err != nil {
|
||||
t.Fatalf("pgtest: open admin connection: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
if _, err := admin.Exec(`CREATE DATABASE ` + name); err != nil {
|
||||
t.Fatalf("pgtest: create database %s: %v", name, err)
|
||||
}
|
||||
return strings.Replace(adminURL, "/postgres?", "/"+name+"?", 1)
|
||||
}
|
||||
|
||||
// start launches the container and waits for it to accept queries, returning
|
||||
// its id and a connection URL for the default database.
|
||||
func start() (id, url string, err error) {
|
||||
out, err := exec.Command("docker", "run", "-d", "--rm",
|
||||
"-e", "POSTGRES_PASSWORD=pgtest",
|
||||
"-P", image,
|
||||
// Durability buys nothing for a database that dies with the test
|
||||
// binary, and turning it off is most of the container's start-up cost.
|
||||
"-c", "fsync=off", "-c", "full_page_writes=off",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("docker run %s: %w", image, err)
|
||||
}
|
||||
id = strings.TrimSpace(string(out))
|
||||
|
||||
port, err := exec.Command("docker", "port", id, "5432/tcp").Output()
|
||||
if err != nil {
|
||||
exec.Command("docker", "rm", "-f", id).Run()
|
||||
return "", "", fmt.Errorf("docker port: %w", err)
|
||||
}
|
||||
// "0.0.0.0:32768" (and possibly a second, IPv6 line); the port is all we want.
|
||||
first, _, _ := strings.Cut(strings.TrimSpace(string(port)), "\n")
|
||||
url = fmt.Sprintf("postgres://postgres:pgtest@127.0.0.1:%s/postgres?sslmode=disable",
|
||||
first[strings.LastIndex(first, ":")+1:])
|
||||
|
||||
if err := waitReady(url); err != nil {
|
||||
exec.Command("docker", "rm", "-f", id).Run()
|
||||
return "", "", err
|
||||
}
|
||||
return id, url, nil
|
||||
}
|
||||
|
||||
func waitReady(url string) error {
|
||||
db, err := sql.Open("pgx", url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
deadline := time.Now().Add(readyLimit)
|
||||
for {
|
||||
if err = db.Ping(); err == nil {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("postgres not ready after %s: %w", readyLimit, err)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Series filter names (issue #140), ordered permanent-then-fixable — the
|
||||
// repairs nothing will ever undo first, the ones a Poll can make right after.
|
||||
// A name is the repair a row needs, not the SQL that finds it; the values are
|
||||
// the wire form the Series list URL carries (#142). "all" is the absent and
|
||||
// unknown case: every Series.
|
||||
const (
|
||||
SeriesFilterAll = "all"
|
||||
SeriesFilterNoURL = "no_series_url"
|
||||
SeriesFilterNoChapter = "never_read_a_chapter"
|
||||
SeriesFilterNoReaders = "no_readers"
|
||||
SeriesFilterNeverChecked = "never_checked"
|
||||
SeriesFilterStale = "stale"
|
||||
SeriesFilterNoCover = "no_cover"
|
||||
SeriesFilterReaderReport = "reader_report"
|
||||
)
|
||||
|
||||
// SeriesFilter is one named hygiene predicate over the whole library. Site
|
||||
// and Kind narrow the row read; Name picks the predicate; Cutoff is the
|
||||
// staleness boundary the "stale" filter compares against, supplied by the
|
||||
// caller's clock — the store has no clock; Page is 1-based.
|
||||
type SeriesFilter struct {
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both library buckets' series
|
||||
Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll
|
||||
Cutoff int64 // unix ms; "stale" reads it, the store never does
|
||||
Page int // 1-based page of the row read; default 1
|
||||
}
|
||||
|
||||
// adminSeriesColumns is the owner's library-wide Series projection in
|
||||
// scanAdminSeries order. It is the privacy boundary: a Series' row carries
|
||||
// the Reader id that raised its Latest Chapter (latest_raised_by), and that id
|
||||
// must never leave the store package — so the projection does not select it,
|
||||
// and only the anonymous boolean in raisedByReaderAnswer crosses it.
|
||||
const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at`
|
||||
|
||||
// raisedByReaderAnswer answers "did a Reader's report set this number" without
|
||||
// naming which Reader. Kept apart from adminSeriesColumns so the column list —
|
||||
// the shape scanAdminSeries is fed — stays free of the Sighting-raiser
|
||||
// identity, and the owner learns which rows to act on and nothing about the
|
||||
// Reader behind them.
|
||||
const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader`
|
||||
|
||||
// seriesPageSize is the row read's page length. The tie-break in the query's
|
||||
// ORDER BY is what makes this a stable page boundary — see SeriesPage.
|
||||
const seriesPageSize = 50
|
||||
|
||||
// AdminSeries is one Series as the owner's library-wide view sees it: a
|
||||
// Series-level fact plus an anonymous Reader count. ReaderCount being zero is
|
||||
// the orphan marker. RaisedByReader is the only trace of the Sighting
|
||||
// mechanism here; the Reader id behind it never reaches this type.
|
||||
type AdminSeries struct {
|
||||
Site string
|
||||
SeriesID string
|
||||
Title string
|
||||
SeriesURL string
|
||||
CoverAddress string // "" = no Cover yet
|
||||
Kind string
|
||||
LatestChapter string
|
||||
LatestChapterNum *float64 // nil until first captured
|
||||
LatestCheckedAt int64
|
||||
// ForcePollAt is the owner's "check now" request stamp (issue #146), zero
|
||||
// meaning never asked. Pending is derived, never stored: a request is
|
||||
// pending while ForcePollAt is newer than LatestCheckedAt.
|
||||
ForcePollAt int64
|
||||
ReaderCount int
|
||||
RaisedByReader bool // a Reader's report set LatestChapterNum
|
||||
}
|
||||
|
||||
// SeriesPage is one page of the owner's filtered Series list plus the count
|
||||
// of every Series matching the same filter — a window number, not the page's
|
||||
// len, so the landing page's figure and the list heading come from one query.
|
||||
type SeriesPage struct {
|
||||
Rows []AdminSeries
|
||||
Total int
|
||||
}
|
||||
|
||||
// SiteSeriesShape is one Site's share of the Series matching a filter: how
|
||||
// many, and the manga/novel split. One grouped pass, then library-wide totals
|
||||
// are summed in Go over the rows — the landing page's per-Site table reads
|
||||
// this and never pays for the rows the list discards.
|
||||
type SiteSeriesShape struct {
|
||||
Site string
|
||||
Total int
|
||||
Manga int
|
||||
Novel int
|
||||
}
|
||||
|
||||
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>").
|
||||
func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
|
||||
|
||||
// adminFilter maps a filter's named predicate to its compile-time WHERE and
|
||||
// HAVING clauses and their bound parameters — the name never reaches query
|
||||
// text, and Site and Kind bind as parameters. Shared by the row read and the
|
||||
// per-Site aggregate so the two cannot disagree on what a filter means.
|
||||
//
|
||||
// The WHERE set is: no URL (an empty URL only — the host-failing-the-fetch-
|
||||
// gate case is invisible to SQL, needs the Site registry in Go, and belongs to
|
||||
// a later repair), never-read-a-chapter and never-checked as disjoint halves
|
||||
// (non-zero versus zero check stamp), stale, no cover, and Reader-report.
|
||||
// no_readers is the one HAVING predicate: it is the orphan test, an aggregate
|
||||
// over the LEFT JOIN, where a bare WHERE has no row to test.
|
||||
//
|
||||
// stale is the checked-but-old half of the stamp partition — because the
|
||||
// verdict line wants "not checked in twelve hours" as one figure, and a never
|
||||
// checked Series is already counted on its own "waiting"/never-checked
|
||||
// filter, folding it in would double-report it. The landing page computes the
|
||||
// inclusive number as stale + never-checked.
|
||||
func adminFilter(f SeriesFilter) (where, having string, args []any, err error) {
|
||||
var clauses []string
|
||||
if f.Kind != "" {
|
||||
args = append(args, f.Kind)
|
||||
clauses = append(clauses, "s.kind = $"+strconv.Itoa(len(args)))
|
||||
}
|
||||
switch f.Name {
|
||||
case "", SeriesFilterAll:
|
||||
case SeriesFilterNoURL:
|
||||
clauses = append(clauses, `s.series_url = ''`)
|
||||
case SeriesFilterNoChapter:
|
||||
clauses = append(clauses, `s.latest_checked_at <> 0 AND s.latest_chapter_num IS NULL`)
|
||||
case SeriesFilterNeverChecked:
|
||||
clauses = append(clauses, `s.latest_checked_at = 0`)
|
||||
case SeriesFilterStale:
|
||||
clauses = append(clauses, `s.latest_checked_at > 0 AND s.latest_checked_at < $`+strconv.Itoa(len(args)+1))
|
||||
args = append(args, f.Cutoff)
|
||||
case SeriesFilterNoCover:
|
||||
clauses = append(clauses, `s.cover_address = ''`)
|
||||
case SeriesFilterReaderReport:
|
||||
clauses = append(clauses, `s.latest_raised_by IS NOT NULL`)
|
||||
case SeriesFilterNoReaders:
|
||||
having = `HAVING COUNT(b.reader_id) = 0`
|
||||
default:
|
||||
return "", "", nil, fmt.Errorf("unknown series filter %q", f.Name)
|
||||
}
|
||||
if len(clauses) > 0 {
|
||||
where = "WHERE " + strings.Join(clauses, " AND ")
|
||||
}
|
||||
return where, having, args, nil
|
||||
}
|
||||
|
||||
// SeriesPage returns one page of the Series matching the filter, least
|
||||
// recently checked first. The LEFT JOIN to Bookmarks is what surfaces the
|
||||
// orphans that hygiene has to find — an inner join would hide them, exactly
|
||||
// as the Lane's join does. ReaderCount is a plain count of every Bookmark on
|
||||
// the Series, which knowingly disagrees with the two Lane queries for as long
|
||||
// as the finished lifecycle bucket exists (#140).
|
||||
//
|
||||
// The tie-break is mandatory, not decorative: every unpollable Series shares a
|
||||
// zero check stamp, so ordering on that column alone gives no stable page
|
||||
// boundary and rows would repeat or vanish across pages. (site, series_id) is
|
||||
// the primary key, hence total. The filtered total is a window count in the
|
||||
// same query — window functions run after grouping and before the limit, so
|
||||
// one where-clause cannot disagree with a second copy of itself.
|
||||
func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return SeriesPage{}, err
|
||||
}
|
||||
if f.Page < 1 {
|
||||
f.Page = 1
|
||||
}
|
||||
// Site narrowing is the row read's own; the aggregate must see every Site.
|
||||
if f.Site != "" {
|
||||
args = append(args, f.Site)
|
||||
clause := "s.site = $" + strconv.Itoa(len(args))
|
||||
if where == "" {
|
||||
where = "WHERE " + clause
|
||||
} else {
|
||||
where += " AND " + clause
|
||||
}
|
||||
}
|
||||
base := len(args)
|
||||
args = append(args, seriesPageSize, seriesPageSize*(f.Page-1))
|
||||
rows, err := s.db.Query(`
|
||||
SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`,
|
||||
COUNT(b.reader_id) AS reader_count,
|
||||
COUNT(*) OVER () AS filtered_total
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at, s.latest_raised_by
|
||||
`+having+`
|
||||
ORDER BY s.latest_checked_at, s.site, s.series_id
|
||||
LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("query series page: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := SeriesPage{}
|
||||
for rows.Next() {
|
||||
a, total, err := scanAdminSeries(rows.Scan)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("scan series page: %w", err)
|
||||
}
|
||||
out.Rows = append(out.Rows, a)
|
||||
out.Total = total
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SeriesShapes returns each Site's share of the Series matching the filter,
|
||||
// one grouped pass. Site and Page are row-read concerns and are ignored; the
|
||||
// Landing page reads this per Site and sums the totals in Go for the
|
||||
// library-wide figure.
|
||||
func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE kind = 'manga') AS manga,
|
||||
COUNT(*) FILTER (WHERE kind = 'novel') AS novel
|
||||
FROM (
|
||||
SELECT s.site, s.kind
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.kind
|
||||
`+having+`
|
||||
) shape
|
||||
GROUP BY site
|
||||
ORDER BY site`, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query series shapes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []SiteSeriesShape{}
|
||||
for rows.Next() {
|
||||
var sh SiteSeriesShape
|
||||
if err := rows.Scan(&sh.Site, &sh.Total, &sh.Manga, &sh.Novel); err != nil {
|
||||
return nil, fmt.Errorf("scan series shape: %w", err)
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer
|
||||
// order, plus the query's reader_count and filtered_total columns, and returns
|
||||
// the window total alongside the row. latest_chapter_num is NULL until first
|
||||
// captured — the "never read a chapter" state. The Sighting-raiser column is
|
||||
// never among the scanned columns.
|
||||
func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
|
||||
var (
|
||||
a AdminSeries
|
||||
latestChapterNum sql.NullFloat64
|
||||
total int
|
||||
)
|
||||
if err := scan(
|
||||
&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
|
||||
&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
|
||||
&a.ForcePollAt,
|
||||
&a.RaisedByReader, &a.ReaderCount, &total,
|
||||
); err != nil {
|
||||
return AdminSeries{}, 0, err
|
||||
}
|
||||
if latestChapterNum.Valid {
|
||||
a.LatestChapterNum = &latestChapterNum.Float64
|
||||
}
|
||||
return a, total, nil
|
||||
}
|
||||
@@ -1,396 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seriesSeed describes one Series (and optionally its bookmarks) to stand up
|
||||
// for an admin filter test. Direct SQL, because the filters separate rows the
|
||||
// Upsert path could not produce together: an orphan has no bookmark, and a
|
||||
// Reader-raised Latest Chapter needs a Sighting the store does not create.
|
||||
type seriesSeed struct {
|
||||
key string
|
||||
kind string
|
||||
url string
|
||||
cover string // cover_address
|
||||
checkedAt int64
|
||||
latestNum *float64
|
||||
bookmarks int // readers that hold it; 0 = orphan
|
||||
raisedBy bool // a Reader's report is attributed as the raiser
|
||||
}
|
||||
|
||||
// seedAdminSeries inserts one series row and its bookmarks (owner first, then
|
||||
// fresh readers), with the exact admin-relevant facts a test needs.
|
||||
func seedAdminSeries(t *testing.T, s *Store, seed seriesSeed) {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(seed.key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", seed.key)
|
||||
}
|
||||
if seed.kind == "" {
|
||||
seed.kind = "manga"
|
||||
}
|
||||
var latestChapter any = ""
|
||||
if seed.latestNum != nil {
|
||||
latestChapter = "Chapter " + strconv.FormatFloat(*seed.latestNum, 'f', -1, 64)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO series (site, series_id, title, kind, series_url, cover_address,
|
||||
latest_checked_at, latest_chapter, latest_chapter_num)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||
site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover,
|
||||
seed.checkedAt, latestChapter, seed.latestNum); err != nil {
|
||||
t.Fatalf("seed series %q: %v", seed.key, err)
|
||||
}
|
||||
for i := range seed.bookmarks {
|
||||
var readerID int64 = s.OwnerID()
|
||||
if i > 0 {
|
||||
readerID = secondReader(t, s)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO bookmarks (reader_id, site, series_id,
|
||||
last_chapter, last_chapter_num, last_chapter_url,
|
||||
favorite, status, updated_at)
|
||||
VALUES ($1, $2, $3, '', 0, '', false, 'reading', $4)`,
|
||||
readerID, site, seriesID, seed.checkedAt); err != nil {
|
||||
t.Fatalf("seed bookmark %q: %v", seed.key, err)
|
||||
}
|
||||
}
|
||||
if seed.raisedBy {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET latest_raised_by = $1 WHERE site = $2 AND series_id = $3`,
|
||||
s.OwnerID(), site, seriesID); err != nil {
|
||||
t.Fatalf("seed raised-by %q: %v", seed.key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pageKeys(t *testing.T, s *Store, f SeriesFilter) map[string]bool {
|
||||
t.Helper()
|
||||
page, err := s.SeriesPage(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(%+v): %v", f, err)
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, a := range page.Rows {
|
||||
keys[a.Key()] = true
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Each filter must return the rows it names and no others, over one shared
|
||||
// seeded mix where every healthy neighbour is present to be wrongly returned.
|
||||
// The stale cutoff is 5000: a Series checked at 9000 is current, at 2000 stale.
|
||||
func TestAdminSeriesFilters(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "https://asurascans.com/comics/healthy", cover: "aaa", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nourl", url: "", cover: "bbb", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "https://asurascans.com/comics/nochapter", cover: "ccc", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "https://asurascans.com/comics/neverchecked", cover: "ddd", checkedAt: 0, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:orphan", url: "https://asurascans.com/comics/orphan", cover: "eee", checkedAt: 9000, latestNum: new(7.0), bookmarks: 0})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:stale", url: "https://asurascans.com/comics/stale", cover: "fff", checkedAt: 2000, latestNum: new(4.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nocover", url: "https://asurascans.com/comics/nocover", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:report", url: "https://asurascans.com/comics/report", cover: "ggg", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
f SeriesFilter
|
||||
want []string
|
||||
}{
|
||||
{"all", SeriesFilter{}, []string{"asura:healthy", "asura:nourl", "asura:nochapter", "asura:neverchecked", "asura:orphan", "asura:stale", "asura:nocover", "asura:report"}},
|
||||
{"no series url", SeriesFilter{Name: SeriesFilterNoURL}, []string{"asura:nourl"}},
|
||||
{"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}},
|
||||
{"never checked", SeriesFilter{Name: SeriesFilterNeverChecked}, []string{"asura:neverchecked"}},
|
||||
{"no readers", SeriesFilter{Name: SeriesFilterNoReaders}, []string{"asura:orphan"}},
|
||||
{"stale", SeriesFilter{Name: SeriesFilterStale, Cutoff: 5000}, []string{"asura:stale"}},
|
||||
{"no cover", SeriesFilter{Name: SeriesFilterNoCover}, []string{"asura:nocover"}},
|
||||
{"reader report", SeriesFilter{Name: SeriesFilterReaderReport}, []string{"asura:report"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := pageKeys(t, s, tc.f)
|
||||
want := map[string]bool{}
|
||||
for _, k := range tc.want {
|
||||
want[k] = true
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want)
|
||||
}
|
||||
for k := range want {
|
||||
if !got[k] {
|
||||
t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// "Never read a chapter" and "never checked" are disjoint by construction:
|
||||
// the first requires a non-zero check stamp, the second a zero one. Over a
|
||||
// mix that should satisfy both, no row may be counted twice.
|
||||
func TestAdminNeverChapterAndNeverCheckedAreDisjoint(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
// A zero-stamp, no-chapter row is never-checked only: if never-read-a-
|
||||
// chapter ever lost its non-zero-stamp guard, it would claim this row too
|
||||
// and the two counts would double-report it.
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:both", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter})
|
||||
neverChecked := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNeverChecked})
|
||||
for k := range noChapter {
|
||||
if neverChecked[k] {
|
||||
t.Fatalf("row %q matches both never-read-a-chapter and never-checked", k)
|
||||
}
|
||||
}
|
||||
if !noChapter["asura:nochapter"] || !neverChecked["asura:neverchecked"] {
|
||||
t.Fatalf("disjoint split lost its own rows: no-chapter=%v never-checked=%v", noChapter, neverChecked)
|
||||
}
|
||||
}
|
||||
|
||||
// Several rows share a zero check stamp, so ordering on latest_checked_at
|
||||
// alone gives no stable page boundary. The (site, series_id) tie-break must
|
||||
// make page 2 a strict continuation of page 1: no repeat, no vanishing row.
|
||||
func TestAdminSeriesPageTieBreakIsStable(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const total = 53 // > one page, < two (page size 50)
|
||||
for i := range total {
|
||||
id := "tie" + strconv.Itoa(i)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:" + id, url: "u", checkedAt: 0, bookmarks: 1})
|
||||
}
|
||||
// A second Site's zero-stamp row is part of the same all-filter list, and
|
||||
// must land on a valid page boundary rather than duplicating or dropping
|
||||
// one of asura's rows: the tie-break is global (site, series_id).
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:z", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
wantTotal := total + 1
|
||||
|
||||
p1, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage page 1: %v", err)
|
||||
}
|
||||
p2, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage page 2: %v", err)
|
||||
}
|
||||
if len(p1.Rows) != seriesPageSize {
|
||||
t.Fatalf("page 1 has %d rows, want %d", len(p1.Rows), seriesPageSize)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range append(append([]AdminSeries{}, p1.Rows...), p2.Rows...) {
|
||||
if seen[a.Key()] {
|
||||
t.Fatalf("row %q repeats across pages", a.Key())
|
||||
}
|
||||
seen[a.Key()] = true
|
||||
}
|
||||
if len(seen) != wantTotal {
|
||||
t.Fatalf("%d distinct rows across pages, want %d (a row vanished)", len(seen), wantTotal)
|
||||
}
|
||||
if p1.Total != wantTotal {
|
||||
t.Fatalf("page total = %d, want %d (the window count must span pages)", p1.Total, wantTotal)
|
||||
}
|
||||
// A page beyond the end is empty, not an error (the list re-reads page 1).
|
||||
// The window count runs over the rows present in the result set, so an
|
||||
// overflow page has no rows and therefore no total — the caller must not
|
||||
// render it, which is exactly why the list re-reads page 1.
|
||||
pFinal, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 99})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage beyond end: %v", err)
|
||||
}
|
||||
if len(pFinal.Rows) != 0 {
|
||||
t.Fatalf("page beyond end = %d rows, want 0", len(pFinal.Rows))
|
||||
}
|
||||
}
|
||||
|
||||
// The filtered total is the window number over the same filter the rows use,
|
||||
// and the per-Site aggregate sums to the same figure — so the landing page's
|
||||
// count and the list's heading can never disagree, whoever computes them.
|
||||
func TestAdminTotalAgreesWithRowsAndShapes(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", cover: "a", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:b", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:c", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(2.0), raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:d", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
filters := []SeriesFilter{
|
||||
{},
|
||||
{Name: SeriesFilterNoCover},
|
||||
{Name: SeriesFilterReaderReport},
|
||||
{Name: SeriesFilterNoChapter},
|
||||
}
|
||||
for _, f := range filters {
|
||||
page, err := s.SeriesPage(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(%+v): %v", f, err)
|
||||
}
|
||||
want := len(page.Rows)
|
||||
if f.Page == 0 && want == seriesPageSize {
|
||||
t.Fatalf("seed produced a full page; bump the seed or drop page size in the test")
|
||||
}
|
||||
if page.Total != want {
|
||||
t.Fatalf("%+v total = %d, want %d (window count disagrees with row count)", f, page.Total, want)
|
||||
}
|
||||
shapes, err := s.SeriesShapes(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes(%+v): %v", f, err)
|
||||
}
|
||||
sum := 0
|
||||
for _, sh := range shapes {
|
||||
sum += sh.Total
|
||||
}
|
||||
if sum != want {
|
||||
t.Fatalf("%+v aggregate sum = %d, want %d (aggregate disagrees with row query)", f, sum, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The default filter's aggregate carries the library shape: per-Site
|
||||
// totals and the manga/novel split, summed in Go for library wide.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes default: %v", err)
|
||||
}
|
||||
if len(shapes) != 2 || shapes[0].Site != "asura" || shapes[1].Site != "demonic" {
|
||||
t.Fatalf("shapes = %+v, want asura then demonic", shapes)
|
||||
}
|
||||
if shapes[0].Total != 3 || shapes[0].Manga != 3 || shapes[0].Novel != 0 {
|
||||
t.Fatalf("asura shape = %+v, want 3 manga, 0 novel", shapes[0])
|
||||
}
|
||||
if shapes[1].Total != 1 || shapes[1].Manga != 0 || shapes[1].Novel != 1 {
|
||||
t.Fatalf("demonic shape = %+v, want 1 novel", shapes[1])
|
||||
}
|
||||
}
|
||||
|
||||
// Site and Library narrowing stack on a named filter without changing what
|
||||
// the filter means.
|
||||
func TestAdminFilterSiteAndKindNarrow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:aa", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(1.0)})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:ab", url: "", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:aa", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:ab", url: "", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
got := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Site: "asura"})
|
||||
if len(got) != 1 || !got["asura:ab"] {
|
||||
t.Fatalf("site+nourl = %v, want only asura:ab", got)
|
||||
}
|
||||
got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"})
|
||||
if len(got) != 1 || !got["demonic:ab"] {
|
||||
t.Fatalf("kind+nourl = %v, want only demonic:ab", got)
|
||||
}
|
||||
got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterAll, Site: "demonic", Kind: "novel"})
|
||||
if len(got) != 2 || !got["demonic:aa"] || !got["demonic:ab"] {
|
||||
t.Fatalf("site+kind+all = %v, want both demonic rows", got)
|
||||
}
|
||||
|
||||
// The aggregate ignores the Site narrowing (it is per-Site by shape), but
|
||||
// honours the Library narrowing: asura's missing-URL row is manga, so the
|
||||
// novel no-URL list is demonic alone.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes: %v", err)
|
||||
}
|
||||
if len(shapes) != 1 || shapes[0].Site != "demonic" ||
|
||||
shapes[0].Total != 1 || shapes[0].Novel != 1 {
|
||||
t.Fatalf("novel no-URL aggregate = %+v, want demonic {Total:1 Novel:1}", shapes)
|
||||
}
|
||||
}
|
||||
|
||||
// The projection is the privacy boundary: a Series whose Latest Chapter was
|
||||
// raised by a Reader's report reads back with the anonymous boolean set, not
|
||||
// with the Reader's id, and no Reader id travels in any returned row.
|
||||
func TestAdminSeriesReportsAnonymously(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:raised", url: "u", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1, raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1})
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
byKey := map[string]AdminSeries{}
|
||||
for _, a := range page.Rows {
|
||||
byKey[a.Key()] = a
|
||||
}
|
||||
if !byKey["asura:raised"].RaisedByReader {
|
||||
t.Fatal("Reader-raised Series read back RaisedByReader=false")
|
||||
}
|
||||
if byKey["asura:polled"].RaisedByReader {
|
||||
t.Fatal("Poll-raised Series read back RaisedByReader=true")
|
||||
}
|
||||
}
|
||||
|
||||
// The privacy test that cannot rot into a template-only guarantee: assert the
|
||||
// admin column constant does not carry the Sighting-raiser column and that the
|
||||
// admin row type has no field for it, modelled on the guard on the Bookmark
|
||||
// column list.
|
||||
func TestAdminProjectionHidesSightingRaiser(t *testing.T) {
|
||||
if strings.Contains(adminSeriesColumns, "latest_raised_by") {
|
||||
t.Fatal("admin column list carries latest_raised_by: the Sighting-raiser id would reach the owner")
|
||||
}
|
||||
if _, ok := reflect.TypeOf(AdminSeries{}).FieldByName("LatestRaisedBy"); ok {
|
||||
t.Fatal("AdminSeries carries a field for the Sighting-raiser id")
|
||||
}
|
||||
}
|
||||
|
||||
// An unknown filter name is rejected rather than silently meaning "all" —
|
||||
// otherwise a mistyped URL would present an empty page as the whole library.
|
||||
func TestAdminFilterUnknownNameRejected(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
for name, call := range map[string]func() error{
|
||||
"page": func() error { _, err := s.SeriesPage(SeriesFilter{Name: "bogus"}); return err },
|
||||
"shape": func() error { _, err := s.SeriesShapes(SeriesFilter{Name: "bogus"}); return err },
|
||||
} {
|
||||
if err := call(); err == nil || !strings.Contains(err.Error(), "unknown series filter") {
|
||||
t.Fatalf("%s with bogus filter = %v, want unknown-filter error", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ForceSeriesPoll is the idempotent stamp write: a second press overwrites
|
||||
// the request time, and touching a missing series is not an error.
|
||||
func TestForceSeriesPollStampsIdempotently(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
if err := s.ForceSeriesPoll("asura", "x", 42); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
if err := s.ForceSeriesPoll("asura", "x", 99); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll re-stamp: %v", err)
|
||||
}
|
||||
// Touching a missing series is not an error: the row may have been
|
||||
// orphaned, and the caller's read decides what exists.
|
||||
if err := s.ForceSeriesPoll("asura", "ghost", 99); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll missing: %v", err)
|
||||
}
|
||||
|
||||
var got int64
|
||||
if err := s.db.QueryRow(
|
||||
`SELECT force_poll_at FROM series WHERE site = 'asura' AND series_id = 'x'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read force_poll_at: %v", err)
|
||||
}
|
||||
if got != 99 {
|
||||
t.Fatalf("force_poll_at = %d, want 99 (the later press wins)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The admin projection carries the force stamp so the web layer can derive
|
||||
// the pending flag without a second read.
|
||||
func TestAdminSeriesCarriesForcePollAt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 1000, bookmarks: 1})
|
||||
if err := s.ForceSeriesPoll("asura", "x", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
if len(page.Rows) != 1 || page.Rows[0].ForcePollAt != 5000 {
|
||||
t.Fatalf("row = %+v, want ForcePollAt 5000", page.Rows)
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
-- One row per tracked series, keyed "<site>:<series_id>".
|
||||
--
|
||||
-- Everything is NOT NULL with a default except latest_chapter_num, where NULL
|
||||
-- is a distinct state: nothing has been captured yet, which is not the same as
|
||||
-- chapter zero.
|
||||
--
|
||||
-- Timestamps are unix milliseconds as bigint, not timestamptz: the userscripts
|
||||
-- send Date.now() over the wire and the ordering rule compares them directly.
|
||||
CREATE TABLE bookmarks (
|
||||
key text PRIMARY KEY,
|
||||
site text NOT NULL,
|
||||
series_id text NOT NULL,
|
||||
title text NOT NULL DEFAULT '',
|
||||
series_url text NOT NULL DEFAULT '',
|
||||
cover text NOT NULL DEFAULT '',
|
||||
last_chapter text NOT NULL DEFAULT '',
|
||||
last_chapter_num double precision NOT NULL DEFAULT 0,
|
||||
last_chapter_url text NOT NULL DEFAULT '',
|
||||
favorite boolean NOT NULL DEFAULT false,
|
||||
latest_chapter text NOT NULL DEFAULT '',
|
||||
latest_chapter_num double precision,
|
||||
-- When the server last polled this series, unix ms; 0 means never, and sorts
|
||||
-- first so a new bookmark is picked up on the next tick with no special case.
|
||||
latest_checked_at bigint NOT NULL DEFAULT 0,
|
||||
status text NOT NULL DEFAULT 'reading',
|
||||
kind text NOT NULL DEFAULT 'manga',
|
||||
updated_at bigint NOT NULL
|
||||
);
|
||||
@@ -1,45 +0,0 @@
|
||||
-- One row per distinct work, shared by every bookmark that tracks it
|
||||
-- (ADR-0003). Keyed (site, series_id), the pair a bookmark key decomposes
|
||||
-- into. title/series_url/cover are written once, at creation, and never
|
||||
-- again: client-supplied values are ignored once the row exists and the
|
||||
-- poller is the only party that may change them. kind and the latest-chapter
|
||||
-- fields are last-write-wins like the bookmark's own fields.
|
||||
CREATE TABLE series (
|
||||
site text NOT NULL,
|
||||
series_id text NOT NULL,
|
||||
title text NOT NULL DEFAULT '',
|
||||
series_url text NOT NULL DEFAULT '',
|
||||
cover text NOT NULL DEFAULT '',
|
||||
kind text NOT NULL DEFAULT 'manga',
|
||||
latest_chapter text NOT NULL DEFAULT '',
|
||||
latest_chapter_num double precision,
|
||||
-- When the server last polled this series, unix ms; 0 means never, and sorts
|
||||
-- first so a new bookmark is picked up on the next tick with no special case.
|
||||
latest_checked_at bigint NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (site, series_id)
|
||||
);
|
||||
|
||||
-- Backfill from today's rows. The bookmark key's uniqueness makes
|
||||
-- (site, series_id) unique in practice; DISTINCT is belt and braces.
|
||||
INSERT INTO series (site, series_id, title, series_url, cover, kind,
|
||||
latest_chapter, latest_chapter_num, latest_checked_at)
|
||||
SELECT DISTINCT site, series_id, title, series_url, cover, kind,
|
||||
latest_chapter, latest_chapter_num, latest_checked_at
|
||||
FROM bookmarks;
|
||||
|
||||
-- The bookmark keeps only what differs between readers (ADR-0003): progress,
|
||||
-- favourite, lifecycle bucket. The dropped columns now live on series.
|
||||
ALTER TABLE bookmarks
|
||||
DROP COLUMN title,
|
||||
DROP COLUMN series_url,
|
||||
DROP COLUMN cover,
|
||||
DROP COLUMN kind,
|
||||
DROP COLUMN latest_chapter,
|
||||
DROP COLUMN latest_chapter_num,
|
||||
DROP COLUMN latest_checked_at;
|
||||
|
||||
-- A bookmark may not point at a series that does not exist. No cascade: a
|
||||
-- series outlives its last bookmark, and deleting one is not a store operation.
|
||||
ALTER TABLE bookmarks
|
||||
ADD CONSTRAINT bookmarks_series_fk
|
||||
FOREIGN KEY (site, series_id) REFERENCES series (site, series_id);
|
||||
@@ -1,17 +0,0 @@
|
||||
-- One row per person. Keyed by their Discord user ID; carries the SHA-256 of
|
||||
-- their userscript token and when they were created. Hashed because a token
|
||||
-- in the database is a token anyone with the database can replay; SHA-256 is
|
||||
-- enough because the tokens are high-entropy random values with nothing to
|
||||
-- brute-force. No one can register yet, so this table holds exactly the one
|
||||
-- owner row the seed creates at startup (see Store.Open).
|
||||
CREATE TABLE readers (
|
||||
id bigserial PRIMARY KEY,
|
||||
discord_id text NOT NULL UNIQUE,
|
||||
token_sha256 bytea NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Every bookmark now belongs to a reader. Added nullable: rows created before
|
||||
-- this migration have no owner yet — 0004 attaches them to the seeded owner
|
||||
-- before NOT NULL and the composite key land.
|
||||
ALTER TABLE bookmarks ADD COLUMN reader_id bigint;
|
||||
@@ -1,19 +0,0 @@
|
||||
-- Attach every pre-existing bookmark to the owner reader, seeded between the
|
||||
-- two migrate passes (Store.Open). The oldest reader is the owner by
|
||||
-- construction: only the seed creates readers, and it runs once per database.
|
||||
-- Run-once via the version table, like every migration.
|
||||
UPDATE bookmarks SET reader_id = (SELECT id FROM readers ORDER BY id LIMIT 1);
|
||||
|
||||
-- Ownership lands structurally: reader_id becomes part of the key, so a
|
||||
-- bookmark is one Reader's progress on one Series and a duplicate for the
|
||||
-- same pair is impossible at the database level. Deleting a Reader takes
|
||||
-- their bookmarks with them. The old text key is gone — the wire "key" is
|
||||
-- derived as site:series_id on read, and nothing references the column.
|
||||
-- Dropping it drops the primary key it carried; the composite key replaces
|
||||
-- it, and the FK index the series constraint needs is created automatically.
|
||||
ALTER TABLE bookmarks
|
||||
ALTER COLUMN reader_id SET NOT NULL,
|
||||
DROP COLUMN key,
|
||||
ADD PRIMARY KEY (reader_id, site, series_id),
|
||||
ADD CONSTRAINT bookmarks_reader_fk
|
||||
FOREIGN KEY (reader_id) REFERENCES readers (id) ON DELETE CASCADE;
|
||||
@@ -1,11 +0,0 @@
|
||||
-- One row per browser session. The id is an opaque random value the cookie
|
||||
-- carries verbatim; a request is authenticated by looking the row up, and
|
||||
-- deleting the row is how a session is revoked. Expired rows are removed
|
||||
-- lazily on lookup and swept by the next login, so nothing runs a background
|
||||
-- cleanup.
|
||||
CREATE TABLE sessions (
|
||||
id text PRIMARY KEY,
|
||||
reader_id bigint NOT NULL REFERENCES readers (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
@@ -1,7 +0,0 @@
|
||||
-- Rotation is an epoch bump: a Reader's credential is derived from the
|
||||
-- deployment secret, their Discord id and this epoch, so bumping it issues a
|
||||
-- new credential and the rewritten token_sha256 invalidates the old one the
|
||||
-- moment the transaction commits. The seed's ON CONFLICT refresh (Store.Open)
|
||||
-- is gated on this being 0, so a restart can never undo a rotation by
|
||||
-- restoring the epoch-0 hash.
|
||||
ALTER TABLE readers ADD COLUMN token_epoch bigint NOT NULL DEFAULT 0;
|
||||
@@ -1,8 +0,0 @@
|
||||
-- Kagane cover bytes belong in their own table so image blobs never enter the
|
||||
-- series queries that drive the latest-chapter poller.
|
||||
CREATE TABLE covers (
|
||||
image_id text PRIMARY KEY,
|
||||
body bytea NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
fetched_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -1,9 +0,0 @@
|
||||
-- Cover bytes move out of Postgres. Existing rows are intentionally dropped:
|
||||
-- the old kagane path already refetches missing Covers on demand.
|
||||
DROP TABLE covers;
|
||||
|
||||
CREATE TABLE covers (
|
||||
address text PRIMARY KEY,
|
||||
path text NOT NULL,
|
||||
content_type text NOT NULL
|
||||
);
|
||||
@@ -1,8 +0,0 @@
|
||||
-- The Cover splits into two facts. `cover` keeps the third-party address the
|
||||
-- bytes come from, which is what the acquisition path refetches and dedupes
|
||||
-- on; `cover_address` is the content address of the bytes once they are
|
||||
-- actually stored, and is what the wire's absolute URL is built from.
|
||||
--
|
||||
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
|
||||
-- that 404s", which is the distinction the API and the UI both depend on.
|
||||
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
|
||||
@@ -1,6 +0,0 @@
|
||||
-- The owner's administrative page (issue #102) renders these counters and
|
||||
-- offers a control to clear them, deliberately shipped before the Sighting
|
||||
-- feature (issue #103) that fills them, so a false mark never needs SQL
|
||||
-- against production. Zero counters mean a trusted Reader.
|
||||
ALTER TABLE readers ADD COLUMN sighting_agreements integer NOT NULL DEFAULT 0;
|
||||
ALTER TABLE readers ADD COLUMN sighting_disagreements integer NOT NULL DEFAULT 0;
|
||||
@@ -1,10 +0,0 @@
|
||||
-- Sighting deferral (issue #103). latest_sighted_at is when a Reader's report
|
||||
-- last stood in for a Poll; it is separate from latest_checked_at because the
|
||||
-- six-hour ceiling has to know when the Series was last really fetched, and a
|
||||
-- Sighting writing the Poll's own column would erase that.
|
||||
-- latest_raised_by is attribution: whoever last raised this Series' Latest
|
||||
-- Chapter by Sighting, so a Poll that contradicts the value downwards names a
|
||||
-- Reader rather than flagging a row. Cleared by the Poll that judges it, NULL
|
||||
-- whenever the stored value is the Poll's own.
|
||||
ALTER TABLE series ADD COLUMN latest_sighted_at bigint NOT NULL DEFAULT 0;
|
||||
ALTER TABLE series ADD COLUMN latest_raised_by bigint REFERENCES readers(id) ON DELETE SET NULL;
|
||||
@@ -1,6 +0,0 @@
|
||||
-- One durable state row per Poll Lane. Zero means no pause or refusal is set.
|
||||
CREATE TABLE poll_lanes (
|
||||
site text NOT NULL PRIMARY KEY,
|
||||
paused_until bigint NOT NULL DEFAULT 0,
|
||||
refuse_until bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
@@ -1,16 +0,0 @@
|
||||
-- Append-only Lane Pass log. Timestamps are unix milliseconds from the poller's clock.
|
||||
CREATE TABLE poll_passes (
|
||||
site text NOT NULL,
|
||||
ran_at bigint NOT NULL,
|
||||
skip text NOT NULL,
|
||||
due integer NOT NULL,
|
||||
checked integer NOT NULL,
|
||||
gap_ms bigint NOT NULL,
|
||||
clamped boolean NOT NULL,
|
||||
refused integer NOT NULL,
|
||||
unreachable integer NOT NULL,
|
||||
no_chapter integer NOT NULL,
|
||||
unfetchable integer NOT NULL,
|
||||
errors integer NOT NULL,
|
||||
PRIMARY KEY (site, ran_at)
|
||||
);
|
||||
@@ -1,11 +0,0 @@
|
||||
-- Admin read-model foundation (#140). The Series list's default order is
|
||||
-- least-recently-checked first, so the table — which has only its primary key
|
||||
-- today — gets an index that can serve it. A grouped query over a join may
|
||||
-- ignore the index, so this is a judgement, not a measurement: re-time on real
|
||||
-- data before adding a second.
|
||||
CREATE INDEX series_latest_checked_at_idx ON series (latest_checked_at);
|
||||
|
||||
-- force_poll_at is the "ask for one Series to be checked now" stamp (#146).
|
||||
-- Zero means never forced; nothing reads the column before that ticket wires
|
||||
-- it, so it lands here unused.
|
||||
ALTER TABLE series ADD COLUMN force_poll_at bigint NOT NULL DEFAULT 0;
|
||||
@@ -1,80 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Session is one browser login: an opaque id the cookie carries verbatim,
|
||||
// the Reader it belongs to, and when it stops being valid.
|
||||
type Session struct {
|
||||
ID string
|
||||
ReaderID int64
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// CreateSession stores a new session row for reader. The id is generated by
|
||||
// the caller (session.NewID) — the store only persists it. Expired rows that
|
||||
// were never looked up are swept in the same transaction: this is the one
|
||||
// write every login makes, so the table stays bounded without a background
|
||||
// job.
|
||||
func (s *Store) CreateSession(id string, readerID int64, ttl time.Duration) (Session, error) {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
expires := time.Now().Add(ttl)
|
||||
if _, err := tx.Exec(`INSERT INTO sessions (id, reader_id, expires_at) VALUES ($1, $2, $3)`,
|
||||
id, readerID, expires); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM sessions WHERE expires_at < now()`); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
return Session{ID: id, ReaderID: readerID, ExpiresAt: expires}, nil
|
||||
}
|
||||
|
||||
// GetSession returns the live session row for id, or ok=false when the id is
|
||||
// unknown or expired. An expired row is deleted on the way out, so the table
|
||||
// never grows past sessions that are still valid.
|
||||
func (s *Store) GetSession(id string, now time.Time) (Session, bool, error) {
|
||||
var sess Session
|
||||
err := s.db.QueryRow(
|
||||
`SELECT id, reader_id, expires_at FROM sessions WHERE id = $1`, id,
|
||||
).Scan(&sess.ID, &sess.ReaderID, &sess.ExpiresAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return Session{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Session{}, false, err
|
||||
}
|
||||
if !sess.ExpiresAt.After(now) {
|
||||
// Best-effort: the row is dead either way; failing the request over a
|
||||
// cleanup delete would only hide the real error. CreateSession's
|
||||
// sweep catches anything this misses.
|
||||
_, _ = s.db.Exec(`DELETE FROM sessions WHERE id = $1`, id)
|
||||
return Session{}, false, nil
|
||||
}
|
||||
return sess, true, nil
|
||||
}
|
||||
|
||||
// DeleteSession revokes one session. Deleting an unknown id is not an error.
|
||||
func (s *Store) DeleteSession(id string) error {
|
||||
_, err := s.db.Exec(`DELETE FROM sessions WHERE id = $1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteReaderSessions revokes every session one Reader holds — the owner's
|
||||
// remedy when a Reader's browser must be logged out everywhere at once. The
|
||||
// next request carrying any of those cookies finds no row and is rejected.
|
||||
func (s *Store) DeleteReaderSessions(readerID int64) error {
|
||||
if _, err := s.db.Exec(`DELETE FROM sessions WHERE reader_id = $1`, readerID); err != nil {
|
||||
return fmt.Errorf("delete sessions for reader %d: %w", readerID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCreateAndGetSession(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
owner := s.OwnerID()
|
||||
|
||||
sess, err := s.CreateSession("sess-1", owner, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
if sess.ID != "sess-1" || sess.ReaderID != owner {
|
||||
t.Fatalf("CreateSession returned %+v, want id sess-1 reader %d", sess, owner)
|
||||
}
|
||||
|
||||
got, ok, err := s.GetSession("sess-1", time.Now())
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetSession: ok=%v err=%v, want ok", ok, err)
|
||||
}
|
||||
if got.ReaderID != owner {
|
||||
t.Fatalf("session reader = %d, want %d", got.ReaderID, owner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSessionUnknownID(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if _, ok, err := s.GetSession("nope", time.Now()); err != nil || ok {
|
||||
t.Fatalf("GetSession(unknown) = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiredSessionIsGone(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
owner := s.OwnerID()
|
||||
if _, err := s.CreateSession("sess-exp", owner, -time.Minute); err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if _, ok, err := s.GetSession("sess-exp", now); err != nil || ok {
|
||||
t.Fatalf("GetSession(expired) = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
// The expired row is deleted on lookup, so the next call cannot revive it.
|
||||
if _, ok, err := s.GetSession("sess-exp", now.Add(-time.Hour)); err != nil || ok {
|
||||
t.Fatalf("GetSession(expired again) = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSessionRevokes(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
owner := s.OwnerID()
|
||||
if _, err := s.CreateSession("sess-del", owner, time.Hour); err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
if err := s.DeleteSession("sess-del"); err != nil {
|
||||
t.Fatalf("DeleteSession: %v", err)
|
||||
}
|
||||
if _, ok, err := s.GetSession("sess-del", time.Now()); err != nil || ok {
|
||||
t.Fatalf("GetSession after delete = ok=%v err=%v, want ok=false", ok, err)
|
||||
}
|
||||
// Deleting twice is not an error.
|
||||
if err := s.DeleteSession("sess-del"); err != nil {
|
||||
t.Fatalf("DeleteSession twice: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSessionIsPerReader(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
other := secondReader(t, s)
|
||||
if _, err := s.CreateSession("sess-other", other, time.Hour); err != nil {
|
||||
t.Fatalf("CreateSession: %v", err)
|
||||
}
|
||||
got, ok, err := s.GetSession("sess-other", time.Now())
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetSession: ok=%v err=%v, want ok", ok, err)
|
||||
}
|
||||
if got.ReaderID != other {
|
||||
t.Fatalf("session reader = %d, want %d", got.ReaderID, other)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,37 +0,0 @@
|
||||
package token
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Token derives one Reader's userscript credential from the deployment
|
||||
// secret, the Reader's Discord id and their token epoch.
|
||||
//
|
||||
// The credential is deterministic rather than stored random because the
|
||||
// server must be able to rebuild the install URL after a restart while the
|
||||
// database holds only hashes: a random token with no plaintext copy anywhere
|
||||
// would be unreconstructible, and keeping plaintext in memory would break
|
||||
// every install link on restart. HMAC output is high-entropy, indistinguishable
|
||||
// from random to anyone without the secret, and changes whenever the epoch
|
||||
// does — which is what rotation is. The stored form is Hash of this value,
|
||||
// so a database leak yields nothing but hashes of unguessable strings.
|
||||
func Token(key []byte, discordID string, epoch int64) string {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
// The separator is unambiguous: discord ids are decimal snowflakes and
|
||||
// epochs are plain integers, so no two (id, epoch) pairs can collide.
|
||||
mac.Write([]byte(discordID))
|
||||
mac.Write([]byte{0})
|
||||
mac.Write([]byte(strconv.FormatInt(epoch, 10)))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// Hash is the SHA-256 of a credential — the only form that ever touches the
|
||||
// database (readers.token_sha256). SHA-256 rather than a password hash is
|
||||
// deliberate: these are unguessable values with nothing to brute-force, so a
|
||||
// slow hash would only add per-request cost.
|
||||
func Hash(cred string) [32]byte {
|
||||
return sha256.Sum256([]byte(cred))
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package token
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTokenDeterministicPerReaderAndEpoch(t *testing.T) {
|
||||
key := []byte("deployment-secret")
|
||||
a := Token(key, "reader-1", 0)
|
||||
b := Token(key, "reader-1", 0)
|
||||
if a != b {
|
||||
t.Fatal("same (reader, epoch) derived different credentials")
|
||||
}
|
||||
if a == Token(key, "reader-2", 0) {
|
||||
t.Fatal("different readers derived the same credential")
|
||||
}
|
||||
if a == Token(key, "reader-1", 1) {
|
||||
t.Fatal("rotation epoch derived the same credential")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenChangesWithSecret(t *testing.T) {
|
||||
a := Token([]byte("key-1"), "reader-1", 0)
|
||||
b := Token([]byte("key-2"), "reader-1", 0)
|
||||
if a == b {
|
||||
t.Fatal("different secrets derived the same credential")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenFormat(t *testing.T) {
|
||||
cred := Token([]byte("key"), "reader-1", 0)
|
||||
// 32 bytes of HMAC-SHA256, hex-encoded: the length the install URL and
|
||||
// the committed placeholder both assume.
|
||||
if len(cred) != 64 {
|
||||
t.Fatalf("credential length = %d, want 64", len(cred))
|
||||
}
|
||||
for _, c := range cred {
|
||||
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
|
||||
t.Fatalf("credential contains non-hex byte %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashIsSha256OfCredential(t *testing.T) {
|
||||
cred := Token([]byte("key"), "reader-1", 0)
|
||||
got := Hash(cred)
|
||||
want := sha256.Sum256([]byte(cred))
|
||||
if !bytes.Equal(got[:], want[:]) {
|
||||
t.Fatal("Hash is not the SHA-256 of the credential")
|
||||
}
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package userscript
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/httpmw"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// tokenPlaceholder is what the bindmounted userscript carries where the
|
||||
// Reader's credential goes: in the API_TOKEN constant and in the @downloadURL
|
||||
// and @updateURL metadata lines. The handler substitutes the requesting
|
||||
// Reader's credential for it at serve time, so no credential literal is ever
|
||||
// committed or deployed, and each Reader's copy carries exactly their own.
|
||||
var tokenPlaceholder = []byte("__API_TOKEN__")
|
||||
|
||||
// versionLine matches the userscript metadata block's @version directive.
|
||||
var versionLine = regexp.MustCompile(`(?m)^// @version[ \t]+.*$`)
|
||||
|
||||
// stampVersion replaces the served @version with one derived from the file's
|
||||
// mtime, discarding whatever the file body says.
|
||||
//
|
||||
// Violentmonkey only updates when the served version sorts higher than the
|
||||
// installed one. Deriving it from the body means one accidental downgrade or
|
||||
// typo freezes updates forever; an mtime-derived version is monotonic by
|
||||
// construction, so any later write always outranks any earlier one.
|
||||
//
|
||||
// A file with no @version line is returned untouched: such a script never
|
||||
// auto-updates anyway, and inventing a metadata block is not this handler's job.
|
||||
func stampVersion(src []byte, mod time.Time) []byte {
|
||||
return versionLine.ReplaceAll(src, []byte("// @version "+mod.UTC().Format("2006.01.02.1504")))
|
||||
}
|
||||
|
||||
// substituteToken replaces every tokenPlaceholder with the Reader's
|
||||
// credential. A file without the placeholder is returned unchanged so Render
|
||||
// can warn about it rather than silently serving a credential-less script.
|
||||
func substituteToken(src []byte, credential string) []byte {
|
||||
return bytes.ReplaceAll(src, tokenPlaceholder, []byte(credential))
|
||||
}
|
||||
|
||||
// Render writes one userscript file with the credential substituted and the
|
||||
// mtime-derived version stamped. Shared by the download path (Handler) and
|
||||
// the web UI's install endpoints, so both serve byte-identical scripts.
|
||||
//
|
||||
// The file is read per request — that is what lets a bindmounted copy be
|
||||
// edited on the host without a restart. It is ~50 KB and polled about once a
|
||||
// day.
|
||||
func Render(w http.ResponseWriter, r *http.Request, path, credential string) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
log.Printf("userscript: stat %s: %v", path, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
log.Printf("userscript: read %s: %v", path, err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
rendered := substituteToken(src, credential)
|
||||
if bytes.Equal(rendered, src) {
|
||||
// The bindmounted file was not built for per-Reader rendering. Serving
|
||||
// it as written is the operator's freedom, but a credential-less copy
|
||||
// is a deployment bug worth one log line — the symptom (silent 401s on
|
||||
// every device) is otherwise indistinguishable from a network fault.
|
||||
log.Printf("userscript: %s has no %s placeholder; serving as written", path, tokenPlaceholder)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Write(stampVersion(rendered, info.ModTime()))
|
||||
}
|
||||
|
||||
// Handler serves the userscript to Violentmonkey's updater, rendered for the
|
||||
// Reader whose credential is in the path.
|
||||
//
|
||||
// The credential lives in the path because the update poll sends no
|
||||
// Authorization header, and the rendered file embeds the credential in
|
||||
// plaintext, so an open path would hand it to anyone who guessed the URL. A
|
||||
// mismatch answers 404 rather than 401: a prober learns nothing about whether
|
||||
// the route exists. The same credential authenticates the API bearer header,
|
||||
// so the two are one secret with one blast radius.
|
||||
//
|
||||
// The path segment is the credential itself, so once it resolves it is also
|
||||
// exactly what the served copy must carry — no re-derivation needed.
|
||||
func Handler(s *store.Store, path string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
cred := r.PathValue("token")
|
||||
if _, ok := httpmw.ResolveReader(s, cred); !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
Render(w, r, path, cred)
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package userscript
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// sampleScript is a stand-in for the real userscript: a metadata block with a
|
||||
// @version line, the credential placeholder in its metadata and body, plus
|
||||
// content that must survive the rewrites untouched.
|
||||
const sampleScript = `// ==UserScript==
|
||||
// @name Manga Bookmark Sync
|
||||
// @version 1.5.0
|
||||
// @downloadURL https://api.example/u/__API_TOKEN__/manga-bookmark.user.js
|
||||
// @match https://asurascans.com/*
|
||||
// ==/UserScript==
|
||||
(function () { "use strict";
|
||||
const API_TOKEN = "__API_TOKEN__";
|
||||
})();
|
||||
`
|
||||
|
||||
func TestStampVersionReplacesVersionLineOnly(t *testing.T) {
|
||||
mod := time.Date(2026, 7, 28, 16, 42, 0, 0, time.UTC)
|
||||
got := string(stampVersion([]byte(sampleScript), mod))
|
||||
|
||||
if !strings.Contains(got, "// @version "+mod.UTC().Format("2006.01.02.1504")) {
|
||||
t.Errorf("body has no stamped version:\n%s", got)
|
||||
}
|
||||
if strings.Contains(got, "1.5.0") {
|
||||
t.Errorf("body still carries the file's own version:\n%s", got)
|
||||
}
|
||||
// Everything outside the @version line is served verbatim, including the
|
||||
// placeholder — stamping must not do the substitution's job.
|
||||
if !strings.Contains(got, `const API_TOKEN = "__API_TOKEN__";`) {
|
||||
t.Errorf("body was altered beyond the version line:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStampVersionWithoutVersionLineServedUnmodified(t *testing.T) {
|
||||
const noVersion = "// ==UserScript==\n// @name x\n// ==/UserScript==\nconsole.log(1);\n"
|
||||
if got := string(stampVersion([]byte(noVersion), time.Now())); got != noVersion {
|
||||
t.Errorf("stampVersion altered a file with no @version line:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubstituteTokenReplacesEveryPlaceholder(t *testing.T) {
|
||||
got := string(substituteToken([]byte(sampleScript), "abc123"))
|
||||
|
||||
if strings.Contains(got, "__API_TOKEN__") {
|
||||
t.Errorf("placeholder survived substitution:\n%s", got)
|
||||
}
|
||||
// The credential lands in the constant and in both metadata lines.
|
||||
if want := `const API_TOKEN = "abc123";`; !strings.Contains(got, want) {
|
||||
t.Errorf("no substituted constant %q:\n%s", want, got)
|
||||
}
|
||||
if want := "https://api.example/u/abc123/manga-bookmark.user.js"; !strings.Contains(got, want) {
|
||||
t.Errorf("no substituted download URL %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubstituteTokenWithoutPlaceholderServedUnmodified(t *testing.T) {
|
||||
const noPlaceholder = "// ==UserScript==\n// @name x\n// ==/UserScript==\n"
|
||||
if got := string(substituteToken([]byte(noPlaceholder), "abc123")); got != noPlaceholder {
|
||||
t.Errorf("substituteToken altered a file without the placeholder:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// ownerWindow is the staleness boundary the Series list's "not checked in
|
||||
// 12h" filter compares against. Declared once; later admin tickets read it.
|
||||
const ownerWindow = 12 * time.Hour
|
||||
|
||||
// adminView is the shared shell data for an administrative page and the roster
|
||||
// fragment returned after a Reader action.
|
||||
type adminView struct {
|
||||
Page string
|
||||
Readers []store.ReaderSummary
|
||||
// OwnerID travels with the roster so it can tell the owner's own row from
|
||||
// the Readers they may act on.
|
||||
OwnerID int64
|
||||
Lanes lanesView
|
||||
SeriesList seriesListView
|
||||
// Detail is the per-Series page data; zero on every other page.
|
||||
Detail seriesDetailView
|
||||
// Overview is the landing page data; zero on every other page.
|
||||
Overview overviewView
|
||||
}
|
||||
|
||||
// adminRoute pairs a route pattern with its handler so the route list and the
|
||||
// gate cannot drift apart.
|
||||
type adminRoute struct {
|
||||
pattern string
|
||||
handler http.HandlerFunc
|
||||
}
|
||||
|
||||
// adminRoutes is every route that reaches past the acting Reader. Register
|
||||
// wraps each one in requireOwner, so a new administrative route is gated by
|
||||
// being listed here rather than by remembering to write a check inside it.
|
||||
func (h *Handler) adminRoutes() []adminRoute {
|
||||
return []adminRoute{
|
||||
{"GET /admin", h.admin},
|
||||
{"GET /admin/lanes", h.adminLanes},
|
||||
{"GET /admin/readers", h.adminReaders},
|
||||
{"GET /admin/series", h.adminSeries},
|
||||
{"GET /admin/series/{key}", h.adminSeriesDetail},
|
||||
{"POST /admin/series/{key}/poll", h.adminSeriesPoll},
|
||||
{"POST /admin/lanes/{site}/pause", h.adminLanePause},
|
||||
{"POST /admin/lanes/{site}/resume", h.adminLaneResume},
|
||||
{"GET /ui/admin/lanes", h.uiLanes},
|
||||
{"POST /readers/{id}/revoke", h.revokeReaderSessions},
|
||||
{"POST /readers/{id}/clear-marks", h.clearReaderMarks},
|
||||
}
|
||||
}
|
||||
|
||||
// AdminPatterns names every administrative route, so one test can prove the
|
||||
// owner gate covers all of them rather than one test per route. The receiver is
|
||||
// nil because only the patterns are read; the bound handlers are never called.
|
||||
func AdminPatterns() []string {
|
||||
routes := (*Handler)(nil).adminRoutes()
|
||||
out := make([]string, 0, len(routes))
|
||||
for _, rt := range routes {
|
||||
out = append(out, rt.pattern)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// requireOwner is the owner test, in one place, layered on the session gate: no
|
||||
// session is still 401, and a signed-in Reader who is not the owner gets 404
|
||||
// rather than 403 — a refusal that confirms the address exists is a refusal
|
||||
// that helps whoever is probing for it.
|
||||
func (h *Handler) requireOwner(next http.HandlerFunc) http.HandlerFunc {
|
||||
return h.requireSession(func(w http.ResponseWriter, r *http.Request) {
|
||||
if readerOf(r) != h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// admin renders the Overview landing page: a verdict line, a stats block
|
||||
// where every figure is a door into the list it counts, and the per-Site
|
||||
// library shape table — all read from the database, never from a poller.
|
||||
func (h *Handler) admin(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.overviewView()
|
||||
if err != nil {
|
||||
log.Printf("admin overview: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "overview", Overview: view})
|
||||
}
|
||||
|
||||
// adminReaders renders the Reader roster on its own bookmarkable page.
|
||||
func (h *Handler) adminReaders(w http.ResponseWriter, r *http.Request) {
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
log.Printf("admin readers: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "readers", Readers: readers, OwnerID: h.store.OwnerID()})
|
||||
}
|
||||
|
||||
func (h *Handler) renderAdmin(w http.ResponseWriter, view adminView) {
|
||||
h.render(w, http.StatusOK, "admin", view)
|
||||
}
|
||||
|
||||
// revokeReaderSessions logs one Reader out of every browser they are signed in
|
||||
// on. The owner gate is the route's, not this handler's.
|
||||
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
|
||||
target, ok := readerPathID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// The owner is not one of the Readers this endpoint reaches: revoking
|
||||
// themselves would sign out the browser making the request, which is what
|
||||
// logout is for. The roster hides the button; this refuses the hand-rolled
|
||||
// POST behind it.
|
||||
if target == h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.DeleteReaderSessions(target); err != nil {
|
||||
log.Printf("revoke sessions: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderRoster(w, "revoke sessions")
|
||||
}
|
||||
|
||||
// clearReaderMarks zeroes one Reader's Sighting counters. The guard those
|
||||
// counters feed has one known false positive — a Site changing its page shape
|
||||
// makes a correct adapter read a wrong high number and marks every honest
|
||||
// Reader of that Site at once (issue #103) — and this is its remedy. It
|
||||
// restores a privilege rather than destroying anything, so the control is
|
||||
// confirmed but never wears the destruction accent.
|
||||
func (h *Handler) clearReaderMarks(w http.ResponseWriter, r *http.Request) {
|
||||
target, ok := readerPathID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := h.store.ClearReaderMarks(target); err != nil {
|
||||
log.Printf("clear marks: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderRoster(w, "clear marks")
|
||||
}
|
||||
|
||||
// readerPathID reads the Reader a route names, answering the request itself
|
||||
// when there is nobody to act on.
|
||||
func readerPathID(w http.ResponseWriter, r *http.Request) (int64, bool) {
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "bad reader id", http.StatusBadRequest)
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
// renderRoster answers an action with the whole roster, so the counts and marks
|
||||
// it shows cannot describe the state before the tap.
|
||||
func (h *Handler) renderRoster(w http.ResponseWriter, what string) {
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
log.Printf("%s: %v", what, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "readers", adminView{Readers: readers, OwnerID: h.store.OwnerID()})
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// lanesView is the Lane status block: one row per Site's latest durable pass,
|
||||
// plus the browser fact derived from that same log. No poller is consulted —
|
||||
// the page answers from the database, so it is complete thirty seconds after
|
||||
// a deploy (issue #145).
|
||||
type lanesView struct {
|
||||
Rows []laneRow
|
||||
// PollerOff means latest-chapter polling is switched off in this
|
||||
// deployment (LATEST_CHAPTER_POLL_ENABLED). It is a config fact, not a
|
||||
// poller answering "absent": the browser line must not blame the sidecar
|
||||
// when nothing polls.
|
||||
PollerOff bool
|
||||
BrowserConfigured bool
|
||||
BrowserReachable bool
|
||||
}
|
||||
|
||||
// laneRow is one Lane formatted for reading rather than for arithmetic: the
|
||||
// template renders strings and flags, and every judgement about what they
|
||||
// mean is made here.
|
||||
type laneRow struct {
|
||||
Site string
|
||||
Due int
|
||||
Checked int
|
||||
// Gap is the last pass's pace, or "—" when no pass has reached one yet —
|
||||
// a refused Lane still reports the pace its last real pass chose, so a
|
||||
// zero here would be a figure the row never measured.
|
||||
Gap string
|
||||
Ran string
|
||||
// Chips are the named outcome counts over the owner's window, in the
|
||||
// taxonomy's fixed order. Empty writes "none observed".
|
||||
Chips []chip
|
||||
HasChips bool
|
||||
// StatePhrase is the reason this Lane declined to work: a skipped pass's
|
||||
// own sentence, or the one true stall. Empty means the pass reached its
|
||||
// loop and read normally. StateGood marks a healthy way to do nothing
|
||||
// (paused, browser asleep, nothing eligible) rather than a fault.
|
||||
StatePhrase string
|
||||
StateGood bool
|
||||
// Attention is the one flag the template colours on, so a Lane that
|
||||
// needs the owner is found at a glance rather than read for.
|
||||
Attention bool
|
||||
// Paused is the live pause state — the poll_lanes stamp the pass row
|
||||
// joins on, still in the future — not the pass's skip: the control must
|
||||
// offer Resume from the moment the owner presses Pause, with no pass
|
||||
// having run to record it (issue #147).
|
||||
Paused bool
|
||||
}
|
||||
|
||||
// chip is one named outcome count over the owner's window.
|
||||
type chip struct {
|
||||
Name string
|
||||
Count int
|
||||
}
|
||||
|
||||
// adminLanes renders the page that hosts the live Lane fragment.
|
||||
func (h *Handler) adminLanes(w http.ResponseWriter, r *http.Request) {
|
||||
h.renderAdmin(w, adminView{Page: "lanes", Lanes: h.lanesView()})
|
||||
}
|
||||
|
||||
// uiLanes answers the status block's own refresh. Only the block refreshes on
|
||||
// a timer; the roster re-renders after an action, as it always has.
|
||||
func (h *Handler) uiLanes(w http.ResponseWriter, r *http.Request) {
|
||||
h.render(w, http.StatusOK, "lanes", h.lanesView())
|
||||
}
|
||||
|
||||
// pauseDurations are the offered pause lengths, by their wire value. A fixed
|
||||
// allow-list rather than time.ParseDuration: the unoffered value must be
|
||||
// refused, and a permissive parser turns the offered set into "anything Go
|
||||
// can read" (issue #147).
|
||||
var pauseDurations = map[string]time.Duration{
|
||||
"1h": time.Hour,
|
||||
"6h": 6 * time.Hour,
|
||||
"24h": 24 * time.Hour,
|
||||
}
|
||||
|
||||
// laneSite reads the Site a lane route names, answering the request itself
|
||||
// when it is not a registry Site. The path value is client-supplied, so it
|
||||
// is checked against the registry before it reaches the store.
|
||||
func laneSite(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
site := r.PathValue("site")
|
||||
if !slices.Contains(latest.SiteNames(), site) {
|
||||
http.Error(w, "unknown site", http.StatusBadRequest)
|
||||
return "", false
|
||||
}
|
||||
return site, true
|
||||
}
|
||||
|
||||
// adminLanePause writes a bounded pause for one Site and answers with the
|
||||
// freshly rendered Lanes block, so the figures describe the state after the
|
||||
// press. The pause is a fact about the Site — the Lane's next pass reads it
|
||||
// from the durable row, never from this process — so it survives a restart.
|
||||
// The owner gate is the route's, not this handler's; the body is capped like
|
||||
// the API path caps its bodies; the Site and the duration are validated
|
||||
// here, before the store sees them (issue #147).
|
||||
func (h *Handler) adminLanePause(w http.ResponseWriter, r *http.Request) {
|
||||
site, ok := laneSite(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
d, ok := pauseDurations[r.PostFormValue("duration")]
|
||||
if !ok {
|
||||
http.Error(w, "unknown pause duration", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.PauseLane(site, time.Now().Add(d).UnixMilli()); err != nil {
|
||||
log.Printf("pause lane %s: %v", site, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "lanes", h.lanesView())
|
||||
}
|
||||
|
||||
// adminLaneResume zeroes one Site's pause and answers with the freshly
|
||||
// rendered Lanes block. Resume is the reversal of a bounded pause, so it
|
||||
// fires instantly with no confirm row (issue #147).
|
||||
func (h *Handler) adminLaneResume(w http.ResponseWriter, r *http.Request) {
|
||||
site, ok := laneSite(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.ResumeLane(site); err != nil {
|
||||
log.Printf("resume lane %s: %v", site, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "lanes", h.lanesView())
|
||||
}
|
||||
|
||||
// lanesView builds the Lane status block from the durable pass log. Both
|
||||
// reads are the store's latest-per-Site projection, so the page's seam is a
|
||||
// seeded row rather than a fake poller; errors degrade to the empty state and
|
||||
// are logged, never shown to the owner in detail.
|
||||
func (h *Handler) lanesView() lanesView {
|
||||
v := lanesView{
|
||||
PollerOff: !h.pollerEnabled,
|
||||
BrowserConfigured: h.browserConfigured,
|
||||
}
|
||||
passes, err := h.store.LatestLanePasses()
|
||||
if err != nil {
|
||||
log.Printf("admin lanes: latest passes: %v", err)
|
||||
// No evidence of a lost sidecar reads as reachable, per the same rule
|
||||
// browserReachable applies: a store failure must not condemn the
|
||||
// browser. The empty table already says no Lane has recorded a pass.
|
||||
v.BrowserReachable = true
|
||||
return v
|
||||
}
|
||||
now := time.Now()
|
||||
outcomes, err := h.store.LanePassOutcomes(now.Add(-ownerWindow).UnixMilli())
|
||||
if err != nil {
|
||||
// The rows are complete without the chips, so a failed outcome sum
|
||||
// must not blank the table into "no data yet" — that is the confident
|
||||
// wrong statement the page exists to avoid. Every row renders "none
|
||||
// observed" instead, which is honest.
|
||||
log.Printf("admin lanes: outcomes: %v", err)
|
||||
outcomes = nil
|
||||
}
|
||||
bySite := make(map[string]store.SiteOutcomes, len(outcomes))
|
||||
for _, o := range outcomes {
|
||||
bySite[o.Site] = o
|
||||
}
|
||||
v.Rows = make([]laneRow, 0, len(passes))
|
||||
v.BrowserReachable = browserReachable(passes, now)
|
||||
for _, p := range passes {
|
||||
v.Rows = append(v.Rows, buildLaneRow(p, bySite[p.Site], now))
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// browserReachable derives the sidecar's reachability from the pass log: a
|
||||
// browser Site is down when its latest pass inside the refusal backoff is a
|
||||
// sidecar loss, a missing fetcher, or an interrupted read. Only browser Sites
|
||||
// ever produce those signals, so no Site registry leaks into the web layer.
|
||||
// A configured browser with no such evidence reads as reachable; an unset
|
||||
// BROWSER_WS_URL degrades identically to a browser that is down.
|
||||
func browserReachable(passes []store.LanePass, now time.Time) bool {
|
||||
backoff := latest.RefuseBackoff
|
||||
for _, p := range passes {
|
||||
ran := time.UnixMilli(p.RanAt)
|
||||
if now.Sub(ran) >= backoff || ran.After(now) {
|
||||
continue
|
||||
}
|
||||
if p.Skip == latest.SkipSidecarDown || p.Skip == latest.SkipNoFetcher || p.Unreachable > 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// buildLaneRow turns one Site's latest pass and window outcome sums into the
|
||||
// row the template prints. The skip column is the authority on why a pass did
|
||||
// nothing; the outcomes render named and unlinked, because the pass row holds
|
||||
// counts and never identities.
|
||||
func buildLaneRow(p store.LanePass, o store.SiteOutcomes, now time.Time) laneRow {
|
||||
row := laneRow{
|
||||
Site: p.Site,
|
||||
Due: p.Due,
|
||||
Checked: p.Checked,
|
||||
Gap: "—",
|
||||
Ran: since(now, time.UnixMilli(p.RanAt)),
|
||||
}
|
||||
if p.GapMS > 0 {
|
||||
row.Gap = (time.Duration(p.GapMS) * time.Millisecond).Truncate(time.Second).String()
|
||||
}
|
||||
row.Chips = outcomeChips(o)
|
||||
row.HasChips = len(row.Chips) > 0
|
||||
row.StatePhrase, row.StateGood, row.Attention = laneState(p, now)
|
||||
row.Paused = time.UnixMilli(p.PausedUntil).After(now)
|
||||
return row
|
||||
}
|
||||
|
||||
// outcomeChips lists a Site's nonzero window sums in the taxonomy's fixed
|
||||
// order, so the chips never reorder as the window changes. None observed is
|
||||
// written by the template, not drawn as a confident zero count.
|
||||
func outcomeChips(o store.SiteOutcomes) []chip {
|
||||
fixed := []struct {
|
||||
name string
|
||||
count int
|
||||
}{
|
||||
{"refused", o.Refused},
|
||||
{"unreachable", o.Unreachable},
|
||||
{"no chapter", o.NoChapter},
|
||||
{"unfetchable", o.Unfetchable},
|
||||
{"errors", o.Errors},
|
||||
}
|
||||
var out []chip
|
||||
for _, f := range fixed {
|
||||
if f.count > 0 {
|
||||
out = append(out, chip{Name: f.name, Count: f.count})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// laneState renders the reason a Lane's last pass did nothing, in one sentence
|
||||
// per skip value with the one true stall kept apart from every Lane that
|
||||
// declined and said why. Good states — a pause, a sleeping browser, nothing
|
||||
// eligible — carry no Attention: the mark must stay spendable on the faults
|
||||
// that actually need the owner.
|
||||
func laneState(p store.LanePass, now time.Time) (phrase string, good, attention bool) {
|
||||
// The pause phrase reads the live poll_lanes stamp the pass row joins
|
||||
// on, not the pass's skip: the owner's press must render as paused on
|
||||
// the very answer it gets, with no pass having run to record it. The
|
||||
// pause is a fact about the Site, and the join delivers it (issue #147).
|
||||
if pausedUntil := time.UnixMilli(p.PausedUntil); pausedUntil.After(now) {
|
||||
phrase = "paused · resumes in " + humanDuration(pausedUntil.Sub(now))
|
||||
good = true
|
||||
return phrase, good, attention
|
||||
}
|
||||
switch p.Skip {
|
||||
case latest.SkipPaused:
|
||||
// A paused pass whose stamp has since lapsed: the Lane still
|
||||
// declined with a reason, so it is never the one true stall.
|
||||
phrase = "paused · resumes in " + humanDuration(time.UnixMilli(p.PausedUntil).Sub(now))
|
||||
good = true
|
||||
case latest.SkipRefusing:
|
||||
phrase = "refusing"
|
||||
if until := time.UnixMilli(p.RefuseUntil); until.After(now) {
|
||||
phrase += " · backs off until " + until.Format("15:04")
|
||||
}
|
||||
attention = true
|
||||
case latest.SkipSidecarDown, latest.SkipNoFetcher:
|
||||
// Known false positive shipped per spec: a sibling Lane's Chrome loss
|
||||
// stamps this Site too, and the enum deliberately has no tenth value
|
||||
// to separate it (issue #141). Render it as written.
|
||||
phrase = "no browser"
|
||||
attention = true
|
||||
case latest.SkipAsleep:
|
||||
phrase = "browser asleep"
|
||||
good = true
|
||||
case latest.SkipDueQuery:
|
||||
phrase = "due query failed"
|
||||
attention = true
|
||||
case latest.SkipEligibleCount:
|
||||
phrase = "eligible count failed"
|
||||
attention = true
|
||||
case latest.SkipNothingEligible:
|
||||
phrase = "nothing eligible"
|
||||
good = true
|
||||
}
|
||||
if phrase == "" && p.Due > 0 && p.Checked == 0 {
|
||||
// The one true stall: the pass reached its loop, Series were waiting,
|
||||
// and none were read. Every skip above is a Lane that said why.
|
||||
phrase = "not checking"
|
||||
attention = true
|
||||
}
|
||||
return phrase, good, attention
|
||||
}
|
||||
|
||||
// humanDuration renders a positive duration compactly for a "resumes in" clue
|
||||
// at the pause and refusal scales — minutes under an hour, then h and h+m.
|
||||
func humanDuration(d time.Duration) string {
|
||||
d = d.Round(time.Minute)
|
||||
if d <= 0 {
|
||||
return "soon"
|
||||
}
|
||||
if d < time.Hour {
|
||||
return fmt.Sprintf("%dm", int(d/time.Minute))
|
||||
}
|
||||
h := int(d / time.Hour)
|
||||
if m := int(d%time.Hour) / int(time.Minute); m == 0 {
|
||||
return fmt.Sprintf("%dh", h)
|
||||
} else {
|
||||
return fmt.Sprintf("%dh%dm", h, m)
|
||||
}
|
||||
}
|
||||
|
||||
// since formats how long ago a Lane last ran, at second resolution: the block
|
||||
// refreshes every thirty seconds, so anything finer is noise the owner would
|
||||
// have to ignore.
|
||||
func since(now, then time.Time) string {
|
||||
d := now.Sub(then).Truncate(time.Second)
|
||||
if d < time.Second {
|
||||
return "just now"
|
||||
}
|
||||
return d.String() + " ago"
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// overviewView is the Overview landing page's data: one verdict line, the
|
||||
// hygiene and library stats blocks, and the per-Site library shape table.
|
||||
// Every judgement — the verdict state, which figures link, what a Lane's
|
||||
// state means — is made here; the template only prints.
|
||||
type overviewView struct {
|
||||
// Verdict is the attention phrase that leads the page.
|
||||
Verdict string
|
||||
// HasCounts is false on a virgin pass log: the waiting figure would be a
|
||||
// confident zero, and "nothing has happened" must not render as health.
|
||||
HasCounts bool
|
||||
// Waiting is the sum of Due over the latest pass per Site.
|
||||
Waiting int
|
||||
// Unchecked is the number of Series not checked in the window, computed
|
||||
// as stale + never_checked: a never-checked Series is already counted on
|
||||
// its own filter, and the verdict wants the inclusive number.
|
||||
Unchecked int
|
||||
// Hygiene is the seven problem filters in the Series list's own render
|
||||
// order; Library is the library split plus the roster. Every figure is a
|
||||
// door into the list that counts it, except a zero.
|
||||
Hygiene []fig
|
||||
Library []fig
|
||||
// Sites is the per-Site library shape table, one row per Site with any
|
||||
// Series, in the store's Site order.
|
||||
Sites []siteRow
|
||||
}
|
||||
|
||||
// fig is one stats figure: its label, the list it counts, and the count
|
||||
// itself. Href empty means the count is zero: a measured zero is a real
|
||||
// figure that stays on the page, but it is not a door, because following it
|
||||
// lands on an empty list.
|
||||
type fig struct {
|
||||
Label string
|
||||
Href string
|
||||
Count int
|
||||
}
|
||||
|
||||
// siteRow is one Site's share of the library: the Series total and the three
|
||||
// hygiene counts the per-Site table carries, each a door to the list narrowed
|
||||
// to that Site, plus the Lane state phrase derived from its latest pass. The
|
||||
// table is library shape only — the Poll outcome sums live on the Lanes page.
|
||||
type siteRow struct {
|
||||
Site string
|
||||
SiteHref string
|
||||
Figs []fig
|
||||
// State is the Lane's own sentence; "" means the last pass read normally.
|
||||
// StateGood / StateBad pick the ok / bad second class.
|
||||
State string
|
||||
StateGood bool
|
||||
StateBad bool
|
||||
}
|
||||
|
||||
// overviewView assembles the landing page from the store's read model: one
|
||||
// SeriesShapes pass per filter summed in Go (the shipped surface offers eight
|
||||
// grouped passes, not a stats query — #140), the pass log's latest pass per
|
||||
// Site, and the roster. A failure in any read is a 500 with a logged reason,
|
||||
// never a page of silent zeroes.
|
||||
func (h *Handler) overviewView() (overviewView, error) {
|
||||
now := time.Now()
|
||||
cutoff := now.Add(-ownerWindow).UnixMilli()
|
||||
|
||||
shapes := make(map[string][]store.SiteSeriesShape, len(seriesFilterOrder))
|
||||
totals := make(map[string]int, len(seriesFilterOrder))
|
||||
for _, name := range seriesFilterOrder {
|
||||
rows, err := h.store.SeriesShapes(store.SeriesFilter{Name: name, Cutoff: cutoff})
|
||||
if err != nil {
|
||||
return overviewView{}, err
|
||||
}
|
||||
shapes[name] = rows
|
||||
for _, sh := range rows {
|
||||
totals[name] += sh.Total
|
||||
}
|
||||
}
|
||||
|
||||
passes, err := h.store.LatestLanePasses()
|
||||
if err != nil {
|
||||
return overviewView{}, err
|
||||
}
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
return overviewView{}, err
|
||||
}
|
||||
|
||||
view := overviewView{Waiting: waiting(passes)}
|
||||
view.Unchecked = totals[store.SeriesFilterStale] + totals[store.SeriesFilterNeverChecked]
|
||||
view.Verdict, view.HasCounts = overviewVerdict(passes, now)
|
||||
|
||||
// The seven problem filters, in seriesFilterOrder's permanent-then-fixable
|
||||
// order; the All filter's count belongs to the Library block, not to a
|
||||
// "hygiene" figure.
|
||||
hygiene := make([]fig, 0, len(seriesFilterOrder)-1)
|
||||
for _, name := range seriesFilterOrder[1:] {
|
||||
hygiene = append(hygiene, door(seriesFilterLabels[name], totals[name], seriesListHref(name, "", "", 0)))
|
||||
}
|
||||
view.Hygiene = hygiene
|
||||
|
||||
var manga, novel int
|
||||
for _, sh := range shapes[store.SeriesFilterAll] {
|
||||
manga += sh.Manga
|
||||
novel += sh.Novel
|
||||
}
|
||||
view.Library = []fig{
|
||||
door("Series", totals[store.SeriesFilterAll], seriesListHref("", "", "", 0)),
|
||||
door("Manga", manga, seriesListHref("", "", store.KindManga, 0)),
|
||||
door("Novels", novel, seriesListHref("", "", store.KindNovel, 0)),
|
||||
door("Readers", len(readers), "/admin/readers"),
|
||||
}
|
||||
|
||||
// One row per Site with any Series, from the All shapes; the hygiene
|
||||
// counts come from the same per-Site projection so the table cannot
|
||||
// disagree with the library-wide figures above it.
|
||||
siteCounts := make(map[string]map[string]int, len(shapes))
|
||||
for name, rows := range shapes {
|
||||
m := make(map[string]int, len(rows))
|
||||
for _, sh := range rows {
|
||||
m[sh.Site] = sh.Total
|
||||
}
|
||||
siteCounts[name] = m
|
||||
}
|
||||
passBySite := make(map[string]store.LanePass, len(passes))
|
||||
for _, p := range passes {
|
||||
passBySite[p.Site] = p
|
||||
}
|
||||
view.Sites = make([]siteRow, 0, len(shapes[store.SeriesFilterAll]))
|
||||
for _, sh := range shapes[store.SeriesFilterAll] {
|
||||
row := siteRow{
|
||||
Site: sh.Site,
|
||||
SiteHref: seriesListHref("", sh.Site, "", 0),
|
||||
Figs: []fig{
|
||||
door("", sh.Total, seriesListHref("", sh.Site, "", 0)),
|
||||
door("", siteCounts[store.SeriesFilterNoCover][sh.Site], seriesListHref(store.SeriesFilterNoCover, sh.Site, "", 0)),
|
||||
door("", siteCounts[store.SeriesFilterNeverChecked][sh.Site], seriesListHref(store.SeriesFilterNeverChecked, sh.Site, "", 0)),
|
||||
door("", siteCounts[store.SeriesFilterStale][sh.Site], seriesListHref(store.SeriesFilterStale, sh.Site, "", 0)),
|
||||
},
|
||||
}
|
||||
if p, ok := passBySite[sh.Site]; ok {
|
||||
row.State, row.StateGood, row.StateBad = laneState(p, now)
|
||||
} else {
|
||||
row.State = "no pass yet"
|
||||
}
|
||||
view.Sites = append(view.Sites, row)
|
||||
}
|
||||
return view, nil
|
||||
}
|
||||
|
||||
// door is one figure with its door: the list that counts it. A measured zero
|
||||
// is still a real figure, but the door closes — following it would land on an
|
||||
// empty list. The count is written once so the figure and what it links to
|
||||
// cannot drift apart.
|
||||
func door(label string, count int, href string) fig {
|
||||
if count == 0 {
|
||||
href = ""
|
||||
}
|
||||
return fig{Label: label, Href: href, Count: count}
|
||||
}
|
||||
|
||||
// overviewVerdict decides the landing page's one line from the latest pass
|
||||
// per Site: no passes at all is "no Lane has reported yet" — never confident
|
||||
// zeroes; otherwise the count of Lanes whose last pass needs the owner, or
|
||||
// "all lanes healthy". The count comes from the same laneState judgement the
|
||||
// Lanes page colours on, so the two pages cannot disagree on what a fault is.
|
||||
func overviewVerdict(passes []store.LanePass, now time.Time) (phrase string, counts bool) {
|
||||
if len(passes) == 0 {
|
||||
return "no Lane has reported yet", false
|
||||
}
|
||||
attention := 0
|
||||
for _, p := range passes {
|
||||
if _, _, attn := laneState(p, now); attn {
|
||||
attention++
|
||||
}
|
||||
}
|
||||
if attention == 0 {
|
||||
return "all lanes healthy", true
|
||||
}
|
||||
if attention == 1 {
|
||||
return "1 lane needs a look", true
|
||||
}
|
||||
return fmt.Sprintf("%d lanes need a look", attention), true
|
||||
}
|
||||
|
||||
// waiting sums Due over the latest pass per Site: how many Series the Lanes
|
||||
// found waiting, from the durable log rather than a running poller.
|
||||
func waiting(passes []store.LanePass) int {
|
||||
n := 0
|
||||
for _, p := range passes {
|
||||
n += p.Due
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -1,398 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// seriesPageSize matches the store's row-read page length: the pager's range
|
||||
// must agree with the LIMIT the store applies or the "of N" figure describes
|
||||
// the wrong page. The store does not export it (#140).
|
||||
const seriesPageSize = 50
|
||||
|
||||
// seriesFilterLabels names every hygiene filter for the Series list select,
|
||||
// keyed by the wire constant the URL carries. The render order is
|
||||
// seriesFilterOrder; the labels are read by later admin tickets too, so the
|
||||
// map and the constants cannot drift apart.
|
||||
var seriesFilterLabels = map[string]string{
|
||||
store.SeriesFilterAll: "All series",
|
||||
store.SeriesFilterNoURL: "No series URL",
|
||||
store.SeriesFilterNoChapter: "Never read a chapter",
|
||||
store.SeriesFilterNoReaders: "No Readers",
|
||||
store.SeriesFilterNeverChecked: "Never checked",
|
||||
store.SeriesFilterStale: "Not checked in 12h",
|
||||
store.SeriesFilterNoCover: "No cover",
|
||||
store.SeriesFilterReaderReport: "Latest from a Reader",
|
||||
}
|
||||
|
||||
// seriesFilterOrder is the select's render order: All first, then the
|
||||
// permanent repairs, then the fixable ones (issue #140).
|
||||
var seriesFilterOrder = []string{
|
||||
store.SeriesFilterAll,
|
||||
store.SeriesFilterNoURL,
|
||||
store.SeriesFilterNoChapter,
|
||||
store.SeriesFilterNoReaders,
|
||||
store.SeriesFilterNeverChecked,
|
||||
store.SeriesFilterStale,
|
||||
store.SeriesFilterNoCover,
|
||||
store.SeriesFilterReaderReport,
|
||||
}
|
||||
|
||||
// seriesListView is the Series list page's data. The template renders strings
|
||||
// and flags, and every judgement about what a value means is made here.
|
||||
type seriesListView struct {
|
||||
Filters []seriesFilterOption
|
||||
Sites []string
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both libraries
|
||||
FilterLabel string
|
||||
Rows []seriesRowView
|
||||
Total int
|
||||
// KindBoth / KindManga / KindNovel are the Library segment links, and
|
||||
// PrevHref / NextHref the pager's, all carrying the active filter, Site
|
||||
// and Kind so narrowing never drops state.
|
||||
KindBoth string
|
||||
KindManga string
|
||||
KindNovel string
|
||||
PrevHref string
|
||||
NextHref string
|
||||
Range string
|
||||
}
|
||||
|
||||
// seriesFilterOption is one entry of the Show select: its wire value, its
|
||||
// rendered label with the library-wide count, and whether it is the active
|
||||
// filter.
|
||||
type seriesFilterOption struct {
|
||||
Name string
|
||||
Label string
|
||||
Count int
|
||||
Selected bool
|
||||
}
|
||||
|
||||
// seriesRowView is one Series row formatted for the template. Band carries
|
||||
// the alternating row tint by class rather than nth-of-type, so the confirm
|
||||
// rows later tickets add are row siblings without breaking the alternation.
|
||||
// Attention tints the title patina: a row with any hygiene chip needs one.
|
||||
//
|
||||
// CanPoll is the Check now control's visibility: absent on a Series with no
|
||||
// page to fetch and on an orphan, so the owner is never offered a button that
|
||||
// can never do anything. Pending is derived — the request stamp is newer than
|
||||
// the check stamp — and Requested is its ageing label.
|
||||
type seriesRowView struct {
|
||||
Key string
|
||||
Title string
|
||||
Site string
|
||||
Ch string // chapter number; "—" until first captured
|
||||
Age string // checked age; "never" until first check
|
||||
Readers int
|
||||
Notes []string // chips, capped at two
|
||||
More int // chips past the cap, rendered as a +N tail
|
||||
Band bool
|
||||
Attention bool
|
||||
CanPoll bool
|
||||
Pending bool
|
||||
Requested string // "requested 3m ago", rendered only while pending
|
||||
}
|
||||
|
||||
// adminSeries renders the filterable, bookmarkable Series list: filter, Site,
|
||||
// Library and page all live in the query string, so the list's state is an
|
||||
// address rather than a click path.
|
||||
func (h *Handler) adminSeries(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.seriesListView(r)
|
||||
if err != nil {
|
||||
log.Printf("admin series: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "series", SeriesList: view})
|
||||
}
|
||||
|
||||
// adminSeriesPoll is the Check now action: it stamps the Series' force_poll_at
|
||||
// and answers with the freshly rendered row, so the figures describe the
|
||||
// state after the press. The control never commands the poller — the request
|
||||
// is a fact about the Series, and the Lane's next pass reads it through
|
||||
// DueForLatestCheck (ADR-0013). The owner gate is the route's, not this
|
||||
// handler's; the body is capped like the API path caps its bodies; the key is
|
||||
// validated here — a malformed key is a 400 and an unknown one a 404.
|
||||
func (h *Handler) adminSeriesPoll(w http.ResponseWriter, r *http.Request) {
|
||||
site, seriesID, ok := strings.Cut(r.PathValue("key"), ":")
|
||||
if !ok || site == "" || seriesID == "" {
|
||||
http.Error(w, "bad series key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if _, found, err := h.adminSeriesByKey(site, seriesID); err != nil {
|
||||
log.Printf("series poll %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
} else if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.ForceSeriesPoll(site, seriesID, time.Now().UnixMilli()); err != nil {
|
||||
log.Printf("series poll %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Re-read after the stamp: the answer must describe the state after the
|
||||
// press. The detail page's control swaps its meta in place and the list
|
||||
// row's swaps the row; htmx names an id target in HX-Target, so the
|
||||
// response matches the surface it came from. The row's band parity travels
|
||||
// with the press (hx-vals), so the swap keeps the zebra alternation.
|
||||
a, found, err := h.adminSeriesByKey(site, seriesID)
|
||||
if err != nil {
|
||||
log.Printf("series poll %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("HX-Target") == "detail-meta" {
|
||||
h.render(w, http.StatusOK, "series-detail-meta", h.seriesDetailView(a))
|
||||
return
|
||||
}
|
||||
band := 0
|
||||
if r.PostFormValue("band") == "1" {
|
||||
band = 1
|
||||
}
|
||||
h.render(w, http.StatusOK, "series-row", seriesRow(a, band, time.Now()))
|
||||
}
|
||||
|
||||
// seriesListView assembles one Series list view from the request's query
|
||||
// string. An unknown filter value is the absent All case, never an error: the
|
||||
// select's options are not the only way this URL can be reached.
|
||||
func (h *Handler) seriesListView(r *http.Request) (seriesListView, error) {
|
||||
q := r.URL.Query()
|
||||
filter := q.Get("filter")
|
||||
if _, ok := seriesFilterLabels[filter]; !ok {
|
||||
filter = store.SeriesFilterAll
|
||||
}
|
||||
site := q.Get("site")
|
||||
kind := q.Get("kind")
|
||||
if kind != store.KindManga && kind != store.KindNovel {
|
||||
kind = ""
|
||||
}
|
||||
page := 1
|
||||
if p, err := strconv.Atoi(q.Get("page")); err == nil && p > 1 {
|
||||
page = p
|
||||
}
|
||||
|
||||
sf := store.SeriesFilter{
|
||||
Site: site,
|
||||
Kind: kind,
|
||||
Name: filter,
|
||||
Cutoff: time.Now().Add(-ownerWindow).UnixMilli(),
|
||||
Page: page,
|
||||
}
|
||||
data, err := h.store.SeriesPage(sf)
|
||||
if err != nil {
|
||||
return seriesListView{}, err
|
||||
}
|
||||
// A page past the end is not an empty list: the store's window count runs
|
||||
// over the rows the result set carries, so an overflow page reports zero
|
||||
// rows and zero total, and the list re-reads at page 1 to know the truth.
|
||||
if len(data.Rows) == 0 && page > 1 {
|
||||
page = 1
|
||||
sf.Page = 1
|
||||
data, err = h.store.SeriesPage(sf)
|
||||
if err != nil {
|
||||
return seriesListView{}, err
|
||||
}
|
||||
}
|
||||
|
||||
view := seriesListView{
|
||||
Site: site,
|
||||
Kind: kind,
|
||||
FilterLabel: seriesFilterLabels[filter],
|
||||
Rows: make([]seriesRowView, 0, len(data.Rows)),
|
||||
Total: data.Total,
|
||||
Sites: latest.SiteNames(),
|
||||
}
|
||||
now := time.Now()
|
||||
for i, a := range data.Rows {
|
||||
view.Rows = append(view.Rows, seriesRow(a, i, now))
|
||||
}
|
||||
view.Filters, err = h.seriesFilterOptions(filter, sf.Cutoff)
|
||||
if err != nil {
|
||||
return seriesListView{}, err
|
||||
}
|
||||
view.KindBoth = seriesListHref(filter, site, "", 0)
|
||||
view.KindManga = seriesListHref(filter, site, store.KindManga, 0)
|
||||
view.KindNovel = seriesListHref(filter, site, store.KindNovel, 0)
|
||||
if page > 1 {
|
||||
view.PrevHref = seriesListHref(filter, site, kind, page-1)
|
||||
}
|
||||
if last := (data.Total + seriesPageSize - 1) / seriesPageSize; page < last {
|
||||
view.NextHref = seriesListHref(filter, site, kind, page+1)
|
||||
}
|
||||
view.Range = pagerRange(data.Total, len(data.Rows), page)
|
||||
return view, nil
|
||||
}
|
||||
|
||||
// seriesFilterOptions renders every hygiene filter with its library-wide
|
||||
// count, one SeriesShapes pass per filter summed in Go — the shipped surface
|
||||
// offers eight grouped passes, not a single stats query (#140). The counts
|
||||
// are library-wide because the select sits next to the Site narrowing and
|
||||
// must not shift as the owner narrows the list itself. Cutoff travels with
|
||||
// the stale filter, or its count would always be zero.
|
||||
func (h *Handler) seriesFilterOptions(selected string, cutoff int64) ([]seriesFilterOption, error) {
|
||||
out := make([]seriesFilterOption, 0, len(seriesFilterOrder))
|
||||
for _, name := range seriesFilterOrder {
|
||||
shapes, err := h.store.SeriesShapes(store.SeriesFilter{Name: name, Cutoff: cutoff})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count := 0
|
||||
for _, sh := range shapes {
|
||||
count += sh.Total
|
||||
}
|
||||
out = append(out, seriesFilterOption{
|
||||
Name: name,
|
||||
Label: seriesFilterLabels[name],
|
||||
Count: count,
|
||||
Selected: name == selected,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seriesRow shapes one store row for the template, capping its chips at two
|
||||
// plus a +N tail; attention marks a row that carries any.
|
||||
// pollState derives the Check now control and the pending marker (issue
|
||||
// #146), shared by the list row and the detail page: CanPoll is false on a
|
||||
// Series with no page to fetch and on an orphan, so the owner is never
|
||||
// offered a button that can never do anything. Pending is derived — the
|
||||
// request stamp is newer than the check stamp — and requested is its ageing
|
||||
// label, which never expires.
|
||||
func pollState(a store.AdminSeries, now time.Time) (canPoll, pending bool, requested string) {
|
||||
canPoll = a.SeriesURL != "" && a.ReaderCount > 0
|
||||
if a.ForcePollAt > a.LatestCheckedAt {
|
||||
pending = true
|
||||
requested = requestedAge(now, a.ForcePollAt)
|
||||
}
|
||||
return canPoll, pending, requested
|
||||
}
|
||||
|
||||
func seriesRow(a store.AdminSeries, i int, now time.Time) seriesRowView {
|
||||
canPoll, pending, requested := pollState(a, now)
|
||||
row := seriesRowView{
|
||||
Key: a.Key(),
|
||||
Site: a.Site,
|
||||
Title: a.Title,
|
||||
Readers: a.ReaderCount,
|
||||
Band: i%2 == 1,
|
||||
CanPoll: canPoll,
|
||||
Pending: pending,
|
||||
Requested: requested,
|
||||
}
|
||||
if a.LatestChapterNum != nil {
|
||||
row.Ch = strconv.FormatFloat(*a.LatestChapterNum, 'f', -1, 64)
|
||||
} else {
|
||||
row.Ch = "—"
|
||||
}
|
||||
row.Age = checkedAge(now, a.LatestCheckedAt)
|
||||
notes := seriesNotes(a, now)
|
||||
if n := len(notes); n > 2 {
|
||||
row.Notes, row.More = notes[:2], n-2
|
||||
} else {
|
||||
row.Notes = notes
|
||||
}
|
||||
row.Attention = len(notes) > 0
|
||||
return row
|
||||
}
|
||||
|
||||
// seriesNotes are a row's hygiene chips in the design's order: no URL, no
|
||||
// cover, orphan, stale, reader sighting.
|
||||
func seriesNotes(a store.AdminSeries, now time.Time) []string {
|
||||
notes := []string{}
|
||||
if a.SeriesURL == "" {
|
||||
notes = append(notes, "no URL")
|
||||
}
|
||||
if a.CoverAddress == "" {
|
||||
notes = append(notes, "no cover")
|
||||
}
|
||||
if a.ReaderCount == 0 {
|
||||
notes = append(notes, "orphan")
|
||||
}
|
||||
if a.LatestCheckedAt > 0 && a.LatestCheckedAt < now.Add(-ownerWindow).UnixMilli() {
|
||||
notes = append(notes, "stale")
|
||||
}
|
||||
if a.RaisedByReader {
|
||||
notes = append(notes, "reader sighting")
|
||||
}
|
||||
return notes
|
||||
}
|
||||
|
||||
// checkedAge formats how long ago a Series was last checked, at the
|
||||
// granularity the list reads at — minutes, hours, days. Zero means never.
|
||||
func checkedAge(now time.Time, ts int64) string {
|
||||
if ts == 0 {
|
||||
return "never"
|
||||
}
|
||||
d := now.Sub(time.UnixMilli(ts))
|
||||
switch {
|
||||
case d < time.Hour:
|
||||
m := int(d / time.Minute)
|
||||
if m < 1 {
|
||||
m = 1
|
||||
}
|
||||
return fmt.Sprintf("%dm ago", m)
|
||||
case d < 24*time.Hour:
|
||||
return fmt.Sprintf("%dh ago", int(d/time.Hour))
|
||||
default:
|
||||
return fmt.Sprintf("%dd ago", int(d/(24*time.Hour)))
|
||||
}
|
||||
}
|
||||
|
||||
// requestedAge is the pending marker's text: how long ago the owner asked,
|
||||
// and nothing about when the request will run — the page does not know when a
|
||||
// sleeping browser will wake (issue #146). An unanswered request ages forever;
|
||||
// there is no expiry.
|
||||
func requestedAge(now time.Time, ts int64) string {
|
||||
return "requested " + checkedAge(now, ts)
|
||||
}
|
||||
|
||||
// pagerRange is the pager's "1–50 of 120" line. The template renders the
|
||||
// pager only over rows (the empty state replaces it), so it is never asked
|
||||
// to describe an empty list.
|
||||
func pagerRange(total, rows, page int) string {
|
||||
from := (page-1)*seriesPageSize + 1
|
||||
return fmt.Sprintf("%d–%d of %d", from, from+rows-1, total)
|
||||
}
|
||||
|
||||
// seriesListHref is one Series list address carrying the filter, Site, Kind
|
||||
// and page. The All filter and page 1 are the absent cases and stay out of
|
||||
// the URL, so the default address is the shortest one.
|
||||
func seriesListHref(filter, site, kind string, page int) string {
|
||||
q := url.Values{}
|
||||
if filter != "" && filter != store.SeriesFilterAll {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if site != "" {
|
||||
q.Set("site", site)
|
||||
}
|
||||
if kind != "" {
|
||||
q.Set("kind", kind)
|
||||
}
|
||||
if page > 1 {
|
||||
q.Set("page", strconv.Itoa(page))
|
||||
}
|
||||
if len(q) == 0 {
|
||||
return "/admin/series"
|
||||
}
|
||||
return "/admin/series?" + q.Encode()
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// seriesDetailView is one Series' page as the owner sees it: strings and
|
||||
// flags, every judgement made here, the template left to print. ReaderCount
|
||||
// is the only figure that crosses the privacy boundary — the owner learns how
|
||||
// many Readers hold the Series, never which Reader reads what.
|
||||
type seriesDetailView struct {
|
||||
Key string // "<site>:<series_id>", the page's address and the Series' identity
|
||||
Site string
|
||||
Kind string
|
||||
// Title, Cover and Chapter come from the shared Series row; the Cover is
|
||||
// the wire URL of the stored bytes, "" before any exist.
|
||||
Title string
|
||||
Cover string
|
||||
Chapter string // Latest Chapter number, or "—" before the first capture
|
||||
Checked string // how long ago the poller last checked, or "never"
|
||||
Readers int
|
||||
|
||||
// Marks, one per hygiene fact, rendered only while it holds.
|
||||
Unpollable bool // no SeriesURL to fetch
|
||||
NoCover bool
|
||||
Orphan bool // no Reader holds the Series
|
||||
SightingRaised bool // a Reader's Sighting set the Latest Chapter
|
||||
|
||||
// Poll is the Check now control and the pending marker (issue #146): the
|
||||
// same derivation and visibility as the list row. CanPoll is false on a
|
||||
// Series with no page to fetch and on an orphan; Pending is derived —
|
||||
// the request stamp is newer than the check stamp — and Requested is its
|
||||
// ageing label.
|
||||
CanPoll bool
|
||||
Pending bool
|
||||
Requested string
|
||||
}
|
||||
|
||||
// adminSeriesDetail renders one Series' page, keyed by the composite
|
||||
// "<site>:<series_id>" the list row already shows. The row is read through
|
||||
// the list's own SeriesPage read narrowed to the key's Site: the admin
|
||||
// projection is the privacy boundary, and a dedicated single-row read would
|
||||
// be a second definition of it.
|
||||
func (h *Handler) adminSeriesDetail(w http.ResponseWriter, r *http.Request) {
|
||||
site, seriesID, ok := strings.Cut(r.PathValue("key"), ":")
|
||||
if !ok || site == "" || seriesID == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
a, found, err := h.adminSeriesByKey(site, seriesID)
|
||||
if err != nil {
|
||||
log.Printf("series detail %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "series-detail", Detail: h.seriesDetailView(a)})
|
||||
}
|
||||
|
||||
// adminSeriesByKey reads one Series through the list's own SeriesPage read
|
||||
// narrowed to the key's Site: the admin projection is the privacy boundary,
|
||||
// and a dedicated single-row read would be a second definition of it. Absence
|
||||
// is reported with found=false, never an error.
|
||||
// ponytail: a page scan per keyed read, one query per page of the Site's rows
|
||||
// up to the window total; a keyed read alongside SeriesPage when the library
|
||||
// outgrows the page size.
|
||||
func (h *Handler) adminSeriesByKey(site, seriesID string) (store.AdminSeries, bool, error) {
|
||||
seen := 0
|
||||
for page := 1; ; page++ {
|
||||
p, err := h.store.SeriesPage(store.SeriesFilter{Site: site, Page: page})
|
||||
if err != nil {
|
||||
return store.AdminSeries{}, false, err
|
||||
}
|
||||
seen += len(p.Rows)
|
||||
for i := range p.Rows {
|
||||
if p.Rows[i].SeriesID == seriesID {
|
||||
return p.Rows[i], true, nil
|
||||
}
|
||||
}
|
||||
if seen >= p.Total {
|
||||
break
|
||||
}
|
||||
}
|
||||
return store.AdminSeries{}, false, nil
|
||||
}
|
||||
|
||||
// seriesDetailView shapes one AdminSeries row for display: every judgement in
|
||||
// Go, the template left to print strings and flags.
|
||||
func (h *Handler) seriesDetailView(a store.AdminSeries) seriesDetailView {
|
||||
canPoll, pending, requested := pollState(a, time.Now())
|
||||
v := seriesDetailView{
|
||||
Key: a.Key(),
|
||||
Site: a.Site,
|
||||
Kind: a.Kind,
|
||||
Title: a.Title,
|
||||
Cover: h.store.CoverWireURL(a.CoverAddress),
|
||||
Readers: a.ReaderCount,
|
||||
Unpollable: a.SeriesURL == "",
|
||||
NoCover: a.CoverAddress == "",
|
||||
Orphan: a.ReaderCount == 0,
|
||||
SightingRaised: a.RaisedByReader,
|
||||
CanPoll: canPoll,
|
||||
Pending: pending,
|
||||
Requested: requested,
|
||||
}
|
||||
if a.LatestChapterNum == nil {
|
||||
v.Chapter = "—"
|
||||
} else {
|
||||
v.Chapter = strconv.FormatFloat(*a.LatestChapterNum, 'f', -1, 64)
|
||||
}
|
||||
if a.LatestCheckedAt == 0 {
|
||||
v.Checked = "never"
|
||||
} else {
|
||||
v.Checked = since(time.Now(), time.UnixMilli(a.LatestCheckedAt))
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/session"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
const (
|
||||
// oauthStateTTL bounds how long a started sign-in stays valid. Ten
|
||||
// minutes is generous for Discord's round trip and short enough that a
|
||||
// captured state is stale before it is worth replaying.
|
||||
oauthStateTTL = 10 * time.Minute
|
||||
// maxStates caps the state map so a flood of /auth/discord hits cannot
|
||||
// grow memory; past the cap the oldest state is evicted, which at worst
|
||||
// invalidates an in-flight sign-in.
|
||||
maxStates = 256
|
||||
// maxResponseBytes caps Discord API bodies; they are small, and an
|
||||
// unbounded read is an OOM handed to Discord's CDN.
|
||||
maxResponseBytes = 1 << 20
|
||||
// discordTimeout keeps a hung Discord request from hanging the login
|
||||
// callback forever.
|
||||
discordTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// DiscordConfig is the OAuth application this service registers as, plus the
|
||||
// guild that gates access.
|
||||
type DiscordConfig struct {
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
GuildID string
|
||||
// RequiredRole, when non-empty, is a role ID a member must hold on top of
|
||||
// guild membership. Empty by default: membership alone suffices.
|
||||
RequiredRole string
|
||||
// APIBase is the Discord API root; configurable so tests run the whole
|
||||
// flow against a local stub.
|
||||
APIBase string
|
||||
// RedirectURI is the full public URL of the callback — Discord requires
|
||||
// the exact string, so it is configured, never derived from headers.
|
||||
RedirectURI string
|
||||
}
|
||||
|
||||
// oauthStates stores one-time sign-in states. A state is generated at
|
||||
// /auth/discord, echoed back by Discord at the callback, and consumed there.
|
||||
type oauthStates struct {
|
||||
mu sync.Mutex
|
||||
expiry map[string]time.Time
|
||||
}
|
||||
|
||||
func newOAuthStates() *oauthStates {
|
||||
return &oauthStates{expiry: make(map[string]time.Time)}
|
||||
}
|
||||
|
||||
func (s *oauthStates) put(state string, expires time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now()
|
||||
for k, at := range s.expiry {
|
||||
if !at.After(now) {
|
||||
delete(s.expiry, k)
|
||||
}
|
||||
}
|
||||
// Evict the state closest to expiring when full, so a flood of starts
|
||||
// cannot grow memory; at worst it invalidates an in-flight sign-in.
|
||||
if len(s.expiry) >= maxStates {
|
||||
var oldest string
|
||||
var oldestAt time.Time
|
||||
for k, at := range s.expiry {
|
||||
if oldest == "" || at.Before(oldestAt) {
|
||||
oldest, oldestAt = k, at
|
||||
}
|
||||
}
|
||||
delete(s.expiry, oldest)
|
||||
}
|
||||
s.expiry[state] = expires
|
||||
}
|
||||
|
||||
// take validates and consumes a state in one step: a state works exactly
|
||||
// once, which is what makes a replayed callback useless.
|
||||
func (s *oauthStates) take(state string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
expires, ok := s.expiry[state]
|
||||
if !ok || !expires.After(time.Now()) {
|
||||
return false
|
||||
}
|
||||
delete(s.expiry, state)
|
||||
return true
|
||||
}
|
||||
|
||||
// discordStart begins the authorization code grant: a fresh state, then a
|
||||
// redirect to Discord's authorize page.
|
||||
func (h *Handler) discordStart(w http.ResponseWriter, r *http.Request) {
|
||||
state := session.NewID()
|
||||
h.states.put(state, time.Now().Add(oauthStateTTL))
|
||||
u := h.discord.APIBase + "/oauth2/authorize?" + url.Values{
|
||||
"client_id": {h.discord.ClientID},
|
||||
"redirect_uri": {h.discord.RedirectURI},
|
||||
"response_type": {"code"},
|
||||
"scope": {"identify guilds.members.read"},
|
||||
"state": {state},
|
||||
}.Encode()
|
||||
http.Redirect(w, r, u, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// discordCallback completes the grant: exchange the code, verify identity,
|
||||
// membership and role, then mint a session. Every failure path renders the
|
||||
// login page with an author-written message — nothing Discord supplied is
|
||||
// ever interpolated into a page, and no secret reaches a log line.
|
||||
func (h *Handler) discordCallback(w http.ResponseWriter, r *http.Request) {
|
||||
ip := session.ClientIP(r)
|
||||
if wait := h.limiter.RetryAfter(ip, time.Now()); wait > 0 {
|
||||
secs := int(wait.Seconds()) + 1
|
||||
w.Header().Set("Retry-After", strconv.Itoa(secs))
|
||||
h.renderLogin(w, http.StatusTooManyRequests,
|
||||
"Too many attempts. Try again in "+strconv.Itoa((secs+59)/60)+" min.")
|
||||
return
|
||||
}
|
||||
|
||||
// Discord refuses the grant (the reader hit cancel, or the application
|
||||
// was misconfigured). The state is consumed so the flow is cleanly over;
|
||||
// this makes no Discord calls, so it is not a failure the limiter counts.
|
||||
if oerr := r.URL.Query().Get("error"); oerr != "" {
|
||||
h.states.take(r.URL.Query().Get("state"))
|
||||
h.renderLogin(w, http.StatusBadRequest, "Sign-in was cancelled.")
|
||||
return
|
||||
}
|
||||
|
||||
code := r.URL.Query().Get("code")
|
||||
if code == "" || !h.states.take(r.URL.Query().Get("state")) {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
h.renderLogin(w, http.StatusBadRequest,
|
||||
"This sign-in link was invalid or already used. Start again.")
|
||||
return
|
||||
}
|
||||
|
||||
tok, err := h.exchangeToken(r.Context(), code)
|
||||
if err != nil {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
log.Printf("discord token exchange: %v", err)
|
||||
h.renderLogin(w, http.StatusBadGateway,
|
||||
"Discord sign-in is unavailable right now. Try again in a moment.")
|
||||
return
|
||||
}
|
||||
|
||||
userID, err := h.discordUserID(r.Context(), tok.AccessToken)
|
||||
if err != nil {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
log.Printf("discord users/@me: %v", err)
|
||||
h.renderLogin(w, http.StatusBadGateway,
|
||||
"Discord sign-in is unavailable right now. Try again in a moment.")
|
||||
return
|
||||
}
|
||||
member, isMember, err := h.discordMember(r.Context(), tok.AccessToken)
|
||||
if err != nil {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
log.Printf("discord member check: %v", err)
|
||||
h.renderLogin(w, http.StatusBadGateway,
|
||||
"Discord sign-in is unavailable right now. Try again in a moment.")
|
||||
return
|
||||
}
|
||||
// The refusal is the same for a non-member and a member without the
|
||||
// required role, and it names neither the guild nor its id: an outsider
|
||||
// cannot tell whether the guild exists, let alone which one gates.
|
||||
//
|
||||
// It also returns before EnsureReader, so a refused sign-in leaves no
|
||||
// Reader row behind — the gate is the only thing standing between guild
|
||||
// membership and a library.
|
||||
if !isMember || (h.discord.RequiredRole != "" && !slices.Contains(member.Roles, h.discord.RequiredRole)) {
|
||||
h.limiter.Fail(ip, time.Now())
|
||||
h.renderLogin(w, http.StatusForbidden,
|
||||
"This Discord account is not a member of this community.")
|
||||
return
|
||||
}
|
||||
|
||||
// Registration is the login (issue #27): first sight of a guild member
|
||||
// creates their Reader, every later sight returns the same one. Their
|
||||
// userscript credential is derived at epoch 0 the way the owner's is, so
|
||||
// the install links work before they have read anything.
|
||||
readerID, err := h.store.EnsureReader(userID, token.Hash(token.Token(h.tokenKey, userID, 0)))
|
||||
if err != nil {
|
||||
log.Printf("register reader: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
h.limiter.Reset(ip)
|
||||
sess, err := h.store.CreateSession(session.NewID(), readerID, session.SessionTTL)
|
||||
if err != nil {
|
||||
log.Printf("create session: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
session.SetCookie(w, r, sess.ID)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// exchangeToken trades an authorization code for an access token. The body is
|
||||
// form-encoded because that is what Discord accepts — it rejects a JSON
|
||||
// payload — so the wire format is fixed here, not in a client library.
|
||||
func (h *Handler) exchangeToken(ctx context.Context, code string) (discordToken, error) {
|
||||
form := url.Values{
|
||||
"client_id": {h.discord.ClientID},
|
||||
"client_secret": {h.discord.ClientSecret},
|
||||
"grant_type": {"authorization_code"},
|
||||
"code": {code},
|
||||
"redirect_uri": {h.discord.RedirectURI},
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
h.discord.APIBase+"/oauth2/token", strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return discordToken{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := h.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return discordToken{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return discordToken{}, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
var tok discordToken
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&tok); err != nil {
|
||||
return discordToken{}, err
|
||||
}
|
||||
if tok.AccessToken == "" {
|
||||
return discordToken{}, errors.New("empty access token")
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
// discordUserID fetches the signed-in user's id via the identify scope.
|
||||
func (h *Handler) discordUserID(ctx context.Context, accessToken string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
h.discord.APIBase+"/users/@me", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+accessToken)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := h.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
var u struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&u); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if u.ID == "" {
|
||||
return "", errors.New("empty user id")
|
||||
}
|
||||
return u.ID, nil
|
||||
}
|
||||
|
||||
type discordMember struct {
|
||||
Roles []string `json:"roles"`
|
||||
}
|
||||
|
||||
// discordMember fetches the current user's membership in the configured guild.
|
||||
//
|
||||
// This is the OAuth endpoint (Get Current User Guild Member), the one the
|
||||
// guilds.members.read scope grants. Its bot-side twin, GET /guilds/{id}/
|
||||
// members/{user}, reads almost identically and is the wrong one: it wants a
|
||||
// Bot token and the application present in the guild, and answers a user
|
||||
// Bearer token with 401 — which fails as an outage rather than a refusal, so
|
||||
// nobody could sign in at all.
|
||||
//
|
||||
// A 404 or 403 (not in the guild, or the token lacks the scope) is a
|
||||
// non-member, not an error.
|
||||
func (h *Handler) discordMember(ctx context.Context, accessToken string) (discordMember, bool, error) {
|
||||
u := h.discord.APIBase + "/users/@me/guilds/" +
|
||||
url.PathEscape(h.discord.GuildID) + "/member"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return discordMember{}, false, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+accessToken)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := h.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return discordMember{}, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden {
|
||||
return discordMember{}, false, nil
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return discordMember{}, false, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
var m discordMember
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&m); err != nil {
|
||||
return discordMember{}, false, err
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
type discordToken struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOAuthStateSingleUse(t *testing.T) {
|
||||
s := newOAuthStates()
|
||||
s.put("st", time.Now().Add(time.Minute))
|
||||
if !s.take("st") {
|
||||
t.Fatal("take of a fresh state = false, want true")
|
||||
}
|
||||
if s.take("st") {
|
||||
t.Fatal("take of a consumed state = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOAuthStateUnknownOrExpired(t *testing.T) {
|
||||
s := newOAuthStates()
|
||||
if s.take("never-seen") {
|
||||
t.Fatal("take of an unknown state = true, want false")
|
||||
}
|
||||
s.put("stale", time.Now().Add(-time.Minute))
|
||||
if s.take("stale") {
|
||||
t.Fatal("take of an expired state = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// The map is capped: a flood of starts evicts older states instead of growing,
|
||||
// and consumed states must not change that.
|
||||
func TestOAuthStateEviction(t *testing.T) {
|
||||
s := newOAuthStates()
|
||||
key := func(i, salt int) string {
|
||||
return string(rune('a'+i%26)) + string(rune('0'+i/26+salt*16))
|
||||
}
|
||||
now := time.Now().Add(time.Hour)
|
||||
for i := 0; i < maxStates*2; i++ {
|
||||
s.put(key(i, 0), now)
|
||||
}
|
||||
if got := len(s.expiry); got != maxStates {
|
||||
t.Fatalf("states after a flood = %d, want %d", got, maxStates)
|
||||
}
|
||||
|
||||
// Consume everything, then flood again: the map stays bounded.
|
||||
for state := range s.expiry {
|
||||
s.take(state)
|
||||
}
|
||||
for i := 0; i < maxStates; i++ {
|
||||
s.put(key(i, 1), now)
|
||||
}
|
||||
if got := len(s.expiry); got != maxStates {
|
||||
t.Fatalf("states after consume+flood = %d, want %d", got, maxStates)
|
||||
}
|
||||
}
|
||||
@@ -1,830 +0,0 @@
|
||||
:root {
|
||||
--measure-wide: 1080px;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root {
|
||||
--measure-wide: 1080px;
|
||||
}
|
||||
}
|
||||
|
||||
.admin-sheet {
|
||||
max-width: var(--measure-wide);
|
||||
}
|
||||
|
||||
.admin-sheet .brand em {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .brand .mark > g > g:last-child {
|
||||
stroke: var(--patina);
|
||||
}
|
||||
|
||||
.topbar-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 18px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.navrow {
|
||||
display: flex;
|
||||
gap: 18px;
|
||||
padding: 2px 20px 0;
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
scrollbar-width: none;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.navrow::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.navrow a {
|
||||
flex: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-width: 44px;
|
||||
padding: 8px 0 12px;
|
||||
color: var(--mute);
|
||||
font: 400 17px var(--font-display);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.navrow a:hover {
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.navrow a.active {
|
||||
color: var(--paper);
|
||||
border-bottom: 2px solid var(--paper);
|
||||
margin-bottom: -1px;
|
||||
}
|
||||
|
||||
.admin-page {
|
||||
padding: 0 20px 40px;
|
||||
}
|
||||
|
||||
.admin-page > .sec,
|
||||
.admin-sheet .readers h2,
|
||||
.admin-sheet .lanes h2 {
|
||||
position: relative;
|
||||
margin: 0;
|
||||
padding: 26px 0 8px;
|
||||
font: 500 11px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-page > .sec::before,
|
||||
.admin-sheet .readers h2::before,
|
||||
.admin-sheet .lanes h2::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 14px;
|
||||
left: 0;
|
||||
width: 34px;
|
||||
height: 2px;
|
||||
background: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .readers,
|
||||
.admin-sheet .lanes {
|
||||
margin: 0;
|
||||
padding: 0 0 16px;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.admin-sheet .readerlist,
|
||||
.admin-sheet .lanelist {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.admin-sheet .readerlist li,
|
||||
.admin-sheet .lanelist li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 16px;
|
||||
min-height: 48px;
|
||||
padding: 10px 0;
|
||||
border-top: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-actions {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
gap: 18px;
|
||||
margin-left: auto;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .reader-actions .ghost,
|
||||
.admin-sheet .c-act .ghost {
|
||||
font-size: 12px;
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-actions .ghost.danger,
|
||||
.admin-sheet .c-act .ghost.danger {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .readerlist form {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.admin-sheet .lane-browser {
|
||||
padding: 12px 0 0;
|
||||
}
|
||||
|
||||
.admin-sheet .ghost.danger {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .ghost.danger:hover {
|
||||
color: var(--danger);
|
||||
border-bottom-color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-id {
|
||||
font: 500 15px/1.5 var(--font-mono);
|
||||
letter-spacing: .01em;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-sessions,
|
||||
.admin-sheet .reader-sightings,
|
||||
.admin-sheet .reader-blocked,
|
||||
.admin-sheet .lane-fact,
|
||||
.admin-sheet .lane-mark {
|
||||
font: 500 13px/1.4 var(--font-mono);
|
||||
letter-spacing: .06em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .reader-sessions {
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-sightings,
|
||||
.admin-sheet .lane-fact {
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-blocked,
|
||||
.admin-sheet .lane-mark {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .lane-site {
|
||||
font: 400 19px/1.2 var(--font-display);
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .lanelist li.attention .lane-site {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
/* A single grid keeps row rules continuous; cell padding supplies gutters. */
|
||||
.admin-sheet .sechead {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.admin-sheet .sechead .statusline {
|
||||
padding: 0 0 8px;
|
||||
font-size: 11px;
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .statusline {
|
||||
margin: 0;
|
||||
padding: 0 0 10px;
|
||||
font: 500 12px/1 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
/* The verdict line is set in the data face, not the display face: it is
|
||||
three counts, not a page title. The judgement is the only bright thing. */
|
||||
.admin-sheet .verdict {
|
||||
padding: 16px 0 12px;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
font: 500 15px/1.6 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .verdict .attn {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .verdict .counts b {
|
||||
color: var(--paper);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(240px, 1fr) 156px 92px 110px 76px minmax(150px, 220px) 140px;
|
||||
column-gap: 0;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead > * {
|
||||
padding: 10px 14px 8px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
font: 500 12px/1 var(--font-mono);
|
||||
letter-spacing: .12em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow > * {
|
||||
padding: 11px 14px 11px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
font: 500 15px/1.5 var(--font-mono);
|
||||
letter-spacing: .01em;
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead > *:last-child,
|
||||
.admin-sheet .tbl .trow > *:last-child {
|
||||
padding-right: 0;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-title,
|
||||
.admin-sheet .tbl .c-site {
|
||||
font: 400 18px/1.35 var(--font-display);
|
||||
letter-spacing: 0;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-title a:hover,
|
||||
.admin-sheet .tbl .c-act .ghost:hover {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow.attention .c-title {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-ch,
|
||||
.admin-sheet .tbl .c-rd {
|
||||
text-align: right;
|
||||
padding-right: 26px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow .c-note .mark {
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-act {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-act .ghost + .ghost {
|
||||
margin-left: 12px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow > .confirm-row {
|
||||
grid-column: 1 / -1;
|
||||
padding: 10px 12px;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.admin-sheet .stats {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(232px, 1fr));
|
||||
margin: 4px 0 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .stat {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: baseline;
|
||||
gap: 16px;
|
||||
padding: 11px 24px 11px 0;
|
||||
}
|
||||
|
||||
.admin-sheet .stat .lbl {
|
||||
font: 500 13px/1.3 var(--font-mono);
|
||||
letter-spacing: .1em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .stat .fig {
|
||||
font: 500 19px/1 var(--font-mono);
|
||||
font-variant-numeric: tabular-nums;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .stat .fig.zero {
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .stat a.fig {
|
||||
/* The hover rule below sets the border on :hover only, which shifts the
|
||||
row a pixel on hover; the transparent baseline holds the layout. */
|
||||
border-bottom: 1px solid transparent;
|
||||
}
|
||||
|
||||
.admin-sheet .stat a.fig:hover {
|
||||
color: var(--patina);
|
||||
border-bottom: 1px solid var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .mark,
|
||||
.admin-sheet .mark-faint {
|
||||
font: 500 13px/1 var(--font-mono);
|
||||
letter-spacing: .06em;
|
||||
text-transform: uppercase;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .mark {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .mark-faint {
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .mark.mark-strong {
|
||||
font-size: 13px;
|
||||
letter-spacing: .14em;
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .mark.mark-strong::before {
|
||||
content: "";
|
||||
display: inline-block;
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
background: var(--patina);
|
||||
margin-right: 8px;
|
||||
vertical-align: .08em;
|
||||
}
|
||||
|
||||
.admin-sheet .mark.bad {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites {
|
||||
grid-template-columns: 170px repeat(4, 106px) minmax(180px, 1fr);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .c-site,
|
||||
.admin-sheet .tbl.lanes .c-site {
|
||||
font: 400 18px/1.35 var(--font-display);
|
||||
letter-spacing: 0;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .c-state {
|
||||
color: var(--patina);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .c-state.bad,
|
||||
.admin-sheet .tbl.lanes .c-skip .bad,
|
||||
.admin-sheet .tbl.lanes .trow.attention .c-site {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-skip .ok {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .thead > *:nth-child(n+2):nth-child(-n+5),
|
||||
.admin-sheet .tbl.sites .trow > *:nth-child(n+2):nth-child(-n+5) {
|
||||
padding-right: 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes {
|
||||
grid-template-columns: 150px 62px 88px 66px 118px minmax(0, 1fr) 168px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-skip {
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-skip > * {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-ctrl {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .thead > *:nth-child(n+2):nth-child(-n+4),
|
||||
.admin-sheet .tbl.lanes .trow > *:nth-child(n+2):nth-child(-n+4) {
|
||||
padding-right: 26px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series {
|
||||
grid-template-columns: 150px 84px 104px 76px minmax(120px, 1fr) 212px;
|
||||
row-gap: 4px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .thead > *:first-child {
|
||||
padding-left: 20px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .thead > *:last-child {
|
||||
padding-right: 20px;
|
||||
}
|
||||
|
||||
/* The site cell's colour is a class, never an inline style: site is
|
||||
client-supplied and unvalidated, and a hostile value reaching a CSS
|
||||
context would render ZgotmplZ, while an unknown class degrades to the
|
||||
unstyled cell. */
|
||||
.admin-sheet .tbl .c-site.site-asura {
|
||||
color: var(--asura);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-demonic {
|
||||
color: var(--demonic);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-comix {
|
||||
color: var(--comix);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-kagane {
|
||||
color: var(--kagane);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-novelfull {
|
||||
color: var(--novelfull);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-lightnovelworld {
|
||||
color: var(--lightnovelworld);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow {
|
||||
display: grid;
|
||||
grid-column: 1 / -1;
|
||||
grid-template-columns: subgrid;
|
||||
padding: 13px 20px 14px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow.band {
|
||||
background: var(--hover);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow > * {
|
||||
padding: 0 14px 0 0;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .c-title {
|
||||
display: flex;
|
||||
grid-column: 1 / -1;
|
||||
align-items: baseline;
|
||||
gap: 16px;
|
||||
padding: 0 0 4px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .c-title .mark {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.admin-sheet .filterbar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 14px 22px;
|
||||
padding: 16px 0 14px;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .fsel {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.admin-sheet .fsel > span {
|
||||
font: 500 11px/1 var(--font-mono);
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .fsel select,
|
||||
.admin-sheet .pausebar select {
|
||||
color-scheme: dark;
|
||||
background: var(--ink);
|
||||
border: none;
|
||||
border-bottom: 1px solid var(--field-line);
|
||||
color: var(--paper);
|
||||
font: 400 16px/1.4 var(--font-display);
|
||||
padding: 4px 4px 5px 0;
|
||||
}
|
||||
|
||||
.admin-sheet .fsel select:hover,
|
||||
.admin-sheet .pausebar select:hover {
|
||||
border-bottom-color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .fsel select option,
|
||||
.admin-sheet .pausebar select option {
|
||||
background: var(--ink);
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .segrow {
|
||||
display: inline-flex;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.admin-sheet .segrow a {
|
||||
padding: 6px 11px 7px;
|
||||
border-bottom: 2px solid transparent;
|
||||
color: var(--mute);
|
||||
font: 500 14px/1 var(--font-mono);
|
||||
}
|
||||
|
||||
.admin-sheet .segrow a.active {
|
||||
border-bottom-color: var(--patina);
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .listhead {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
padding: 16px 0 4px;
|
||||
color: var(--paper);
|
||||
font: 400 18px/1.2 var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .listhead .lbl {
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .listhead .lbl em {
|
||||
color: var(--patina);
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
.admin-sheet .pager {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
padding: 14px 0 0;
|
||||
color: var(--mute-2);
|
||||
font: 500 11px/1 var(--font-mono);
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .pager .pg {
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .pager .pg:hover {
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .pager .pg.disabled {
|
||||
color: var(--faint);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.admin-sheet .empty {
|
||||
padding: 28px 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.admin-sheet .empty strong {
|
||||
color: var(--paper);
|
||||
font: 400 20px var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .empty p {
|
||||
margin: 6px 0 0;
|
||||
font: 14px/1.5 var(--font-body);
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .confirm-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 10px 12px;
|
||||
background: var(--danger-wash);
|
||||
}
|
||||
|
||||
.admin-sheet .confirm-row span {
|
||||
flex: 1 1 16ch;
|
||||
color: var(--danger-soft);
|
||||
font: 400 15px/1.3 var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .confirm-row div {
|
||||
display: flex;
|
||||
flex: none;
|
||||
gap: 12px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-back {
|
||||
display: inline-block;
|
||||
margin: 18px 0 0;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-title {
|
||||
margin: 10px 0 2px;
|
||||
color: var(--paper);
|
||||
font: 400 28px/1.25 var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .detail-key {
|
||||
margin: 0;
|
||||
color: var(--mute-2);
|
||||
font: 500 11px/1.4 var(--font-mono);
|
||||
letter-spacing: .08em;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 14px;
|
||||
margin: 10px 0 0;
|
||||
color: var(--mute-2);
|
||||
font: 500 12px/1.5 var(--font-mono);
|
||||
letter-spacing: .08em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .cover {
|
||||
width: 160px;
|
||||
aspect-ratio: 3 / 4;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin: 18px 0 4px;
|
||||
background: var(--hatch);
|
||||
color: var(--mute-2);
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 0 28px;
|
||||
}
|
||||
|
||||
.admin-sheet .dform {
|
||||
padding: 14px 0 0;
|
||||
}
|
||||
|
||||
.admin-sheet .dform h3 {
|
||||
margin: 0 0 4px;
|
||||
color: var(--mute-2);
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .dform .field {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.admin-sheet .pausebar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
@media (max-width: 719px) {
|
||||
.admin-sheet .topbar {
|
||||
flex-wrap: wrap;
|
||||
row-gap: 12px;
|
||||
}
|
||||
|
||||
.admin-sheet .brand {
|
||||
flex: 1 1 100%;
|
||||
}
|
||||
|
||||
.admin-sheet .topbar-actions {
|
||||
margin-left: auto;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 1019px) {
|
||||
.admin-sheet .tbl.lanes {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .thead {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .trow {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: baseline;
|
||||
gap: 4px 16px;
|
||||
padding: 12px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .trow > * {
|
||||
padding: 0;
|
||||
border-bottom: none;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-site {
|
||||
width: 100%;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-ctrl {
|
||||
margin-left: auto;
|
||||
text-align: right;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 899px) {
|
||||
.admin-sheet .tbl,
|
||||
.admin-sheet .tbl.lanes {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 2px 10px;
|
||||
padding: 11px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow > * {
|
||||
padding: 0;
|
||||
border-bottom: none;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-ch,
|
||||
.admin-sheet .tbl .c-rd {
|
||||
padding-right: 0;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-title {
|
||||
width: 100%;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow > * {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-act {
|
||||
margin-left: auto;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.7 MiB |
@@ -1,23 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 200 172" role="img" aria-label="BookmarkManager">
|
||||
<title>BookmarkManager</title>
|
||||
<g fill="#100f0e" stroke="#f2ece5" stroke-width="5" stroke-linejoin="round" stroke-linecap="round">
|
||||
<path fill="none" d="M28 36H4v114h192V36h-24"></path>
|
||||
<path fill="none" d="M28 23H17v127h166V23h-11"></path>
|
||||
<g id="mb-half">
|
||||
<path d="M28 7 88 55v97L28 138z"></path>
|
||||
<g fill="#f2ece5" stroke="none">
|
||||
<path d="M37 25 55 39v41L37 66z"></path>
|
||||
<path d="M60 42 79 57v42L60 84z"></path>
|
||||
<path d="M37 75 79 108v13L37 88z"></path>
|
||||
<path d="M37 98 79 129v11L37 131z"></path>
|
||||
</g>
|
||||
</g>
|
||||
<use href="#mb-half" transform="matrix(-1 0 0 1 200 0)"></use>
|
||||
<g stroke="#e0452c">
|
||||
<path d="M100 4l9 5v11l-9 5-9-5V9z"></path>
|
||||
<path d="M94 24h12v24H94z"></path>
|
||||
<path d="M70 47h60v14H70z"></path>
|
||||
<path d="M91 61h18v87l-9 20-9-20z"></path>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 977 B |
@@ -1,50 +0,0 @@
|
||||
{{/* Every owner-only address shares this shell; page content stays behind its
|
||||
bookmarkable route so the active tab survives a reload. */}}
|
||||
{{define "admin"}}
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||
<meta name="color-scheme" content="dark light">
|
||||
<title>BookmarkManager — Admin</title>
|
||||
<link rel="icon" href="/static/logo.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
<link rel="stylesheet" href="/static/admin.css">
|
||||
<link rel="preload" href="/static/fonts/instrument-serif-400-latin.woff2" as="font" type="font/woff2" crossorigin>
|
||||
<script src="/static/htmx.min.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="sheet admin-sheet">
|
||||
<header class="topbar">
|
||||
<h1 class="brand">{{template "mark" .}}<span>Bookmark<em>Manager</em></span></h1>
|
||||
<span class="topbar-actions">
|
||||
<a class="ghost" href="/">Library</a>
|
||||
<form method="post" action="/logout">
|
||||
<button type="submit" class="ghost">Log out</button>
|
||||
</form>
|
||||
</span>
|
||||
</header>
|
||||
|
||||
<nav class="navrow" aria-label="Admin pages">
|
||||
<a href="/admin" class="{{if eq .Page "overview"}}active{{end}}" {{if eq .Page "overview"}}aria-current="page"{{end}}>Overview</a>
|
||||
<a href="/admin/lanes" class="{{if eq .Page "lanes"}}active{{end}}" {{if eq .Page "lanes"}}aria-current="page"{{end}}>Lanes</a>
|
||||
<a href="/admin/readers" class="{{if eq .Page "readers"}}active{{end}}" {{if eq .Page "readers"}}aria-current="page"{{end}}>Readers</a>
|
||||
<a href="/admin/series" class="{{if or (eq .Page "series") (eq .Page "series-detail")}}active{{end}}" {{if or (eq .Page "series") (eq .Page "series-detail")}}aria-current="page"{{end}}>Series</a>
|
||||
</nav>
|
||||
|
||||
<main class="page admin-page">
|
||||
{{if eq .Page "lanes"}}
|
||||
<div aria-live="polite">{{template "lanes" .Lanes}}</div>
|
||||
{{else if eq .Page "readers"}}
|
||||
{{template "readers" .}}
|
||||
{{else if eq .Page "series"}}
|
||||
{{template "series-list" .SeriesList}}
|
||||
{{else if eq .Page "series-detail"}}
|
||||
{{template "series-detail" .Detail}}
|
||||
{{else}}{{template "overview" .Overview}}{{end}}
|
||||
</main>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
{{end}}
|
||||
@@ -1,93 +0,0 @@
|
||||
{{define "app"}}
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||
<meta name="color-scheme" content="dark light">
|
||||
<title>BookmarkManager</title>
|
||||
<link rel="icon" href="/static/logo.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
<link rel="preload" href="/static/fonts/instrument-serif-400-latin.woff2" as="font" type="font/woff2" crossorigin>
|
||||
{{/* Body text before meta lines: DM Sans is the biggest face and the one
|
||||
most of the page is set in; the mono is small and arrives from CSS. */}}
|
||||
<link rel="preload" href="/static/fonts/dm-sans-var-latin.woff2" as="font" type="font/woff2" crossorigin>
|
||||
<script src="/static/htmx.min.js" defer></script>
|
||||
<script src="/static/filter.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
{{template "icons" .}}
|
||||
<div class="sheet">
|
||||
<header class="topbar">
|
||||
<h1 class="brand">{{template "mark" .}}<span>Bookmark<em>Manager</em></span></h1>
|
||||
{{/* Plain full-page links, not htmx swaps: switching library replaces the
|
||||
tab row and the chrome, which is a page, not a fragment. */}}
|
||||
<nav class="libswitch" aria-label="Library">
|
||||
<a href="/?tab=all" class="{{if eq .Lib "manga"}}active{{end}}"
|
||||
{{if eq .Lib "manga"}}aria-current="page"{{end}}>Manga</a>
|
||||
<a href="/?lib=novel&tab=all" class="{{if eq .Lib "novel"}}active{{end}}"
|
||||
{{if eq .Lib "novel"}}aria-current="page"{{end}}>Novels</a>
|
||||
</nav>
|
||||
{{/* The owner's only difference on this page: a link out to the
|
||||
administrative one. It sits beside Log out rather than in the library
|
||||
switch — that switch says which library, not which page. */}}
|
||||
{{if .Owner}}<a class="ghost" href="/admin">Admin</a>{{end}}
|
||||
<form method="post" action="/logout">
|
||||
<button type="submit" class="ghost">Log out</button>
|
||||
</form>
|
||||
</header>
|
||||
|
||||
{{/* Search sits above the tabs on a phone and folds into the tab row on a
|
||||
wider screen — one flex container, order swapped in CSS. */}}
|
||||
<div class="chrome">
|
||||
<div class="searchbar">
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-search"/></svg>
|
||||
<input id="search" class="search" type="search" placeholder="Find a title"
|
||||
autocomplete="off" aria-label="Search titles">
|
||||
</div>
|
||||
|
||||
{{/* These are real links with real hrefs that change the URL, so they are
|
||||
navigation, not an ARIA tablist — aria-current carries "which bucket am
|
||||
I in" without owing a tabpanel contract we do not implement. */}}
|
||||
<nav class="tabs" aria-label="Bookmark buckets">
|
||||
<a href="{{.PageURL "all"}}" class="{{if eq .Tab "all"}}active{{end}}"
|
||||
{{if eq .Tab "all"}}aria-current="page"{{end}}
|
||||
hx-get="{{.ListURL "all"}}" hx-target="#list" hx-swap="innerHTML"
|
||||
hx-push-url="{{.PageURL "all"}}" hx-on::after-request="setActiveTab(this)">All</a>
|
||||
{{/* The one bucket novels do not have: without a poller-fed "what is out
|
||||
that I have not read", the tab would only ever restate All. */}}
|
||||
{{if eq .Lib "manga"}}
|
||||
<a href="{{.PageURL "new"}}" class="tab-new {{if eq .Tab "new"}}active{{end}}"
|
||||
{{if eq .Tab "new"}}aria-current="page"{{end}}
|
||||
hx-get="{{.ListURL "new"}}" hx-target="#list" hx-swap="innerHTML"
|
||||
hx-push-url="{{.PageURL "new"}}" hx-on::after-request="setActiveTab(this)">Updated
|
||||
{{template "newcount" .}}</a>
|
||||
{{end}}
|
||||
<a href="{{.PageURL "fav"}}" class="{{if eq .Tab "fav"}}active{{end}}"
|
||||
{{if eq .Tab "fav"}}aria-current="page"{{end}}
|
||||
hx-get="{{.ListURL "fav"}}" hx-target="#list" hx-swap="innerHTML"
|
||||
hx-push-url="{{.PageURL "fav"}}" hx-on::after-request="setActiveTab(this)">Favourites</a>
|
||||
<a href="{{.PageURL "archived"}}" class="{{if eq .Tab "archived"}}active{{end}}"
|
||||
{{if eq .Tab "archived"}}aria-current="page"{{end}}
|
||||
hx-get="{{.ListURL "archived"}}" hx-target="#list" hx-swap="innerHTML"
|
||||
hx-push-url="{{.PageURL "archived"}}" hx-on::after-request="setActiveTab(this)">Archived</a>
|
||||
<a href="{{.PageURL "finished"}}" class="{{if eq .Tab "finished"}}active{{end}}"
|
||||
{{if eq .Tab "finished"}}aria-current="page"{{end}}
|
||||
hx-get="{{.ListURL "finished"}}" hx-target="#list" hx-swap="innerHTML"
|
||||
hx-push-url="{{.PageURL "finished"}}" hx-on::after-request="setActiveTab(this)">Finished</a>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
{{template "setup" .}}
|
||||
|
||||
{{template "keyrow" .}}
|
||||
|
||||
{{template "recent" .}}
|
||||
|
||||
<main id="list" class="list">
|
||||
{{template "list" .}}
|
||||
</main>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
{{end}}
|
||||
@@ -1,78 +0,0 @@
|
||||
{{/* The regions that live outside the swapped #list: the "Continue reading"
|
||||
strip, the Updated badge and the action key. All are rendered inline by
|
||||
app.html and again, out of band, on every /ui/ response — a mutation must
|
||||
not leave them describing the library as it was before the tap.
|
||||
|
||||
All always render, hidden when they have nothing to say, so an out-of-band
|
||||
swap always has an element with the right id to replace. */}}
|
||||
|
||||
{{define "recent"}}
|
||||
<section class="recent" id="recent"{{if .OOB}} hx-swap-oob="true"{{end}}{{if not .Recent}} hidden{{end}}>
|
||||
<h2>Continue reading</h2>
|
||||
<div class="recent-strip">
|
||||
{{range .Recent}}
|
||||
<a class="recent-card {{if .HasNewChapter}}is-new{{end}}" href="{{.ContinueURL}}"
|
||||
target="_blank" rel="noopener noreferrer">
|
||||
<span class="recent-cover">
|
||||
{{if .Cover}}<img src="{{.Cover}}" alt="" loading="lazy">
|
||||
{{else}}<span class="monogram" aria-hidden="true">{{.Initial}}</span>{{end}}
|
||||
{{if .HasNewChapter}}<span class="foot-rule"></span>
|
||||
{{else if .Favorite}}<span class="foot-rule brass"></span>{{end}}
|
||||
</span>
|
||||
<span class="recent-title">{{.Title}}</span>
|
||||
<span class="recent-chapter">{{.DisplayChapter}}{{if .HasNewChapter}} · New{{end}}</span>
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
{{/* The action key. The icon strip on a card is unlabelled, so one permanent
|
||||
line under the tabs names every glyph. It follows the tab rather than the
|
||||
row: the archived and finished buckets swap Archive for Restore, and a
|
||||
finished series has no Done to offer. */}}
|
||||
{{define "keyrow"}}
|
||||
<div class="keyrow" id="keyrow" aria-label="Action key"{{if .OOB}} hx-swap-oob="true"{{end}}>
|
||||
<span class="pair"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-play"/></svg><span>Read</span></span>
|
||||
<span class="pair brass"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-star"/></svg><span>Fav</span></span>
|
||||
<span class="pair"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-pencil"/></svg><span>Chapter</span></span>
|
||||
{{if or (eq .Tab "archived") (eq .Tab "finished")}}
|
||||
<span class="pair"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-undo"/></svg><span>Restore</span></span>
|
||||
{{else}}
|
||||
<span class="pair"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-box"/></svg><span>Archive</span></span>
|
||||
{{end}}
|
||||
{{if ne .Tab "finished"}}
|
||||
<span class="pair"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-check"/></svg><span>Done</span></span>
|
||||
{{end}}
|
||||
<span class="pair trash"><svg viewBox="0 0 24 24" aria-hidden="true"><use href="#i-trash"/></svg><span>Delete</span></span>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{/* The brand mark, inline so it takes the page's ink and ember rather than the
|
||||
fixed palette of /static/logo.svg (which the favicon needs). */}}
|
||||
{{define "mark"}}
|
||||
<svg class="mark" viewBox="0 0 200 172" aria-hidden="true">
|
||||
<g fill="var(--ink)" stroke="currentColor" stroke-width="5" stroke-linejoin="round" stroke-linecap="round">
|
||||
<path fill="none" d="M28 36H4v114h192V36h-24"></path>
|
||||
<path fill="none" d="M28 23H17v127h166V23h-11"></path>
|
||||
<g id="mb-half">
|
||||
<path d="M28 7 88 55v97L28 138z"></path>
|
||||
<g fill="currentColor" stroke="none">
|
||||
<path d="M37 25 55 39v41L37 66z"></path>
|
||||
<path d="M60 42 79 57v42L60 84z"></path>
|
||||
<path d="M37 75 79 108v13L37 88z"></path>
|
||||
<path d="M37 98 79 129v11L37 131z"></path>
|
||||
</g>
|
||||
</g>
|
||||
<use href="#mb-half" transform="matrix(-1 0 0 1 200 0)"></use>
|
||||
<g stroke="var(--ember)">
|
||||
<path d="M100 4l9 5v11l-9 5-9-5V9z"></path>
|
||||
<path d="M94 24h12v24H94z"></path>
|
||||
<path d="M70 47h60v14H70z"></path>
|
||||
<path d="M91 61h18v87l-9 20-9-20z"></path>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
{{end}}
|
||||
|
||||
{{define "newcount"}}<span class="count" id="new-count"{{if .OOB}} hx-swap-oob="true"{{end}}{{if not .NewCount}} hidden{{end}}>{{.NewCount}}</span>{{end}}
|
||||
@@ -1,60 +0,0 @@
|
||||
{{/* Poll Lane status: one row per Site's latest durable pass, refreshing
|
||||
itself so a run can be watched rather than sampled by reloading. The
|
||||
refresh is one attribute on the fragment root and the endpoint answers
|
||||
with this same fragment, so the swap replaces the element that asked.
|
||||
|
||||
Every figure is read from poll_passes, never from a running poller: a
|
||||
restart answers from the database the moment it is up (issue #145). The
|
||||
browser fact is a deployment-config fact plus a reachability derived from
|
||||
the pass log; the cause chips and the state phrase are decided in Go,
|
||||
this template only prints them. */}}
|
||||
{{define "lanes"}}
|
||||
<section class="lanes" id="lanes"
|
||||
hx-get="/ui/admin/lanes" hx-trigger="every 30s" hx-swap="outerHTML">
|
||||
<div class="sechead">
|
||||
<h2 class="sec">Poll Lanes</h2>
|
||||
<p class="statusline">
|
||||
{{if .PollerOff}}Polling: <span class="mark-faint">off</span>
|
||||
{{else}}Browser: {{if not .BrowserConfigured}}<span class="mark-faint">not configured</span>{{else if .BrowserReachable}}<span class="mark mark-strong">reachable</span>{{else}}<span class="mark bad">unreachable</span>{{end}}{{end}}
|
||||
</p>
|
||||
</div>
|
||||
{{if .Rows}}
|
||||
<div class="tbl lanes">
|
||||
<div class="thead">
|
||||
<span>Site</span><span>Due</span><span>Checked</span><span>Gap</span>
|
||||
<span>Last pass</span><span>Outcomes · state</span><span></span>
|
||||
</div>
|
||||
{{range .Rows}}
|
||||
<div class="trow{{if .Attention}} attention{{end}}">
|
||||
<span class="c-site">{{.Site}}</span>
|
||||
<span>{{.Due}}</span>
|
||||
<span>{{.Checked}}</span>
|
||||
<span>{{.Gap}}</span>
|
||||
<span>ran {{.Ran}}</span>
|
||||
<span class="c-skip">{{if .HasChips}}{{range $i, $c := .Chips}}{{if $i}}<span class="mark-faint"> · </span>{{end}}<span class="mark">{{$c.Name}} {{$c.Count}}</span>{{end}}{{else}}<span class="mark-faint">none observed</span>{{end}}{{if .StatePhrase}} · <span class="{{if .StateGood}}ok{{else}}bad{{end}}">{{.StatePhrase}}</span>{{end}}</span>
|
||||
{{/* The pause control lives in the one slot the design leaves for it:
|
||||
a running Lane offers the three durations and Pause; a paused Lane
|
||||
offers Resume in the same place. Pause is not destruction — it
|
||||
takes nothing away and reverses in one press — so neither wears a
|
||||
confirm row or the danger accent. The form wraps the select so the
|
||||
offered duration travels with the press. */}}
|
||||
<span class="c-ctrl">{{if .Paused}}<span class="pausebar">
|
||||
<form hx-post="/admin/lanes/{{.Site}}/resume" hx-target="#lanes" hx-swap="outerHTML">
|
||||
<button type="submit" class="ghost">Resume</button>
|
||||
</form>
|
||||
</span>{{else}}<span class="pausebar">
|
||||
<form hx-post="/admin/lanes/{{.Site}}/pause" hx-target="#lanes" hx-swap="outerHTML">
|
||||
<select name="duration" aria-label="Pause duration">
|
||||
<option>1h</option><option selected>6h</option><option>24h</option>
|
||||
</select>
|
||||
<button type="submit" class="ghost">Pause</button>
|
||||
</form>
|
||||
</span>{{end}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<p class="empty">No data yet — no Lane has recorded a pass.</p>
|
||||
{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
@@ -1,14 +0,0 @@
|
||||
{{/* The Overview landing page: one verdict line leading, then a stats block
|
||||
where every figure is a door into the list that counts it, and the
|
||||
per-Site library shape table. Every judgement — the verdict state, which
|
||||
figures link, what a Lane's state means — is made in Go; this template
|
||||
only prints. */}}
|
||||
{{define "overview"}}
|
||||
<p class="verdict"><span class="attn">{{.Verdict}}</span> {{if .HasCounts}}<span class="counts">· <b>{{.Waiting}}</b> series waiting · <b>{{.Unchecked}}</b> unchecked over 12h</span>{{end}}</p>
|
||||
<h2 class="sec">Hygiene</h2>
|
||||
<div class="stats">{{range .Hygiene}}<div class="stat"><span class="lbl">{{.Label}}</span>{{if .Href}}<a class="fig" href="{{.Href}}">{{.Count}}</a>{{else}}<span class="fig zero">{{.Count}}</span>{{end}}</div>{{end}}</div>
|
||||
<h2 class="sec">Library</h2>
|
||||
<div class="stats">{{range .Library}}<div class="stat"><span class="lbl">{{.Label}}</span>{{if .Href}}<a class="fig" href="{{.Href}}">{{.Count}}</a>{{else}}<span class="fig zero">{{.Count}}</span>{{end}}</div>{{end}}</div>
|
||||
<h2 class="sec">Sites · last 12h</h2>
|
||||
<div class="tbl sites"><div class="thead"><span>Site</span><span>Series</span><span>No cover</span><span>Never chk</span><span>Stale</span><span>State</span></div>{{range .Sites}}<div class="trow"><a class="c-site site-{{.Site}}" href="{{.SiteHref}}">{{.Site}}</a>{{range .Figs}}{{if .Href}}<a class="fig" href="{{.Href}}">{{.Count}}</a>{{else}}<span class="fig zero">{{.Count}}</span>{{end}}{{end}}<span class="c-state{{if .StateGood}} ok{{end}}{{if .StateBad}} bad{{end}}">{{.State}}</span></div>{{end}}</div>
|
||||
{{end}}
|
||||
@@ -1,48 +0,0 @@
|
||||
{{/* The Reader roster, on the owner's administrative page. Re-rendered whole
|
||||
as the response to an action so the counts and marks it shows cannot
|
||||
describe the state before the tap. Both actions are confirm-gated: one
|
||||
signs a Reader out of every device at once, the other wipes a record.
|
||||
|
||||
Owner-only at route registration, so nothing here re-tests who is asking. */}}
|
||||
{{define "readers"}}
|
||||
<section class="readers" id="readers">
|
||||
<h2>Readers</h2>
|
||||
<p class="setup-copy">Everyone who has signed in through Discord. Revoking
|
||||
signs a Reader out of every device; their library and bookmarks are
|
||||
untouched, and they can sign in again. The Sighting counters record how
|
||||
often a later Poll confirmed or contradicted what that Reader's browser
|
||||
reported; enough contradictions stop their reports deferring a Poll, and
|
||||
clearing the marks gives that back.</p>
|
||||
<ul class="readerlist">
|
||||
{{range .Readers}}
|
||||
<li>
|
||||
<span class="reader-id">{{.DiscordID}}</span>
|
||||
<span class="reader-sessions">{{.Sessions}} session{{if ne .Sessions 1}}s{{end}}</span>
|
||||
<span class="reader-sightings">{{.Agreements}} confirmed / {{.Disagreements}} contradicted</span>
|
||||
{{/* Blocked is spelled out rather than left to be worked out from two
|
||||
numbers and a threshold. */}}
|
||||
{{if .Blocked}}<span class="reader-blocked">deferral blocked</span>{{end}}
|
||||
<span class="reader-actions">
|
||||
{{/* Clearing restores a privilege, so it is a plain ghost button —
|
||||
the destruction accent belongs to revocation alone. It is offered
|
||||
on every row, including one reading zero: the remedy must be
|
||||
findable before the counters climb, not after. */}}
|
||||
<form hx-post="/readers/{{.ID}}/clear-marks" hx-target="#readers" hx-swap="outerHTML"
|
||||
hx-confirm="Clearing wipes this Reader's whole Sighting record, confirmations included. Clear?">
|
||||
<button type="submit" class="ghost">Clear marks</button>
|
||||
</form>
|
||||
{{/* The owner's own row never offers Revoke: it is the one row where the
|
||||
button would sign the tapping browser out, and the endpoint refuses
|
||||
it anyway. Logout is the deliberate way to do that. */}}
|
||||
{{if and .Sessions (ne .ID $.OwnerID)}}
|
||||
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
|
||||
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
|
||||
<button type="submit" class="ghost danger">Revoke sessions</button>
|
||||
</form>
|
||||
{{end}}
|
||||
</span>
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
</section>
|
||||
{{end}}
|
||||
@@ -1,35 +0,0 @@
|
||||
{{/* Per-Series page: one address per Series, keyed "<site>:<series_id>" so the
|
||||
list row is one hop from it. Everything here is a Series-level fact plus
|
||||
the anonymous Reader count. The Check now control lands in its own .dform
|
||||
below the (empty) .detail-grid; the pending marker rides the meta line
|
||||
with the other marks. */}}
|
||||
{{define "series-detail"}}
|
||||
<a class="ghost detail-back" href="/admin/series">← Series</a>
|
||||
<h1 class="detail-title">{{.Title}}</h1>
|
||||
<p class="detail-key">{{.Key}} · {{.Site}} · {{.Kind}}</p>
|
||||
{{if .Cover}}<div class="cover"><img src="{{.Cover}}" alt="" loading="lazy"></div>
|
||||
{{else}}<div class="cover"></div>{{end}}
|
||||
{{template "series-detail-meta" .}}
|
||||
<div class="detail-grid"></div>
|
||||
{{if .CanPoll}}
|
||||
<div class="dform">
|
||||
<div class="field"><a class="ghost act" hx-post="/admin/series/{{.Key}}/poll" hx-target="#detail-meta" hx-swap="outerHTML" href="#">Check now</a></div>
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{/* series-detail-meta is the meta line, and the answer a Check now press on
|
||||
the detail page swaps into its place: the same marks, re-rendered after
|
||||
the stamp so the pending marker shows. */}}
|
||||
{{define "series-detail-meta"}}
|
||||
<div class="detail-meta" id="detail-meta">
|
||||
<span>ch {{.Chapter}}</span>
|
||||
<span>checked {{.Checked}}</span>
|
||||
<span>{{.Readers}} readers</span>
|
||||
{{if .Pending}}<span class="mark">{{.Requested}}</span>{{end}}
|
||||
{{if .Unpollable}}<span class="mark">unpollable</span>{{end}}
|
||||
{{if .NoCover}}<span class="mark">no cover</span>{{end}}
|
||||
{{if .Orphan}}<span class="mark">orphan</span>{{end}}
|
||||
{{if .SightingRaised}}<span class="mark">sighting-raised</span>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -1,53 +0,0 @@
|
||||
{{/* The Series list: every Series across every Reader's library, filtered by
|
||||
one hygiene rule and narrowed by Site and Library. Filter, Site, Library
|
||||
and page all live in the query string, so the list's state is an address
|
||||
that can be bookmarked: the two selects submit the GET form, and the
|
||||
Library segment links and the pager preserve the filter and Site. */}}
|
||||
{{define "series-list"}}
|
||||
<form class="filterbar" method="get" action="/admin/series">
|
||||
<input type="hidden" name="kind" value="{{.Kind}}">
|
||||
<label class="fsel"><span>Show</span><select name="filter" onchange="this.form.submit()">
|
||||
{{range .Filters}}<option value="{{.Name}}"{{if .Selected}} selected{{end}}>{{.Label}} ({{.Count}})</option>{{end}}
|
||||
</select></label>
|
||||
<label class="fsel"><span>Site</span><select name="site" onchange="this.form.submit()">
|
||||
<option value=""{{if not .Site}} selected{{end}}>All sites</option>
|
||||
{{range .Sites}}<option value="{{.}}"{{if eq $.Site .}} selected{{end}}>{{.}}</option>{{end}}
|
||||
</select></label>
|
||||
<span class="segrow">
|
||||
<a href="{{.KindBoth}}"{{if not .Kind}} class="active"{{end}}>both</a>
|
||||
<a href="{{.KindManga}}"{{if eq .Kind "manga"}} class="active"{{end}}>manga</a>
|
||||
<a href="{{.KindNovel}}"{{if eq .Kind "novel"}} class="active"{{end}}>novels</a>
|
||||
</span>
|
||||
</form>
|
||||
<div class="listhead">{{.Total}} series <span class="lbl">· <em>{{.FilterLabel}}</em></span></div>
|
||||
{{if .Rows}}
|
||||
<div class="tbl series">
|
||||
<div class="thead"><span>Site</span><span class="c-ch">Ch</span><span>Checked</span><span class="c-rd">Readers</span><span>Notes</span><span></span></div>
|
||||
{{range .Rows}}{{template "series-row" .}}{{end}}
|
||||
</div>
|
||||
<div class="pager">
|
||||
{{if .PrevHref}}<a class="pg" href="{{.PrevHref}}">‹ prev</a>{{else}}<span class="pg disabled">‹ prev</span>{{end}}
|
||||
<span>{{.Range}}</span>
|
||||
{{if .NextHref}}<a class="pg" href="{{.NextHref}}">next ›</a>{{else}}<span class="pg disabled">next ›</span>{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="empty"><strong>No series</strong><p>Nothing matches <em>{{.FilterLabel}}</em>.</p></div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{/* series-row is one Series list row, and the answer a Check now press swaps
|
||||
into the row's place (hx-target="closest .trow"): it must render the
|
||||
pending marker the press created. The control is absent on a Series with
|
||||
no page to fetch and on an orphan, so the owner is never offered a button
|
||||
that can never do anything. */}}
|
||||
{{define "series-row"}}
|
||||
<div class="trow{{if .Attention}} attention{{end}}{{if .Band}} band{{end}}">
|
||||
<span class="c-title"><a href="/admin/series/{{.Key}}">{{.Title}}</a>{{if .Pending}}<span class="mark">{{.Requested}}</span>{{end}}</span>
|
||||
<span class="c-site site-{{.Site}}">{{.Site}}</span>
|
||||
<span class="c-ch">{{.Ch}}</span>
|
||||
<span>{{.Age}}</span>
|
||||
<span class="c-rd">{{.Readers}}</span>
|
||||
<span class="c-note">{{range .Notes}}<span class="mark">{{.}}</span>{{end}}{{if .More}}<span class="mark mark-faint">+{{.More}}</span>{{end}}</span>
|
||||
<span class="c-act">{{if .CanPoll}}<a class="ghost act" hx-post="/admin/series/{{.Key}}/poll" hx-target="closest .trow" hx-swap="outerHTML" hx-vals='{"band":{{if .Band}}1{{else}}0{{end}}}' href="#">Check now</a>{{end}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -1,34 +0,0 @@
|
||||
{{/* The userscript install panel. Each link serves the script rendered
|
||||
with the acting Reader's credential inside it, so the credential never
|
||||
appears in this page's markup, the address bar, or a redirect. Rotation
|
||||
is confirm-gated because it invalidates every installed copy at once;
|
||||
the response swaps this same panel open with the reinstall warning. */}}
|
||||
{{define "setup"}}
|
||||
<details class="setup" id="setup"{{if .Rotated}} open{{end}}>
|
||||
<summary>Userscripts</summary>
|
||||
<p class="setup-copy">Install each script once per device. They keep your
|
||||
bookmarks in sync across every site and update themselves from here.</p>
|
||||
<p class="setup-links">
|
||||
<a class="ghost" href="/install/manga-bookmark.user.js">Install Manga script</a>
|
||||
<a class="ghost" href="/install/novel-bookmark.user.js">Install Novels script</a>
|
||||
</p>
|
||||
<p class="setup-copy">On mobile, Violentmonkey does not pick up the install
|
||||
links — the script opens as text. Download the file instead, then add it
|
||||
from Violentmonkey's own menu.</p>
|
||||
<p class="setup-links">
|
||||
<a class="ghost" href="/install/manga-bookmark.user.js?download=1">Download Manga script</a>
|
||||
<a class="ghost" href="/install/novel-bookmark.user.js?download=1">Download Novels script</a>
|
||||
</p>
|
||||
{{if .Rotated}}
|
||||
<p class="setup-warn" role="status">Credential rotated — the old one no
|
||||
longer works. Reinstall both scripts on every device now, or they will
|
||||
silently stop syncing.</p>
|
||||
{{else}}
|
||||
<form class="setup-rotate" hx-post="/rotate-token" hx-target="#setup"
|
||||
hx-swap="outerHTML"
|
||||
hx-confirm="Rotation invalidates the current credential on every device immediately. You will have to reinstall both scripts everywhere. Rotate?">
|
||||
<button type="submit" class="ghost">Rotate credential</button>
|
||||
</form>
|
||||
{{end}}
|
||||
</details>
|
||||
{{end}}
|
||||
@@ -1,627 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"html/template"
|
||||
"io/fs"
|
||||
"log"
|
||||
"math"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/session"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
"bookmarkmanager/backend/internal/userscript"
|
||||
)
|
||||
|
||||
//go:embed templates
|
||||
var templateFS embed.FS
|
||||
|
||||
//go:embed static
|
||||
var staticFS embed.FS
|
||||
|
||||
// RecentCount is how many series the "Continue reading" strip shows.
|
||||
const RecentCount = 5
|
||||
|
||||
// Handler serves the browser UI: full pages at / and htmx fragments at /ui/.
|
||||
// It is a separate handler from api.Handler because the two speak different
|
||||
// representations (HTML versus JSON) to different clients under different auth.
|
||||
type Handler struct {
|
||||
store *store.Store
|
||||
// tokenKey derives Readers' userscript credentials (internal/token): the
|
||||
// install endpoints render the scripts with the credential inside, which
|
||||
// is the one place the UI needs the secret.
|
||||
tokenKey []byte
|
||||
// mangaUserscriptPath / novelUserscriptPath are the bindmounted script
|
||||
// files the install endpoints render — the same files the /u/ download
|
||||
// paths serve.
|
||||
mangaUserscriptPath string
|
||||
novelUserscriptPath string
|
||||
tmpl *template.Template
|
||||
discord DiscordConfig
|
||||
states *oauthStates
|
||||
limiter *session.LoginLimiter
|
||||
// httpClient is the plain stdlib client that talks to Discord. It is not
|
||||
// an injected interface: tests point APIBase at a stub server instead.
|
||||
httpClient *http.Client
|
||||
// pollerEnabled reports whether latest-chapter polling is switched on in
|
||||
// this deployment (LATEST_CHAPTER_POLL_ENABLED) and browserConfigured
|
||||
// whether a browser sidecar is configured (BROWSER_WS_URL set). Both are
|
||||
// deployment facts resolved by the composition root; the Lanes page (issue
|
||||
// #145) reports them from config and derives reachability from the pass
|
||||
// log rather than from whether a poller goroutine happened to start.
|
||||
pollerEnabled bool
|
||||
browserConfigured bool
|
||||
}
|
||||
|
||||
// listView is what every list-rendering template receives.
|
||||
type listView struct {
|
||||
// Lib is the library this view renders: store.KindManga or store.KindNovel.
|
||||
// Manga is the default and carries no query parameter, so every pre-novel
|
||||
// URL keeps meaning exactly what it did.
|
||||
Lib string
|
||||
Tab string // "all", "fav", or "new"
|
||||
Recent []store.Bookmark
|
||||
Items []store.Bookmark
|
||||
// NewCount is the badge on the Updated tab: how many series being read
|
||||
// have a chapter out that has not been read. It is counted over the whole
|
||||
// reading set, not the active tab, so the badge does not change meaning as
|
||||
// the user moves between tabs.
|
||||
NewCount int
|
||||
// OOB marks a render of the chrome partials as an out-of-band swap rather
|
||||
// than the inline copy app.html lays out.
|
||||
OOB bool
|
||||
// Rotated marks the setup panel as having just rotated the credential:
|
||||
// it swaps the reinstall warning in over the button row.
|
||||
Rotated bool
|
||||
// EmptyLibrary means this Reader holds no bookmarks in either library, so
|
||||
// the empty state can offer the installs instead of reporting on a filter.
|
||||
// It is not "newly registered": a Reader who deletes their last bookmark is
|
||||
// in the same position and needs the same links.
|
||||
EmptyLibrary bool
|
||||
// Owner marks the acting Reader as the deployment's owner, which offers
|
||||
// the link to the administrative page. Nothing else in the UI differs.
|
||||
Owner bool
|
||||
}
|
||||
|
||||
// PageURL and ListURL are the two link shapes every tab needs. Building them
|
||||
// here rather than concatenating in the template is what keeps the library
|
||||
// parameter from being dropped on one link out of ten.
|
||||
func (v listView) PageURL(tab string) string {
|
||||
if v.Lib == store.KindNovel {
|
||||
return "/?lib=novel&tab=" + tab
|
||||
}
|
||||
return "/?tab=" + tab
|
||||
}
|
||||
|
||||
func (v listView) ListURL(tab string) string {
|
||||
if v.Lib == store.KindNovel {
|
||||
return "/ui/list?lib=novel&tab=" + tab
|
||||
}
|
||||
return "/ui/list?tab=" + tab
|
||||
}
|
||||
|
||||
// loginView is what the login template receives.
|
||||
type loginView struct {
|
||||
Error string
|
||||
}
|
||||
|
||||
// New parses every template up front so a broken one kills the process at
|
||||
// startup rather than the first request that touches it.
|
||||
//
|
||||
// pollerEnabled and browserConfigured are deployment facts the composition
|
||||
// root resolves from LATEST_CHAPTER_POLL_ENABLED and BROWSER_WS_URL: the Lanes
|
||||
// page (issue #145) reports them and derives browser reachability from the
|
||||
// pass log, so no running poller is wired through here at all.
|
||||
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string, pollerEnabled, browserConfigured bool) (*Handler, error) {
|
||||
tmpl, err := template.ParseFS(templateFS, "templates/*.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Handler{
|
||||
store: s,
|
||||
tokenKey: tokenKey,
|
||||
mangaUserscriptPath: mangaPath,
|
||||
novelUserscriptPath: novelPath,
|
||||
tmpl: tmpl,
|
||||
discord: discord,
|
||||
states: newOAuthStates(),
|
||||
limiter: session.NewLoginLimiter(),
|
||||
httpClient: &http.Client{Timeout: discordTimeout},
|
||||
pollerEnabled: pollerEnabled,
|
||||
browserConfigured: browserConfigured,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /{$}", h.index)
|
||||
mux.HandleFunc("GET /auth/discord", h.discordStart)
|
||||
mux.HandleFunc("GET /auth/discord/callback", h.discordCallback)
|
||||
mux.HandleFunc("POST /logout", h.logout)
|
||||
mux.Handle("GET /static/", staticHandler())
|
||||
|
||||
mux.HandleFunc("GET /ui/list", h.requireSession(h.uiList))
|
||||
mux.HandleFunc("POST /ui/bookmarks/{key}/favorite", h.requireSession(h.uiFavorite))
|
||||
mux.HandleFunc("POST /ui/bookmarks/{key}/status", h.requireSession(h.uiStatus))
|
||||
mux.HandleFunc("POST /ui/bookmarks/{key}/chapter", h.requireSession(h.uiChapter))
|
||||
mux.HandleFunc("DELETE /ui/bookmarks/{key}", h.requireSession(h.uiDelete))
|
||||
|
||||
// Install endpoints render the script directly under the session: the
|
||||
// credential travels inside the served bytes, never in the address bar or
|
||||
// the page markup. Updates after install use the credential-bearing /u/
|
||||
// path the script embeds, which needs no session.
|
||||
mux.HandleFunc("GET /install/manga-bookmark.user.js", h.requireSession(h.installUserscript("manga-bookmark.user.js")))
|
||||
mux.HandleFunc("GET /install/novel-bookmark.user.js", h.requireSession(h.installUserscript("novel-bookmark.user.js")))
|
||||
mux.HandleFunc("POST /rotate-token", h.requireSession(h.rotateToken))
|
||||
// Owner-only: every route that reaches past the acting Reader is gated in
|
||||
// one place, so a missing gate is visible in the route list.
|
||||
for _, rt := range h.adminRoutes() {
|
||||
mux.HandleFunc(rt.pattern, h.requireOwner(rt.handler))
|
||||
}
|
||||
}
|
||||
|
||||
// staticHandler serves the embedded assets. An hour, not longer: assets are
|
||||
// not fingerprinted, and embed.FS reports a zero ModTime, so http.FileServer
|
||||
// emits no Last-Modified or ETag and a client has no way to revalidate a
|
||||
// cached copy after a deploy short of waiting out max-age.
|
||||
func staticHandler() http.Handler {
|
||||
sub, err := fs.Sub(staticFS, "static")
|
||||
if err != nil {
|
||||
panic("embed static: " + err.Error())
|
||||
}
|
||||
// Go's built-in table has no .woff2 and the scratch image has no
|
||||
// /etc/mime.types, so without this the fonts go out as
|
||||
// application/octet-stream.
|
||||
if err := mime.AddExtensionType(".woff2", "font/woff2"); err != nil {
|
||||
panic("woff2 mime: " + err.Error())
|
||||
}
|
||||
files := http.FileServer(http.FS(sub))
|
||||
return http.StripPrefix("/static/", http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
files.ServeHTTP(w, r)
|
||||
}))
|
||||
}
|
||||
|
||||
type ctxKey int
|
||||
|
||||
// readerCtxKey is where requireSession stashes the authenticated Reader id.
|
||||
const readerCtxKey ctxKey = iota
|
||||
|
||||
// sessionReader reports whether the request carries a live session, and for
|
||||
// whom. The cookie holds only the session id; the row behind it is looked up
|
||||
// on every request, so deleting a session takes effect immediately. Expiry is
|
||||
// enforced here, in the store, which also removes rows that have lapsed.
|
||||
func (h *Handler) sessionReader(r *http.Request) (int64, bool) {
|
||||
c, err := r.Cookie(session.CookieName)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
sess, ok, err := h.store.GetSession(c.Value, time.Now())
|
||||
if err != nil {
|
||||
log.Printf("session lookup: %v", err)
|
||||
return 0, false
|
||||
}
|
||||
return sess.ReaderID, ok
|
||||
}
|
||||
|
||||
// requireSession guards the fragment endpoints. It answers 401 rather than
|
||||
// redirecting, because htmx swaps whatever body it receives into the page and a
|
||||
// redirected login page would be spliced into the card list.
|
||||
func (h *Handler) requireSession(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
readerID, ok := h.sessionReader(r)
|
||||
if !ok {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next(w, r.WithContext(context.WithValue(r.Context(), readerCtxKey, readerID)))
|
||||
}
|
||||
}
|
||||
|
||||
// readerOf returns the authenticated Reader id requireSession stashed.
|
||||
func readerOf(r *http.Request) int64 { return r.Context().Value(readerCtxKey).(int64) }
|
||||
|
||||
func (h *Handler) render(w http.ResponseWriter, status int, name string, data any) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
if err := h.tmpl.ExecuteTemplate(w, name, data); err != nil {
|
||||
// The status line is already sent, so this can only be logged.
|
||||
log.Printf("render %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// index renders the list, or the login page when there is no session. The login
|
||||
// page is served at / with status 200 rather than as a redirect to a separate
|
||||
// URL: one page, no redirect loop to reason about.
|
||||
func (h *Handler) index(w http.ResponseWriter, r *http.Request) {
|
||||
readerID, ok := h.sessionReader(r)
|
||||
if !ok {
|
||||
h.render(w, http.StatusOK, "login", loginView{})
|
||||
return
|
||||
}
|
||||
view, err := h.buildListView(readerID, libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
|
||||
if err != nil {
|
||||
log.Printf("index: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// The owner's page differs only by the link to the administrative page:
|
||||
// the roster lives there now, so the page read every day is only reading.
|
||||
view.Owner = readerID == h.store.OwnerID()
|
||||
h.render(w, http.StatusOK, "app", view)
|
||||
}
|
||||
|
||||
// filterBookmarks returns the subset keep reports true for, preserving order.
|
||||
// It always returns a non-nil slice so an empty tab renders its empty state.
|
||||
func filterBookmarks(all []store.Bookmark, keep func(store.Bookmark) bool) []store.Bookmark {
|
||||
out := []store.Bookmark{}
|
||||
for _, b := range all {
|
||||
if keep(b) {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// kindOf reads a bookmark's library. A row cached or written before the kind
|
||||
// column existed has none; every one of those is manga, which is what the
|
||||
// column default says too.
|
||||
func kindOf(b store.Bookmark) string {
|
||||
if b.Kind == "" {
|
||||
return store.KindManga
|
||||
}
|
||||
return b.Kind
|
||||
}
|
||||
|
||||
// libOf normalises the query parameter. Anything that is not the novel library
|
||||
// is the manga one, so a typo lands on the default page rather than an empty
|
||||
// list.
|
||||
func libOf(q string) string {
|
||||
if q == store.KindNovel {
|
||||
return store.KindNovel
|
||||
}
|
||||
return store.KindManga
|
||||
}
|
||||
|
||||
// buildListView loads one reader's list once and derives both the tab-filtered
|
||||
// items and the recent strip from it.
|
||||
//
|
||||
// Archived and finished series appear in their own tab and nowhere else — not
|
||||
// in All, not in Updated, not in Favourites, and not in the recent strip. An
|
||||
// archived favourite therefore shows only under Archived: Favourites means
|
||||
// "favourites I am currently reading".
|
||||
func (h *Handler) buildListView(readerID int64, lib, tab string) (listView, error) {
|
||||
all, err := h.store.List(readerID) // already ordered updated_at DESC
|
||||
if err != nil {
|
||||
return listView{}, err
|
||||
}
|
||||
// Taken before the filter narrows the slice: a Reader with novels but no
|
||||
// manga has a working install already, and does not need to be told to go
|
||||
// and get one.
|
||||
emptyLibrary := len(all) == 0
|
||||
// Narrow to one library first: reading, withNew and recent all derive from
|
||||
// this slice, so doing it later would let the other library's rows into the
|
||||
// strip and the Updated badge.
|
||||
all = filterBookmarks(all, func(b store.Bookmark) bool { return kindOf(b) == lib })
|
||||
|
||||
// Novels do not offer an Updated tab, so a hand-typed one lands on All.
|
||||
if lib == store.KindNovel && tab == "new" {
|
||||
tab = "all"
|
||||
}
|
||||
reading := filterBookmarks(all, func(b store.Bookmark) bool { return b.Status == store.StatusReading })
|
||||
|
||||
withNew := filterBookmarks(reading, func(b store.Bookmark) bool { return b.HasNewChapter() })
|
||||
|
||||
var items []store.Bookmark
|
||||
switch tab {
|
||||
case "fav":
|
||||
items = filterBookmarks(reading, func(b store.Bookmark) bool { return b.Favorite })
|
||||
case "new":
|
||||
items = withNew
|
||||
case "archived":
|
||||
items = filterBookmarks(all, func(b store.Bookmark) bool { return b.Status == store.StatusArchived })
|
||||
case "finished":
|
||||
items = filterBookmarks(all, func(b store.Bookmark) bool { return b.Status == store.StatusFinished })
|
||||
default:
|
||||
tab = "all"
|
||||
items = reading
|
||||
}
|
||||
// The strip is scoped to series with a chapter waiting, which is the one
|
||||
// question the list below it does not already answer: the list is ordered by
|
||||
// reading recency, so the head of it *is* the strip whenever the strip is
|
||||
// just "the most recent rows". Only on All — on Updated it would render the
|
||||
// same set twice, and on the other tabs it would contradict the bucket.
|
||||
//
|
||||
// It therefore disappears entirely on a library with nothing new. That is
|
||||
// the intended reading: an empty strip has nothing to say, and the ~240px it
|
||||
// costs on a phone belongs to the list.
|
||||
var recent []store.Bookmark
|
||||
if tab == "all" {
|
||||
recent = withNew
|
||||
if len(recent) > RecentCount {
|
||||
recent = recent[:RecentCount]
|
||||
}
|
||||
}
|
||||
return listView{Lib: lib, Tab: tab, Recent: recent, Items: items,
|
||||
NewCount: len(withNew), EmptyLibrary: emptyLibrary}, nil
|
||||
}
|
||||
|
||||
func (h *Handler) uiList(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.buildListView(readerOf(r), libOf(r.URL.Query().Get("lib")), r.URL.Query().Get("tab"))
|
||||
if err != nil {
|
||||
log.Printf("ui list: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "list", view)
|
||||
// The chrome is outside this response's swap target, so without this a tab
|
||||
// switch would leave the strip and badge from whichever tab the page was
|
||||
// loaded on — the same URL would render differently depending on how the
|
||||
// reader got there.
|
||||
h.writeChromeOOB(w, view)
|
||||
}
|
||||
|
||||
// currentTab is the tab the reader is looking at, read from htmx's own header,
|
||||
// so out-of-band chrome is rebuilt for that view rather than for a default.
|
||||
func currentTab(r *http.Request) string {
|
||||
u, err := url.Parse(r.Header.Get("HX-Current-URL"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return u.Query().Get("tab")
|
||||
}
|
||||
|
||||
// currentLib is the library the reader is looking at, read from htmx's own
|
||||
// header for the same reason currentTab is: out-of-band chrome must be rebuilt
|
||||
// for that view rather than for the default one.
|
||||
func currentLib(r *http.Request) string {
|
||||
u, err := url.Parse(r.Header.Get("HX-Current-URL"))
|
||||
if err != nil {
|
||||
return store.KindManga
|
||||
}
|
||||
return libOf(u.Query().Get("lib"))
|
||||
}
|
||||
|
||||
// writeChromeOOB appends the regions that live outside #list — the recent
|
||||
// strip, the Updated badge and the action key — as out-of-band swaps, so a
|
||||
// mutation cannot leave them describing the library as it was before the tap.
|
||||
// The key is in here because it is tab-shaped too: archived and finished swap
|
||||
// Archive for Restore.
|
||||
func (h *Handler) writeChromeOOB(w http.ResponseWriter, view listView) {
|
||||
view.OOB = true
|
||||
names := []string{"recent", "keyrow"}
|
||||
if view.Lib == store.KindManga {
|
||||
names = append(names, "newcount")
|
||||
}
|
||||
for _, name := range names {
|
||||
if err := h.tmpl.ExecuteTemplate(w, name, view); err != nil {
|
||||
// The card is already written; stale chrome beats a torn response.
|
||||
log.Printf("render %s oob: %v", name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// refreshChrome rebuilds the chrome for the reader's current tab after a
|
||||
// mutation and appends it to the response.
|
||||
func (h *Handler) refreshChrome(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.buildListView(readerOf(r), currentLib(r), currentTab(r))
|
||||
if err != nil {
|
||||
log.Printf("ui chrome: %v", err)
|
||||
return
|
||||
}
|
||||
h.writeChromeOOB(w, view)
|
||||
}
|
||||
|
||||
// renderLogin renders the login page with an error message, for refused or
|
||||
// failed sign-ins. Every message is author-written text — nothing Discord
|
||||
// supplied is ever interpolated into a page.
|
||||
func (h *Handler) renderLogin(w http.ResponseWriter, status int, msg string) {
|
||||
h.render(w, status, "login", loginView{Error: msg})
|
||||
}
|
||||
|
||||
// logout revokes the session row and clears the cookie in one step: the next
|
||||
// request finds no row and is rejected.
|
||||
func (h *Handler) logout(w http.ResponseWriter, r *http.Request) {
|
||||
if c, err := r.Cookie(session.CookieName); err == nil {
|
||||
if err := h.store.DeleteSession(c.Value); err != nil {
|
||||
log.Printf("delete session: %v", err)
|
||||
}
|
||||
}
|
||||
session.ClearCookie(w, r)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// loadForMutation fetches the row a mutation targets, writing the error
|
||||
// response itself when there is nothing to mutate.
|
||||
func (h *Handler) loadForMutation(w http.ResponseWriter, r *http.Request) (store.Bookmark, bool) {
|
||||
key := r.PathValue("key")
|
||||
if key == "" {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return store.Bookmark{}, false
|
||||
}
|
||||
b, ok, err := h.store.Get(readerOf(r), key)
|
||||
if err != nil {
|
||||
log.Printf("ui get %q: %v", key, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return store.Bookmark{}, false
|
||||
}
|
||||
if !ok {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return store.Bookmark{}, false
|
||||
}
|
||||
return b, true
|
||||
}
|
||||
|
||||
// saveAndRenderCard upserts and renders the row as stored, then refreshes the
|
||||
// chrome. Upsert decides whether updated_at moves, so the argument's timestamp
|
||||
// is only a candidate and the response must come from the return value.
|
||||
//
|
||||
// ponytail: the swapped card stays put even when its new status no longer
|
||||
// matches the active tab. That much is deliberate — the card showing its new
|
||||
// state is the feedback for the tap. The strip and the badge are not: they
|
||||
// describe the whole library, so they are rebuilt out of band on every
|
||||
// mutation, at the cost of one extra list read per toggle.
|
||||
func (h *Handler) saveAndRenderCard(w http.ResponseWriter, r *http.Request, b store.Bookmark) {
|
||||
stored, err := h.store.Upsert(readerOf(r), b)
|
||||
if err != nil {
|
||||
log.Printf("ui upsert %q: %v", b.Key, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "card", stored)
|
||||
h.refreshChrome(w, r)
|
||||
}
|
||||
|
||||
// uiFavorite flips the favourite flag. last_chapter_num is untouched, so
|
||||
// Upsert keeps the stored updated_at and the list does not reorder.
|
||||
func (h *Handler) uiFavorite(w http.ResponseWriter, r *http.Request) {
|
||||
b, ok := h.loadForMutation(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
b.Favorite = !b.Favorite
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
h.saveAndRenderCard(w, r, b)
|
||||
}
|
||||
|
||||
// uiStatus moves a bookmark between lifecycle buckets. This is the only place
|
||||
// a series can be marked finished — the JSON API refuses that value, so the
|
||||
// userscript cannot set it even by accident.
|
||||
//
|
||||
// last_chapter_num is untouched, so Upsert keeps the stored updated_at and the
|
||||
// list does not reorder.
|
||||
func (h *Handler) uiStatus(w http.ResponseWriter, r *http.Request) {
|
||||
b, ok := h.loadForMutation(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
switch s := r.PostFormValue("status"); s {
|
||||
case store.StatusReading, store.StatusArchived, store.StatusFinished:
|
||||
b.Status = s
|
||||
default:
|
||||
http.Error(w, "invalid status", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
h.saveAndRenderCard(w, r, b)
|
||||
}
|
||||
|
||||
// uiChapter forces the read chapter to a value the user typed.
|
||||
//
|
||||
// Writing the number also clears last_chapter_url: that URL points at the
|
||||
// chapter actually read, and once the number is forced elsewhere it would send
|
||||
// the reader backwards. ContinueURL then falls back to the series page, which
|
||||
// is always right.
|
||||
//
|
||||
// A submit that does not change the number touches nothing. The form is
|
||||
// pre-filled, so a bare tap of Save is an easy accidental submit; it must not
|
||||
// destroy last_chapter_url, nor rewrite the last_chapter display string ("45.0"
|
||||
// to "45") behind a frozen updated_at.
|
||||
func (h *Handler) uiChapter(w http.ResponseWriter, r *http.Request) {
|
||||
b, ok := h.loadForMutation(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
raw := strings.TrimSpace(r.PostFormValue("chapter"))
|
||||
num, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil || num < 0 || math.IsNaN(num) || math.IsInf(num, 0) {
|
||||
http.Error(w, "chapter must be a non-negative number", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if num != b.LastChapterNum {
|
||||
b.LastChapterURL = ""
|
||||
b.LastChapter = raw
|
||||
b.LastChapterNum = num
|
||||
}
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
h.saveAndRenderCard(w, r, b)
|
||||
}
|
||||
|
||||
// uiDelete removes the row and answers with an empty body, which htmx swaps in
|
||||
// place of the card — removing it from the page.
|
||||
func (h *Handler) uiDelete(w http.ResponseWriter, r *http.Request) {
|
||||
key := r.PathValue("key")
|
||||
if key == "" {
|
||||
http.Error(w, "missing key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.Delete(readerOf(r), key); err != nil {
|
||||
log.Printf("ui delete %q: %v", key, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
// The empty body is what removes the card; the chrome still has to be told
|
||||
// the library got smaller.
|
||||
h.refreshChrome(w, r)
|
||||
}
|
||||
|
||||
// installUserscript renders the bindmounted script with the acting Reader's
|
||||
// derived credential substituted in. The credential is derived, not stored,
|
||||
// so installs work after any restart; the Reader never types or copies it —
|
||||
// clicking Install is the whole setup.
|
||||
//
|
||||
// ?download=1 forces a save instead. Mobile Violentmonkey (Chromium) does not
|
||||
// intercept navigation to a .user.js URL, so the Install link only renders the
|
||||
// source as text there; the Reader needs the file on disk to add it by hand.
|
||||
func (h *Handler) installUserscript(name string) http.HandlerFunc {
|
||||
path := h.mangaUserscriptPath
|
||||
if name == "novel-bookmark.user.js" {
|
||||
path = h.novelUserscriptPath
|
||||
}
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
discordID, epoch, err := h.store.ReaderTokenInfo(readerOf(r))
|
||||
if err != nil {
|
||||
log.Printf("install %s: %v", name, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if r.URL.Query().Has("download") {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`)
|
||||
}
|
||||
userscript.Render(w, r, path, token.Token(h.tokenKey, discordID, epoch))
|
||||
}
|
||||
}
|
||||
|
||||
// rotateToken issues the acting Reader a new credential: the epoch bumps and
|
||||
// the stored hash is rewritten, so the old credential stops authenticating
|
||||
// the moment the statement commits. Every device must reinstall, or its
|
||||
// script keeps failing silently — the setup panel states that warning next
|
||||
// to the button, and the response repeats it as confirmation.
|
||||
func (h *Handler) rotateToken(w http.ResponseWriter, r *http.Request) {
|
||||
readerID := readerOf(r)
|
||||
discordID, epoch, err := h.store.ReaderTokenInfo(readerID)
|
||||
if err != nil {
|
||||
log.Printf("rotate token: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// The hash is computed for epoch+1 and guarded by it in the store, so a
|
||||
// concurrent rotation cannot leave the stored hash describing another
|
||||
// epoch.
|
||||
if err := h.store.RotateToken(readerID, epoch, token.Hash(token.Token(h.tokenKey, discordID, epoch+1))); err != nil {
|
||||
log.Printf("rotate token: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
view := listView{Lib: store.KindManga, Rotated: true}
|
||||
h.render(w, http.StatusOK, "setup", view)
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fetcher retrieves a series page. It exists as an interface so tests can inject
|
||||
// a fake: nothing in the test suite may touch the network or the TLS client.
|
||||
type fetcher interface {
|
||||
Get(ctx context.Context, url string) (body string, status int, err error)
|
||||
}
|
||||
|
||||
// latestPoller re-checks each bookmarked series' newest published chapter on a
|
||||
// schedule, independent of the userscript's own in-browser checks. The two run
|
||||
// in parallel and report the same observable fact, so whichever writes last wins
|
||||
// and neither needs to know about the other.
|
||||
//
|
||||
// Two clocks, deliberately independent:
|
||||
//
|
||||
// - interval is how often this goroutine wakes up and looks.
|
||||
// - cooldown is how long one bookmark rests since its own last check.
|
||||
//
|
||||
// Only the cooldown is per bookmark, and it is enforced by the WHERE clause in
|
||||
// DueForLatestCheck rather than by any timer. Shortening interval therefore
|
||||
// cannot shorten anyone's cooldown; it only makes the poller wake up and find
|
||||
// nothing due more often.
|
||||
type latestPoller struct {
|
||||
store *Store
|
||||
fetch fetcher
|
||||
now func() time.Time // injected so tests can freeze it
|
||||
cooldown time.Duration
|
||||
interval time.Duration
|
||||
stagger time.Duration
|
||||
batch int
|
||||
}
|
||||
|
||||
// Run polls until ctx is cancelled.
|
||||
//
|
||||
// runOnce is called synchronously, so a batch that overruns the tick delays the
|
||||
// next one instead of stacking a second batch on top of it. That is the intended
|
||||
// failure mode for a misconfigured batch x stagger: a slower cadence, never
|
||||
// concurrent fetch storms.
|
||||
func (p *latestPoller) Run(ctx context.Context) {
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s",
|
||||
p.interval, p.cooldown, p.batch, p.stagger)
|
||||
t := time.NewTicker(p.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("latest-chapter poller: stopped")
|
||||
return
|
||||
case <-t.C:
|
||||
p.runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runOnce processes one batch of due bookmarks.
|
||||
func (p *latestPoller) runOnce(ctx context.Context) {
|
||||
cutoff := p.now().Add(-p.cooldown).UnixMilli()
|
||||
due, err := p.store.DueForLatestCheck(cutoff, p.batch)
|
||||
if err != nil {
|
||||
log.Printf("latest poll: due query: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
checked := 0
|
||||
for i, b := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
// Staggered rather than fired together: a burst of simultaneous requests
|
||||
// from one server IP is the traffic shape most likely to move that IP's
|
||||
// bot score. This is the server-side analogue of the userscript's "one
|
||||
// series per navigation ... indistinguishable from browsing" (L455-456).
|
||||
stopped := false
|
||||
if i > 0 && p.stagger > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
stopped = true
|
||||
case <-time.After(p.stagger):
|
||||
}
|
||||
}
|
||||
if stopped {
|
||||
break
|
||||
}
|
||||
p.checkOne(ctx, b)
|
||||
checked++
|
||||
}
|
||||
// due vs checked is how you tell which constraint is binding: ticks that
|
||||
// report due=0 mean the cooldown is the limit, ticks that report due==batch
|
||||
// every time mean throughput is.
|
||||
log.Printf("latest poll: due=%d checked=%d", len(due), checked)
|
||||
}
|
||||
|
||||
// checkOne re-checks one series. Every failure path here is "log and move on":
|
||||
// the poller is a best-effort enhancement, and no single bad series may stall a
|
||||
// batch or take down the process.
|
||||
func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("latest poll %q: recovered from panic: %v", b.Key, r)
|
||||
}
|
||||
}()
|
||||
|
||||
// Stamped before the fetch, not after, so an error, a timeout, or a shutdown
|
||||
// mid-request still consumes the cooldown. Otherwise a renamed or deleted
|
||||
// series would be retried on every single tick forever. The userscript
|
||||
// stamps in the same order and for the same reason (L471-473).
|
||||
if err := p.store.MarkLatestChecked(b.Key, p.now().UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %q: mark checked: %v", b.Key, err)
|
||||
return
|
||||
}
|
||||
|
||||
// series_url is client-supplied (PUT /bookmarks/{key} accepts any string),
|
||||
// so this is not just an optimisation against burning a request on an
|
||||
// unknown site: without it, the server would issue a GET from its own
|
||||
// network position to whatever URL a token-holder writes, including
|
||||
// link-local/internal addresses or non-https schemes. The cooldown above
|
||||
// is already consumed, so a row that never passes this check is retried at
|
||||
// cooldown pace rather than hot-looping.
|
||||
if !fetchableSeriesURL(b.Site, b.SeriesURL) {
|
||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", b.Key, b.Site, b.SeriesURL)
|
||||
return
|
||||
}
|
||||
|
||||
body, status, err := p.fetch.Get(ctx, b.SeriesURL)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
|
||||
return
|
||||
}
|
||||
if status != 200 {
|
||||
log.Printf("latest poll %q: fetch %s: status %d", b.Key, b.SeriesURL, status)
|
||||
return
|
||||
}
|
||||
|
||||
latest, ok := latestChapterFrom(b.Site, b.SeriesURL, body)
|
||||
if !ok {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||
log.Printf("latest poll %q: no chapter links in %d bytes", b.Key, len(body))
|
||||
return
|
||||
}
|
||||
|
||||
// Re-read: the row may have been updated or deleted while the fetch was in
|
||||
// flight, and writing b back wholesale would undo that.
|
||||
//
|
||||
// ponytail: non-transactional read-modify-write, wrap Get+Upsert in a tx if
|
||||
// this ever runs for more than one user. A client PUT that commits between
|
||||
// these two statements is lost to the stale re-read — reverting read
|
||||
// progress or a status change, and moving updated_at because the stored
|
||||
// value now differs. Accepted for a single-user deployment: the window is
|
||||
// milliseconds and the loser is one poll cycle.
|
||||
cur, found, err := p.store.Get(b.Key)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: reread: %v", b.Key, err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
// Equality, not >, mirroring the userscript (L427): a site that retracts a
|
||||
// chapter should correct the stored number downward.
|
||||
if cur.LatestChapterNum != nil && *cur.LatestChapterNum == latest.Num {
|
||||
return
|
||||
}
|
||||
|
||||
num := latest.Num
|
||||
cur.LatestChapter = latest.Label
|
||||
cur.LatestChapterNum = &num
|
||||
// A candidate only. last_chapter_num is untouched, so the CASE in Upsert
|
||||
// keeps the stored updated_at and the bookmark list does not reorder.
|
||||
cur.UpdatedAt = p.now().UnixMilli()
|
||||
if _, err := p.store.Upsert(cur); err != nil {
|
||||
log.Printf("latest poll %q: upsert: %v", b.Key, err)
|
||||
return
|
||||
}
|
||||
log.Printf("latest poll %q: latest is now %s", b.Key, latest.Label)
|
||||
}
|
||||
|
||||
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
|
||||
// to parse and seriesURL is safe to hand to the fetcher: an https URL with a
|
||||
// non-empty host. series_url comes from client-supplied PUT bodies, so this is
|
||||
// a defence against the poller being used to probe arbitrary hosts from the
|
||||
// server's own network position, not just a check against wasted requests.
|
||||
func fetchableSeriesURL(site, seriesURL string) bool {
|
||||
switch site {
|
||||
case "asura", "demonic":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return u.Scheme == "https" && u.Host != ""
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package latest
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,9 +11,8 @@ import (
|
||||
)
|
||||
|
||||
// maxBodyBytes caps what a single series page can cost in memory. Real pages
|
||||
// measured 100-400 KB on 2026-07-26; lightnovelworld runs larger — 685 KB and
|
||||
// 1.18 MB measured 2026-08-11 — so the headroom there is roughly 3.5x, and the
|
||||
// cap mostly guards against a proxy handing back something enormous.
|
||||
// measured 100-400 KB on 2026-07-26, so this is roughly 10x headroom and mostly
|
||||
// guards against a proxy handing back something enormous.
|
||||
const maxBodyBytes = 4 << 20
|
||||
|
||||
// chromeUA matches the client profile below. A Chrome fingerprint paired with a
|
||||
@@ -21,20 +20,20 @@ const maxBodyBytes = 4 << 20
|
||||
const chromeUA = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36"
|
||||
|
||||
// TLSFetcher fetches series pages with a Chrome TLS fingerprint.
|
||||
// tlsFetcher fetches series pages with a Chrome TLS fingerprint.
|
||||
//
|
||||
// Plain net/http was verified working against both sites on 2026-07-26, so this
|
||||
// is not fixing an observed block — it is deliberate defence-in-depth against a
|
||||
// future fingerprint-based one, chosen up front rather than reacted to later.
|
||||
// The library is pure Go, so CGO_ENABLED=0, the static binary, and the
|
||||
// distroless image are all unaffected.
|
||||
type TLSFetcher struct {
|
||||
type tlsFetcher struct {
|
||||
client tls_client.HttpClient
|
||||
}
|
||||
|
||||
var _ Fetcher = (*TLSFetcher)(nil)
|
||||
var _ fetcher = (*tlsFetcher)(nil)
|
||||
|
||||
func NewTLSFetcher() (*TLSFetcher, error) {
|
||||
func newTLSFetcher() (*tlsFetcher, error) {
|
||||
c, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(),
|
||||
tls_client.WithTimeoutSeconds(30),
|
||||
tls_client.WithClientProfile(profiles.Chrome_133),
|
||||
@@ -42,13 +41,13 @@ func NewTLSFetcher() (*TLSFetcher, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new tls client: %w", err)
|
||||
}
|
||||
return &TLSFetcher{client: c}, nil
|
||||
return &tlsFetcher{client: c}, nil
|
||||
}
|
||||
|
||||
// Get fetches url and returns the body and status. Redirects are followed: the
|
||||
// demonic chapter anchors are a redirect form, and asura has moved domains
|
||||
// before.
|
||||
func (f *TLSFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
||||
func (f *tlsFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
||||
req, err := fhttp.NewRequest(fhttp.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("build request %q: %w", url, err)
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// latestChapter is the newest chapter a series page advertises.
|
||||
type latestChapter struct {
|
||||
Num float64
|
||||
Label string
|
||||
}
|
||||
|
||||
// asuraSlugRe pulls the series slug out of a stored series_url.
|
||||
// Shape verified live 2026-07-26: https://asurascans.com/comics/<slug>, where
|
||||
// the slug carries a trailing build-hash suffix (e.g. "-f886a8af") that
|
||||
// rotates on every site redeploy — callers must strip it (asuraBuildHash)
|
||||
// before using the slug to scope anything.
|
||||
var asuraSlugRe = regexp.MustCompile(`/comics/([^/?#]+)`)
|
||||
|
||||
// demonicChapterRe matches the pre-redirect anchors demonic series pages link
|
||||
// through. Both the raw "&" and the HTML-escaped "&" forms occur.
|
||||
var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?chapter=([0-9.]+)`)
|
||||
|
||||
// latestChapterFrom returns the highest chapter number body advertises for this
|
||||
// series. ok is false when the body yields nothing usable — an unknown site, an
|
||||
// empty body, a Cloudflare challenge page, and a site redesign all land here,
|
||||
// and the caller treats all four identically.
|
||||
//
|
||||
// Ported from the userscript's latestChapterFromAnchors (asura L123-133,
|
||||
// demonic L183-193), including its reason for taking a maximum rather than a
|
||||
// first or last: neither site lists chapters in a dependable order.
|
||||
//
|
||||
// The userscript's asura rule additionally requires the anchor text to match
|
||||
// /Chapter\s+[\d.]+/i. That check exists only to skip the "First Chapter"
|
||||
// shortcut, which points at chapter/1 and therefore can never win a maximum, so
|
||||
// it is redundant here. For asura, scoping the pattern to this series' own slug
|
||||
// replaces it with a stronger guarantee: a chapter link belonging to some other
|
||||
// series cannot contribute even if the page starts carrying them. demonic has no
|
||||
// such guarantee — demonicChapterRe matches any chaptered.php?manga=<id> anchor
|
||||
// with no per-series scoping, because the stored series_id for demonic is a
|
||||
// slug, not the numeric id the URL carries, so it cannot easily be scoped.
|
||||
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
|
||||
var re *regexp.Regexp
|
||||
switch site {
|
||||
case "asura":
|
||||
m := asuraSlugRe.FindStringSubmatch(seriesURL)
|
||||
if m == nil {
|
||||
return latestChapter{}, false
|
||||
}
|
||||
// Stored URLs predating a redeploy may carry a stale build hash;
|
||||
// chapter hrefs in the fetched body carry the current one. Strip to
|
||||
// the stable ID (same rule as migrateAsuraKeys) and make the hash
|
||||
// optional in the pattern, so scoping survives rotations.
|
||||
slug := asuraBuildHash.ReplaceAllString(m[1], "")
|
||||
// Compiled per call rather than cached: this runs once per fetch, which
|
||||
// is at most a few times a minute, and the slug varies per series.
|
||||
re = regexp.MustCompile(`/comics/` + regexp.QuoteMeta(slug) + `(?:-[0-9a-f]{8})?/chapter/([0-9.]+)`)
|
||||
case "demonic":
|
||||
re = demonicChapterRe
|
||||
default:
|
||||
return latestChapter{}, false
|
||||
}
|
||||
|
||||
var best latestChapter
|
||||
found := false
|
||||
for _, m := range re.FindAllStringSubmatch(body, -1) {
|
||||
// [0-9.]+ can swallow a trailing separator, e.g. "chapter/12." in a
|
||||
// sentence; ParseFloat would reject the whole match.
|
||||
raw := strings.Trim(m[1], ".")
|
||||
num, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !found || num > best.Num {
|
||||
best = latestChapter{Num: num, Label: "Chapter " + raw}
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
|
||||
// fetched 2026-07-26. The first anchor is the "First Chapter" shortcut: it is a
|
||||
// real chapter link with no "Chapter N" text, and it must not be mistaken for
|
||||
// the latest just because it parses.
|
||||
const asuraSeriesFixture = `
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/1" class="py-3 rounded-md bg-[#E8E8E8]"><svg class="w-4 h-4"></svg>First Chapter</a>
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/179" data-astro-prefetch="hover" class="group flex"><span class="font-medium">Chapter 179</span></a>
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/181" data-astro-prefetch="hover" class="group flex"><span class="font-medium">Chapter 181</span></a>
|
||||
<a href="/comics/chronicles-of-the-demon-faction-f886a8af/chapter/180" data-astro-prefetch="hover" class="group flex"><span class="font-medium">Chapter 180</span></a>
|
||||
`
|
||||
|
||||
// A chapter link belonging to a different series, of the kind a "you might also
|
||||
// like" strip would introduce. Slug scoping must exclude it.
|
||||
const asuraCrossSeriesFixture = asuraSeriesFixture + `
|
||||
<a href="/comics/some-other-series-aabbccdd/chapter/999" class="group flex"><span>Chapter 999</span></a>
|
||||
`
|
||||
|
||||
// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer fetched
|
||||
// 2026-07-26. Note the raw "&", the doubled space after <a, and the decimal
|
||||
// chapters, all as they appear live.
|
||||
const demonicSeriesFixture = `
|
||||
<a href="/chaptered.php?manga=11799&chapter=0.5" class="chplinks" title="Catastrophic Necromancer 0.5">Chapter 0.5</a>
|
||||
<a href="/chaptered.php?manga=11799&chapter=294" class="chplinks" title="Catastrophic Necromancer 294">Chapter 294</a>
|
||||
<a href="/chaptered.php?manga=11799&chapter=296" class="chplinks" title="Catastrophic Necromancer 296">Chapter 296</a>
|
||||
<a href="/chaptered.php?manga=11799&chapter=295" class="chplinks" title="Catastrophic Necromancer 295">Chapter 295</a>
|
||||
`
|
||||
|
||||
// What Cloudflare serves instead of the page when an IP's bot score flips.
|
||||
const challengeFixture = `<!DOCTYPE html><html><head><title>Just a moment...</title>
|
||||
<script src="/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page/v1"></script></head>
|
||||
<body><div id="challenge-running">Checking your browser</div></body></html>`
|
||||
|
||||
func TestLatestChapterFrom(t *testing.T) {
|
||||
const asuraURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
const demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
site string
|
||||
seriesURL string
|
||||
body string
|
||||
wantOK bool
|
||||
wantNum float64
|
||||
wantLabel string
|
||||
}{
|
||||
{
|
||||
name: "asura takes the max, not the last listed",
|
||||
site: "asura", seriesURL: asuraURL, body: asuraSeriesFixture,
|
||||
wantOK: true, wantNum: 181, wantLabel: "Chapter 181",
|
||||
},
|
||||
{
|
||||
name: "asura ignores another series' chapter links",
|
||||
site: "asura", seriesURL: asuraURL, body: asuraCrossSeriesFixture,
|
||||
wantOK: true, wantNum: 181, wantLabel: "Chapter 181",
|
||||
},
|
||||
{
|
||||
name: "asura scoping survives a build-hash rotation",
|
||||
site: "asura",
|
||||
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-059befe1",
|
||||
body: asuraCrossSeriesFixture,
|
||||
wantOK: true, wantNum: 181, wantLabel: "Chapter 181",
|
||||
},
|
||||
{
|
||||
name: "asura with an unparseable series url",
|
||||
site: "asura", seriesURL: "https://asurascans.com/", body: asuraSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "demonic takes the max across raw and escaped ampersands",
|
||||
site: "demonic", seriesURL: demonicURL, body: demonicSeriesFixture,
|
||||
wantOK: true, wantNum: 296, wantLabel: "Chapter 296",
|
||||
},
|
||||
{
|
||||
name: "demonic keeps decimal chapters parseable",
|
||||
site: "demonic", seriesURL: demonicURL,
|
||||
body: `<a href="/chaptered.php?manga=11799&chapter=0.5">Chapter 0.5</a>`,
|
||||
wantOK: true, wantNum: 0.5, wantLabel: "Chapter 0.5",
|
||||
},
|
||||
{
|
||||
name: "empty body",
|
||||
site: "asura", seriesURL: asuraURL, body: "",
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "cloudflare challenge page",
|
||||
site: "asura", seriesURL: asuraURL, body: challengeFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "demonic markup handed to the asura rule",
|
||||
site: "asura", seriesURL: asuraURL, body: demonicSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "unknown site",
|
||||
site: "mangadex", seriesURL: "https://example.com/x", body: asuraSeriesFixture,
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := latestChapterFrom(tt.site, tt.seriesURL, tt.body)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("ok = %v, want %v (got %+v)", ok, tt.wantOK, got)
|
||||
}
|
||||
if !tt.wantOK {
|
||||
return
|
||||
}
|
||||
if got.Num != tt.wantNum {
|
||||
t.Errorf("Num = %v, want %v", got.Num, tt.wantNum)
|
||||
}
|
||||
if got.Label != tt.wantLabel {
|
||||
t.Errorf("Label = %q, want %q", got.Label, tt.wantLabel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeFetcher stands in for the network. Every poller test uses it, so nothing
|
||||
// in this file can reach tls-client or a real site.
|
||||
type fakeFetcher struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
body string
|
||||
status int
|
||||
err error
|
||||
// perURL overrides body/status/err for specific URLs.
|
||||
perURL map[string]fakeResponse
|
||||
}
|
||||
|
||||
type fakeResponse struct {
|
||||
body string
|
||||
status int
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, url)
|
||||
f.mu.Unlock()
|
||||
if r, ok := f.perURL[url]; ok {
|
||||
return r.body, r.status, r.err
|
||||
}
|
||||
return f.body, f.status, f.err
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) callCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.calls)
|
||||
}
|
||||
|
||||
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
|
||||
// instantly and deterministically.
|
||||
func newTestPoller(t *testing.T, s *Store, f fetcher, at time.Time) *latestPoller {
|
||||
t.Helper()
|
||||
return &latestPoller{
|
||||
store: s,
|
||||
fetch: f,
|
||||
now: func() time.Time { return at },
|
||||
cooldown: time.Hour,
|
||||
interval: 10 * time.Minute,
|
||||
stagger: 0,
|
||||
batch: 14,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
|
||||
|
||||
now := time.UnixMilli(5_000_000)
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
b, ok, err := s.Get("asura:chronicles-of-the-demon-faction-f886a8af")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get: %v ok=%v", err, ok)
|
||||
}
|
||||
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
||||
}
|
||||
if b.LatestChapter != "Chapter 181" {
|
||||
t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
|
||||
}
|
||||
if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point of the updated_at CASE in Upsert: a newly published chapter is
|
||||
// not reading progress and must not move the series up the list.
|
||||
func TestRunOnceDoesNotReorderList(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||
const key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||
|
||||
// "other" is the most recently read, so it must stay at the top of List().
|
||||
if _, err := s.Upsert(Bookmark{
|
||||
Key: "asura:other", Site: "asura", SeriesID: "other",
|
||||
SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed other: %v", err)
|
||||
}
|
||||
seedForCheck(t, s, key, url, 0)
|
||||
before, _, err := s.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get before: %v", err)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
|
||||
|
||||
after, _, err := s.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after: %v", err)
|
||||
}
|
||||
if after.UpdatedAt != before.UpdatedAt {
|
||||
t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
|
||||
before.UpdatedAt, after.UpdatedAt)
|
||||
}
|
||||
list, err := s.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if list[0].Key != "asura:other" {
|
||||
t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed fetch must still consume the cooldown, or a renamed series gets
|
||||
// retried on every tick forever.
|
||||
func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resp fakeResponse
|
||||
}{
|
||||
{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
|
||||
{"non-200", fakeResponse{body: "nope", status: 503}},
|
||||
{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
|
||||
{"empty body", fakeResponse{body: "", status: 200}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/x"
|
||||
seedForCheck(t, s, "asura:x", url, 0)
|
||||
|
||||
now := time.UnixMilli(7_000_000)
|
||||
f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
||||
}
|
||||
b, _, err := s.Get("asura:x")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if b.LatestChapterNum != nil {
|
||||
t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceRespectsBatchLimit(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for i := 0; i < 20; i++ {
|
||||
key := "asura:s" + string(rune('a'+i))
|
||||
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: "", status: 200}
|
||||
p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
|
||||
p.batch = 5
|
||||
p.runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 5 {
|
||||
t.Fatalf("fetched %d series, want 5 (batch limit)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// One unreachable series must not abandon the rest of the batch.
|
||||
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
|
||||
for _, k := range keys {
|
||||
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
||||
}
|
||||
|
||||
now := time.UnixMilli(6_000_000)
|
||||
f := &fakeFetcher{
|
||||
body: "", status: 200,
|
||||
perURL: map[string]fakeResponse{
|
||||
"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
|
||||
},
|
||||
}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 5 {
|
||||
t.Fatalf("fetched %d series, want all 5 attempted", got)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
|
||||
t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The cooldown is enforced by the due query, so a second immediate pass must do
|
||||
// nothing at all — this is what makes the tick interval independent of it.
|
||||
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const url = "https://asurascans.com/comics/x"
|
||||
seedForCheck(t, s, "asura:x", url, 0)
|
||||
|
||||
now := time.UnixMilli(8_000_000)
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("first pass fetched %d, want 1", got)
|
||||
}
|
||||
// Same instant, and again 59 minutes later: both inside the 1h cooldown.
|
||||
p.runOnce(context.Background())
|
||||
p.now = func() time.Time { return now.Add(59 * time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times inside the cooldown, want 1", got)
|
||||
}
|
||||
// Past the cooldown, it is due again.
|
||||
p.now = func() time.Time { return now.Add(61 * time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 2 {
|
||||
t.Fatalf("fetched %d times after the cooldown, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A site that retracts a chapter should correct the stored number downward,
|
||||
// mirroring the userscript's equality check (L427) rather than a >.
|
||||
func TestRunOnceCorrectsDownward(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
||||
const key = "demonic:Catastrophic-Necromancer"
|
||||
|
||||
high := 400.0
|
||||
if _, err := s.Upsert(Bookmark{
|
||||
Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
|
||||
SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
|
||||
UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
||||
|
||||
b, _, err := s.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
|
||||
}
|
||||
}
|
||||
|
||||
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
|
||||
// reject anything that is not a known site with an https URL before spending a
|
||||
// request on it — the cooldown still gets consumed either way.
|
||||
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
site string
|
||||
seriesURL string
|
||||
wantCalls int
|
||||
}{
|
||||
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
|
||||
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
|
||||
{"unparseable url", "asura", "http://[::1", 0},
|
||||
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
key := tt.site + ":x"
|
||||
if _, err := s.Upsert(Bookmark{
|
||||
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
||||
UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
now := time.UnixMilli(4_000_000)
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).checkOne(context.Background(), Bookmark{
|
||||
Key: key, Site: tt.site, SeriesURL: tt.seriesURL,
|
||||
})
|
||||
|
||||
if got := f.callCount(); got != tt.wantCalls {
|
||||
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
|
||||
}
|
||||
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A cancelled context must abandon the batch rather than run it to completion.
|
||||
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
|
||||
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
f := &fakeFetcher{body: "", status: 200}
|
||||
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)
|
||||
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("fetched %d series with a cancelled context, want 0", got)
|
||||
}
|
||||
}
|
||||
+125
-219
@@ -7,73 +7,51 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/api"
|
||||
"bookmarkmanager/backend/internal/httpmw"
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
"bookmarkmanager/backend/internal/userscript"
|
||||
"bookmarkmanager/backend/internal/web"
|
||||
)
|
||||
|
||||
// Config holds all runtime settings, sourced from environment variables.
|
||||
type Config struct {
|
||||
// TokenKey derives every Reader's userscript credential (internal/token).
|
||||
// Required: without it no install URL can ever be built.
|
||||
TokenKey string
|
||||
Token string
|
||||
AllowedOrigins []string
|
||||
// DatabaseURL is the Postgres connection URL; required, no default,
|
||||
// because a wrong guess would silently start on an empty database.
|
||||
DatabaseURL string
|
||||
// CoverDir is the filesystem volume for immutable cover bytes. Required:
|
||||
// serving a stored address without durable bytes would be worse than a
|
||||
// startup failure.
|
||||
CoverDir string
|
||||
// PublicBaseURL is the origin this deployment answers on, e.g.
|
||||
// "https://bookmarks.example.com". Required: cover URLs go out absolute
|
||||
// because the userscript renders them on third-party origins, where a
|
||||
// relative path would resolve against the Site (ADR-0007), and there is
|
||||
// no way to guess it from a request the poller never sees.
|
||||
PublicBaseURL string
|
||||
Port string
|
||||
// OwnerDiscordID identifies the seeded owner Reader (issue #22). Required:
|
||||
// bookmarks are scoped to a Reader, and a fresh deployment needs one
|
||||
// before anybody logs in. The owner is also the only Reader who can revoke
|
||||
// another Reader's sessions.
|
||||
OwnerDiscordID string
|
||||
// Discord is the OAuth application the browser UI signs in with.
|
||||
Discord web.DiscordConfig
|
||||
DBPath string
|
||||
Port string
|
||||
// WebPassword gates the browser UI. Empty disables the web routes entirely.
|
||||
WebPassword string
|
||||
// UserscriptPath is the file served at /u/{token}/manga-bookmark.user.js.
|
||||
// Supplied by a bindmount so the script can be edited without a rebuild.
|
||||
UserscriptPath string
|
||||
// NovelUserscriptPath is the file served at
|
||||
// /u/{token}/novel-bookmark.user.js. Same bindmount, second script: the
|
||||
// two libraries are separate installs.
|
||||
NovelUserscriptPath string
|
||||
// BrowserWSURL is the CDP websocket the poller's browser Sites read
|
||||
// through. Set means a browser sidecar is configured in this deployment —
|
||||
// the Lanes page reports the fact and derives reachability from the pass
|
||||
// log rather than asking the poller (issue #145).
|
||||
BrowserWSURL string
|
||||
// LatestPoll configures the background latest-chapter fetcher.
|
||||
LatestPoll LatestPoll
|
||||
}
|
||||
|
||||
// LatestPoll configures the background latest-chapter poller.
|
||||
//
|
||||
// Only the kill switch lives here. Pace is per Site — rest time and gap are
|
||||
// registry properties (internal/latest/sites.go, issue #100), because each
|
||||
// Lane has to be able to differ from the others. The five environment
|
||||
// settings that used to size a shared pace (cooldown, browser cooldown,
|
||||
// interval, stagger, batch) are gone with it: no deployed .env may carry them.
|
||||
// Sizing: batch x (cooldown / interval) is how many series hold a true cooldown
|
||||
// cadence — 14 x (1h / 10m) = 84 with these defaults, which covers this
|
||||
// deployment. Past that nothing breaks; the effective cadence stretches to
|
||||
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
|
||||
type LatestPoll struct {
|
||||
Enabled bool
|
||||
Enabled bool
|
||||
Cooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
}
|
||||
|
||||
const (
|
||||
defaultPollCooldown = time.Hour
|
||||
defaultPollInterval = 10 * time.Minute
|
||||
defaultPollStagger = 20 * time.Second
|
||||
defaultPollBatch = 14
|
||||
// minPollCooldown keeps a typo from turning a polite background check into
|
||||
// a hammer against sites that are already bot-scoring us.
|
||||
minPollCooldown = 15 * time.Minute
|
||||
)
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
@@ -96,33 +74,69 @@ func envBool(key string, def bool) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// loadLatestPoll reads the poller's settings. The pace knobs that used to be
|
||||
// clamped here are registry properties now (issue #100), so there is nothing
|
||||
// left to clamp.
|
||||
// envDuration reads a duration env var. An unparseable or non-positive value
|
||||
// falls back to def and logs rather than failing startup: the poller is an
|
||||
// enhancement, and a typo in one of its knobs must not stop bookmark sync.
|
||||
func envDuration(key string, def time.Duration) time.Duration {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
d, err := time.ParseDuration(raw)
|
||||
if err != nil || d <= 0 {
|
||||
log.Printf("config: %s=%q is not a positive duration, using %s", key, raw, def)
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// envInt reads a positive integer env var, with the same fallback policy.
|
||||
func envInt(key string, def int) int {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
log.Printf("config: %s=%q is not a positive integer, using %d", key, raw, def)
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// loadLatestPoll reads the poller's settings, clamping anything that would make
|
||||
// it antisocial.
|
||||
func loadLatestPoll() LatestPoll {
|
||||
return LatestPoll{Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true)}
|
||||
p := LatestPoll{
|
||||
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
|
||||
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
|
||||
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
|
||||
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
|
||||
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
|
||||
}
|
||||
if p.Cooldown < minPollCooldown {
|
||||
log.Printf("config: cooldown %s is below the %s floor, clamping", p.Cooldown, minPollCooldown)
|
||||
p.Cooldown = minPollCooldown
|
||||
}
|
||||
// batch x stagger has to fit inside one tick or a batch is still running
|
||||
// when the next one is due. Run() serialises them, so this degrades to a
|
||||
// slower cadence rather than to overlapping fetches — worth a warning, not
|
||||
// a failure.
|
||||
if span := time.Duration(p.Batch) * p.Stagger; span > p.Interval {
|
||||
log.Printf("config: batch(%d) x stagger(%s) = %s exceeds interval %s; batches will overrun their tick",
|
||||
p.Batch, p.Stagger, span, p.Interval)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func loadConfig() Config {
|
||||
c := Config{
|
||||
TokenKey: os.Getenv("TOKEN_KEY"),
|
||||
DatabaseURL: os.Getenv("DATABASE_URL"),
|
||||
CoverDir: os.Getenv("COVER_DIR"),
|
||||
PublicBaseURL: os.Getenv("PUBLIC_BASE_URL"),
|
||||
Port: envOr("PORT", "8080"),
|
||||
OwnerDiscordID: os.Getenv("OWNER_DISCORD_ID"),
|
||||
UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"),
|
||||
NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"),
|
||||
BrowserWSURL: os.Getenv("BROWSER_WS_URL"),
|
||||
LatestPoll: loadLatestPoll(),
|
||||
}
|
||||
c.Discord = web.DiscordConfig{
|
||||
ClientID: os.Getenv("DISCORD_CLIENT_ID"),
|
||||
ClientSecret: os.Getenv("DISCORD_CLIENT_SECRET"),
|
||||
GuildID: os.Getenv("DISCORD_GUILD_ID"),
|
||||
RequiredRole: os.Getenv("DISCORD_REQUIRED_ROLE"),
|
||||
APIBase: envOr("DISCORD_API_BASE", "https://discord.com/api/v10"),
|
||||
RedirectURI: os.Getenv("DISCORD_REDIRECT_URI"),
|
||||
Token: os.Getenv("API_TOKEN"),
|
||||
DBPath: envOr("DB_PATH", "/data/bookmarks.db"),
|
||||
Port: envOr("PORT", "8080"),
|
||||
WebPassword: os.Getenv("WEB_PASSWORD"),
|
||||
UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"),
|
||||
LatestPoll: loadLatestPoll(),
|
||||
}
|
||||
for _, o := range strings.Split(os.Getenv("ALLOWED_ORIGINS"), ",") {
|
||||
if o = strings.TrimSpace(o); o != "" {
|
||||
@@ -135,51 +149,37 @@ func loadConfig() Config {
|
||||
// newRouter wires routes and middleware. CORS is the outermost layer so
|
||||
// preflight OPTIONS short-circuits before auth; /bookmarks* is auth-protected,
|
||||
// /healthz is public.
|
||||
//
|
||||
// The web layer learns the deployment's poller and browser config from cfg —
|
||||
// nothing of the running poller is wired through here; the Lanes page reads
|
||||
// the database (issue #145).
|
||||
func newRouter(s *store.Store, cfg Config) http.Handler {
|
||||
func newRouter(store *Store, cfg Config) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
h := &api.Handler{Store: s}
|
||||
mux.HandleFunc("GET /healthz", api.Healthz)
|
||||
// Public: cover bytes are rendered by the userscript on origins that may
|
||||
// not send our credentials, and the address is the hash of a URL the Site
|
||||
// already publishes (ADR-0007).
|
||||
mux.HandleFunc("GET /covers/{address}", h.Cover)
|
||||
mux.HandleFunc("GET /healthz", healthz)
|
||||
|
||||
// Outside httpmw.Auth (the updater sends no Authorization header) and
|
||||
// outside the web UI's Discord auth (the script must be installable
|
||||
// without a browser session). The path segment carries the credential
|
||||
// instead, and the script is rendered with the resolved Reader's
|
||||
// credential substituted in.
|
||||
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js",
|
||||
userscript.Handler(s, cfg.UserscriptPath))
|
||||
mux.HandleFunc("GET /u/{token}/novel-bookmark.user.js",
|
||||
userscript.Handler(s, cfg.NovelUserscriptPath))
|
||||
// Outside withAuth (the updater sends no Authorization header) and outside
|
||||
// the WEB_PASSWORD gate (the script must be installable either way). The
|
||||
// path segment carries the token instead.
|
||||
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js", userscriptHandler(cfg.Token, cfg.UserscriptPath))
|
||||
|
||||
h := &bookmarkHandler{store: store}
|
||||
protected := http.NewServeMux()
|
||||
protected.HandleFunc("GET /bookmarks", h.List)
|
||||
protected.HandleFunc("PUT /bookmarks/{key}", h.Put)
|
||||
protected.HandleFunc("DELETE /bookmarks/{key}", h.Delete)
|
||||
protected.HandleFunc("GET /bookmarks", h.list)
|
||||
protected.HandleFunc("PUT /bookmarks/{key}", h.put)
|
||||
protected.HandleFunc("DELETE /bookmarks/{key}", h.delete)
|
||||
|
||||
auth := httpmw.Auth(s, protected)
|
||||
auth := withAuth(cfg.Token, protected)
|
||||
mux.Handle("/bookmarks", auth)
|
||||
mux.Handle("/bookmarks/", auth)
|
||||
|
||||
// The browser UI is always registered; signing in is Discord OAuth, so
|
||||
// there is no password to forget and no gate to leave unset. The poller
|
||||
// and browser facts are config, not the poller's: the Lanes page reads
|
||||
// the pass log and reports the deployment as configured.
|
||||
wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey),
|
||||
cfg.UserscriptPath, cfg.NovelUserscriptPath,
|
||||
cfg.LatestPoll.Enabled, strings.TrimSpace(cfg.BrowserWSURL) != "")
|
||||
if err != nil {
|
||||
log.Fatalf("web handler: %v", err)
|
||||
// The browser UI is registered only when a password is configured, so a
|
||||
// deployment that forgets WEB_PASSWORD exposes nothing rather than
|
||||
// exposing an unprotected list.
|
||||
if cfg.WebPassword != "" {
|
||||
web, err := newWebHandler(store, cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("web handler: %v", err)
|
||||
}
|
||||
web.register(mux)
|
||||
}
|
||||
wh.Register(mux)
|
||||
|
||||
return httpmw.CORS(cfg.AllowedOrigins, httpmw.Gzip(guardEmptyUserscriptToken(mux)))
|
||||
return withCORS(cfg.AllowedOrigins, withGzip(guardEmptyUserscriptToken(mux)))
|
||||
}
|
||||
|
||||
// guardEmptyUserscriptToken heads off ServeMux's own path-cleaning redirect:
|
||||
@@ -199,109 +199,31 @@ func guardEmptyUserscriptToken(next http.Handler) http.Handler {
|
||||
|
||||
func main() {
|
||||
cfg := loadConfig()
|
||||
if cfg.TokenKey == "" {
|
||||
log.Fatal("TOKEN_KEY is required")
|
||||
}
|
||||
if cfg.OwnerDiscordID == "" {
|
||||
log.Fatal("OWNER_DISCORD_ID is required")
|
||||
}
|
||||
if cfg.DatabaseURL == "" {
|
||||
log.Fatal("DATABASE_URL is required")
|
||||
}
|
||||
if cfg.CoverDir == "" {
|
||||
log.Fatal("COVER_DIR is required")
|
||||
}
|
||||
if cfg.PublicBaseURL == "" {
|
||||
log.Fatal("PUBLIC_BASE_URL is required")
|
||||
}
|
||||
// The web UI signs in through Discord, so a deployment without the OAuth
|
||||
// application is misconfigured rather than passwordless.
|
||||
for key, v := range map[string]string{
|
||||
"DISCORD_CLIENT_ID": cfg.Discord.ClientID,
|
||||
"DISCORD_CLIENT_SECRET": cfg.Discord.ClientSecret,
|
||||
"DISCORD_GUILD_ID": cfg.Discord.GuildID,
|
||||
"DISCORD_REDIRECT_URI": cfg.Discord.RedirectURI,
|
||||
} {
|
||||
if v == "" {
|
||||
log.Fatalf("%s is required", key)
|
||||
}
|
||||
}
|
||||
// The owner's userscript credential is derived from TOKEN_KEY at epoch 0
|
||||
// (internal/token); the readers row carries its SHA-256, not the
|
||||
// credential itself.
|
||||
owner := store.Owner{
|
||||
DiscordID: cfg.OwnerDiscordID,
|
||||
TokenHash: token.Hash(token.Token([]byte(cfg.TokenKey), cfg.OwnerDiscordID, 0)),
|
||||
if cfg.Token == "" {
|
||||
log.Fatal("API_TOKEN is required")
|
||||
}
|
||||
|
||||
s, err := store.Open(cfg.DatabaseURL, owner, cfg.CoverDir, cfg.PublicBaseURL)
|
||||
store, err := OpenStore(cfg.DBPath)
|
||||
if err != nil {
|
||||
log.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
defer store.Close()
|
||||
|
||||
// The poller is off the request path entirely: if it cannot start, the
|
||||
// service still serves bookmarks and the userscript still captures latest
|
||||
// chapters on its own.
|
||||
//
|
||||
// One headless browser serves both consumers that need a Cloudflare
|
||||
// challenge cleared: the poller's kagane/novelfull page fetches and
|
||||
// kagane's cover bytes. Optional — unset leaves kagane unpolled and its
|
||||
// Covers blank until the bytes exist.
|
||||
var browser latest.Fetcher
|
||||
pollCtx, stopPoll := context.WithCancel(context.Background())
|
||||
defer stopPoll()
|
||||
if ws := strings.TrimSpace(cfg.BrowserWSURL); ws != "" {
|
||||
bf, err := latest.NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
log.Printf("browser fetcher disabled: %v", err)
|
||||
} else {
|
||||
browser = bf
|
||||
context.AfterFunc(pollCtx, bf.Close)
|
||||
log.Printf("browser fetcher at %s", ws)
|
||||
}
|
||||
}
|
||||
// A Series nobody had bookmarked before gets its Latest Chapter and its
|
||||
// Cover from one fetch, at creation, instead of waiting out a poll queue
|
||||
// ordered by Reader count. Off the write path: the hook returns as soon
|
||||
// as the goroutine is started.
|
||||
var tlsFetch latest.Fetcher
|
||||
if f, err := latest.NewTLSFetcher(); err != nil {
|
||||
log.Printf("creation-time acquisition: plain-TLS Sites disabled, cannot build client: %v", err)
|
||||
} else {
|
||||
tlsFetch = f
|
||||
}
|
||||
var browserCover latest.BrowserCoverFetcher
|
||||
if b, ok := browser.(latest.BrowserCoverFetcher); ok {
|
||||
browserCover = b
|
||||
}
|
||||
// The Acquirer must survive a TLS client failure: kagane needs only the
|
||||
// sidecar, and novelfull degrades to whatever is left.
|
||||
if tlsFetch != nil || browser != nil {
|
||||
acq := &latest.Acquirer{
|
||||
Store: s,
|
||||
Fetch: tlsFetch,
|
||||
BrowserFetch: browser,
|
||||
BrowserCoverFetch: browserCover,
|
||||
Covers: latest.NewCoverFetcher(),
|
||||
Ctx: pollCtx,
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
}
|
||||
// The poller's only connection to the web layer is the database now: it is
|
||||
// started for its own sake, and the Lanes page reads the pass rows it
|
||||
// records (issue #145).
|
||||
startLatestPoller(pollCtx, s, cfg.LatestPoll, browser)
|
||||
startLatestPoller(pollCtx, store, cfg.LatestPoll)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: ":" + cfg.Port,
|
||||
Handler: newRouter(s, cfg),
|
||||
Handler: newRouter(store, cfg),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
go func() {
|
||||
// The connection URL carries a password, so it stays out of the log.
|
||||
log.Printf("listening on :%s (origins=%v)", cfg.Port, cfg.AllowedOrigins)
|
||||
log.Printf("listening on :%s (db=%s, origins=%v)", cfg.Port, cfg.DBPath, cfg.AllowedOrigins)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("serve: %v", err)
|
||||
}
|
||||
@@ -322,44 +244,28 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// newLatestPoller wires the fetcher seams into the poller. Pace is registry
|
||||
// property, not config (issue #100), so there are no knobs to pass through.
|
||||
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
|
||||
var covers latest.BrowserCoverFetcher
|
||||
if f, ok := browser.(latest.BrowserCoverFetcher); ok {
|
||||
covers = f
|
||||
}
|
||||
return &latest.Poller{
|
||||
Store: s,
|
||||
Fetch: fetch,
|
||||
BrowserFetch: browser,
|
||||
CoverFetch: covers,
|
||||
CoverBytesFetch: latest.NewCoverFetcher(),
|
||||
Now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// startLatestPoller launches the background poller unless it is disabled or its
|
||||
// HTTP client cannot be built. Any problem here is logged and skipped: this
|
||||
// feature going missing degrades the service to userscript-only latest-chapter
|
||||
// tracking, which is exactly how it behaved before. It returns the running
|
||||
// Poller, or nil when there is none; the caller starts it for its own sake —
|
||||
// the Lanes page reads the pass log, so no return value is wired anywhere.
|
||||
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) *latest.Poller {
|
||||
// tracking, which is exactly how it behaved before.
|
||||
func startLatestPoller(ctx context.Context, store *Store, cfg LatestPoll) {
|
||||
if !cfg.Enabled {
|
||||
log.Println("latest-chapter poller: disabled by config")
|
||||
return nil
|
||||
return
|
||||
}
|
||||
f, err := latest.NewTLSFetcher()
|
||||
f, err := newTLSFetcher()
|
||||
if err != nil {
|
||||
log.Printf("latest-chapter poller: disabled, cannot build client: %v", err)
|
||||
return nil
|
||||
return
|
||||
}
|
||||
p := &latestPoller{
|
||||
store: store,
|
||||
fetch: f,
|
||||
now: time.Now,
|
||||
cooldown: cfg.Cooldown,
|
||||
interval: cfg.Interval,
|
||||
stagger: cfg.Stagger,
|
||||
batch: cfg.Batch,
|
||||
}
|
||||
// Nil browser: sites behind a JavaScript challenge are simply not polled,
|
||||
// and their latest_chapter comes from the userscript alone — which is how
|
||||
// the service behaved before the sidecar existed.
|
||||
p := newLatestPoller(s, cfg, f, browser)
|
||||
|
||||
go p.Run(ctx)
|
||||
return p
|
||||
}
|
||||
|
||||
+84
-105
@@ -9,22 +9,28 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadLatestPollDefaults(t *testing.T) {
|
||||
t.Setenv("LATEST_CHAPTER_POLL_ENABLED", "")
|
||||
if got := loadLatestPoll(); got != (LatestPoll{Enabled: true}) {
|
||||
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, LatestPoll{Enabled: true})
|
||||
for _, k := range []string{
|
||||
"LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN",
|
||||
"LATEST_CHAPTER_POLL_INTERVAL", "LATEST_CHAPTER_POLL_STAGGER",
|
||||
"LATEST_CHAPTER_POLL_BATCH",
|
||||
} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadsCoverDirectory(t *testing.T) {
|
||||
t.Setenv("COVER_DIR", "/covers")
|
||||
if got := loadConfig().CoverDir; got != "/covers" {
|
||||
t.Fatalf("CoverDir = %q, want /covers", got)
|
||||
got := loadLatestPoll()
|
||||
want := LatestPoll{
|
||||
Enabled: true,
|
||||
Cooldown: time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 20 * time.Second,
|
||||
Batch: 14,
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,25 +53,71 @@ func TestLoadLatestPollEnabledParsing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// newLatestPoller wires the fetcher seams; pace lives in the registry, so
|
||||
// nothing here sizes a cooldown any more.
|
||||
func TestNewLatestPollerWiresFetchers(t *testing.T) {
|
||||
tls := &latest.TLSFetcher{}
|
||||
p := newLatestPoller(nil, LatestPoll{Enabled: true}, tls, nil)
|
||||
if p.Fetch != tls {
|
||||
t.Fatalf("Fetch not wired")
|
||||
func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
wantFrom func(LatestPoll) any
|
||||
want any
|
||||
}{
|
||||
{
|
||||
name: "cooldown below the floor is clamped up",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "1m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Cooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "cooldown at the floor is kept",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "15m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Cooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "a valid override is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Interval },
|
||||
want: 5 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "an unparseable duration falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "ten minutes"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Interval },
|
||||
want: 10 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "a zero duration falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_STAGGER": "0s"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Stagger },
|
||||
want: 20 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "a valid batch is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "30"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Batch },
|
||||
want: 30,
|
||||
},
|
||||
{
|
||||
name: "a negative batch falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "-5"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Batch },
|
||||
want: 14,
|
||||
},
|
||||
{
|
||||
name: "a non-numeric batch falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "lots"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Batch },
|
||||
want: 14,
|
||||
},
|
||||
}
|
||||
if p.BrowserFetch != nil {
|
||||
t.Fatalf("BrowserFetch = %v, want nil for a browser-less deployment", p.BrowserFetch)
|
||||
}
|
||||
if p.CoverFetch != nil {
|
||||
t.Fatalf("CoverFetch = %v, want nil when the browser is absent", p.CoverFetch)
|
||||
}
|
||||
if p.CoverBytesFetch == nil {
|
||||
t.Fatalf("CoverBytesFetch = nil, want the TLS cover fetcher")
|
||||
}
|
||||
if p.Now == nil {
|
||||
t.Fatalf("Now = nil, want the live clock")
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for k, v := range tt.env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
if got := tt.wantFrom(loadLatestPoll()); got != tt.want {
|
||||
t.Fatalf("got %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +148,7 @@ func TestPutStatusValidation(t *testing.T) {
|
||||
if tc.want != http.StatusOK {
|
||||
return
|
||||
}
|
||||
var got store.Bookmark
|
||||
var got Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
@@ -135,7 +187,7 @@ func TestPutOmittedStatusPreservesArchivedAndAppliesProgress(t *testing.T) {
|
||||
t.Fatalf("status = %d, want 200 (body %s)", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got store.Bookmark
|
||||
var got Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
@@ -148,7 +200,7 @@ func TestPutOmittedStatusPreservesArchivedAndAppliesProgress(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGzipCompressesTextNotFonts(t *testing.T) {
|
||||
srv, _ := newWebTestServer(t, testConfig())
|
||||
srv, _ := newWebTestServer(t, webConfig())
|
||||
|
||||
cases := []struct {
|
||||
path string
|
||||
@@ -189,76 +241,3 @@ func TestGzipCompressesTextNotFonts(t *testing.T) {
|
||||
t.Errorf("Content-Encoding without Accept-Encoding = %q, want empty", enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutKindValidation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
kind string
|
||||
want int
|
||||
}{
|
||||
{"empty is no opinion", "", http.StatusOK},
|
||||
{"manga", "manga", http.StatusOK},
|
||||
{"novel", "novel", http.StatusOK},
|
||||
{"garbage", "comic", http.StatusBadRequest},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
body := fmt.Sprintf(`{"title":"Solo","kind":%q}`, tc.kind)
|
||||
req := auth(httptest.NewRequest(http.MethodPut, "/bookmarks/asura:solo",
|
||||
strings.NewReader(body)))
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != tc.want {
|
||||
t.Fatalf("status = %d, want %d (body %s)", rr.Code, tc.want, rr.Body.String())
|
||||
}
|
||||
if tc.want != http.StatusOK {
|
||||
return
|
||||
}
|
||||
var got store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
want := tc.kind
|
||||
if want == "" {
|
||||
want = "manga"
|
||||
}
|
||||
if got.Kind != want {
|
||||
t.Fatalf("stored kind = %q, want %q", got.Kind, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The preserve path: a novel row re-PUT by a client that omits the field
|
||||
// entirely must stay a novel and still record the progress it carried.
|
||||
func TestPutOmittedKindPreservesNovelAndAppliesProgress(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
const key = "/bookmarks/lightnovelworld:a-will-eternal"
|
||||
|
||||
seed := auth(httptest.NewRequest(http.MethodPut, key,
|
||||
strings.NewReader(`{"title":"A Will Eternal","kind":"novel","last_chapter_num":10}`)))
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, seed)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("seed status = %d, want 200 (%s)", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, auth(httptest.NewRequest(http.MethodPut, key,
|
||||
strings.NewReader(`{"title":"A Will Eternal","last_chapter_num":11}`))))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Kind != store.KindNovel {
|
||||
t.Fatalf("Kind = %q, want novel", got.Kind)
|
||||
}
|
||||
if got.LastChapterNum != 11 {
|
||||
t.Fatalf("LastChapterNum = %v, want 11", got.LastChapterNum)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,57 +1,29 @@
|
||||
package httpmw
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"log"
|
||||
"crypto/subtle"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
const bearerPrefix = "Bearer "
|
||||
|
||||
type ctxKey int
|
||||
|
||||
// readerCtxKey is where Auth stashes the authenticated Reader id.
|
||||
const readerCtxKey ctxKey = iota
|
||||
|
||||
// ReaderID returns the Reader id Auth authenticated, for handlers that take
|
||||
// the acting Reader from the request rather than from a fixed field.
|
||||
func ReaderID(r *http.Request) int64 { return r.Context().Value(readerCtxKey).(int64) }
|
||||
|
||||
// ResolveReader maps a presented credential to a Reader. The credential is
|
||||
// hashed and matched against readers.token_sha256 — an equality on 32-byte
|
||||
// values, never a comparison of the credential itself. The same resolution
|
||||
// backs the API bearer header and the userscript download path, so a Reader
|
||||
// has exactly one credential with one blast radius.
|
||||
func ResolveReader(s *store.Store, cred string) (int64, bool) {
|
||||
readerID, ok, err := s.ReaderIDForTokenHash(token.Hash(cred))
|
||||
if err != nil {
|
||||
log.Printf("auth: reader lookup: %v", err)
|
||||
return 0, false
|
||||
}
|
||||
return readerID, ok
|
||||
}
|
||||
|
||||
// Auth guards a handler with a per-Reader bearer credential. The acting
|
||||
// Reader travels in the request context, so a handler scopes every store call
|
||||
// to exactly the Reader that authenticated.
|
||||
func Auth(s *store.Store, next http.Handler) http.Handler {
|
||||
// withAuth guards a handler with a constant-time bearer-token check.
|
||||
func withAuth(token string, next http.Handler) http.Handler {
|
||||
want := []byte(token)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(h, bearerPrefix) {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
readerID, ok := ResolveReader(s, strings.TrimPrefix(h, bearerPrefix))
|
||||
if !ok {
|
||||
got := []byte(strings.TrimPrefix(h, bearerPrefix))
|
||||
if subtle.ConstantTimeCompare(got, want) != 1 {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), readerCtxKey, readerID)))
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -99,11 +71,11 @@ func (w *gzipWriter) Write(b []byte) (int, error) {
|
||||
return w.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
// Gzip compresses text responses for clients that ask. The templates,
|
||||
// withGzip compresses text responses for clients that ask. The templates,
|
||||
// stylesheet and htmx together are ~120 KB uncompressed and roughly a quarter
|
||||
// of that gzipped, which is the difference between a fast and a slow first load
|
||||
// on mobile data.
|
||||
func Gzip(next http.Handler) http.Handler {
|
||||
func withGzip(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
|
||||
next.ServeHTTP(w, r)
|
||||
@@ -120,11 +92,11 @@ func Gzip(next http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// CORS reflects the request Origin only when it is in allowed, answers
|
||||
// withCORS reflects the request Origin only when it is in allowed, answers
|
||||
// preflight OPTIONS with 204, and passes everything else through. It wraps the
|
||||
// auth middleware so preflight (which carries no Authorization header) is never
|
||||
// rejected by auth.
|
||||
func CORS(allowed []string, next http.Handler) http.Handler {
|
||||
func withCORS(allowed []string, next http.Handler) http.Handler {
|
||||
set := make(map[string]struct{}, len(allowed))
|
||||
for _, o := range allowed {
|
||||
set[o] = struct{}{}
|
||||
@@ -1,356 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
"bookmarkmanager/backend/internal/token"
|
||||
)
|
||||
|
||||
// registerReader creates an extra Reader the way a first login does and
|
||||
// returns its id. The credential is derived the same way the owner's is, so it
|
||||
// authenticates through the real router.
|
||||
func registerReader(t *testing.T, s *store.Store, discordID string) int64 {
|
||||
t.Helper()
|
||||
id, err := s.EnsureReader(discordID, token.Hash(readerCredential(discordID)))
|
||||
if err != nil {
|
||||
t.Fatalf("register reader %q: %v", discordID, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// credRequest builds a request authenticated as the Reader whose credential
|
||||
// is passed.
|
||||
func credRequest(method, target, cred string) *http.Request {
|
||||
req := httptest.NewRequest(method, target, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+cred)
|
||||
return req
|
||||
}
|
||||
|
||||
// readerCredential is the epoch-0 derived credential of an arbitrary Reader.
|
||||
func readerCredential(discordID string) string {
|
||||
return token.Token([]byte(testTokenKey), discordID, 0)
|
||||
}
|
||||
|
||||
// withBody attaches a request body, for PUTs that carry a JSON payload.
|
||||
func withBody(req *http.Request, body string) *http.Request {
|
||||
req.Body = io.NopCloser(strings.NewReader(body))
|
||||
req.ContentLength = int64(len(body))
|
||||
return req
|
||||
}
|
||||
|
||||
// A refused credential is refused however plausible it looks: only a hash the
|
||||
// readers table holds authenticates anything.
|
||||
func TestUnknownCredentialRejected(t *testing.T) {
|
||||
srv := newRouter(newTestStore(t), testConfig())
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("never-registered")))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unregistered Reader's credential: status = %d, want 401", rr.Code)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", ownerCredential()))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("owner's derived credential: status = %d, want 200", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A Reader's credential authenticates exactly that Reader: rows written under
|
||||
// one credential are invisible to the other, on the same key.
|
||||
func TestPerReaderIsolation(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
registerReader(t, s, "other-reader")
|
||||
srv := newRouter(s, testConfig())
|
||||
|
||||
ownerKey := "asura:solo"
|
||||
putBookmark(t, srv, ownerKey, store.Bookmark{
|
||||
Key: ownerKey, Site: "asura", SeriesID: "solo",
|
||||
Title: "Solo Leveling", UpdatedAt: 1,
|
||||
})
|
||||
|
||||
// The other Reader's list is empty even though the owner holds the key.
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other reader list: status = %d, want 200", rr.Code)
|
||||
}
|
||||
var theirs []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &theirs); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(theirs) != 0 {
|
||||
t.Fatalf("other reader sees %d bookmarks, want 0 (owner's rows leaked)", len(theirs))
|
||||
}
|
||||
|
||||
// The other Reader writes the same key; both rows coexist, each visible
|
||||
// only to its owner. The series title is shared (ADR-0003) — the
|
||||
// reader-owned fields are progress and updated_at.
|
||||
req := credRequest(http.MethodPut, "/bookmarks/"+ownerKey, readerCredential("other-reader"))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
body := `{"key":"asura:solo","site":"asura","series_id":"solo","title":"Theirs","last_chapter_num":3}`
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, withBody(req, body))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other reader put: status = %d, want 200", rr.Code)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
|
||||
var theirs2 []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &theirs2); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(theirs2) != 1 || theirs2[0].LastChapterNum != 3 {
|
||||
t.Fatalf("other reader list = %+v, want their own row with their progress", theirs2)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", ownerCredential()))
|
||||
var owners []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &owners); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(owners) != 1 || owners[0].Title != "Solo Leveling" || owners[0].LastChapterNum != 0 {
|
||||
t.Fatalf("owner list = %+v, want their own row at their own progress", owners)
|
||||
}
|
||||
|
||||
// The mirror: the owner's write does not move the other Reader's progress
|
||||
// either. Without it, isolation is only asserted in one direction.
|
||||
req = credRequest(http.MethodPut, "/bookmarks/"+ownerKey, ownerCredential())
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, withBody(req, `{"key":"asura:solo","site":"asura","series_id":"solo","title":"Solo Leveling","last_chapter_num":9}`))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("owner put: status = %d, want 200", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("other-reader")))
|
||||
var theirs3 []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &theirs3); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(theirs3) != 1 || theirs3[0].LastChapterNum != 3 {
|
||||
t.Fatalf("other reader list = %+v, want progress 3 after the owner's write", theirs3)
|
||||
}
|
||||
|
||||
// DELETE is scoped to its caller too, asserted in both directions: each
|
||||
// Reader's delete on the shared key takes only their own row.
|
||||
list := func(cred string) []store.Bookmark {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", cred))
|
||||
var got []store.Bookmark
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
del := func(cred string) {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodDelete, "/bookmarks/"+ownerKey, cred))
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("delete: status = %d, want 204", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
del(readerCredential("other-reader"))
|
||||
if got := list(ownerCredential()); len(got) != 1 {
|
||||
t.Fatalf("owner's row was deletable by the other Reader: %+v", got)
|
||||
}
|
||||
if got := list(readerCredential("other-reader")); len(got) != 0 {
|
||||
t.Fatalf("other Reader's own delete left %+v behind", got)
|
||||
}
|
||||
|
||||
// The mirror: the other Reader takes the key again, the owner deletes
|
||||
// theirs, and the other's row is untouched.
|
||||
req = credRequest(http.MethodPut, "/bookmarks/"+ownerKey, readerCredential("other-reader"))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, withBody(req, body))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other reader re-put: status = %d, want 200", rr.Code)
|
||||
}
|
||||
del(ownerCredential())
|
||||
if got := list(readerCredential("other-reader")); len(got) != 1 {
|
||||
t.Fatalf("other Reader's row was deletable by the owner: %+v", got)
|
||||
}
|
||||
if got := list(ownerCredential()); len(got) != 0 {
|
||||
t.Fatalf("owner's own delete left %+v behind", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The install endpoints are session-gated and render the script directly
|
||||
// with the Reader's credential inside: the credential never appears in the
|
||||
// address bar, the page markup, or any Location header.
|
||||
func TestInstallServesScriptWithCredential(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "manga-bookmark.user.js")
|
||||
novelPath := filepath.Join(dir, "novel-bookmark.user.js")
|
||||
for _, p := range []string{path, novelPath} {
|
||||
if err := os.WriteFile(p, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
}
|
||||
cfg.UserscriptPath = path
|
||||
cfg.NovelUserscriptPath = novelPath
|
||||
srv, st := newWebTestServer(t, cfg)
|
||||
|
||||
for _, script := range []string{"manga-bookmark.user.js", "novel-bookmark.user.js"} {
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/install/"+script, nil))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("%s without session: status = %d, want 401", script, rr.Code)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/install/"+script, nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("%s with session: status = %d, want 200", script, rr.Code)
|
||||
}
|
||||
body := rr.Body.String()
|
||||
if strings.Contains(body, "__API_TOKEN__") {
|
||||
t.Fatalf("%s served with an unsubstituted placeholder", script)
|
||||
}
|
||||
// The credential rides inside the served script — nowhere visible in
|
||||
// the UI — and is the session holder's own.
|
||||
if !strings.Contains(body, `API_TOKEN = "`+ownerCredential()+`"`) {
|
||||
t.Fatalf("%s does not carry the owner's credential:\n%s", script, body)
|
||||
}
|
||||
if loc := rr.Header().Get("Location"); loc != "" {
|
||||
t.Fatalf("%s answered with a redirect, credential in Location %q", script, loc)
|
||||
}
|
||||
// The plain link must stay inline: Violentmonkey's updater polls the
|
||||
// /u/ path and an attachment disposition there would break updates.
|
||||
if cd := rr.Header().Get("Content-Disposition"); cd != "" {
|
||||
t.Fatalf("%s served as %q, want inline", script, cd)
|
||||
}
|
||||
|
||||
// ?download=1 is the mobile path: Violentmonkey on Chromium ignores a
|
||||
// .user.js navigation, so the Reader saves the file and adds it by hand.
|
||||
req = httptest.NewRequest(http.MethodGet, "/install/"+script+"?download=1", nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("%s?download=1: status = %d, want 200", script, rr.Code)
|
||||
}
|
||||
if got, want := rr.Header().Get("Content-Disposition"), `attachment; filename="`+script+`"`; got != want {
|
||||
t.Fatalf("%s?download=1: Content-Disposition = %q, want %q", script, got, want)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), `API_TOKEN = "`+ownerCredential()+`"`) {
|
||||
t.Fatalf("%s?download=1 does not carry the owner's credential", script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rotation through the web UI invalidates the old credential immediately,
|
||||
// mints one that authenticates the API and the script path, and warns that
|
||||
// every device must reinstall.
|
||||
func TestRotateCredentialViaWebUI(t *testing.T) {
|
||||
s, _ := newTestStoreURL(t)
|
||||
path := filepath.Join(t.TempDir(), "manga-bookmark.user.js")
|
||||
if err := os.WriteFile(path, []byte("const API_TOKEN = \"__API_TOKEN__\";\n"), 0o644); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
cfg := testConfig()
|
||||
cfg.UserscriptPath = path
|
||||
srv := newRouter(s, cfg)
|
||||
|
||||
oldCred := ownerCredential()
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", oldCred))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("old credential before rotation: status = %d, want 200", rr.Code)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/rotate-token", nil)
|
||||
req.AddCookie(sessionCookie(t, s))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("rotate: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "Credential rotated") {
|
||||
t.Fatalf("rotation response does not warn about reinstall:\n%s", rr.Body.String())
|
||||
}
|
||||
|
||||
// The old credential is dead on the API and on the script path.
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", oldCred))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("old credential after rotation: status = %d, want 401", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+oldCred+"/manga-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("old credential script path after rotation: status = %d, want 404", rr.Code)
|
||||
}
|
||||
|
||||
// The new credential authenticates the API and the script path, and is
|
||||
// substituted into the served script.
|
||||
newCred := token.Token([]byte(testTokenKey), testDiscordID, 1)
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", newCred))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("new credential after rotation: status = %d, want 200", rr.Code)
|
||||
}
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/u/"+newCred+"/manga-bookmark.user.js", nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("new credential script path: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+newCred+`"`) {
|
||||
t.Fatalf("served script does not carry the rotated credential:\n%s", got)
|
||||
}
|
||||
|
||||
// The install link now renders the script with the new credential.
|
||||
req = httptest.NewRequest(http.MethodGet, "/install/manga-bookmark.user.js", nil)
|
||||
req.AddCookie(sessionCookie(t, s))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("install after rotation: status = %d, want 200", rr.Code)
|
||||
}
|
||||
if got := rr.Body.String(); !strings.Contains(got, `API_TOKEN = "`+newCred+`"`) {
|
||||
t.Fatalf("install after rotation does not carry the new credential:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The app page offers the install links; the credential never appears in its
|
||||
// markup.
|
||||
func TestIndexShowsSetupPanelWithoutCredential(t *testing.T) {
|
||||
srv, st := newWebTestServer(t, testConfig())
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.AddCookie(sessionCookie(t, st))
|
||||
rr := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rr, req)
|
||||
|
||||
body := rr.Body.String()
|
||||
for _, want := range []string{
|
||||
`href="/install/manga-bookmark.user.js"`,
|
||||
`href="/install/novel-bookmark.user.js"`,
|
||||
`href="/install/manga-bookmark.user.js?download=1"`,
|
||||
`href="/install/novel-bookmark.user.js?download=1"`,
|
||||
"Rotate credential",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("app page lacks %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, ownerCredential()) {
|
||||
t.Fatal("app page leaks the credential")
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,64 @@
|
||||
package session
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
CookieName = "bmgr_session"
|
||||
sessionCookieName = "mangabm_session"
|
||||
// 60 days: long enough that a phone stays logged in between reading spells.
|
||||
SessionTTL = 60 * 24 * time.Hour
|
||||
sessionTTL = 60 * 24 * time.Hour
|
||||
// Domain separation, so the session key can never collide with any other
|
||||
// use of the secrets it is derived from. Changing this string logs
|
||||
// everyone out.
|
||||
sessionKeyPurpose = "mangabm-web-session-v1"
|
||||
)
|
||||
|
||||
// NewID returns an opaque session id: 32 random bytes, hex-encoded. The id is
|
||||
// all the cookie carries and all the sessions table keys on, so its entropy is
|
||||
// what stops a guessed id from being someone else's session.
|
||||
func NewID() string {
|
||||
var b [32]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
panic("session id: " + err.Error())
|
||||
// sessionKey derives the cookie-signing key from both secrets. Sessions are
|
||||
// stateless — there is no session table — so rotating either API_TOKEN or
|
||||
// WEB_PASSWORD invalidates every outstanding cookie at once. The \x00
|
||||
// separator prevents the concatenation ambiguity a bare apiToken+webPassword
|
||||
// would have (e.g. "ab"+"c" colliding with "a"+"bc").
|
||||
func sessionKey(apiToken, webPassword string) []byte {
|
||||
sum := sha256.Sum256([]byte(apiToken + "\x00" + webPassword + sessionKeyPurpose))
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
// signSession encodes "<expiryMs>.<base64url HMAC(expiryMs)>".
|
||||
func signSession(key []byte, expiryMs int64) string {
|
||||
payload := strconv.FormatInt(expiryMs, 10)
|
||||
return payload + "." + sessionMAC(key, payload)
|
||||
}
|
||||
|
||||
func sessionMAC(key []byte, payload string) string {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(payload))
|
||||
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// verifySession checks shape, then expiry, then the signature — in that order.
|
||||
// The signature comparison is constant-time; the checks before it only look at
|
||||
// data the holder already supplied, so their timing leaks nothing.
|
||||
func verifySession(key []byte, value string, nowMs int64) bool {
|
||||
payload, sig, ok := strings.Cut(value, ".")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return hex.EncodeToString(b[:])
|
||||
expiry, err := strconv.ParseInt(payload, 10, 64)
|
||||
if err != nil || expiry <= nowMs {
|
||||
return false
|
||||
}
|
||||
want := sessionMAC(key, payload)
|
||||
return subtle.ConstantTimeCompare([]byte(sig), []byte(want)) == 1
|
||||
}
|
||||
|
||||
// isHTTPS reports whether the browser's connection is encrypted. Behind Traefik
|
||||
@@ -35,23 +69,21 @@ func isHTTPS(r *http.Request) bool {
|
||||
return r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"
|
||||
}
|
||||
|
||||
// SetCookie writes the session cookie. The value is the session id and nothing
|
||||
// else; the row behind it is looked up on every request.
|
||||
func SetCookie(w http.ResponseWriter, r *http.Request, id string) {
|
||||
func setSessionCookie(w http.ResponseWriter, r *http.Request, key []byte) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: CookieName,
|
||||
Value: id,
|
||||
Name: sessionCookieName,
|
||||
Value: signSession(key, time.Now().Add(sessionTTL).UnixMilli()),
|
||||
Path: "/",
|
||||
MaxAge: int(SessionTTL / time.Second),
|
||||
MaxAge: int(sessionTTL / time.Second),
|
||||
HttpOnly: true,
|
||||
Secure: isHTTPS(r),
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
})
|
||||
}
|
||||
|
||||
func ClearCookie(w http.ResponseWriter, r *http.Request) {
|
||||
func clearSessionCookie(w http.ResponseWriter, r *http.Request) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: CookieName,
|
||||
Name: sessionCookieName,
|
||||
Value: "",
|
||||
Path: "/",
|
||||
MaxAge: -1,
|
||||
@@ -62,11 +94,11 @@ func ClearCookie(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
const (
|
||||
MaxFailures = 10
|
||||
Window = 20 * time.Minute
|
||||
loginMaxFailures = 10
|
||||
loginWindow = 20 * time.Minute
|
||||
)
|
||||
|
||||
// ClientIP returns the address the reverse proxy actually observed.
|
||||
// clientIP returns the address the reverse proxy actually observed.
|
||||
//
|
||||
// Traefik appends the peer address to whatever X-Forwarded-For the client sent,
|
||||
// so the leftmost entry is attacker-controlled and the rightmost is not. Go's
|
||||
@@ -74,7 +106,7 @@ const (
|
||||
// by sending its own; Values covers every line so the true last hop is found.
|
||||
// RemoteAddr is useless behind the proxy — it is always the Traefik container —
|
||||
// so it serves only as the direct-connection fallback for local development.
|
||||
func ClientIP(r *http.Request) string {
|
||||
func clientIP(r *http.Request) string {
|
||||
if vals := r.Header.Values("X-Forwarded-For"); len(vals) > 0 {
|
||||
hops := strings.Split(vals[len(vals)-1], ",")
|
||||
if ip := strings.TrimSpace(hops[len(hops)-1]); ip != "" {
|
||||
@@ -88,46 +120,46 @@ func ClientIP(r *http.Request) string {
|
||||
return host
|
||||
}
|
||||
|
||||
// LoginLimiter throttles failed sign-in attempts: MaxFailures failures inside
|
||||
// a rolling Window blocks further attempts from that IP until the oldest one
|
||||
// loginLimiter throttles password guessing: loginMaxFailures failures inside a
|
||||
// rolling loginWindow blocks further attempts from that IP until the oldest one
|
||||
// ages out. There is no permanent ban and no unlock step.
|
||||
//
|
||||
// Behind carrier-grade NAT this budget is shared with every other subscriber on
|
||||
// the same public address, so a stranger can lock the owner out for up to one
|
||||
// window. That is accepted: the block self-heals, and ten attempts is generous
|
||||
// for the occasional fumbled sign-in.
|
||||
// for a mistyped password.
|
||||
//
|
||||
// State is in memory and per-process, so a restart clears it. Entries are
|
||||
// pruned lazily on access; for a single-user deployment the map cannot grow
|
||||
// past the handful of addresses that ever attempt a login.
|
||||
type LoginLimiter struct {
|
||||
type loginLimiter struct {
|
||||
mu sync.Mutex
|
||||
failures map[string][]time.Time
|
||||
}
|
||||
|
||||
func NewLoginLimiter() *LoginLimiter {
|
||||
return &LoginLimiter{failures: make(map[string][]time.Time)}
|
||||
func newLoginLimiter() *loginLimiter {
|
||||
return &loginLimiter{failures: make(map[string][]time.Time)}
|
||||
}
|
||||
|
||||
// retryAfter returns how long ip must wait, or zero when it may try now.
|
||||
func (l *LoginLimiter) RetryAfter(ip string, now time.Time) time.Duration {
|
||||
func (l *loginLimiter) retryAfter(ip string, now time.Time) time.Duration {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
recent := l.pruneLocked(ip, now)
|
||||
if len(recent) < MaxFailures {
|
||||
if len(recent) < loginMaxFailures {
|
||||
return 0
|
||||
}
|
||||
return recent[0].Add(Window).Sub(now)
|
||||
return recent[0].Add(loginWindow).Sub(now)
|
||||
}
|
||||
|
||||
func (l *LoginLimiter) Fail(ip string, now time.Time) {
|
||||
func (l *loginLimiter) fail(ip string, now time.Time) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.failures[ip] = append(l.pruneLocked(ip, now), now)
|
||||
}
|
||||
|
||||
func (l *LoginLimiter) Reset(ip string) {
|
||||
func (l *loginLimiter) reset(ip string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
delete(l.failures, ip)
|
||||
@@ -135,8 +167,8 @@ func (l *LoginLimiter) Reset(ip string) {
|
||||
|
||||
// pruneLocked drops attempts older than the window and returns what is left.
|
||||
// The caller must hold l.mu.
|
||||
func (l *LoginLimiter) pruneLocked(ip string, now time.Time) []time.Time {
|
||||
cutoff := now.Add(-Window)
|
||||
func (l *loginLimiter) pruneLocked(ip string, now time.Time) []time.Time {
|
||||
cutoff := now.Add(-loginWindow)
|
||||
// In-place filter: kept reuses the backing array of the slice being
|
||||
// ranged over. Safe to alias because append writes at index len(kept),
|
||||
// which is always <= the range index i, and element i is read before
|
||||
@@ -1,4 +1,4 @@
|
||||
package session
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
@@ -9,19 +9,66 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewID(t *testing.T) {
|
||||
a := NewID()
|
||||
b := NewID()
|
||||
if a == b {
|
||||
t.Fatal("NewID returned the same value twice")
|
||||
func TestSessionRoundTrip(t *testing.T) {
|
||||
key := sessionKey("token-abc", "pw-abc")
|
||||
now := time.Now().UnixMilli()
|
||||
value := signSession(key, now+60_000)
|
||||
if !verifySession(key, value, now) {
|
||||
t.Fatal("verifySession = false for a freshly signed cookie, want true")
|
||||
}
|
||||
if len(a) != 64 { // 32 random bytes, hex
|
||||
t.Fatalf("NewID() length = %d, want 64", len(a))
|
||||
}
|
||||
|
||||
func TestSessionRejects(t *testing.T) {
|
||||
key := sessionKey("token-abc", "pw-abc")
|
||||
now := time.Now().UnixMilli()
|
||||
valid := signSession(key, now+60_000)
|
||||
payload, sig, _ := strings.Cut(valid, ".")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{"empty", ""},
|
||||
{"no separator", payload + sig},
|
||||
{"unparseable expiry", "notanumber." + sig},
|
||||
{"expired", signSession(key, now-1)},
|
||||
{"tampered signature", payload + "." + flipLastChar(sig)},
|
||||
{"tampered expiry", "99999999999999." + sig},
|
||||
{"signed with another key", signSession(sessionKey("other-token", "pw-abc"), now+60_000)},
|
||||
}
|
||||
for _, r := range a {
|
||||
if !strings.ContainsRune("0123456789abcdef", r) {
|
||||
t.Fatalf("NewID() = %q, want hex", a)
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if verifySession(key, tc.value, now) {
|
||||
t.Fatalf("verifySession(%q) = true, want false", tc.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func flipLastChar(s string) string {
|
||||
if s == "" {
|
||||
return "x"
|
||||
}
|
||||
last := s[len(s)-1]
|
||||
if last == 'A' {
|
||||
return s[:len(s)-1] + "B"
|
||||
}
|
||||
return s[:len(s)-1] + "A"
|
||||
}
|
||||
|
||||
func TestSessionKeyDependsOnToken(t *testing.T) {
|
||||
a := sessionKey("token-a", "pw-abc")
|
||||
b := sessionKey("token-b", "pw-abc")
|
||||
if string(a) == string(b) {
|
||||
t.Fatal("sessionKey collided for different API tokens")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionKeyDependsOnWebPassword(t *testing.T) {
|
||||
a := sessionKey("token-abc", "pw-a")
|
||||
b := sessionKey("token-abc", "pw-b")
|
||||
if string(a) == string(b) {
|
||||
t.Fatal("sessionKey collided for different web passwords with the same API token")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +86,7 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r := httptest.NewRequest(http.MethodPost, "/login", nil)
|
||||
if tc.tls {
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
}
|
||||
@@ -47,18 +94,15 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
r.Header.Set("X-Forwarded-Proto", tc.forwarded)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
SetCookie(rr, r, "abc123")
|
||||
setSessionCookie(rr, r, sessionKey("token-abc", "pw-abc"))
|
||||
|
||||
cookies := rr.Result().Cookies()
|
||||
if len(cookies) != 1 {
|
||||
t.Fatalf("got %d cookies, want 1", len(cookies))
|
||||
}
|
||||
c := cookies[0]
|
||||
if c.Name != CookieName {
|
||||
t.Fatalf("cookie name = %q, want %q", c.Name, CookieName)
|
||||
}
|
||||
if c.Value != "abc123" {
|
||||
t.Fatalf("cookie value = %q, want the session id verbatim", c.Value)
|
||||
if c.Name != sessionCookieName {
|
||||
t.Fatalf("cookie name = %q, want %q", c.Name, sessionCookieName)
|
||||
}
|
||||
if !c.HttpOnly {
|
||||
t.Fatal("cookie HttpOnly = false, want true")
|
||||
@@ -72,8 +116,8 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
if c.Secure != tc.wantSecure {
|
||||
t.Fatalf("cookie Secure = %v, want %v", c.Secure, tc.wantSecure)
|
||||
}
|
||||
if c.MaxAge != int(SessionTTL/time.Second) {
|
||||
t.Fatalf("cookie MaxAge = %d, want %d", c.MaxAge, int(SessionTTL/time.Second))
|
||||
if c.MaxAge != int(sessionTTL/time.Second) {
|
||||
t.Fatalf("cookie MaxAge = %d, want %d", c.MaxAge, int(sessionTTL/time.Second))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -82,7 +126,7 @@ func TestSetSessionCookieAttributes(t *testing.T) {
|
||||
func TestClearSessionCookie(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/logout", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
ClearCookie(rr, r)
|
||||
clearSessionCookie(rr, r)
|
||||
|
||||
cookies := rr.Result().Cookies()
|
||||
if len(cookies) != 1 {
|
||||
@@ -119,70 +163,70 @@ func TestClientIP(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r := httptest.NewRequest(http.MethodPost, "/login", nil)
|
||||
r.RemoteAddr = tc.remoteAddr
|
||||
for _, v := range tc.xff {
|
||||
r.Header.Add("X-Forwarded-For", v)
|
||||
}
|
||||
if got := ClientIP(r); got != tc.want {
|
||||
t.Fatalf("ClientIP() = %q, want %q", got, tc.want)
|
||||
if got := clientIP(r); got != tc.want {
|
||||
t.Fatalf("clientIP() = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterBlocksAfterMaxFailures(t *testing.T) {
|
||||
l := NewLoginLimiter()
|
||||
l := newLoginLimiter()
|
||||
now := time.Now()
|
||||
for i := 0; i < MaxFailures; i++ {
|
||||
if wait := l.RetryAfter("1.2.3.4", now); wait != 0 {
|
||||
t.Fatalf("blocked after %d failures, want block only after %d", i, MaxFailures)
|
||||
for i := 0; i < loginMaxFailures; i++ {
|
||||
if wait := l.retryAfter("1.2.3.4", now); wait != 0 {
|
||||
t.Fatalf("blocked after %d failures, want block only after %d", i, loginMaxFailures)
|
||||
}
|
||||
l.Fail("1.2.3.4", now)
|
||||
l.fail("1.2.3.4", now)
|
||||
}
|
||||
wait := l.RetryAfter("1.2.3.4", now)
|
||||
wait := l.retryAfter("1.2.3.4", now)
|
||||
if wait <= 0 {
|
||||
t.Fatalf("retryAfter = %v after %d failures, want > 0", wait, MaxFailures)
|
||||
t.Fatalf("retryAfter = %v after %d failures, want > 0", wait, loginMaxFailures)
|
||||
}
|
||||
if wait > Window {
|
||||
t.Fatalf("retryAfter = %v, want <= %v", wait, Window)
|
||||
if wait > loginWindow {
|
||||
t.Fatalf("retryAfter = %v, want <= %v", wait, loginWindow)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterWindowExpires(t *testing.T) {
|
||||
l := NewLoginLimiter()
|
||||
l := newLoginLimiter()
|
||||
start := time.Now()
|
||||
for i := 0; i < MaxFailures; i++ {
|
||||
l.Fail("1.2.3.4", start)
|
||||
for i := 0; i < loginMaxFailures; i++ {
|
||||
l.fail("1.2.3.4", start)
|
||||
}
|
||||
if l.RetryAfter("1.2.3.4", start) == 0 {
|
||||
if l.retryAfter("1.2.3.4", start) == 0 {
|
||||
t.Fatal("expected block immediately after the failures")
|
||||
}
|
||||
later := start.Add(Window + time.Second)
|
||||
if wait := l.RetryAfter("1.2.3.4", later); wait != 0 {
|
||||
later := start.Add(loginWindow + time.Second)
|
||||
if wait := l.retryAfter("1.2.3.4", later); wait != 0 {
|
||||
t.Fatalf("retryAfter = %v once the window passed, want 0", wait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterResetClearsCounter(t *testing.T) {
|
||||
l := NewLoginLimiter()
|
||||
l := newLoginLimiter()
|
||||
now := time.Now()
|
||||
for i := 0; i < MaxFailures; i++ {
|
||||
l.Fail("1.2.3.4", now)
|
||||
for i := 0; i < loginMaxFailures; i++ {
|
||||
l.fail("1.2.3.4", now)
|
||||
}
|
||||
l.Reset("1.2.3.4")
|
||||
if wait := l.RetryAfter("1.2.3.4", now); wait != 0 {
|
||||
l.reset("1.2.3.4")
|
||||
if wait := l.retryAfter("1.2.3.4", now); wait != 0 {
|
||||
t.Fatalf("retryAfter = %v after reset, want 0", wait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginLimiterIsPerIP(t *testing.T) {
|
||||
l := NewLoginLimiter()
|
||||
l := newLoginLimiter()
|
||||
now := time.Now()
|
||||
for i := 0; i < MaxFailures; i++ {
|
||||
l.Fail("1.2.3.4", now)
|
||||
for i := 0; i < loginMaxFailures; i++ {
|
||||
l.fail("1.2.3.4", now)
|
||||
}
|
||||
if wait := l.RetryAfter("5.6.7.8", now); wait != 0 {
|
||||
if wait := l.retryAfter("5.6.7.8", now); wait != 0 {
|
||||
t.Fatalf("retryAfter for a different IP = %v, want 0", wait)
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
// htmx replaces the list on a tab switch, so re-apply to the new cards.
|
||||
document.body.addEventListener("htmx:afterSwap", applyFilter);
|
||||
document.addEventListener("bmgr:refilter", applyFilter);
|
||||
document.addEventListener("mangabm:refilter", applyFilter);
|
||||
})();
|
||||
|
||||
function setActiveTab(el) {
|
||||
@@ -58,7 +58,7 @@ function setActiveTab(el) {
|
||||
});
|
||||
// The strip is outside the swapped region, so its visibility is re-decided
|
||||
// here rather than by the server that just answered.
|
||||
document.dispatchEvent(new Event("bmgr:refilter"));
|
||||
document.dispatchEvent(new Event("mangabm:refilter"));
|
||||
}
|
||||
|
||||
// The chapter-edit form and the archive/finish/remove confirm rows are the
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user