Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 264839e798 | |||
| cbe0a28921 | |||
| e45445cb20 | |||
| 4a95657425 | |||
| 5b7adf5f2c | |||
| 00fa237151 | |||
| fe2cd12049 | |||
| c432a3be30 | |||
| 6f9109c885 | |||
| f5b52e48c8 | |||
| bf08d6e65c | |||
| d72c48295d | |||
| fc8a40cfc6 | |||
| cb2b104e63 | |||
| 134c9307b1 | |||
| 09a094ca67 | |||
| e8a3c5f826 | |||
| 14990bde21 | |||
| 503fb49d0a | |||
| e0b9063d9e | |||
| fd1131d11d | |||
| 030ffdc26e | |||
| 0a245a0dde | |||
| 249f11e1fe | |||
| f5d3fe58ec | |||
| 766aa8f00d | |||
| 550b258c59 | |||
| 3ac865cd08 | |||
| f568fb5e8c | |||
| 20fff588cc | |||
| ba679223b2 | |||
| 1e6f1e985d | |||
| 3303a55b20 | |||
| ddbd57070d | |||
| 3f53c79cf4 | |||
| 17ee0bd3f8 |
+1
-20
@@ -1,24 +1,5 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "CMD=$(python3 -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',d).get('command',''))\" 2>/dev/null || true); case \"$CMD\" in *grep*|*rg\\ *|*ripgrep*|*find\\ *|*fd\\ *|*ack\\ *|*ag\\ *) [ -f graphify-out/graph.json ] && echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"MANDATORY: graphify-out/graph.json exists. You MUST run `graphify query \\\"<question>\\\"` before grepping raw files. Only grep after graphify has oriented you, or to modify/debug specific lines.\"}}' || true ;; esac"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": "Read|Glob",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "HIT=$(python3 -c \"import json,sys;d=json.load(sys.stdin);t=d.get('tool_input',d);exts=('.py','.js','.ts','.tsx','.jsx','.astro','.vue','.svelte','.go','.rs','.java','.rb','.c','.h','.cpp','.hpp','.cc','.cs','.kt','.swift','.php','.scala','.lua','.sh','.md','.rst','.txt','.mdx');vals=[str(t.get('file_path') or ''),str(t.get('pattern') or ''),str(t.get('path') or '')];j=' '.join(vals).lower().replace(chr(92),'/');tails=[('.'+x.rsplit('.',1)[-1]) for v in vals if v for x in [v.lower().replace(chr(92),'/').rsplit('/',1)[-1]] if '.' in x];sys.stdout.write('1' if 'graphify-out/' not in j and any(tl in exts for tl in tails) else '')\" 2>/dev/null || true); if [ \"$HIT\" = 1 ] && [ -f graphify-out/graph.json ]; then echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"MANDATORY: graphify-out/graph.json exists. You MUST run graphify before reading source files. Use: `graphify query \\\"<question>\\\"` (scoped subgraph), `graphify explain \\\"<concept>\\\"`, or `graphify path \\\"<A>\\\" \\\"<B>\\\"`. Only read raw files after graphify has oriented you, or to modify/debug specific lines. This rule applies to subagents too \u2014 include it in every subagent prompt involving code exploration.\"}}'; fi || true"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
"PreToolUse": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
---
|
||||
name: gitea
|
||||
description: Use for every forge operation in this repo — read, create, comment on, label, close, or search an issue; create, review, merge, or check out a PR; and whenever `gh`, `issue://`, or `pr://` fails or a ticket number is ambiguous. This repo's forge is self-hosted Gitea driven by `tea`, not GitHub.
|
||||
---
|
||||
|
||||
# Gitea, not GitHub
|
||||
|
||||
`origin` is the self-hosted Gitea instance `gitea.violetcrown.my.id`, repo
|
||||
`sulthan/mangaBookmark`. Everything past plain git goes through
|
||||
[`tea`](https://gitea.com/gitea/tea) (0.14.2 on this machine).
|
||||
|
||||
**`gh` is not installed**, so the harness's `issue://<n>` and `pr://<n>` URIs
|
||||
error out (`GitHub CLI (gh) is not installed`, measured 2026-08-17). There is no
|
||||
fallback to add — read tickets with `tea`.
|
||||
|
||||
`tea` infers the repo from `origin`; auth lives in `tea login`, never a
|
||||
`GH_TOKEN`. Your Gitea username comes from `tea login list` — `tea` has no `@me`.
|
||||
|
||||
Flags are the environment's job: run `tea <command> --help` rather than trusting
|
||||
a remembered flag. This file carries only what `--help` will not tell you.
|
||||
|
||||
## Commands
|
||||
|
||||
| Job | Command |
|
||||
|---|---|
|
||||
| Read | `tea issue <n> --comments` / `tea pr <n> --comments` — `--comments` is not optional |
|
||||
| List | `tea issue list --state open\|closed\|all -o json --fields index,title,body,labels,state,author` |
|
||||
| Search | `tea issue list -k "<keyword>" -L "<label>" -A "<author>"` (`-K all` also searches PRs) |
|
||||
| Create | `tea issue create -t "..." -d "..."` (`-L`, `-a` optional) |
|
||||
| Comment | `tea comment <n> "..."` |
|
||||
| Label | `tea issue edit <n> --add-labels "..."` / `--remove-labels "..."` |
|
||||
| Close | `tea issue close <n>` / `tea pr close <n>` |
|
||||
| PR | `tea pr create --head <branch> --base main -t "..." -d "..."`, `tea pr checkout <n>`, `tea pr review <n>`, `tea pr merge <n>` |
|
||||
|
||||
## Traps
|
||||
|
||||
- **A bare read is a truncated read.** Without `--comments`, `tea issue <n>` and
|
||||
`tea pr <n>` print the opening body and drop every comment silently — no
|
||||
prompt, no marker, no hint that more exists (measured 2026-08-17: issue #123
|
||||
prints 40 lines bare, 132 with `--comments`). The comments are where the
|
||||
decisions live and the body is usually the stalest part of the ticket, so
|
||||
**every read that exists to understand an issue or PR passes `--comments`**,
|
||||
and understanding means body plus all comments plus whatever ticket they point
|
||||
at. Comment count is `tea issue list --fields index,comments`, so a read that
|
||||
shows fewer than that is incomplete. A PR's review comments are a second
|
||||
stream: `tea pr review-comments <n>`.
|
||||
- **One index space for issues and PRs.** A bare `#42` may be either: try
|
||||
`tea pr 42`, fall back to `tea issue 42`. Say which one you found.
|
||||
- **Output is rendered boxes**, not plain text. Anything you parse needs
|
||||
`-o json`, plus `--fields` to keep the payload small. `tea pr create` prints
|
||||
the PR URL on its last line.
|
||||
- **`close` takes no `--comment`.** Comment with `tea comment <n>`, then close.
|
||||
- **Gitea will not auto-create a label.** `tea labels list` first; missing one
|
||||
gets `tea labels create --name "..." --color "#rrggbb"` before the `edit`.
|
||||
- **Multi-line bodies go through a heredoc**, never inline escapes:
|
||||
```bash
|
||||
tea issue create -t "Title" -d "$(cat <<'EOF'
|
||||
body line one
|
||||
|
||||
- acceptance criterion
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
- **No sub-issue and no dependency command.** Gitea's API has issue
|
||||
dependencies, `tea` does not expose them, so parentage and blocking live as
|
||||
body lines — the shapes are in `docs/agents/issue-tracker.md`.
|
||||
|
||||
## Conventions this repo layers on top
|
||||
|
||||
Ticket bodies, wayfinding issues, and the PR-as-request-surface flag:
|
||||
`docs/agents/issue-tracker.md`. Triage label strings: `docs/agents/triage-labels.md`.
|
||||
A label named there still has to exist in the tracker before `--add-labels`.
|
||||
|
||||
Finish a forge action by stating the number you touched and its state after —
|
||||
"commented and closed #71" — so the write is checkable without a second query.
|
||||
@@ -11,8 +11,7 @@ to the tracker. You do not write the implementation — every line of ticket cod
|
||||
is written by a `ticket-implementer` subagent in its own git worktree. Reach for
|
||||
the editor yourself only for a merge conflict resolution.
|
||||
|
||||
Ticket source and `tea` usage: `docs/agents/issue-tracker.md`. Codebase
|
||||
questions: `graphify query "<question>"` before grepping.
|
||||
Ticket source and tracker conventions: `docs/agents/issue-tracker.md`. `tea` usage: skill `gitea`.
|
||||
|
||||
## 1. Collect the tickets
|
||||
|
||||
|
||||
+5
-18
@@ -73,26 +73,13 @@ DISCORD_REDIRECT_URI=
|
||||
# The backend re-checks each bookmarked series' newest published chapter on its
|
||||
# own schedule, so latest_chapter stays fresh even when you never open the manga
|
||||
# sites. This runs in parallel with the userscript's own in-browser check.
|
||||
# Set to 0 to turn it off entirely.
|
||||
# Set to 0 to turn it off entirely. Pace is per Site (one Poll Lane per Site,
|
||||
# issue #100) and lives in the backend registry, not here — there is nothing
|
||||
# else to configure.
|
||||
# LATEST_CHAPTER_POLL_ENABLED=1
|
||||
#
|
||||
# Two independent clocks. COOLDOWN is how long a plain-TLS series rests between
|
||||
# checks; BROWSER_COOLDOWN is the longer rest for kagane and novelfull. INTERVAL
|
||||
# is how often the poller wakes up and looks for series past their cooldowns.
|
||||
# Shortening INTERVAL cannot shorten either cooldown.
|
||||
LATEST_CHAPTER_POLL_COOLDOWN=1h # plain-TLS per series, floor 15m
|
||||
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN=6h # browser-backed per series, floor 15m
|
||||
LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
|
||||
LATEST_CHAPTER_POLL_BATCH=14 # series per wake
|
||||
LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
|
||||
#
|
||||
# Uses a ticker, not an immediate first run: the first poll happens one
|
||||
# INTERVAL after startup, not at startup. A container restarting more often
|
||||
# than INTERVAL never polls.
|
||||
#
|
||||
# BATCH x (COOLDOWN / INTERVAL) series hold the cooldown cadence — 84 with these
|
||||
# defaults. Beyond that the cadence stretches uniformly rather than breaking;
|
||||
# raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL.
|
||||
# Every Site rests an hour between checks and gaps ten seconds between fetches;
|
||||
# a Site with many Series tightens its own gap. See backend/internal/latest/sites.go.
|
||||
|
||||
# CDP endpoint of the browser, used for the two sites behind a Cloudflare
|
||||
# JavaScript challenge (kagane, novelfull) and by the web UI's kagane cover
|
||||
|
||||
@@ -5,11 +5,6 @@
|
||||
backend/server
|
||||
backend/backend
|
||||
.playwright-mcp/
|
||||
# graphify map is committed; only regenerable/local parts are ignored
|
||||
graphify-out/cost.json
|
||||
graphify-out/cache/
|
||||
graphify-out/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/
|
||||
graphify-out/.rebuild.lock
|
||||
plans/
|
||||
.scratch/
|
||||
docs/superpowers/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for OpenCode (and Claude Code) working in this repo.
|
||||
Repo-wide guidance for coding agents.
|
||||
|
||||
## What this is
|
||||
|
||||
@@ -13,16 +13,19 @@ One backend, one `bookmarks` table: a `kind` column (`manga`|`novel`) splits the
|
||||
|
||||
## Hard constraints (drive design — don't violate)
|
||||
|
||||
Nothing below is derivable from reading the code — it is why the code looks the
|
||||
way it does, plus dated measurements against services we don't control.
|
||||
|
||||
Userscript targets **Violentmonkey**, so `GM_*` APIs available, but stay GM-free where plain web APIs suffice — keeps portability across engines:
|
||||
- **Avoid `GM_*` unless needed.** Prefer page `localStorage` over `GM_setValue`/`GM_getValue`, on-page UI over `GM_registerMenuCommand`, plain `fetch()` over `GM_xmlhttpRequest` for cross-origin.
|
||||
- Cross-origin `fetch()` work **only** against CORS-enabled backend. Manga sites `https://`, so backend **must be HTTPS** (else mixed-content block).
|
||||
- Every site is its **own origin with its own `localStorage`** — a shared remote store is the only way to unify bookmarks. Cloud sync required, not optional.
|
||||
- Userscript run in **isolated world**, so embedded API token safe from site's JS.
|
||||
- Cloudflare's block on manga sites **IP-reputation-based, not universal — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — Cloudflare's bot scoring can flip previously-clean IP without notice. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test.
|
||||
- **kagane.to and novelfull.com are the exception to the above** — both sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`). When that's unset, kagane is skipped entirely (a plain fetch would only retrieve a challenge page) while novelfull pages are still attempted over plain TLS — its challenge is a live time-varying fact and its cover bytes never need the browser. The four other sites poll fine over plain TLS.
|
||||
- Cloudflare's block on manga sites is **per-zone configuration plus request fingerprint, not IP reputation — and not reliably reproducible.** Verified 2026-07-26: plain `curl` from both CGNAT dev machine *and* deployed VPS got clean 200s with real HTML on both asurascans.com and demonicscans.org (homepage, series, chapter pages) — no interactive Turnstile challenge from either IP at test time. Contradicts earlier untested assumption CGNAT dev IP blocked; wasn't, at least this date. Treat "does curl work right now" as live, time-varying fact to re-check, not fixed property of machine — a Site can turn its protection on overnight, which is exactly what comix.to did on 2026-08-12. An earlier version of this line blamed "Cloudflare's bot scoring"; that was wrong. The 1-99 bot score is Enterprise Bot Management only and does not exist for a free-plan zone, and no per-IP request rate is documented as an input to challenge issuance. Backend fetcher still needs graceful-degrade path for when challenged, and adapters should be **verified against live pages** (Playwright MCP, on-device devtools, direct probe) before finalizing, not assumed from single earlier test.
|
||||
- **kagane.to, comix.to and novelfull.com are the exception to the above** — all three sit behind a Cloudflare JavaScript challenge no TLS fingerprint clears, so the backend polls them over CDP (`BROWSER_WS_URL`). When that's unset, kagane and comix are skipped entirely (a plain fetch would only retrieve a challenge page) while novelfull pages are still attempted over plain TLS — its challenge is a live time-varying fact and its cover bytes never need the browser. comix turned hostile on 2026-08-12: its cover host `static.comix.to` is gated too, so its cover bytes go through the browser as well, and its page is read as an in-tab `fetch()` of the series URL rather than a rendered DOM — comix is an SPA, and rendering costs ~65 requests for the same server-rendered HTML one fetch returns. The three other sites poll fine over plain TLS.
|
||||
- **The CDP browser must look like a real browser, and stock headless images don't.** Measured 2026-08-08 against kagane.to, all from the same IP: `chromedp/headless-shell:stable` never cleared the challenge in 90s (`navigator.webdriver` true, empty plugin list, Chromium-branded client hints — suppressing `webdriver` alone changed nothing); `zenika/alpine-chrome` ships Chrome 124, refused outright; real Chrome with the default `--headless=new` UA never cleared, because the UA says `HeadlessChrome`; real Chrome with a stock UA **and** a non-UTC clock zone cleared in ~4s. Hence `chrome/` — a Debian image with `google-chrome-stable`, a version-derived UA, and `TZ`/`BROWSER_TZ`. Chrome reads the zone *name* through ICU from `/etc/localtime`'s symlink target, ignoring the file's contents, so mounting the host's `/etc/localtime` does **not** work; `/etc/timezone` is mounted instead.
|
||||
- **The browser is not in the API stack and must not be put back.** It's its own compose unit (`chrome/docker-compose.yml`) on a second machine, reached over the tailnet — it held 471 MiB on a 1974 MiB swapless VPS, and a residential egress scores better with Cloudflare anyway (ADR-0006). Consequences that constrain code: `BROWSER_WS_URL` must be a tailnet **IP** (a MagicDNS name 500s at `/json/version`, same trap as the old Docker service name); the CDP port binds to the tailnet address only, since CDP authenticates nothing and that host has a real LAN; and the browser is on-demand (ADR-0005), so an unreachable or asleep one must degrade exactly as an unset `BROWSER_WS_URL` — plain-TLS libraries unaffected, kagane/novelfull logged and skipped, stored covers still served. Never add `chromedp.NoModifyURL`: discovery per fetch is what makes a restarted Chrome invisible.
|
||||
- **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, so Cloudflare scores it as one; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one.
|
||||
- **The browser is not in the API stack and must not be put back.** It's its own compose unit (`chrome/docker-compose.yml`) on a second machine, reached over the tailnet — it held 471 MiB on a 1974 MiB swapless VPS, and a residential egress avoids the cloud-hosting-IP signature Bot Fight Mode documentedly challenges (not a better "score" — free-plan zones have no score). Consequences that constrain code: `BROWSER_WS_URL` must be a tailnet **IP** (a MagicDNS name 500s at `/json/version`, same trap as the old Docker service name); the CDP port binds to the tailnet address only, since CDP authenticates nothing and that host has a real LAN; and the browser is on-demand, so an unreachable or asleep one must degrade exactly as an unset `BROWSER_WS_URL` — plain-TLS libraries unaffected, kagane/comix logged and skipped, stored covers still served. Never add `chromedp.NoModifyURL`: discovery per fetch is what makes a restarted Chrome invisible.
|
||||
- **UTC is the tell, not a country mismatch.** A UTC clock is the datacenter default, and the challenge refuses it; any real zone clears. Measured 2026-08-08, identical container, one Indonesian egress IP: UTC never cleared in 60s (twice), while `Asia/Jakarta` **and** `America/New_York` both cleared in 4s. An earlier note here claimed the zone had to match the egress IP's country — that was wrong, inferred from the host clock (`Asia/Bangkok`) rather than the measured egress. A second earlier claim, that Cloudflare "scores" a UTC clock, was also wrong: the measurement is real but the mechanism is not documented anywhere — Cloudflare publishes no timezone signal, and free-plan zones carry no score at all. `BROWSER_TZ` therefore needs a plausible zone, not a geolocated one.
|
||||
- **A challenged page needs the tab kept open.** The interstitial takes seconds to solve and only then writes clearance into the browser's shared cookie jar. Navigate-read-close never clears anything; `BrowserFetcher.run` holds one tab and re-reads until the payload arrives.
|
||||
|
||||
## Architecture
|
||||
@@ -36,7 +39,7 @@ Two Violentmonkey userscripts (isolated world, per-site adapters, localStorage c
|
||||
on-demand Chrome, separate machine (chrome/)
|
||||
```
|
||||
|
||||
Two deployable units on two machines: the API stack (`docker-compose.yml` + `docker-compose.prod.yml`, on the VPS) and the browser (`chrome/docker-compose.yml`, on the home machine). They share nothing but `BROWSER_WS_URL` and update independently. Backend-specific architecture (packages, endpoints, poller, config env vars) lives in `backend/AGENTS.md`. Userscript-specific structure (adapters, retry queue, UI, live URL shapes) lives in `userscript/AGENTS.md`. Deploy order `DEPLOY.md` (§7 for the browser), redeploy `REDEPLOY.md` (§8 for the browser).
|
||||
Two deployable units on two machines: the API stack (`docker-compose.yml` + `docker-compose.prod.yml`, on the VPS) and the browser (`chrome/docker-compose.yml`, on the home machine). They share nothing but `BROWSER_WS_URL` and update independently. Backend-specific detail lives in `backend/AGENTS.md`, userscript-specific detail in `userscript/AGENTS.md`.
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -45,39 +48,41 @@ Backend (`cd backend`):
|
||||
- Single test: `go test -run TestName ./...`
|
||||
- Build static binary: `CGO_ENABLED=0 go build`
|
||||
|
||||
Local stack: `docker compose up` (bookmark-api + postgres only; `postgres-data` named volume, `restart: unless-stopped`). No browser — without `BROWSER_WS_URL` the poller logs and skips kagane and novelfull. To run one: `cd chrome && BROWSER_BIND_ADDR=172.17.0.1 docker compose up -d --build`, then `BROWSER_WS_URL=ws://172.17.0.1:9222` in the root `.env` (bridge gateway, so the API container can name it by IP).
|
||||
Local stack: `docker compose up` (bookmark-api + postgres only; `postgres-data` named volume, `restart: unless-stopped`). No browser — without `BROWSER_WS_URL` the poller logs and skips kagane and comix. To run one: `cd chrome && BROWSER_BIND_ADDR=172.17.0.1 docker compose up -d --build`, then `BROWSER_WS_URL=ws://172.17.0.1:9222` in the root `.env` (bridge gateway, so the API container can name it by IP).
|
||||
|
||||
Live CDP proof (needs that browser and network, skipped otherwise):
|
||||
`SMOKE_BROWSER_WS_URL=ws://<ip>:<port> go test -run TestSmokeKagane ./internal/latest`
|
||||
— fetches a real kagane cover and chapter list. A red run means the challenge is
|
||||
`SMOKE_BROWSER_WS_URL=ws://<ip>:<port> go test -run 'TestSmokeKagane|TestSmokeComix' ./internal/latest`
|
||||
— fetches a real kagane and comix cover and chapter list. A red run means the challenge is
|
||||
not clearing from this IP, which is a live fact to re-check, not necessarily a defect.
|
||||
A red `TestSmokeComix` reporting `ERR_CERT_COMMON_NAME_INVALID` is not the
|
||||
challenge: it means the resolver the browser container uses hijacks `comix.to`.
|
||||
Observed 2026-08-16 on one Indonesian ISP, which CNAMEs it to a block page
|
||||
(`aduankonten.id`). Check with `docker exec <browser> getent hosts comix.to`,
|
||||
and if it is hijacked, run the container with
|
||||
`--add-host comix.to:<ip> --add-host static.comix.to:<ip>` from a DoH lookup
|
||||
(`curl -H 'accept: application/dns-json' 'https://1.1.1.1/dns-query?name=comix.to&type=A'`).
|
||||
Machine-local, so don't put those hosts in `chrome/docker-compose.yml`.
|
||||
|
||||
Smoke test: `curl` endpoints with `Authorization: Bearer <token>`; confirm `OPTIONS` preflight return CORS headers and `/healthz` return 200.
|
||||
|
||||
## Forge: Gitea, not GitHub
|
||||
|
||||
`origin` is self-hosted Gitea instance (`gitea.violetcrown.my.id`), so **`gh` don't work here — use `tea` (Gitea CLI) for anything past plain git.** Common ones:
|
||||
|
||||
- Open PR: `tea pr create --head <branch> --base main --title "..." --description "..."`
|
||||
- List / view / check out: `tea pr list`, `tea pr <n>`, `tea pr checkout <n>`
|
||||
- Issues: `tea issue create`, `tea issue list`
|
||||
- Auth lives in `tea login`, not `GH_TOKEN` env var.
|
||||
|
||||
`tea` print output as rendered boxes rather than plain text; PR URL lands on last line.
|
||||
`origin` is self-hosted Gitea (`gitea.violetcrown.my.id`, repo `sulthan/mangaBookmark`), so **`gh` don't work here and the `issue://`/`pr://` URIs error out — drive the forge with `tea`.** How to run it — commands, traps, JSON output: skill `gitea`. Tracker conventions (ticket bodies, wayfinding, PR-as-request-surface flag): `docs/agents/issue-tracker.md`. Triage label strings: `docs/agents/triage-labels.md`.
|
||||
|
||||
## Design system
|
||||
|
||||
Web UI + userscript panel follow **Cinder**, rules in `docs/design-system.md`
|
||||
— source of truth Claude Design project `BookmarkManager Web UI`
|
||||
(`969ac210-fe02-4c01-ae1b-9a271dcc779a`). Read it before touching
|
||||
`backend/internal/web/static/style.css`, `backend/internal/web/templates/*`, or userscript
|
||||
`TEMPLATE`/`CSS`. Core law: **ember means new chapter only** — no other
|
||||
state (busy, error, destruction) may use `--ember`; destruction gets
|
||||
`--danger`. No cards/corners/shadows, one `--measure: 760px` column, tokens
|
||||
only (never hardcode hex outside `:root`), both colour branches touched
|
||||
together. Any move that pulls series out of list (archive/finish/remove)
|
||||
must be confirm-gated via its own `.confirm-row`; only restore fires
|
||||
instantly.
|
||||
Web UI + userscript panel follow **Cinder**. Tokens are the `:root` block in
|
||||
`backend/internal/web/static/style.css`; that file, `backend/internal/web/templates/*`,
|
||||
and the userscript `TEMPLATE`/`CSS` are the only places it is expressed.
|
||||
Source of truth for the visual language is the Claude Design project
|
||||
`BookmarkManager Web UI` (`969ac210-fe02-4c01-ae1b-9a271dcc779a`).
|
||||
|
||||
Core law: **ember means new chapter only** — no other state (busy, error,
|
||||
destruction) may use `--ember`; destruction gets `--danger`. No
|
||||
cards/corners/shadows, one `--measure: 760px` column, tokens only (never
|
||||
hardcode hex outside `:root`), both colour branches touched together. Any move
|
||||
that pulls a series out of the list (archive/finish/remove) must be
|
||||
confirm-gated via its own `.confirm-row`; only restore fires instantly.
|
||||
|
||||
## Security invariants
|
||||
|
||||
@@ -119,13 +124,19 @@ Review gate: auth, CORS, session, crypto, and the fetch gate are security-critic
|
||||
## Comments
|
||||
|
||||
Comment only if code alone can't carry info. Cost per read — must earn spot.
|
||||
Wrong comment worse than none: it misleads readers and measurably degrades
|
||||
LLM performance on the file. Missing comment costs little. Bias to fewer.
|
||||
|
||||
Write for:
|
||||
- Why not what. Tradeoffs, non-obvious decisions.
|
||||
- Load-bearing detail looking incidental — say so if "simplify" breaks it.
|
||||
Docstring on public/exported surface — exception, near-always worth it.
|
||||
Contract only: what it takes, returns, throws, mutates; units; pre/post
|
||||
conditions. Not a restatement of the body. Skip on private/obvious.
|
||||
|
||||
Inline — write for:
|
||||
- Why not what. Tradeoffs, non-obvious decisions, rejected alternatives.
|
||||
- heavy detail looking incidental — say so if "simplify" breaks it.
|
||||
- Non-local consequence, invisible from function alone.
|
||||
- Wire format / encoding / interface contract — save callers re-deriving.
|
||||
- Gotcha/workaround, with ref if exists.
|
||||
- Wire format / encoding / ordering / invariant — save callers re-deriving.
|
||||
- Gotcha/workaround, with ref (issue, RFC, vendor bug) if exists.
|
||||
- Domain/business rule not derivable from code.
|
||||
|
||||
Skip:
|
||||
@@ -134,34 +145,49 @@ Skip:
|
||||
- Banners, dividers, `// helpers`.
|
||||
- Change narration (`// fix bug`, `// as requested`, `// new impl`) — git's job.
|
||||
- Commented-out code — delete.
|
||||
- TODO without concrete action.
|
||||
- TODO without concrete action + owner.
|
||||
- Narrating the plan you just reasoned through. Plan in prose or in your head;
|
||||
ship the code, not the transcript.
|
||||
- Anything restating a name that could be fixed by renaming instead.
|
||||
|
||||
Style: one dense comment over function beats one per line inside. Tight, no worked example unless bug subtle. Wrong comment worse than none — update/delete on change. Default fewer — sparse+high-signal beats comprehensive.
|
||||
Staleness filter: if the comment describes something likely to change
|
||||
independently of this line, it will rot and start lying. Either anchor it to
|
||||
something stable, assert it in a test, or leave it out.
|
||||
|
||||
Test: "competent reader get this from code in few sec?" Yes → skip. Needs detour through another file/spec/git-blame → write it.
|
||||
Style: one dense comment over a function beats one per line inside. Tight; no
|
||||
worked example unless the bug is subtle. On edit, update or delete stale
|
||||
comments in the code you touch — silence beats a lie.
|
||||
|
||||
## Agent skills
|
||||
Test: "competent reader get this from code in a few sec?" Yes → skip.
|
||||
Needs detour through another file/spec/git-blame/external doc → write it.
|
||||
|
||||
`AGENTS.md` is the single source of truth for agent guidance; every `CLAUDE.md` in this repo is a symlink to the `AGENTS.md` beside it. Edit `AGENTS.md`.
|
||||
## Writing an AGENTS.md
|
||||
|
||||
### Issue tracker
|
||||
`AGENTS.md` is the single source of truth for agent guidance; every `CLAUDE.md`
|
||||
in this repo is a symlink to the `AGENTS.md` beside it. Edit `AGENTS.md`.
|
||||
|
||||
Issues live as Gitea issues on `gitea.violetcrown.my.id` (`sulthan/mangaBookmark`), driven by the `tea` CLI — not `gh`. See `docs/agents/issue-tracker.md`.
|
||||
**Cite code, never docs, issues, or plans.** A spec, ADR, plan file, or Gitea
|
||||
issue records what was true when it was written and then goes stale silently;
|
||||
an agent that follows the pointer reads a decision that may already have been
|
||||
reversed. Code is the only source true at read time — cite a package, file,
|
||||
symbol, env var, or route. The sole non-code exception is a sibling
|
||||
`AGENTS.md`. If a doc holds a fact an agent needs, restate the fact here rather
|
||||
than linking to it.
|
||||
|
||||
### Triage labels
|
||||
**State a fact in prose only if the code cannot answer it.** Split by
|
||||
derivability:
|
||||
|
||||
Default five-role vocabulary, label strings unchanged (`needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`). See `docs/agents/triage-labels.md`.
|
||||
- *Structure* — packages, routes, env vars, columns, struct fields. Rots fast,
|
||||
cheap to re-read. **Name the symbol, write nothing else.**
|
||||
- *Mechanism* — what a function does, how a flow proceeds. **Name the symbol
|
||||
plus at most one line of orientation.**
|
||||
- *Rationale* — why it is this way, what a "simplify" would break, what was
|
||||
tried and rejected. Not in the code and cannot be re-derived. **Write it out.**
|
||||
- *Measurement* — an observation against something we don't control. **Write it
|
||||
out with the date**; a dated fact is honest, an undated one pretends to be
|
||||
permanent.
|
||||
|
||||
### Domain docs
|
||||
|
||||
Single-context: one root `CONTEXT.md` plus `docs/adr/`, both created lazily. See `docs/agents/domain.md`.
|
||||
|
||||
## graphify
|
||||
|
||||
Project has knowledge graph at graphify-out/ with god nodes, community structure, cross-file relationships.
|
||||
|
||||
Rules:
|
||||
- For codebase questions and exploration, always first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. Return scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
|
||||
- If graphify-out/wiki/index.md exists, use for broad navigation instead of raw source browsing.
|
||||
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain don't surface enough context.
|
||||
- After modifying code, run `graphify update .` to keep graph current (AST-only, no API cost).
|
||||
Restating mechanism in prose is how these files rot: the code changes, the
|
||||
paragraph doesn't, and the next agent trusts the paragraph. A pointer degrades
|
||||
more honestly — and every symbol you name must actually exist, since a dead
|
||||
pointer is a bug, not a stale sentence.
|
||||
|
||||
+74
-10
@@ -43,6 +43,13 @@ Readers: Progress, Favourite, Lifecycle bucket. Facts about the Series itself be
|
||||
to the Series, not here.
|
||||
_Avoid_: entry, item, record, subscription
|
||||
|
||||
**Orphan Series**:
|
||||
A Series no Reader bookmarks. Removing a Bookmark never removes the Series, so the row
|
||||
outlives every relationship to it: nothing reads it, no Poll visits it, and it still owns
|
||||
a Cover. A state of the Series, not a Lifecycle bucket — it says how many Readers hold it,
|
||||
never anything about a Reader.
|
||||
_Avoid_: dangling, unused, dead series, stale
|
||||
|
||||
**Library**:
|
||||
One of the two halves of the collection — manga or novel — selected by a Bookmark's
|
||||
`kind`. The web UI and the userscripts each address exactly one Library at a time.
|
||||
@@ -56,12 +63,12 @@ is real activity, so only Progress reorders the list.
|
||||
_Avoid_: position, bookmark (the noun is taken), last read
|
||||
|
||||
**Latest Chapter**:
|
||||
The highest-numbered chapter a Site has published for a Series, discovered without the
|
||||
reader present. The number is what ranks it, never a date and never the Site's own
|
||||
"newest chapter" banner — where a Site disagrees with itself, its list of chapters is
|
||||
the record and its summary of that list is not. Distinct from Progress in every way
|
||||
that matters: it is a fact about the Site, not about the reader, and it must never
|
||||
reorder the list.
|
||||
The highest-numbered chapter a Site has published for a Series. The number is what
|
||||
ranks it, never a date and never the Site's own "newest chapter" banner — where a Site
|
||||
disagrees with itself, its list of chapters is the record and its summary of that list
|
||||
is not. Established by a Poll and, between Polls, by a Sighting. Distinct from Progress
|
||||
in every way that matters: it is a fact about the Site, not about the reader, and it
|
||||
must never reorder the list.
|
||||
_Avoid_: newest, current chapter, update
|
||||
|
||||
**Poll**:
|
||||
@@ -70,15 +77,72 @@ Reader present. Performed once per Series no matter how many Readers bookmarked
|
||||
a Poll is work done on behalf of the Series, never on behalf of a Reader.
|
||||
_Avoid_: scrape, refresh, check, sync
|
||||
|
||||
**Poll Lane**:
|
||||
One Site's own stream of Polls, carrying the pace at which that Site is willing to be
|
||||
asked. Every Site has exactly one and no Lane can slow, block or borrow from another's;
|
||||
a Reader never has one and never influences one.
|
||||
_Avoid_: worker, queue, scheduler, batch, wave
|
||||
|
||||
**Lane Pass**:
|
||||
One sweep of a Poll Lane over the Series due on its Site: what it found waiting, how many it
|
||||
read, and whether it declined to work at all. A fact about the Lane rather than about any
|
||||
Series — a pass that read nothing is still a pass, and one that declined carries the reason it
|
||||
declined, since a Lane resting and a Lane stuck look identical from a count alone. Its record
|
||||
outlives the process that made it: "the poller has done nothing for six hours" is only
|
||||
answerable by something written down.
|
||||
_Avoid_: run, cycle, tick, batch, poll history
|
||||
|
||||
**Forced Poll**:
|
||||
A Poll the owner asks for by hand instead of waiting for the Series's turn. It jumps its
|
||||
Lane's queue and ignores every waiting rule — the rest between Polls, a Sighting standing
|
||||
in for a check, a Series only finished Readers hold — but never overrules a Site that is
|
||||
refusing us, the Lane's spacing between fetches, or a Series with no page to fetch. Asked
|
||||
for by marking the Series, never by commanding the poller, so it happens on the Lane's
|
||||
next pass rather than at the moment of asking.
|
||||
It also takes whatever Cover the Site publishes today: asking for one is asking to accept the
|
||||
page as it now stands, so it is the only read after Acquisition that can replace a Cover.
|
||||
_Avoid_: manual poll, refresh, retry, force refresh
|
||||
|
||||
**Paused Lane**:
|
||||
A Poll Lane the owner has stopped for a bounded time. It makes no Polls until the pause
|
||||
expires, so its Series stay due and unstamped exactly as they do when a Site cannot be
|
||||
reached. Every pause carries an expiry — a Lane cannot be stopped indefinitely — and it
|
||||
outlives a restart, being a fact about the Site rather than about the running process.
|
||||
_Avoid_: disabled, off, stopped, suspended, kill switch (that is the deploy-time switch)
|
||||
|
||||
**Stall**:
|
||||
A Poll Lane that owed Polls, made none, and has nothing to say for it. Distinct from the
|
||||
two conditions it resembles: a Site that refuses is exercising the pace it is entitled to,
|
||||
and a Lane the owner paused was told to stop — a Stall is neither asked for nor explained.
|
||||
It is the one fault no Reader surface can show: every Bookmark still opens, Progress still
|
||||
syncs, and Latest Chapter is quietly wrong for as long as it lasts.
|
||||
_Avoid_: outage, downtime, failure, backlog, lag
|
||||
|
||||
**Sighting**:
|
||||
What a Reader's browser happened to see of a Series's Latest Chapter while that Reader
|
||||
was on the page. It reports the same fact as a Poll but carries none of its authority:
|
||||
a Poll always overrules it, and only a Sighting on a Series no Reader else holds may
|
||||
defer one. A Sighting a later Poll contradicts downwards is a false Sighting, and
|
||||
enough of those cost the Reader the right to defer at all.
|
||||
_Avoid_: client report, user poll, observation, claim
|
||||
|
||||
**Acquisition**:
|
||||
The single read of a Series page made the moment the Series first exists, giving it
|
||||
both its Latest Chapter and its Cover without waiting out the Poll queue. Distinct
|
||||
both its Latest Chapter and its Cover without waiting for the Lane's pace. Distinct
|
||||
from a Poll in the two ways that matter: a Reader is present — it is triggered by
|
||||
their first Bookmark of that Series — and it is the only read that establishes a
|
||||
Cover rather than refreshing facts. It happens once in a Series's life; every later
|
||||
read of the same page is a Poll.
|
||||
their first Bookmark of that Series — and it establishes a Cover rather than refreshing
|
||||
facts, which no Poll does unless the owner forces one. It happens once in a Series's
|
||||
life; every later read of the same page is a Poll.
|
||||
_Avoid_: initial poll, first fetch, prefetch, warm-up
|
||||
|
||||
**Correction**:
|
||||
A Latest Chapter the owner sets by hand, on a Series no Poll can read. It reports the
|
||||
same fact as a Poll and carries even less authority than a Sighting: the next Poll
|
||||
overwrites it, so does any Reader's Sighting, and it is never a floor or a pin. It
|
||||
exists only because the Site page is unreadable — where a Poll can read the page, the
|
||||
Poll is the answer and a Correction is not wanted.
|
||||
_Avoid_: override, pin, manual value, fix
|
||||
|
||||
**New Chapter**:
|
||||
The state where Latest Chapter is ahead of Progress. The single condition the ember
|
||||
accent is permitted to signal.
|
||||
|
||||
@@ -93,6 +93,14 @@ the backend dials but not the password the database expects, and `bookmark-api`
|
||||
crash-loops on `password authentication failed`. Set it before §2 and leave it
|
||||
alone.
|
||||
|
||||
An `.env` written before issue #100 carries the old poll-pace names
|
||||
(`LATEST_CHAPTER_POLL_COOLDOWN`, `_BROWSER_COOLDOWN`, `_INTERVAL`, `_BATCH`,
|
||||
`_STAGGER`). All five are dead configuration now — the pace lives in the Site
|
||||
registry (`backend/internal/latest/sites.go`), so **delete those lines** and
|
||||
keep only the kill switch `LATEST_CHAPTER_POLL_ENABLED`. Leaving them behind
|
||||
is harmless (nothing reads them) but silently misleads the next person who
|
||||
edits the file.
|
||||
|
||||
> Match `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER` to your Traefik's actual
|
||||
> names (check your Traefik static config — common alternatives: `https`,
|
||||
> `myresolver`, `cloudflare`). Wrong names = no certificate issued.
|
||||
@@ -275,7 +283,8 @@ copy immediately — reinstall on all devices, or they silently stop syncing.
|
||||
Kagane and novelfull sit behind a Cloudflare JavaScript challenge no TLS
|
||||
fingerprint clears, so the poller reaches them through a real Chrome over CDP.
|
||||
That browser does **not** run on the VPS: it held 471 MiB of a 1974 MiB box
|
||||
with no swap, and it scores better from a residential IP anyway (ADR-0006). It
|
||||
with no swap, and a residential IP avoids the cloud-hosting-IP signature Bot
|
||||
Fight Mode challenges anyway (ADR-0006). It
|
||||
is its own compose unit, deployed and updated independently of everything
|
||||
above.
|
||||
|
||||
@@ -454,7 +463,8 @@ SMOKE_BROWSER_WS_URL=ws://100.x.y.z:9222 go test -run TestSmokeKagane ./internal
|
||||
```
|
||||
|
||||
A red run means "not clearing from this address right now", which is a live
|
||||
fact to re-check before it is a defect — Cloudflare's scoring moves. Then, from
|
||||
fact to re-check before it is a defect — a Site's Cloudflare settings, and the
|
||||
fingerprint this Chrome presents after an update, both move. Then, from
|
||||
the web UI, open a bookmarked kagane series and confirm the cover renders. Once
|
||||
a cover is stored it is served from Postgres forever after, so the browser being
|
||||
asleep, unreachable, or mid-power-outage costs chapter freshness and nothing
|
||||
@@ -503,7 +513,7 @@ picks up a restarted Chrome's new debugger UUID by itself.
|
||||
| kagane rows never get a `latest_chapter`; log says `browser fetcher disabled` or nothing at all | `BROWSER_WS_URL` unset. Expected before §7 is done. |
|
||||
| kagane polls all fail; log shows a 500 from `/json/version` | `BROWSER_WS_URL` names a MagicDNS hostname (or any name). Chrome's DevTools handler only accepts an IP or `localhost` — use the tailnet IP. |
|
||||
| kagane polls fail with a connection error | Home machine off, off the tailnet, or the unit is down. `tailscale ping <machine>`, then `docker compose ps` in its `chrome/`. Costs freshness only; stored covers keep serving. |
|
||||
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series (up to `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, default 6h). |
|
||||
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series. |
|
||||
| `compose` in `chrome/` errors `set BROWSER_BIND_ADDR to this machine's tailnet IP` | No `chrome/.env`, or the variable is empty. Deliberate — it has no default so an unset value cannot publish CDP to the LAN. |
|
||||
| browser container restarts, or is OOM-killed | `docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'`. The 512 MiB cap is sized against a measured 645 MiB untuned peak; a real breach is a Chrome regression worth reading `docker logs` for, not a number to raise reflexively. |
|
||||
|
||||
|
||||
@@ -53,12 +53,7 @@ covers are stored, so the library renders in full with the browser switched off.
|
||||
| `DISCORD_API_BASE` | `https://discord.com/api/v10` | Test seam — tests point it at a local stub so the real token exchange runs. |
|
||||
| `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. |
|
||||
| `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. |
|
||||
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. |
|
||||
| `LATEST_CHAPTER_POLL_COOLDOWN` | `1h` | Rest between checks of one plain-TLS series; floor `15m`. |
|
||||
| `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN` | `6h` | Rest between checks of one browser-backed series; floor `15m`. |
|
||||
| `LATEST_CHAPTER_POLL_INTERVAL` | `10m` | How often the poller wakes. Cannot shorten either cooldown. |
|
||||
| `LATEST_CHAPTER_POLL_BATCH` | `14` | Series per wake. Keep `BATCH × STAGGER` under `INTERVAL`. |
|
||||
| `LATEST_CHAPTER_POLL_STAGGER` | `20s` | Delay between fetches in a batch — this is the outbound request rate. |
|
||||
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. Pace is per Site in the registry — one Poll Lane per Site, each with its own rest and gap (issue #100) — so no other knobs exist. |
|
||||
|
||||
Compose reads a few more from the same `.env` that the backend never sees:
|
||||
`POSTGRES_PASSWORD` (required — `DATABASE_URL` is built from it, and Postgres
|
||||
|
||||
+2
-2
@@ -436,8 +436,8 @@ free -m # the Gitea runner should still have its headroom
|
||||
|
||||
Nothing here needs doing during an API redeploy. The API stack does not
|
||||
`depends_on` the browser, and an unreachable one degrades exactly as an unset
|
||||
`BROWSER_WS_URL`: plain-TLS libraries unaffected, kagane and novelfull logged
|
||||
and skipped, stored covers still served.
|
||||
`BROWSER_WS_URL`: plain-TLS libraries unaffected, kagane and comix logged
|
||||
and skipped, novelfull attempted over plain TLS, stored covers still served.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+267
-184
@@ -1,186 +1,269 @@
|
||||
Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGENTS.md` for the project-wide architecture diagram, hard constraints, and design system.
|
||||
Scope: `backend/`.
|
||||
|
||||
- **Backend** (`backend/`): stdlib `net/http` (handful routes, no framework) + Postgres over `jackc/pgx/v5` (pure Go, `CGO_ENABLED=0` -> static binary -> distroless/scratch image). Reverse proxy terminates TLS; Go service listens plain `:8080`.
|
||||
Single binary, split into packages under `backend/internal/`: `store`
|
||||
(Bookmark type, Postgres persistence, migration runner), `latest` (background
|
||||
poller, site parsers, TLS fetcher), `session` (cookie signing, login
|
||||
rate limiter), `httpmw` (Auth/Gzip/CORS middleware), `api` (JSON
|
||||
bookmark handlers), `userscript` (userscript-serving handler), `web`
|
||||
(browser UI handler + `templates/` + `static/`, `go:embed`-ed).
|
||||
`backend/main.go` is the composition root — the only place that wires
|
||||
packages together into `newRouter`. Root-level `*_test.go` hold
|
||||
integration tests that exercise the full router; unit tests for a
|
||||
package live beside it under `internal/`.
|
||||
- **Schema is migration-owned.** `internal/store/migrations/*.sql` is
|
||||
`go:embed`-ed and applied on every start by `store.migrate`: one numbered
|
||||
file per change, one transaction each, versions recorded in
|
||||
`schema_migrations`. Files are **append-only** — editing an applied one
|
||||
changes nothing on a database that already ran it. No column probing, no
|
||||
data-fixup migrations: both were SQLite-era machinery and are gone.
|
||||
- **Tests need Docker.** `internal/pgtest` starts one `postgres:17-alpine`
|
||||
container per test binary (`TestMain` -> `pgtest.Main`) and hands each test
|
||||
its own database (`pgtest.URL(t)`). A package whose tests touch the store
|
||||
must have that `TestMain`.
|
||||
- **Reader-owned store, four tables.** `readers` is keyed by Discord user ID
|
||||
and carries the SHA-256 of the Reader's userscript credential plus a
|
||||
`token_epoch` (issue #24). Credentials are derived, never stored: `token.Token(TOKEN_KEY, discord_id, epoch)` (HMAC, `internal/token`), and only its SHA-256 sits in `readers.token_sha256`, so install URLs can be rebuilt after any restart while a database leak yields nothing but hashes. The seed creates the **owner** row at startup; its epoch-0 hash is refreshed on every start **only while the row has never been rotated**, so a restart can never resurrect a rotated-away credential. Every other row is created by that Reader's own first login (`Store.EnsureReader`, idempotent on `discord_id`, and it never rewrites an existing row's hash). Rotation is `Store.RotateToken` (epoch bump + hash rewrite in one transaction), driven by the web UI.
|
||||
`series` keyed `(site, series_id)`
|
||||
(`asura`|`demonic`|`comix`|`kagane`|`novelfull`|`lightnovelworld`) owns the
|
||||
shared facts — title, cover, canonical URL, `kind` (`manga`|`novel`),
|
||||
Latest Chapter, `latest_checked_at` — and `bookmarks` holds only what
|
||||
differs between readers: progress, favourite, lifecycle bucket,
|
||||
`updated_at`. A bookmark is keyed `(reader_id, site, series_id)` — no
|
||||
surrogate id; the wire `key` is derived as `site:series_id` on read — and
|
||||
every store read/write is scoped to the reader it names. Auth resolves the
|
||||
acting Reader from the presented credential (`httpmw.Auth`) and nothing
|
||||
else — there is no unauthenticated-by-Reader route and no global token; the
|
||||
reader id travels in the request context. Sync **last-write-wins**; the wire format
|
||||
stays flat (ADR-0004). `Store.Upsert` decomposes one flat body across two
|
||||
tables and enforces the ownership rule: client `title`/`series_url`/`cover`
|
||||
are written only when the series row is new (ADR-0003).
|
||||
- **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth).
|
||||
- **Web UI:** same binary serve the browser UI on a second
|
||||
hostname — `GET /` (list, or login page when no session),
|
||||
`GET /auth/discord` + `GET /auth/discord/callback` (Discord OAuth,
|
||||
ADR-0002), `POST /logout`, `GET /static/*`, htmx fragment endpoints
|
||||
under `/ui/*`. Templates + assets `go:embed`-ed under
|
||||
`backend/internal/web/`, so `backend/Dockerfile` must copy the whole
|
||||
`internal/` tree, not just `*.go`. Sessions are rows in the `sessions`
|
||||
table: the cookie carries only an opaque id, looked up (and expiry-
|
||||
checked) on every request, and deleting the row revokes the session.
|
||||
Guild membership *is* registration (issue #27): `discordCallback` gates on
|
||||
membership (and `DISCORD_REQUIRED_ROLE` when set) and then calls
|
||||
`Store.EnsureReader`, so a refusal creates nothing and a returning Reader
|
||||
reuses their row. The owner is the only Reader with administrative reach:
|
||||
`POST /readers/{id}/revoke` (404 for anyone else) drops that Reader's
|
||||
sessions, and the `readers` panel renders only on the owner's page.
|
||||
A Reader with no bookmarks at all sees `listView.Fresh`, whose empty state
|
||||
offers both install links instead of describing a filter.
|
||||
UI mutations read-modify-write
|
||||
through `Store.Get` + `Store.Upsert` so `updated_at` rule stays one
|
||||
place. See `docs/superpowers/specs/2026-07-25-web-ui-design.md`.
|
||||
**Design-tool caveat:** templates link `/static/style.css` root-absolutely
|
||||
(correct — served from `/`), but impeccable detector resolves
|
||||
stylesheet href with `path.resolve(fileDir, href)`, drops directory
|
||||
on leading `/` and silently skip file. Relative href don't help
|
||||
either: template's directory isn't its served path. So
|
||||
`detect.mjs backend/internal/web/templates` reports **false clean** —
|
||||
always pass `backend/internal/web/static` too. One finding there,
|
||||
`overused-font` on "Instrument Serif", deliberate identity choice, not debt.
|
||||
- **Every action that moves series out of list is confirm-gated.**
|
||||
Archive, finish, remove each open own `.confirm-row` disclosure
|
||||
(`toggleConfirmRow(key, kind)` in `filter.js`, `kind` ∈
|
||||
`archive|finish|remove`); restore fire instantly since it's the reversal.
|
||||
Remove's row wear ember wash, two reversible ones wear `.calm` grey.
|
||||
`--ember` stay reserved for new-chapter signal: busy bar and inline
|
||||
error use `--mute`.
|
||||
- **Latest-chapter poller:** ticker goroutine in same binary re-check
|
||||
each bookmarked series' newest published chapter from backend's own
|
||||
network access, so `latest_chapter` stay fresh when user not
|
||||
browsing. Second, parallel signal — userscript keep own
|
||||
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged.
|
||||
Two independent clocks: per-series cooldown (`series.latest_checked_at`,
|
||||
enforced by `Store.DueForLatestCheck`'s WHERE clause) and wake interval.
|
||||
The poller walks **Series, not Bookmarks** — a series referenced by several
|
||||
bookmarks is fetched once per cycle, and the due queue orders
|
||||
`reader_count DESC, latest_checked_at ASC` (ADR-0003). Series row stamped
|
||||
*before* fetch so broken series wait out full cooldown instead of retrying
|
||||
every tick; found chapter written straight to the series row via
|
||||
`Store.SetLatestChapter`, so a bookmark's `updated_at` — and the list
|
||||
order — is never touched.
|
||||
Fetches use `bogdanfinn/tls-client` with Chrome profile as defence in depth
|
||||
against fingerprint-based blocking; any failure log and skip. kagane and
|
||||
novelfull sit behind Cloudflare JavaScript challenges the TLS client can't
|
||||
clear, so they are fetched over CDP via `BROWSER_WS_URL`; kagane is simply
|
||||
not polled when that's unset, while novelfull falls back to a plain-TLS
|
||||
attempt — its challenge is a live time-varying fact, and its cover bytes
|
||||
never need the browser. See
|
||||
`docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`.
|
||||
The poller's series write is a single-column UPDATE
|
||||
(`Store.SetLatestChapter`), not a read-modify-write of the whole bookmark:
|
||||
it cannot revert read progress or move `updated_at`, so the old
|
||||
stale-re-read race is gone with the Get+Upsert flow.
|
||||
- **Covers are acquired at creation, then served from our own origin
|
||||
(ADR-0007):** the first Bookmark of a Series fires `Store.OnSeriesCreated`,
|
||||
which `latest.Acquirer` turns into one series-page fetch yielding both the
|
||||
Latest Chapter and the cover URL; the bytes then go through
|
||||
`latest.CoverBytesFetcher` into `Store.SetSeriesCover`. It runs in a
|
||||
goroutine — the Reader's PUT must neither block on a Site nor fail with one
|
||||
— and every failure is logged and dropped, leaving the Bookmark intact. The
|
||||
wire's `cover` is the absolute `PUBLIC_BASE_URL + /covers/{sha256}` once
|
||||
bytes exist and `""` before, never an address that 404s. `GET /covers/{addr}`
|
||||
is public and uncredentialed: the userscript renders it on a Site's origin,
|
||||
where no cookie or token of ours travels. A client-sent `cover` is decoded
|
||||
and discarded, permanently (ADR-0004 compatibility).
|
||||
Browser-backed Sites join the same pipeline (issue #62): kagane pages *and*
|
||||
cover bytes go through the browser sidecar (nothing falls back to a plain
|
||||
fetch, which would only retrieve a challenge page), while novelfull needs
|
||||
the browser only for its HTML — the cover URL comes out of the
|
||||
browser-fetched page and the bytes go over plain TLS. With no browser
|
||||
configured, kagane Covers are simply absent; novelfull still gets one — at
|
||||
creation and on the poll — when its page body happens to answer a plain
|
||||
request (the challenge is a live time-varying fact). The old kagane-only
|
||||
serving path (`/img/kagane/{id}`, template rewrite, `CoverFetcher`) is gone
|
||||
(issue #63): the one public route serves every Site.
|
||||
- **`updated_at` drives list order, so moves only on real reading progress:** server apply its timestamp when row new or `last_chapter_num` changes, else keep stored value — favouriting series or recording newly published chapter must not reorder list. `PUT` therefore returns row **as stored**, clients must adopt that response rather than own payload. See `plans/2026-07-25-bookmark-list-favorites-design.md` §4.
|
||||
- **Lifecycle buckets:** `status` on each bookmark is `reading` | `archived` |
|
||||
`finished`, orthogonal to `favorite`. Archived and finished appear only in
|
||||
own tab — not in All, Updated, Favourites, or recent strip. Poller keeps
|
||||
checking archived series and skip finished ones. `finished` settable
|
||||
only from web UI; `PUT /bookmarks/{key}` reject it with 400.
|
||||
**Empty incoming status means "keep stored one"** — resolved on the
|
||||
`VALUES` side of `Store.Upsert`, not conflict clause, since
|
||||
`excluded.*` is post-evaluation row and default applied there would
|
||||
wipe bucket on every PUT from client that predates column. See
|
||||
`docs/superpowers/specs/2026-07-27-status-buckets-design.md`.
|
||||
- **Config via env:** `TOKEN_KEY` (derives every Reader's userscript credential;
|
||||
required), `OWNER_DISCORD_ID` (seeds the owner Reader — the administrator and
|
||||
the owner of every pre-registration bookmark; required),
|
||||
`ALLOWED_ORIGINS` (comma list),
|
||||
`DATABASE_URL` (Postgres connection URL, required — no default),
|
||||
`COVER_DIR` (required filesystem volume for content-addressed Cover bytes),
|
||||
`PUBLIC_BASE_URL` (required origin this deployment answers on, trailing
|
||||
slash trimmed; every Cover URL on the wire is built from it, absolute
|
||||
because the userscript renders on a Site's origin — ADR-0007),
|
||||
`PORT` (default `8080`), `DISCORD_CLIENT_ID`/`_CLIENT_SECRET`/`_GUILD_ID`/
|
||||
`_REDIRECT_URI` (required; Discord OAuth for the browser UI),
|
||||
`DISCORD_REQUIRED_ROLE` (optional role gate, empty by default),
|
||||
`DISCORD_API_BASE` (default `https://discord.com/api/v10`),
|
||||
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/
|
||||
`_BATCH`/`_STAGGER` (background latest-chapter poller; defaults on,
|
||||
`1h` plain-TLS cooldown, `6h` browser cooldown, `10m`/`14`/`20s`; both
|
||||
cooldowns have a `15m` floor).
|
||||
`USERSCRIPT_PATH` and `NOVEL_USERSCRIPT_PATH` (files served at
|
||||
`/u/{token}/manga-bookmark.user.js` and `/u/{token}/novel-bookmark.user.js`,
|
||||
defaults `/userscript/manga-bookmark.user.js` and
|
||||
`/userscript/novel-bookmark.user.js`, both supplied by bindmount; the
|
||||
Each entry names the code that holds the truth — read that for *what it does*.
|
||||
The prose here is only what code cannot tell you: rationale, rejected
|
||||
alternatives, dated measurements, and invariants a plausible refactor would
|
||||
silently break.
|
||||
|
||||
### Layout
|
||||
|
||||
`backend/main.go` → `newRouter` is the composition root, the only place
|
||||
packages are wired. Packages under `backend/internal/`: `store`, `latest`,
|
||||
`session`, `httpmw`, `api`, `userscript`, `web`, `token`, `pgtest`. Root-level
|
||||
`*_test.go` exercise the full router; unit tests live beside their package.
|
||||
|
||||
Not visible from any single file: stdlib `net/http` with no framework,
|
||||
Postgres over `jackc/pgx/v5`, `CGO_ENABLED=0` static binary into a distroless
|
||||
image, TLS terminated by the reverse proxy so the service listens plain `:8080`.
|
||||
|
||||
### Schema — `internal/store/migrations/*.sql`, run by `store.migrate`
|
||||
|
||||
- Migration files are **append-only**. Editing an applied one changes nothing
|
||||
on a database that already recorded its version in `schema_migrations`, so
|
||||
the fix silently applies to new deployments only.
|
||||
- No column probing, no data-fixup migrations. Both were SQLite-era machinery
|
||||
and were removed deliberately — don't reintroduce either.
|
||||
|
||||
### Tests need Docker — `internal/pgtest`
|
||||
|
||||
`pgtest.Main` from `TestMain` starts one `postgres:17-alpine` per test binary;
|
||||
`pgtest.URL` hands each test its own database. A package whose tests touch the
|
||||
store must have that `TestMain` or it has no database at all.
|
||||
|
||||
### Reader-owned store — `internal/store`, `internal/token`
|
||||
- The Reader-owned tables are `readers`, `bookmarks`, `series`, and `sessions`; auxiliary `covers`, `poll_lanes`, and `poll_passes` are also defined in the migrations.
|
||||
|
||||
- **Credentials are derived, never stored.** `token.Token(TOKEN_KEY, discord_id, epoch)`
|
||||
is an HMAC; only its SHA-256 reaches `readers.token_sha256`. So install URLs
|
||||
can be rebuilt after any restart, and a database leak yields nothing usable.
|
||||
- **The owner's epoch-0 hash is refreshed at startup only while the row has
|
||||
never been rotated.** Drop that condition and a restart resurrects a
|
||||
rotated-away credential.
|
||||
- `Store.EnsureReader` never rewrites an existing row's hash — a returning
|
||||
Reader's login must not invalidate their installed scripts.
|
||||
- **Every read and write is scoped to the acting Reader**, resolved from the
|
||||
presented credential by `httpmw.Auth` and carried in the request context.
|
||||
There is no unauthenticated-by-Reader route and no global token.
|
||||
- **`series` holds what readers share, `bookmarks` only what differs.** A
|
||||
bookmark key is `(reader_id, site, series_id)` with no surrogate id; the wire
|
||||
`key` is derived as `site:series_id` on read.
|
||||
- `Store.Upsert` splits one flat body across both tables and enforces the
|
||||
ownership rule: client `title`/`series_url`/`cover` are written **only when
|
||||
the series row is new**, so one reader cannot retitle a shared series.
|
||||
- Sync is last-write-wins and the wire format stays flat — clients depend on
|
||||
both; neither is an implementation detail to tidy up.
|
||||
|
||||
### Web UI — `internal/web`
|
||||
|
||||
Routes, templates and assets are all in that package; `AdminPatterns()` and
|
||||
`adminRoutes()` enumerate the privileged ones.
|
||||
|
||||
- **`backend/Dockerfile` must copy the whole `internal/` tree**, not just
|
||||
`*.go`: templates and static assets are `go:embed`-ed from
|
||||
`internal/web/`.
|
||||
- **Guild membership *is* registration.** `discordCallback` gates on membership
|
||||
(plus `DISCORD_REQUIRED_ROLE` when set) and only then calls
|
||||
`Store.EnsureReader`, so a refusal creates nothing.
|
||||
- Sessions are rows, not signatures: the cookie carries an opaque id and
|
||||
expiry is checked on lookup, which is what makes deleting the row an instant
|
||||
revocation.
|
||||
- UI mutations go through `Store.Get` + `Store.Upsert` so the `updated_at` rule
|
||||
below stays in exactly one place.
|
||||
- `listView.Fresh` exists because a Reader with no bookmarks at all needs
|
||||
install links, not an empty-filter message.
|
||||
- **Design-tool caveat:** `detect.mjs backend/internal/web/templates` reports a
|
||||
**false clean**. Templates link `/static/style.css` root-absolutely (correct —
|
||||
it is served from `/`), but the detector resolves hrefs with
|
||||
`path.resolve(fileDir, href)`, which drops the directory on a leading `/` and
|
||||
skips the file silently; a relative href doesn't help either, since a
|
||||
template's directory isn't its served path. Always pass
|
||||
`backend/internal/web/static` too. The one finding there, `overused-font` on
|
||||
"Instrument Serif", is a deliberate identity choice, not debt.
|
||||
|
||||
### Confirm gating — `internal/web/static/filter.js`, `toggleConfirmRow(key, kind)`
|
||||
|
||||
Every action that pulls a series out of the list (`archive|finish|remove`) opens
|
||||
its own `.confirm-row`; restore fires instantly because it is the reversal.
|
||||
Remove wears the ember wash, the two reversible ones wear `.calm` grey.
|
||||
**`--ember` is reserved for the new-chapter signal** — the busy bar and inline
|
||||
errors must use `--mute`, or the one colour that means "something to read"
|
||||
stops meaning it.
|
||||
|
||||
### Latest-chapter poller — `internal/latest`, Site registry in `sites.go`
|
||||
|
||||
One goroutine per Site (a Poll Lane) re-checks that Site's bookmarked series
|
||||
from the backend's own network position, so `latest_chapter` stays fresh while
|
||||
nobody is browsing. The userscript's `reportLatestChapter` is a second,
|
||||
parallel signal — it PUTs every read, unchanged numbers included, because an
|
||||
unchanged read is exactly the Sighting worth deferring a Poll on.
|
||||
|
||||
- **Pace lives in the Site registry, not config.** Two clocks: per-series rest
|
||||
(`series.latest_checked_at`, enforced in `Store.DueForLatestCheck`'s WHERE)
|
||||
and per-Lane gap (`effectiveGap`). The five env knobs that used to size one
|
||||
shared pace are gone; don't add them back.
|
||||
- **The poller walks Series, not Bookmarks** — a series several readers hold is
|
||||
fetched once per cycle, and the due queue orders `reader_count DESC,
|
||||
latest_checked_at ASC` so the widely-read ones win contention.
|
||||
- **The series row is stamped *before* the fetch**, so a permanently broken
|
||||
series waits out its rest instead of being retried every tick.
|
||||
- `Store.SetLatestChapter` is a single-column UPDATE, deliberately not a
|
||||
read-modify-write of the bookmark: it therefore cannot revert read progress
|
||||
or move `updated_at`. The old stale-re-read race died with the Get+Upsert
|
||||
flow — don't restore one here.
|
||||
|
||||
**Sightings** (`Store.RecordSighting`, the due query's HAVING clause,
|
||||
`latest.checkOne`) let a Reader's own page read defer a Poll.
|
||||
|
||||
- Recorded by the PUT handler **before** the Upsert, because the raise test
|
||||
needs the row as it stands.
|
||||
- A Series is deferred only while it has exactly one Bookmark, was sighted
|
||||
within one Rest, and is under `sightingCeilingRests` since its last Poll — so
|
||||
a shared Series is never deferred and nothing goes six hours unpolled
|
||||
whatever arrives.
|
||||
- A *higher* report clears the attribution rather than crediting it: the value
|
||||
the Poll then stores is its own, so a later retraction isn't the Reader's
|
||||
fault.
|
||||
- `store.SightingDisagreementLimit` contradictions stop a Reader deferring —
|
||||
their reports still write the Latest Chapter — and
|
||||
`store.SightingAgreementsToClear` agreements forgive them, as does the
|
||||
owner's clear-marks control.
|
||||
- Deferral is recomputed from live facts each round, so nothing needs
|
||||
invalidating when a Series gains a second Bookmark. The one input read
|
||||
earlier is the Reader's marks, so crossing or clearing a threshold takes
|
||||
effect from their next Sighting and the standing already bought lasts out its
|
||||
rest.
|
||||
|
||||
**Refusals and browser loss are Lane-local.** Two `errChallengeHeld` in a pass
|
||||
stop that Site for `RefuseBackoff` while other Lanes continue. An
|
||||
`errBrowserInterrupted` (remote Chrome restarted) sets a shared Poller flag so
|
||||
the *other* browser Lanes skip their passes for the same window — otherwise a
|
||||
restarting Chrome stamps one Series per Lane per pass, burning rests on
|
||||
failures. The flag decays and they probe again.
|
||||
|
||||
- **`isInterstitial` matches the orchestration path
|
||||
`/cdn-cgi/challenge-platform/h/`, never the bare prefix.** Cloudflare injects
|
||||
`/cdn-cgi/challenge-platform/scripts/jsd/main.js` into ordinary 200 pages
|
||||
once a zone turns JS detections on, which demonic did on 2026-08-16: the
|
||||
prefix match read every real demonic page as a refusal and parked the Lane in
|
||||
backoff while plain TLS was returning full series pages.
|
||||
- Fetches use `bogdanfinn/tls-client` with a Chrome profile as defence in depth
|
||||
against fingerprint blocking; any failure logs and skips.
|
||||
- kagane, comix and novelfull sit behind Cloudflare JS challenges the TLS
|
||||
client can't clear, so they go over CDP (`BROWSER_WS_URL`). kagane and comix
|
||||
are simply not polled when it's unset — a plain fetch would only retrieve a
|
||||
challenge page — while novelfull still attempts plain TLS, because its
|
||||
challenge is a live time-varying fact and its cover bytes never need a browser.
|
||||
- **comix's browser read is an in-tab `fetch()` of the Series URL, not a DOM
|
||||
render.** It is an SPA: rendering cost ~65 requests for the same
|
||||
server-rendered HTML one fetch returns (measured 2026-08-12).
|
||||
- Browser Lanes wake Chrome only when 5+ Series are due or one has waited 15m,
|
||||
and cover work runs in the background so a slow CDN can't eat a Lane's gap.
|
||||
|
||||
### Covers — `Store.OnSeriesCreated`, `latest.Acquirer`, `latest.CoverBytesFetcher`, `Store.SetSeriesCover`
|
||||
|
||||
Acquired once when the first Bookmark of a Series is created, then served from
|
||||
our own origin by the public `GET /covers/{addr}`.
|
||||
|
||||
- Acquisition runs in a goroutine: the Reader's PUT must neither block on a
|
||||
Site nor fail with one. Every failure is logged and dropped, leaving the
|
||||
Bookmark intact.
|
||||
- The wire `cover` is the absolute `PUBLIC_BASE_URL + /covers/{sha256}` once
|
||||
bytes exist and `""` before — **never an address that 404s**. Absolute
|
||||
because the userscript renders it on a Site's origin.
|
||||
- `GET /covers/{addr}` is public and uncredentialed by design: no cookie or
|
||||
token of ours may travel to a Site's origin.
|
||||
- A client-sent `cover` is decoded and discarded, permanently — wire
|
||||
compatibility, not an oversight.
|
||||
- **One route serves all six Sites.** No proxy, no per-Site rewrite, no second
|
||||
place that decides a renderable address: the wire `cover` is it. Templates
|
||||
render `.Cover` and nothing else. The old kagane-only serving path
|
||||
(`/img/kagane/{id}` plus a template rewrite) is gone; don't reintroduce a
|
||||
per-Site route because one Site's CDN misbehaves.
|
||||
- The only Site names left in cover code are in `browserOnlyCoverURL`
|
||||
(`internal/latest`): kagane answers a plain fetch with a challenge *and*
|
||||
`cross-origin-resource-policy: same-origin`, and `static.comix.to` answers
|
||||
with the same challenge its pages serve. Every other Site's CDN answers plain
|
||||
TLS.
|
||||
- **comix cover bytes must arrive by direct navigation, not an in-page fetch:**
|
||||
its Series page sets `cross-origin-embedder-policy: require-corp`, which
|
||||
fails a page-context fetch of `static.comix.to`.
|
||||
- With no browser configured, kagane and comix Covers are simply absent;
|
||||
novelfull still gets one whenever its page answers a plain request.
|
||||
|
||||
### `updated_at` drives list order — `Store.Upsert`
|
||||
|
||||
The server applies its own timestamp only when the row is new or
|
||||
`last_chapter_num` changes, else it keeps the stored value. **Favouriting a
|
||||
series, or a newly published chapter arriving, must not reorder the list** —
|
||||
only real reading progress moves a row. Consequently `PUT` returns the row **as
|
||||
stored** and clients must adopt that response rather than their own payload.
|
||||
|
||||
### Lifecycle buckets — `status` on each bookmark
|
||||
|
||||
`reading` | `archived` | `finished`, orthogonal to `favorite`. Archived and
|
||||
finished appear only in their own tab, never in All, Updated, Favourites or the
|
||||
recent strip. The poller keeps checking archived series and skips finished ones.
|
||||
|
||||
- `finished` is settable only from the web UI; `PUT /bookmarks/{key}` rejects
|
||||
it with 400.
|
||||
- **An empty incoming status means "keep the stored one"**, and it is resolved
|
||||
on the `VALUES` side of `Store.Upsert`, not in the conflict clause:
|
||||
`excluded.*` is the post-evaluation row, so a default applied there would
|
||||
wipe the bucket on every PUT from a client predating the column.
|
||||
|
||||
### Config — `Config` / `loadConfig` / `loadLatestPoll` in `backend/main.go`
|
||||
|
||||
That function is the complete list of env vars, their defaults, and which are
|
||||
required. What it can't tell you:
|
||||
|
||||
- `PUBLIC_BASE_URL` must be an absolute origin because every Cover URL on the
|
||||
wire is built from it and the userscript renders on a Site's origin.
|
||||
- `BROWSER_WS_URL` **must be a tailnet IP, never a hostname** — Chrome's
|
||||
DevTools handler 500s `/json/version` for any Host that isn't an IP or
|
||||
`localhost`. Unset (the default) disables browser polling.
|
||||
- `USERSCRIPT_PATH` / `NOVEL_USERSCRIPT_PATH` are bindmounted files; the
|
||||
`__API_TOKEN__` placeholder inside them is substituted with the requesting
|
||||
Reader's credential at serve time).
|
||||
`BROWSER_WS_URL` (CDP endpoint of the browser, which runs on a **separate
|
||||
machine** and is reached over the tailnet — ADR-0006, `chrome/docker-compose.yml`.
|
||||
Used by the poller for kagane and novelfull page fetches and by the cover
|
||||
pipeline for kagane's image bytes (the browser is the only route that clears
|
||||
the challenge kagane serves its covers behind); unset — the default —
|
||||
disables browser polling and leaves kagane Covers blank until stored bytes
|
||||
exist. Must be a tailnet IP, never a hostname: Chrome's DevTools handler 500s
|
||||
`/json/version` for any Host that isn't an IP or `localhost`).
|
||||
- **No per-Site cover path (issue #63):** every Cover — all six Sites — is
|
||||
served by the one public `GET /covers/{addr}` route from content-addressed
|
||||
bytes. There is no proxy, no per-Site rewrite, no second place that decides
|
||||
a Cover's renderable address: the wire `cover` is it. The only place a Site
|
||||
name still appears in cover code is the extraction module (`latest`), where
|
||||
kagane's image URLs are claimed by `browserOnlyCoverURL` — they answer a
|
||||
plain fetch with a challenge and `cross-origin-resource-policy: same-origin`;
|
||||
every other Site's CDN answers plain TLS. Templates render `.Cover` — the
|
||||
wire value — never anything else.
|
||||
- **Web UI also owns:** session-gated `GET /install/{manga,novel}-bookmark.user.js`
|
||||
(renders the bindmounted script with the acting Reader's derived credential
|
||||
substituted in — the credential never appears in page markup, the address
|
||||
bar, or a redirect; `?download=1` adds `Content-Disposition: attachment` for
|
||||
mobile Violentmonkey, which ignores a `.user.js` navigation) and
|
||||
`POST /rotate-token` (atomic epoch bump + hash
|
||||
rewrite; invalidates every installed copy, so the panel warns to reinstall
|
||||
on all devices).
|
||||
Owner-only `POST /readers/{id}/revoke` (drops one Reader's session rows and
|
||||
re-renders the `readers` panel; 404 for any non-owner) is the only route that
|
||||
reaches across Readers.
|
||||
Reader's credential at serve time.
|
||||
- Pace is per Site in the registry, not env. The
|
||||
`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/`_BATCH`/`_STAGGER` knobs are
|
||||
gone on purpose.
|
||||
- The 1h rest for browser Sites is safe on documented grounds: a challenged
|
||||
page costs seconds of a serialized single-tab browser, free-plan zones carry
|
||||
no bot score and no published per-IP rate input, and `cf_clearance` expires
|
||||
in 30 minutes, so every cadence at or above 1h re-solves anyway.
|
||||
|
||||
### Userscript install & rotation — `internal/userscript`, `internal/token`
|
||||
|
||||
Session-gated `GET /install/{manga,novel}-bookmark.user.js` renders the
|
||||
bindmounted script with the acting Reader's derived credential substituted in,
|
||||
so the credential never appears in page markup, the address bar, or a redirect.
|
||||
`?download=1` adds `Content-Disposition: attachment` for mobile Violentmonkey,
|
||||
which ignores a `.user.js` navigation. `POST /rotate-token` is an atomic epoch
|
||||
bump plus hash rewrite and invalidates every installed copy — the panel must
|
||||
keep warning to reinstall on all devices.
|
||||
|
||||
### Owner-only admin — `internal/web/admin.go`
|
||||
|
||||
- **Every route reaching past the acting Reader is listed in `adminRoutes()`
|
||||
and wrapped in `requireOwner` at registration** — add it there, not as a
|
||||
check inside a handler; `web.AdminPatterns()` is what the gate test walks. A
|
||||
non-owner gets 404, never 403.
|
||||
- The one owner comparison left outside the gate is in `index`
|
||||
(`view.Owner = readerID == h.store.OwnerID()`): it gates a link, not an
|
||||
endpoint, so it is a rendering decision a registration-time wrapper cannot
|
||||
express. Do not "unify" it into the gate.
|
||||
- The Lanes page reads the pass log, never a running poller: `lanesView()` in
|
||||
`admin_lanes.go` projects `store.LatestLanePasses()` and
|
||||
`store.LanePassOutcomes()` (ADR-0012), so a restart answers the instant the
|
||||
database is up. Browser configuration is a config fact and reachability is
|
||||
derived from recent browser-Site passes inside `latest.RefuseBackoff` — no
|
||||
reporter interface exists to fake.
|
||||
- A pass that returns before computing figures (refusal backoff, sidecar down)
|
||||
carries the previous pass's numbers forward rather than recording zeroes.
|
||||
- **`Checked` next to `Due` is what separates a stopped Lane from a quiet one**,
|
||||
so neither may be dropped from the row.
|
||||
- Due-without-Checked is **not** by itself a stall: a browser Lane under both
|
||||
wake thresholds records its pass with the `SkipAsleep` skip and renders
|
||||
"browser asleep", and that never counts toward `Attention`. It is the
|
||||
commonest healthy state for kagane, comix and novelfull, so spending the
|
||||
stall mark on it would train the owner to ignore the mark that matters.
|
||||
|
||||
@@ -621,6 +621,46 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Sighting deferral (issue #103) only reaches production through the PUT
|
||||
// handler: the store and poller can be right and the feature still dead if the
|
||||
// handler never records the report. Asserted where a client can see it - the
|
||||
// series stops being due the moment the PUT lands.
|
||||
func TestPutRecordsASighting(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
srv := newRouter(s, testConfig())
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
hour := time.Hour.Milliseconds()
|
||||
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", now-2*hour)
|
||||
due, err := s.DueForLatestCheck("asura", now-hour, now-6*hour)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("due before the PUT = %d series, want 1", len(due))
|
||||
}
|
||||
|
||||
body := `{"key":"asura:x","site":"asura","series_id":"x",
|
||||
"series_url":"https://asurascans.com/comics/x",
|
||||
"last_chapter":"Chapter 5","last_chapter_num":5,
|
||||
"latest_chapter":"Chapter 9","latest_chapter_num":9}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, auth(req))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
due, err = s.DueForLatestCheck("asura", now-hour, now-6*hour)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 0 {
|
||||
t.Fatalf("due after the PUT = %d series, want 0: the handler recorded no Sighting", len(due))
|
||||
}
|
||||
}
|
||||
|
||||
// The userscript route is registered outside the web UI's Discord auth, so it
|
||||
// must keep working whatever the web config — see internal/userscript for the
|
||||
// handler's own behaviour. The credential in the path is the owner's derived
|
||||
|
||||
@@ -99,7 +99,18 @@ func (h *Handler) Put(w http.ResponseWriter, r *http.Request) {
|
||||
// reading progress actually moved. Any client value is ignored.
|
||||
b.UpdatedAt = time.Now().UnixMilli()
|
||||
|
||||
stored, err := h.Store.Upsert(httpmw.ReaderID(r), b)
|
||||
// A userscript PUT is a Sighting: the Reader's browser was on the Series
|
||||
// page and read its Latest Chapter (issue #103). Recorded before the
|
||||
// Upsert, which is what makes the raise comparison possible, and never
|
||||
// from the web UI's own read-modify-write — a Reader toggling a favourite
|
||||
// has not looked at the Site and must not postpone a Poll. A failure here
|
||||
// costs a deferral, not the write, so it is logged and dropped.
|
||||
readerID := httpmw.ReaderID(r)
|
||||
if err := h.Store.RecordSighting(readerID, b.Site, b.SeriesID, b.LatestChapterNum, b.UpdatedAt); err != nil {
|
||||
log.Printf("record sighting: %v", err)
|
||||
}
|
||||
|
||||
stored, err := h.Store.Upsert(readerID, b)
|
||||
if err != nil {
|
||||
log.Printf("upsert: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
|
||||
@@ -120,7 +120,7 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
|
||||
// Stamped after success — the reverse of the poller, which stamps before
|
||||
// the fetch: the Reader is here, watching the Series they just created, so
|
||||
// a failed acquisition must leave the row due for a fast retry rather than
|
||||
// consuming the cooldown. The stamp happens even when the page read
|
||||
// consuming the rest. The stamp happens even when the page read
|
||||
// succeeded but produced no facts to persist.
|
||||
if err := a.Store.MarkLatestChecked(sr.Site, sr.SeriesID, time.Now().UnixMilli()); err != nil {
|
||||
log.Printf("acquire %q: mark checked: %v", sr.Key(), err)
|
||||
|
||||
@@ -135,6 +135,33 @@ func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A pause governs the Lane only: a Reader's first bookmark of a Series on a
|
||||
// paused Site still reads the page, because acquisition is the creation-time
|
||||
// fetch, not the poll queue (issue #147).
|
||||
func TestAcquireIgnoresLanePause(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
if err := s.PauseLane("asura", time.Now().Add(6*time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200}
|
||||
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
||||
acq := newAcquirer(s, page, covers)
|
||||
|
||||
bookmarkNewSeries(t, s, acquireSeriesURL)
|
||||
acq.Wait()
|
||||
|
||||
if got := page.callCount(); got != 1 {
|
||||
t.Fatalf("series page fetches on a paused Site = %d, want 1", got)
|
||||
}
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, acquireKey)
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
||||
}
|
||||
}
|
||||
|
||||
// A Series that already exists is not re-acquired: no fetch, and the Cover it
|
||||
// already has is left alone.
|
||||
func TestAcquireSkipsAnExistingSeries(t *testing.T) {
|
||||
|
||||
@@ -18,18 +18,22 @@ import (
|
||||
|
||||
// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
|
||||
// challenge clears in a few seconds when it clears at all; anything longer is a
|
||||
// challenge that is not going to pass, and the caller's cooldown was already
|
||||
// challenge that is not going to pass, and the caller's rest was already
|
||||
// stamped before this ran.
|
||||
const challengeTimeout = 45 * time.Second
|
||||
|
||||
var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
||||
|
||||
// comixSeriesPathRe matches the one path shape comixRead will open: a Series
|
||||
// page, "/title/<id>-<slug>". Verified live 2026-08-12.
|
||||
var comixSeriesPathRe = regexp.MustCompile(`^/title/[^/?#]+/?$`)
|
||||
|
||||
// BrowserFetcher retrieves pages through a remote headless Chrome over the
|
||||
// DevTools Protocol.
|
||||
//
|
||||
// It exists for one reason: kagane.to and novelfull.com sit behind a
|
||||
// Cloudflare JavaScript challenge. Verified 2026-08-03 (kagane) and 2026-08-05
|
||||
// (novelfull) from the deployment host, plain HTTP and bogdanfinn/tls-client
|
||||
// It exists for one reason: kagane.to, novelfull.com and comix.to sit behind a
|
||||
// Cloudflare JavaScript challenge. Verified 2026-08-03 (kagane), 2026-08-05
|
||||
// (novelfull) and 2026-08-12 (comix), plain HTTP and bogdanfinn/tls-client
|
||||
// with a Chrome_133 profile both get 403 with cf-mitigated: challenge on every
|
||||
// path, including the API, robots.txt and images. Clearing it requires
|
||||
// executing the challenge script, which only a real browser does.
|
||||
@@ -40,10 +44,11 @@ var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
||||
// sync that break silently and separately. The browser's own cookie jar
|
||||
// persists across polls, so the challenge is solved once every few hours.
|
||||
//
|
||||
// The two sites differ in how the chapter list is read: kagane serves it from
|
||||
// a JSON API that must be called from inside the page (so the request carries
|
||||
// the clearance cookie), while novelfull renders it into the HTML so the
|
||||
// cleared DOM is the payload.
|
||||
// The three sites differ in what a cleared tab is asked for: kagane fetches a
|
||||
// JSON API from inside the page (the list exists nowhere else), comix fetches
|
||||
// its own Series URL from inside the page (the served HTML carries the facts,
|
||||
// and rendering the SPA costs ~65 requests instead of one), and novelfull
|
||||
// renders its list into the HTML so the cleared DOM is the payload.
|
||||
type BrowserFetcher struct {
|
||||
allocCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
@@ -87,10 +92,9 @@ func (f *BrowserFetcher) Close() {
|
||||
}
|
||||
|
||||
// Get navigates to seriesURL, lets any challenge resolve, then reads the
|
||||
// payload the Site's registry entry describes — kagane's chapter-list API from
|
||||
// inside the page so the request carries the clearance cookie, novelfull's
|
||||
// served HTML. The returned body is whatever the Site's chapter list lives in,
|
||||
// which is what the entry's LatestChapter parse expects.
|
||||
// payload the Site's registry entry describes (the shapes are listed on
|
||||
// BrowserFetcher). The returned body is whatever the Site's chapter list lives
|
||||
// in, which is what the entry's LatestChapter parse expects.
|
||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||
var body string
|
||||
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
|
||||
@@ -141,29 +145,47 @@ func novelfullRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
return chromedp.OuterHTML("html", out, chromedp.ByQuery), true
|
||||
}
|
||||
|
||||
// comixRead fetches the Series page from inside the cleared tab. comix is an
|
||||
// SPA: rendering the page costs ~65 requests, while one same-origin fetch of
|
||||
// the same address returns the server-rendered HTML — 24.5 KB, ~480 ms,
|
||||
// carrying both parser anchors (measured 2026-08-12, issue #98). So this is
|
||||
// kaganeRead's shape, not novelfullRead's, even though the payload is HTML.
|
||||
// Refusing any other address is the per-Site half of the SSRF gate.
|
||||
func comixRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
pageURL, ok := comixSeriesPageURL(seriesURL)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return chromedp.Evaluate(
|
||||
`fetch(`+jsString(pageURL)+`).then(r => r.ok ? r.text() : "")`,
|
||||
out, awaitPromise), true
|
||||
}
|
||||
|
||||
// Image retrieves one cover's bytes through the browser sidecar, and its
|
||||
// content type.
|
||||
//
|
||||
// It exists because kagane serves covers behind the same challenge as its
|
||||
// pages *and* with `cross-origin-resource-policy: same-origin`, so the bytes
|
||||
// are only reachable from inside a browser that already holds the clearance
|
||||
// cookie (verified 2026-08-08). Acquisition through the sidecar is the only
|
||||
// route.
|
||||
// It exists because kagane and comix serve covers behind the same challenge as
|
||||
// their pages — kagane additionally with
|
||||
// `cross-origin-resource-policy: same-origin` — so the bytes are only
|
||||
// reachable from inside a browser that already holds the clearance cookie
|
||||
// (verified 2026-08-08 for kagane, 2026-08-12 for comix). Acquisition through
|
||||
// the sidecar is the only route.
|
||||
//
|
||||
// The image URL is navigated to rather than fetched from some other kagane
|
||||
// page: the challenge only runs on a top-level navigation, and once it clears
|
||||
// The image URL is navigated to rather than fetched from another page of the
|
||||
// Site: the challenge only runs on a top-level navigation, and once it clears
|
||||
// the document *is* the image, so a same-origin fetch of location.href reads
|
||||
// it straight back out of the cache.
|
||||
// it straight back out of the cache. For comix the navigation is also the only
|
||||
// route that works at all — its Series page sets
|
||||
// `cross-origin-embedder-policy: require-corp`, which fails a page-context
|
||||
// fetch of the cover host.
|
||||
//
|
||||
// The challenge is not solved by the first read: WaitReady("body") is satisfied
|
||||
// by the interstitial too. run holds the tab open until the in-page fetch
|
||||
// succeeds, which is what gives the challenge script the seconds it needs.
|
||||
func (f *BrowserFetcher) Image(ctx context.Context, imageURL string) ([]byte, string, error) {
|
||||
m := kaganeImageURLRe.FindStringSubmatch(imageURL)
|
||||
if m == nil {
|
||||
if !browserOnlyCoverURL(imageURL) {
|
||||
return nil, "", fmt.Errorf("not a browser-fetchable cover url: %q", imageURL)
|
||||
}
|
||||
imageID := m[1]
|
||||
var dataURL string
|
||||
err := f.run(ctx, imageURL,
|
||||
chromedp.Evaluate(`fetch(location.href).then(r => r.ok
|
||||
@@ -175,23 +197,23 @@ func (f *BrowserFetcher) Image(ctx context.Context, imageURL string) ([]byte, st
|
||||
: "")`, &dataURL, awaitPromise),
|
||||
func() bool { return dataURL != "" })
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageID, err)
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageURL, err)
|
||||
}
|
||||
// "data:image/webp;base64,<payload>".
|
||||
head, payload, ok := strings.Cut(dataURL, ";base64,")
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("browser image %s: not a data url", imageID)
|
||||
return nil, "", fmt.Errorf("browser image %s: not a data url", imageURL)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(payload)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageID, err)
|
||||
return nil, "", fmt.Errorf("browser image %s: %w", imageURL, err)
|
||||
}
|
||||
return raw, strings.TrimPrefix(head, "data:"), nil
|
||||
}
|
||||
|
||||
// errChallengeHeld reports that the budget ran out with the interstitial still
|
||||
// up. Distinct from a transport failure: it means "this site said no", which
|
||||
// the poller answers with a 403 and its ordinary cooldown.
|
||||
// the poller answers with a refusal backoff for that Site's Lane (issue #100).
|
||||
var errChallengeHeld = errors.New("challenge held")
|
||||
|
||||
// errBrowserInterrupted distinguishes a remote Chrome restart from the
|
||||
@@ -228,11 +250,18 @@ func browserConnectionLost(ctx context.Context) bool {
|
||||
const challengePollInterval = 2 * time.Second
|
||||
|
||||
// isInterstitial reports whether html is Cloudflare's challenge page rather
|
||||
// than the site's own. Matched on the challenge runtime's script path, which is
|
||||
// stable across the interstitial's wording and locale — the visible "Just a
|
||||
// than the site's own. Matched on the challenge orchestration path
|
||||
// (/cdn-cgi/challenge-platform/h/<b|g|x>/orchestrate/...), which is stable
|
||||
// across the interstitial's wording and locale — the visible "Just a
|
||||
// moment..." title is neither.
|
||||
//
|
||||
// The bare "/cdn-cgi/challenge-platform/" prefix is NOT enough: Cloudflare
|
||||
// injects /cdn-cgi/challenge-platform/scripts/jsd/main.js into ordinary 200
|
||||
// pages when JS detections are on, so matching the prefix declared every real
|
||||
// demonic page a refusal and parked that Lane in 15m backoff (observed
|
||||
// 2026-08-16, demonic turned detections on).
|
||||
func isInterstitial(html string) bool {
|
||||
return strings.Contains(html, "/cdn-cgi/challenge-platform/")
|
||||
return strings.Contains(html, "/cdn-cgi/challenge-platform/h/")
|
||||
}
|
||||
|
||||
// run navigates to target and re-reads until done reports an answer, bounded by
|
||||
@@ -323,6 +352,19 @@ func novelfullSeriesURL(seriesURL string) bool {
|
||||
strings.HasSuffix(u.Path, ".html")
|
||||
}
|
||||
|
||||
// comixSeriesPageURL returns the address comixRead fetches inside the tab: the
|
||||
// Series page itself, rebuilt from the pinned host and path so nothing else
|
||||
// travels. Host-pinned here for the same reason kagane's is — series_url is
|
||||
// client-supplied and a headless browser is a strong SSRF primitive.
|
||||
func comixSeriesPageURL(seriesURL string) (string, bool) {
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil || u.Scheme != "https" || u.Hostname() != "comix.to" ||
|
||||
!comixSeriesPathRe.MatchString(u.Path) {
|
||||
return "", false
|
||||
}
|
||||
return "https://comix.to" + u.Path, true
|
||||
}
|
||||
|
||||
// awaitPromise makes Evaluate resolve the promise rather than returning a
|
||||
// serialised Promise object.
|
||||
func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
|
||||
|
||||
@@ -61,6 +61,81 @@ func TestNovelfullSeriesURL(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestComixSeriesPageURL(t *testing.T) {
|
||||
const series = "https://comix.to/title/n8we-dungeons-and-crayons"
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
want string
|
||||
}{
|
||||
{"series page", series, series},
|
||||
{"trailing slash kept", series + "/", series + "/"},
|
||||
// Query and fragment are dropped: only the pinned path travels.
|
||||
{"query dropped", series + "?tab=chapters", series},
|
||||
{"foreign host", "https://evil.example/title/x", ""},
|
||||
{"lookalike host", "https://comix.to.evil.example/title/x", ""},
|
||||
{"not https", "http://comix.to/title/x", ""},
|
||||
{"not a series path", "https://comix.to/search", ""},
|
||||
{"chapter page", series + "/11139891-chapter-80", ""},
|
||||
{"garbage", "://nope", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, ok := comixSeriesPageURL(tc.url)
|
||||
if ok != (tc.want != "") || got != tc.want {
|
||||
t.Fatalf("comixSeriesPageURL(%q) = %q, %v; want %q", tc.url, got, ok, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The browser is an SSRF primitive and a cover address can originate in a
|
||||
// client-supplied PUT body, so this gate decides what it may navigate to.
|
||||
func TestBrowserOnlyCoverURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
url string
|
||||
want bool
|
||||
}{
|
||||
{"https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg", true},
|
||||
{"https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed", true},
|
||||
// Every other Site's CDN answers plain TLS.
|
||||
{"https://gg.asuracomic.net/covers/x.webp", false},
|
||||
{"http://static.comix.to/039d/x.jpg", false},
|
||||
{"https://static.comix.to.evil.example/039d/x.jpg", false},
|
||||
{"https://evil.example/static.comix.to/x.jpg", false},
|
||||
{"https://static.comix.to/039d/x.jpg?next=http://169.254.169.254/", false},
|
||||
{"https://static.comix.to/039d/x.svg", false},
|
||||
{"https://static.comix.to/../etc/passwd.jpg", false},
|
||||
{"https://static.comix.to/", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.url, func(t *testing.T) {
|
||||
if got := browserOnlyCoverURL(tc.url); got != tc.want {
|
||||
t.Fatalf("browserOnlyCoverURL(%q) = %v, want %v", tc.url, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The jsd script is injected into ordinary 200 pages when a zone turns JS
|
||||
// detections on; only the orchestration path means the page itself is the
|
||||
// challenge. Conflating the two parked the demonic Lane in refusal backoff
|
||||
// while every fetch was in fact the real series page (observed 2026-08-16).
|
||||
func TestIsInterstitial(t *testing.T) {
|
||||
if !isInterstitial(challengeFixture) {
|
||||
t.Fatal("challenge page not detected as interstitial")
|
||||
}
|
||||
const jsdInjected = `<html><head><title>The Possessed Grappler</title>
|
||||
<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head>
|
||||
<body><a href="/chaptered.php?manga=13721&chapter=22">Chapter 22</a></body></html>`
|
||||
if isInterstitial(jsdInjected) {
|
||||
t.Fatal("real page carrying the injected jsd script misread as interstitial")
|
||||
}
|
||||
if got, ok := demonicLatestChapter("", jsdInjected); !ok || got.Label != "Chapter 22" {
|
||||
t.Fatalf("demonicLatestChapter = %+v, ok = %v, want Chapter 22", got, ok)
|
||||
}
|
||||
}
|
||||
func TestClassifyBrowserInterruption(t *testing.T) {
|
||||
if err := classifyBrowserError(context.Background(), true, context.Canceled); !errors.Is(err, errBrowserInterrupted) {
|
||||
t.Fatalf("classifyBrowserError(context.Canceled) = %v, want browser interruption", err)
|
||||
|
||||
@@ -25,7 +25,8 @@ type CoverBytesFetcher interface {
|
||||
// fetchCoverBytes routes a cover's byte retrieval by URL shape, not by Site
|
||||
// name: the browser fetcher's module claims the addresses only it can fetch
|
||||
// (kagane's image route answers a plain fetch with a challenge and
|
||||
// `cross-origin-resource-policy: same-origin`), and everything else goes over
|
||||
// `cross-origin-resource-policy: same-origin`, static.comix.to answers one with
|
||||
// the same challenge its pages serve), and everything else goes over
|
||||
// plain TLS. Missing fetchers degrade to an error the caller logs, never a
|
||||
// fallback onto a path that cannot succeed. One routing rule for the poll and
|
||||
// the acquirer, so the two cannot drift apart.
|
||||
@@ -46,6 +47,15 @@ func fetchCoverBytes(ctx context.Context, cover string, browser BrowserCoverFetc
|
||||
// inject it to exercise hostile DNS results without touching the live network.
|
||||
type CoverResolver func(context.Context, string) ([]netip.Addr, error)
|
||||
|
||||
// maxCoverBytes caps one cover, separately from the series-page maxBodyBytes:
|
||||
// a cover is a bounded binary asset, not a text page, and 4 MiB rejected 12%
|
||||
// of asurascans covers measured 2026-08-17 (p90 4.52 MB, max 8.57 MB — two of
|
||||
// the three over-cap files were JPEGs, not the animated GIF of issue #71).
|
||||
// 10 MiB is ~18% headroom over that worst case and matches the GitHub and
|
||||
// Discord image limits; see docs/research/gif-maximum-byte-size.md. GIF itself
|
||||
// has no maximum size, so this number is policy, not format.
|
||||
const maxCoverBytes = 10 << 20
|
||||
|
||||
// TLSCoverFetcher retrieves image bytes with the standard HTTPS client. Unlike
|
||||
// TLSFetcher, it does not need a browser fingerprint: cover hosts are public
|
||||
// CDNs and the response is accepted only after the destination gate passes.
|
||||
@@ -151,15 +161,15 @@ func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte,
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", raw)
|
||||
}
|
||||
if resp.ContentLength > maxBodyBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
|
||||
if resp.ContentLength > maxCoverBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxCoverBytes)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes+1))
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverBytes+1))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("read cover: %w", err)
|
||||
}
|
||||
if len(body) > maxBodyBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
|
||||
if len(body) > maxCoverBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxCoverBytes)
|
||||
}
|
||||
return body, contentType, nil
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ func TestCoverFetcherRejectsOversizedBody(t *testing.T) {
|
||||
var calls int
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxBodyBytes+1))
|
||||
response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxCoverBytes+1))
|
||||
response.ContentLength = -1
|
||||
return response, nil
|
||||
})}
|
||||
@@ -187,6 +187,29 @@ func TestCoverFetcherRejectsOversizedBody(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Covers between the series-page cap and the cover cap must be accepted: the
|
||||
// 4 MiB page cap rejected 12% of asurascans covers (issue #71).
|
||||
func TestCoverFetcherAcceptsCoverOverPageCap(t *testing.T) {
|
||||
body := bytes.Repeat([]byte("x"), maxBodyBytes+1)
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return coverResponse(http.StatusOK, "image/gif", "", body), nil
|
||||
})}
|
||||
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
|
||||
})
|
||||
|
||||
got, contentType, err := fetcher.Fetch(context.Background(), "https://cdn.example/big.gif")
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch rejected a %d-byte cover: %v", len(body), err)
|
||||
}
|
||||
if len(got) != len(body) {
|
||||
t.Fatalf("body = %d bytes, want %d", len(got), len(body))
|
||||
}
|
||||
if contentType != "image/gif" {
|
||||
t.Fatalf("content type = %q, want image/gif", contentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverFetcherRejectsNonImage(t *testing.T) {
|
||||
var calls int
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"log"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
@@ -17,8 +18,8 @@ type Fetcher interface {
|
||||
}
|
||||
|
||||
// BrowserCoverFetcher retrieves one cover's bytes through the browser-backed
|
||||
// path — the only route that clears the challenge kagane's image URLs answer
|
||||
// a plain fetch with. Satisfied by BrowserFetcher.
|
||||
// path — the only route that clears the challenge kagane's and comix's image
|
||||
// URLs answer a plain fetch with. Satisfied by BrowserFetcher.
|
||||
type BrowserCoverFetcher interface {
|
||||
Image(ctx context.Context, imageURL string) (body []byte, contentType string, err error)
|
||||
}
|
||||
@@ -28,15 +29,11 @@ type BrowserCoverFetcher interface {
|
||||
// in parallel and report the same observable fact, so whichever writes last wins
|
||||
// and neither needs to know about the other.
|
||||
//
|
||||
// Two clocks, deliberately independent:
|
||||
//
|
||||
// - Interval is how often this goroutine wakes up and looks.
|
||||
// - Cooldowns are how long a series rests since its own last check. Browser-
|
||||
// backed sites use the longer BrowserCooldown.
|
||||
//
|
||||
// Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
|
||||
// by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
|
||||
// it only makes the poller wake up and find nothing due more often.
|
||||
// Every Site gets its own Poll Lane: one independent stream of Polls with its
|
||||
// own pace, running concurrently with every other Site's (issue #100). Rest
|
||||
// time and gap live in the Site registry, not here — see sites.go. Rest is
|
||||
// enforced by the WHERE clause in DueForLatestCheck rather than by any timer;
|
||||
// the gap is enforced by the Lane sleeping between fetches.
|
||||
type Poller struct {
|
||||
Store *store.Store
|
||||
Fetch Fetcher
|
||||
@@ -50,11 +47,26 @@ type Poller struct {
|
||||
// same failure-isolated prefetch path.
|
||||
CoverBytesFetch CoverBytesFetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
Cooldown time.Duration
|
||||
BrowserCooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
// eligibleCount reports how many of a Site's Series are eligible for
|
||||
// polling, defaulting to Store.EligibleSeriesCount. Injected so tests can
|
||||
// fail the count alone: the eligible query shares the due query's tables,
|
||||
// so no real store failure can reach this path without breaking the due
|
||||
// query first (issue #141).
|
||||
eligibleCount func(site string) (int, error)
|
||||
|
||||
// refuseUntil gates a Site's Lane after it refused twice in one run: no
|
||||
// Series of that Site is attempted again before this time (issue #100).
|
||||
// The stamp is durable — the pass gate reads it from the store, so a
|
||||
// restart does not forget the refusal; nothing of it lives in memory.
|
||||
// browserDownAt is when a browser Lane last lost the sidecar; the other
|
||||
// browser Lanes skip their passes for the next RefuseBackoff, so a
|
||||
// restarting Chrome does not stamp one Series per pass per Lane (story 20).
|
||||
mu sync.Mutex
|
||||
browserDownAt time.Time
|
||||
// coverWG tracks in-flight cover work. Covers heal in the background so a
|
||||
// slow cover host cannot delay the next Series-page Poll; tests join it
|
||||
// before asserting on cover fetches.
|
||||
coverWG sync.WaitGroup
|
||||
}
|
||||
|
||||
// fillBlankCover gives a Series its Cover when it has none. The blank state is
|
||||
@@ -76,7 +88,14 @@ func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, cover stri
|
||||
if cover == "" {
|
||||
return
|
||||
}
|
||||
p.storeCover(ctx, sr, cover)
|
||||
// Like healCover, the fill runs in the background: a large import of
|
||||
// blanks would otherwise pay one og:image fetch per Series against the
|
||||
// Lane's gap (issue #100, story 12).
|
||||
p.coverWG.Add(1)
|
||||
go func() {
|
||||
defer p.coverWG.Done()
|
||||
p.storeCover(ctx, sr, cover)
|
||||
}()
|
||||
}
|
||||
|
||||
// prefetchCover heals Series that already carry a third-party source URL but
|
||||
@@ -119,9 +138,9 @@ func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL stri
|
||||
|
||||
// fetcherFor returns the fetcher a site's page needs, or nil when the site
|
||||
// cannot be fetched at all right now. A Site whose registry entry carries a
|
||||
// Browser read — kagane and novelfull, both behind a Cloudflare JavaScript
|
||||
// challenge no TLS fingerprint clears — prefers the browser; when it is
|
||||
// absent, the entry's Fallback decides whether plain TLS may take over. One
|
||||
// Browser read — kagane, comix and novelfull, all behind a Cloudflare
|
||||
// JavaScript challenge no TLS fingerprint clears — prefers the browser; when it
|
||||
// is absent, the entry's Fallback decides whether plain TLS may take over. One
|
||||
// routing rule for the poll and the acquirer, so the two cannot drift apart.
|
||||
func fetcherFor(site string, browser, tls Fetcher) Fetcher {
|
||||
s, known := sites[site]
|
||||
@@ -143,125 +162,484 @@ func fetcherFor(site string, browser, tls Fetcher) Fetcher {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run polls until ctx is cancelled.
|
||||
//
|
||||
// runOnce is called synchronously, so a batch that overruns the tick delays the
|
||||
// next one instead of stacking a second batch on top of it. That is the intended
|
||||
// failure mode for a misconfigured batch x stagger: a slower cadence, never
|
||||
// concurrent fetch storms.
|
||||
// Run polls until ctx is cancelled: one goroutine per Site Lane, each pacing
|
||||
// itself by the Site's effective gap. Lanes share nothing but the store and
|
||||
// the browser fetcher's single tab (BrowserFetcher serializes itself), so one
|
||||
// hostile Site burns only its own budget.
|
||||
// laneNames returns every registry Site in the deterministic order both Run
|
||||
// and runOnce iterate: sorted, so lane behaviour and its tests agree on who
|
||||
// runs first.
|
||||
func laneNames() []string { return SiteNames() }
|
||||
|
||||
func (p *Poller) Run(ctx context.Context) {
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
|
||||
p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
|
||||
t := time.NewTicker(p.Interval)
|
||||
defer t.Stop()
|
||||
names := laneNames()
|
||||
log.Printf("latest-chapter poller: %d lanes, rest=%s gap=%s", len(names), defaultRest, defaultGap)
|
||||
for _, name := range names {
|
||||
go p.lane(ctx, name)
|
||||
}
|
||||
<-ctx.Done()
|
||||
log.Println("latest-chapter poller: stopped")
|
||||
}
|
||||
|
||||
// lane is one Site's Poll Lane: one pass, then sleep the pace the pass
|
||||
// reported, then another pass, until ctx is cancelled. The sleep is the whole
|
||||
// pace discipline — a pass that fetched nothing still reports its gap so the
|
||||
// Lane wakes often enough to notice Series as they become due. The pass shares
|
||||
// the Poller's browser-down state, so a sidecar loss is noticed once and the
|
||||
// other browser Lanes skip passes until the backoff window decays.
|
||||
func (p *Poller) lane(ctx context.Context, name string) {
|
||||
for {
|
||||
pace := p.runLanePass(ctx, name, true)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("latest-chapter poller: stopped")
|
||||
return
|
||||
case <-t.C:
|
||||
p.runOnce(ctx)
|
||||
case <-time.After(pace):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runOnce processes one batch of due series.
|
||||
// runOnce processes one round: one pass of every Lane, back to back, no real
|
||||
// time passing. This is the deterministic entry point the test suite drives a
|
||||
// round at a time. The production Run loop does the same work paced by its own
|
||||
// sleeps; pacing is the only difference.
|
||||
func (p *Poller) runOnce(ctx context.Context) {
|
||||
for _, name := range laneNames() {
|
||||
p.runLanePass(ctx, name, false)
|
||||
}
|
||||
}
|
||||
|
||||
// One skip value per way a Lane Pass can return before its loop (issue #141);
|
||||
// empty means the pass reached the loop. The values are wire strings — stored
|
||||
// in poll_passes and read by the Lanes page — so they are stable, not prose.
|
||||
const (
|
||||
// Exported so the web layer renders a skip's reason without retyping the
|
||||
// wire string (issue #145); the values are storage and page-stable.
|
||||
SkipPaused = "paused" // the pause row was read at the top
|
||||
SkipRefusing = "refusing" // refusal backoff
|
||||
SkipSidecarDown = "sidecar-down" // a sibling browser Lane lost Chrome
|
||||
SkipNoFetcher = "no-fetcher" // browser Site, no browser configured, no fallback
|
||||
SkipDueQuery = "due-query" // the due query failed
|
||||
SkipAsleep = "asleep" // under both browser wake thresholds
|
||||
SkipEligibleCount = "eligible-count" // the eligible count failed
|
||||
SkipNothingEligible = "nothing-eligible" // nothing eligible; sleeps a full rest
|
||||
)
|
||||
|
||||
// readOutcome classifies one Series read for the pass row's outcome counts
|
||||
// (issue #141). The classification the read already makes is counted, never a
|
||||
// second taxonomy: refused is the Site holding a challenge, unreachable the
|
||||
// browser interrupting, noChapter a 200 with real HTML but no chapter links,
|
||||
// unfetchable the host pin or a missing fetcher, and errors everything else.
|
||||
type readOutcome int
|
||||
|
||||
const (
|
||||
outcomeSuccess readOutcome = iota
|
||||
outcomeRefused
|
||||
outcomeUnreachable
|
||||
outcomeNoChapter
|
||||
outcomeUnfetchable
|
||||
outcomeError
|
||||
)
|
||||
|
||||
// outcomeCounts are the five named outcome counts of one pass. A success
|
||||
// count is derived, never stored: checked minus the four, with unreachable
|
||||
// excluded because the sidecar-loss path returns before the checked counter
|
||||
// increments (issue #141).
|
||||
type outcomeCounts struct {
|
||||
refused, unreachable, noChapter, unfetchable, errors int
|
||||
}
|
||||
|
||||
func (c *outcomeCounts) add(o readOutcome) {
|
||||
switch o {
|
||||
case outcomeRefused:
|
||||
c.refused++
|
||||
case outcomeUnreachable:
|
||||
c.unreachable++
|
||||
case outcomeNoChapter:
|
||||
c.noChapter++
|
||||
case outcomeUnfetchable:
|
||||
c.unfetchable++
|
||||
case outcomeError:
|
||||
c.errors++
|
||||
}
|
||||
}
|
||||
|
||||
// passRecord is what one pass's durable row will be: the skip value and
|
||||
// outcome counts filled in along the pass's return path. recordPass assembles
|
||||
// the row, so every exit records exactly once.
|
||||
type passRecord struct {
|
||||
site string
|
||||
ranAt int64
|
||||
skip string
|
||||
counts outcomeCounts
|
||||
}
|
||||
|
||||
// lanePassRetention is how far back a Lane's pass log is kept. It is not the
|
||||
// display window: retention is how far back a question can reach, and the
|
||||
// window is what the owner is shown (issue #139).
|
||||
const lanePassRetention = 14 * 24 * time.Hour
|
||||
|
||||
// runLanePass processes one pass of one Site's Lane: select the due Series,
|
||||
// pace through them, and report how long the Lane should wait before its next
|
||||
// pass. paced spaces consecutive fetches by the Site's effective gap — the
|
||||
// production Lane's rate limit; the deterministic test entry runs back to back.
|
||||
func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.Duration {
|
||||
now := p.Now()
|
||||
cutoff := now.Add(-p.Cooldown).UnixMilli()
|
||||
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
|
||||
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites(), p.Batch)
|
||||
// Durable pass log (issue #141): one row per exit. The figures are filled
|
||||
// in as the pass measures them; a pass that returns before measuring
|
||||
// carries the previous pass's forward inside recordPass.
|
||||
fig := passFigures{}
|
||||
rec := passRecord{site: name, ranAt: now.UnixMilli()}
|
||||
defer func() { p.recordPass(rec, fig) }()
|
||||
|
||||
// One Lane row read at the top of a pass, serving two gates (issue #139).
|
||||
// Both stamps outlive our process, so the gates read the durable row
|
||||
// rather than memory: a refusal is the Site's mood and a pause the
|
||||
// owner's order, and neither is lost to a restart.
|
||||
pausedUntil, refuseUntil, err := p.Store.LaneGates(name)
|
||||
if err != nil {
|
||||
log.Printf("latest poll: due query: %v", err)
|
||||
return
|
||||
// Fail open: a store that cannot answer the gate cannot record the
|
||||
// pass either, and one Lane must not stall on its own gate read.
|
||||
log.Printf("latest poll %s: lane gates: %v", name, err)
|
||||
}
|
||||
if pausedUntil > now.UnixMilli() {
|
||||
// Paused ahead of the refusal check: no Series is touched, so the
|
||||
// queue stays intact for when the pause lifts (issue #141, #147).
|
||||
rec.skip = SkipPaused
|
||||
log.Printf("latest poll %s: paused until %s, skipping pass", name, time.UnixMilli(pausedUntil).Format(time.RFC3339))
|
||||
return time.Duration(pausedUntil-now.UnixMilli()) * time.Millisecond
|
||||
}
|
||||
if refuseUntil > now.UnixMilli() {
|
||||
// Cooling down after a refusal: do not attempt this Site at all.
|
||||
rec.skip = SkipRefusing
|
||||
return time.Duration(refuseUntil-now.UnixMilli()) * time.Millisecond
|
||||
}
|
||||
if isBrowserSite(name) {
|
||||
if downFor, down := p.browserDownFor(now); down && downFor < RefuseBackoff {
|
||||
// A sibling browser Lane lost the sidecar within the backoff
|
||||
// window: skip this pass, so a restarting Chrome does not stamp
|
||||
// this Site's Series one pass at a time. After RefuseBackoff the
|
||||
// flag decays and the Lane probes again (issue #100, story 20).
|
||||
rec.skip = SkipSidecarDown
|
||||
log.Printf("latest poll %s: browser lane skipping pass (sidecar down %s ago)", name, downFor)
|
||||
return RefuseBackoff - downFor
|
||||
}
|
||||
}
|
||||
s := sites[name]
|
||||
f := fetcherFor(name, p.BrowserFetch, p.Fetch)
|
||||
if f == nil {
|
||||
// No fetcher at all right now (browser absent, no fallback): every
|
||||
// Series stays unstamped and due, so a browser that appears after a
|
||||
// restart finds its full queue waiting (issue #100).
|
||||
rec.skip = SkipNoFetcher
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
|
||||
checked := 0
|
||||
due, err := p.Store.DueForLatestCheck(name, now.Add(-s.Rest).UnixMilli(),
|
||||
now.Add(-sightingCeilingRests*s.Rest).UnixMilli())
|
||||
if err != nil {
|
||||
rec.skip = SkipDueQuery
|
||||
log.Printf("latest poll %s: due query: %v", name, err)
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
fig.Due = len(due)
|
||||
if s.Browser != nil && f == p.BrowserFetch && !browserWakeDue(due, now, s.Rest) && !anyForced(due) {
|
||||
// Below both thresholds Chrome stays asleep (ADR-0005 on-demand
|
||||
// browser): waking it for a single Poll would cost a challenge solve
|
||||
// per request. A forced Series is the one exception — a human asking
|
||||
// is not the machine waking itself (issue #146). The Lane still paces
|
||||
// at the default gap, which is what the owner's page must show rather
|
||||
// than a zero.
|
||||
rec.skip = SkipAsleep
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
if s.Browser != nil {
|
||||
// Browser Lanes share one tab, so their combined ceiling is about 360
|
||||
// Polls an hour. When they cannot keep up, the wait past the rest time
|
||||
// grows — log by how much, every pass, so the decision to give them
|
||||
// more pages is made from a measurement rather than a guess.
|
||||
if behind := maxSeriesWait(due, now, s.Rest) - s.Rest; behind > 0 {
|
||||
log.Printf("latest poll %s: browser lane behind by %s (browser Sites cannot keep up with the hour)", name, behind)
|
||||
}
|
||||
}
|
||||
|
||||
eligible, err := p.countEligible(name)
|
||||
if err != nil {
|
||||
rec.skip = SkipEligibleCount
|
||||
log.Printf("latest poll %s: eligible count: %v", name, err)
|
||||
fig.Gap = defaultGap
|
||||
return defaultGap
|
||||
}
|
||||
gap, clamped := effectiveGap(s, eligible)
|
||||
fig.Gap, fig.Clamped = gap, clamped
|
||||
if clamped {
|
||||
log.Printf("latest poll %s: gap clamped to %s floor (eligible series=%d)", name, minGap, eligible)
|
||||
}
|
||||
if eligible == 0 {
|
||||
// Nothing to poll for the foreseeable future; sleep a full rest instead
|
||||
// of re-querying every gap.
|
||||
rec.skip = SkipNothingEligible
|
||||
return s.Rest
|
||||
}
|
||||
|
||||
refusals := 0
|
||||
for i, sr := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
// Staggered rather than fired together: a burst of simultaneous requests
|
||||
// from one server IP is the traffic shape most likely to move that IP's
|
||||
// bot score. This is the server-side analogue of the userscript's "one
|
||||
// series per navigation ... indistinguishable from browsing" (L455-456).
|
||||
stopped := false
|
||||
if i > 0 && p.Stagger > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
stopped = true
|
||||
case <-time.After(p.Stagger):
|
||||
}
|
||||
}
|
||||
if stopped {
|
||||
if refusals >= 2 {
|
||||
// This Site refused twice in a row: the remaining Series are left
|
||||
// unstamped and due, and the Lane waits RefuseBackoff before
|
||||
// trying it again.
|
||||
break
|
||||
}
|
||||
p.checkOne(ctx, sr)
|
||||
checked++
|
||||
if paced && i > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break
|
||||
case <-time.After(gap):
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
outcome := p.checkOne(ctx, sr)
|
||||
if outcome == outcomeUnreachable {
|
||||
// The mid-loop browser loss writes an empty skip on purpose: the
|
||||
// pass returns before the checked counter increments, so its row
|
||||
// is stall-shaped (due > 0, checked 0, skip ''), and a stall is
|
||||
// the exact signal this exit produces. A tenth skip value would
|
||||
// make it legible but is deliberately not invented here.
|
||||
rec.counts.add(outcome)
|
||||
p.setBrowserDown(now)
|
||||
log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, RefuseBackoff)
|
||||
return gap
|
||||
}
|
||||
if outcome == outcomeRefused {
|
||||
refusals++
|
||||
} else {
|
||||
refusals = 0
|
||||
}
|
||||
rec.counts.add(outcome)
|
||||
fig.Checked++
|
||||
}
|
||||
// due vs checked is how you tell which constraint is binding: ticks that
|
||||
// report due=0 mean the cooldown is the limit, ticks that report due==batch
|
||||
// every time mean throughput is.
|
||||
log.Printf("latest poll: due=%d checked=%d", len(due), checked)
|
||||
if fig.Checked > 0 {
|
||||
log.Printf("latest poll %s: due=%d checked=%d", name, len(due), fig.Checked)
|
||||
}
|
||||
if refusals >= 2 {
|
||||
// The refusal outlives the process: the durable stamp gates a restart,
|
||||
// so a Site that just told us to back off is not re-probed.
|
||||
if err := p.Store.SetLaneRefusal(name, now.Add(RefuseBackoff).UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %s: persist refusal: %v", name, err)
|
||||
}
|
||||
log.Printf("latest poll %s: refused twice this run, waiting %s", name, RefuseBackoff)
|
||||
return RefuseBackoff
|
||||
}
|
||||
return gap
|
||||
}
|
||||
|
||||
// passFigures are the numbers one pass measured for its durable row (issue
|
||||
// #141): due and checked as the pass saw them, the pace it chose, and whether
|
||||
// the gap sat on the floor. A pass that returned before measuring keeps the
|
||||
// previous pass's figures via carry-forward in recordPass; the in-memory
|
||||
// snapshot those once mirrored into is gone — the page reads the durable row
|
||||
// now (issue #145).
|
||||
type passFigures struct {
|
||||
Due, Checked int
|
||||
Gap time.Duration
|
||||
Clamped bool
|
||||
}
|
||||
|
||||
// recordPass writes the durable row for one pass (issue #141). Called deferred
|
||||
// from runLanePass so every return path records exactly one row. A pass that
|
||||
// never computed its own figures — its gap is zero — carries the previous
|
||||
// pass's due, gap, clamped and checked forward rather than stating zeroes it
|
||||
// did not measure; the skip column says why it declined, so the zeroes that
|
||||
// remain (due-query, no-fetcher) read as explanations rather than
|
||||
// measurements.
|
||||
func (p *Poller) recordPass(rec passRecord, fig passFigures) {
|
||||
row := store.LanePass{
|
||||
Site: rec.site,
|
||||
RanAt: rec.ranAt,
|
||||
Skip: rec.skip,
|
||||
Due: fig.Due,
|
||||
Checked: fig.Checked,
|
||||
GapMS: fig.Gap.Milliseconds(),
|
||||
Clamped: fig.Clamped,
|
||||
Refused: rec.counts.refused,
|
||||
Unreachable: rec.counts.unreachable,
|
||||
NoChapter: rec.counts.noChapter,
|
||||
Unfetchable: rec.counts.unfetchable,
|
||||
Errors: rec.counts.errors,
|
||||
}
|
||||
if row.GapMS == 0 {
|
||||
// The pass never computed a gap, so it has no figures of its own:
|
||||
// carry the previous pass's, in one latest-per-Site read — the
|
||||
// recorder needs one Site, not six (issue #139).
|
||||
if prev, ok, err := p.Store.LatestLanePass(rec.site); err != nil {
|
||||
log.Printf("latest poll %s: previous pass: %v", rec.site, err)
|
||||
} else if ok {
|
||||
row.Due, row.Checked = prev.Due, prev.Checked
|
||||
row.GapMS, row.Clamped = prev.GapMS, prev.Clamped
|
||||
}
|
||||
}
|
||||
if err := p.Store.RecordLanePass(row, rec.ranAt-lanePassRetention.Milliseconds()); err != nil {
|
||||
log.Printf("latest poll %s: record lane pass: %v", rec.site, err)
|
||||
}
|
||||
}
|
||||
|
||||
// countEligible routes the eligible count through the test seam when one is
|
||||
// set, else the store.
|
||||
func (p *Poller) countEligible(site string) (int, error) {
|
||||
if p.eligibleCount != nil {
|
||||
return p.eligibleCount(site)
|
||||
}
|
||||
return p.Store.EligibleSeriesCount(site)
|
||||
}
|
||||
|
||||
// setBrowserDown records when a browser Lane lost the sidecar. It is Poller
|
||||
// state rather than pass state so the other browser Lanes see it too.
|
||||
func (p *Poller) setBrowserDown(now time.Time) {
|
||||
p.mu.Lock()
|
||||
p.browserDownAt = now
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// browserDownFor reports how long the sidecar has been down and that it is
|
||||
// down at all — the zero time means never down, which must not read as a
|
||||
// zero-duration loss. The window decays: once RefuseBackoff passes without a
|
||||
// fresh loss, Lanes probe again.
|
||||
func (p *Poller) browserDownFor(now time.Time) (time.Duration, bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.browserDownAt.IsZero() {
|
||||
return 0, false
|
||||
}
|
||||
return now.Sub(p.browserDownAt), true
|
||||
}
|
||||
|
||||
// isBrowserSite reports whether the registry routes this Site's page through
|
||||
// the browser sidecar.
|
||||
func isBrowserSite(name string) bool {
|
||||
return sites[name].Browser != nil
|
||||
}
|
||||
|
||||
// browserWakeDue reports whether a browser Lane may start a run: five or more
|
||||
// of its Series are due, or any one of them has been due for browserWakeAge.
|
||||
// Below both thresholds the Lane leaves Chrome asleep — Series Polled together
|
||||
// become due together, so the group naturally stays clustered, and the age
|
||||
// rule exists to stop a Series that drifted out of the group from starving.
|
||||
func browserWakeDue(due []store.Series, now time.Time, rest time.Duration) bool {
|
||||
if len(due) >= browserWakeCount {
|
||||
return true
|
||||
}
|
||||
return maxSeriesWait(due, now, rest) >= browserWakeAge
|
||||
}
|
||||
|
||||
// anyForced reports whether the due list holds a forced Series: one whose
|
||||
// owner check-now request (issue #146) has not been answered yet. A human
|
||||
// asking wakes a sleeping Chrome even below the wake thresholds; the request
|
||||
// itself still ages visibly if the home machine is off.
|
||||
func anyForced(due []store.Series) bool {
|
||||
for _, sr := range due {
|
||||
if sr.Forced {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// maxSeriesWait returns how long the most-overdue of the due Series has been
|
||||
// waiting past its due moment (0 when due is empty).
|
||||
func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.Duration {
|
||||
var oldest time.Duration
|
||||
for _, sr := range due {
|
||||
if w := now.Sub(time.UnixMilli(sr.LatestCheckedAt).Add(rest)); w > oldest {
|
||||
oldest = w
|
||||
}
|
||||
}
|
||||
return oldest
|
||||
}
|
||||
|
||||
// checkOne re-checks one series. Every failure path here is "log and move on":
|
||||
// the poller is a best-effort enhancement, and no single bad series may stall a
|
||||
// batch or take down the process.
|
||||
func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
||||
// Lane or take down the process. The returned outcome classifies the read for
|
||||
// the pass row (issue #141), so the Lane can count a refusal, a lost browser,
|
||||
// a chapter-less page, an unfetchable address or a transport error without
|
||||
// re-deriving the taxonomy.
|
||||
func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("latest poll %q: recovered from panic: %v", sr.Key(), r)
|
||||
outcome = outcomeError
|
||||
}
|
||||
}()
|
||||
|
||||
// Stamped before the fetch, not after, so an error, a timeout, or a shutdown
|
||||
// mid-request still consumes the cooldown. Otherwise a renamed or deleted
|
||||
// series would be retried on every single tick forever. The userscript
|
||||
// stamps in the same order and for the same reason (L471-473).
|
||||
// mid-request still consumes the rest. Otherwise a renamed or deleted
|
||||
// series would be retried on every single pass forever. The userscript
|
||||
// stamps in the same order and for the same reason (L471-473). A Series
|
||||
// never reaches checkOne without a fetcher — runLanePass skips those — so
|
||||
// the stamp means "attempted", and an untried Series stays due.
|
||||
if err := p.Store.MarkLatestChecked(sr.Site, sr.SeriesID, p.Now().UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %q: mark checked: %v", sr.Key(), err)
|
||||
return
|
||||
return outcomeError
|
||||
}
|
||||
|
||||
facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, p.BrowserFetch, p.Fetch)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, errNotFetchable):
|
||||
// The cooldown above is already consumed, so a row that never
|
||||
// passes the gate is retried at cooldown pace rather than
|
||||
// The rest above is already consumed, so a row that never
|
||||
// passes the gate is retried at rest pace rather than
|
||||
// hot-looping.
|
||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
||||
return
|
||||
return outcomeUnfetchable
|
||||
case errors.Is(err, errNoFetcher):
|
||||
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||
return
|
||||
return outcomeUnfetchable
|
||||
}
|
||||
// A legacy cover heals independently of the page read: its source may
|
||||
// answer — a CDN — while the origin does not, so a fetch failure does
|
||||
// not skip the heal, matching the order the shared read replaced.
|
||||
p.prefetchCover(ctx, sr)
|
||||
p.healCover(ctx, sr)
|
||||
log.Printf("latest poll %q: %v", sr.Key(), err)
|
||||
return
|
||||
if errors.Is(err, errChallengeHeld) {
|
||||
return outcomeRefused
|
||||
}
|
||||
if errors.Is(err, errBrowserInterrupted) {
|
||||
return outcomeUnreachable
|
||||
}
|
||||
return outcomeError
|
||||
}
|
||||
// A legacy cover source is healed independently of the page read.
|
||||
p.prefetchCover(ctx, sr)
|
||||
p.healCover(ctx, sr)
|
||||
// Cover fill is independent of the chapter signal: a page that lost its
|
||||
// chapter list may keep its og:image, and a blank Series heals either way.
|
||||
p.fillBlankCover(ctx, sr, facts.Cover)
|
||||
if !facts.HasLatest {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||
// already stamped, so this waits out a rest instead of hot-looping.
|
||||
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
|
||||
return
|
||||
return outcomeNoChapter
|
||||
}
|
||||
|
||||
// The Poll is the oracle for whatever Sighting last raised this Series
|
||||
// (issue #103), and the judgement is free: the comparison below already
|
||||
// exists, and no extra request is made to reach it.
|
||||
p.judgeSighting(sr, facts.Latest.Num)
|
||||
|
||||
// Equality, not >, mirroring the userscript (L427): a site that retracts a
|
||||
// chapter should correct the stored number downward. The comparison is
|
||||
// against the due-query snapshot; a concurrent write in between only costs
|
||||
// one redundant UPDATE of the same absolute value, never a wrong one.
|
||||
if sr.LatestChapterNum != nil && *sr.LatestChapterNum == facts.Latest.Num {
|
||||
return
|
||||
return outcomeSuccess
|
||||
}
|
||||
|
||||
// Series-level write: the row is shared, so one update refreshes every
|
||||
@@ -270,9 +648,66 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
||||
// the list.
|
||||
if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil {
|
||||
log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err)
|
||||
return
|
||||
return outcomeError
|
||||
}
|
||||
log.Printf("latest poll %q: latest is now %s", sr.Key(), facts.Latest.Label)
|
||||
return outcomeSuccess
|
||||
}
|
||||
|
||||
// judgeSighting settles the Sighting the Series' stored Latest Chapter is owed
|
||||
// to, if any, against what the Site actually publishes. The asymmetry is the
|
||||
// whole of the detection rule and is what keeps it free of false alarms: a Poll
|
||||
// finding a *lower* number than stored means the Reader who raised it reported
|
||||
// a chapter that does not exist, while a Poll finding a higher one is only the
|
||||
// Site publishing since and means nothing about the report. Equality confirms
|
||||
// the report, which is how an honest Reader earns back a mark.
|
||||
//
|
||||
// A Series with no attribution — the stored value is a Poll's own, or a
|
||||
// previous Poll already judged the report — is nobody's to answer for.
|
||||
func (p *Poller) judgeSighting(sr store.Series, found float64) {
|
||||
if sr.LatestRaisedBy == nil || sr.LatestChapterNum == nil {
|
||||
return
|
||||
}
|
||||
stored := *sr.LatestChapterNum
|
||||
if found > stored {
|
||||
// The report is neither confirmed nor contradicted, but it is answered:
|
||||
// the value about to be stored is the Poll's own, so leaving the
|
||||
// attribution would credit this Reader with the next Poll's agreement
|
||||
// and blame them if the Site later retracts.
|
||||
if err := p.Store.ClearSightingAttribution(sr.Site, sr.SeriesID, *sr.LatestRaisedBy); err != nil {
|
||||
log.Printf("latest poll %q: clear sighting attribution: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if found < stored {
|
||||
// Logged with both numbers and the Reader, because that is what tells a
|
||||
// broken Site adapter (which marks every Reader of that Site at once)
|
||||
// from one Reader deliberately lying.
|
||||
log.Printf("latest poll %q: sighting contradicted: reader %d raised it to %v, site publishes %v",
|
||||
sr.Key(), *sr.LatestRaisedBy, stored, found)
|
||||
}
|
||||
if err := p.Store.RecordSightingOutcome(sr.Site, sr.SeriesID, *sr.LatestRaisedBy, found == stored); err != nil {
|
||||
log.Printf("latest poll %q: record sighting outcome: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// healCover runs prefetchCover in the background. Cover bytes come from a
|
||||
// different host — often a CDN — and heal once in a Series's life, so they
|
||||
// must not consume a Lane's gap: a large import with many blanks would
|
||||
// otherwise make every Latest Chapter go stale behind a slow image host
|
||||
// (issue #100).
|
||||
func (p *Poller) healCover(ctx context.Context, sr store.Series) {
|
||||
p.coverWG.Add(1)
|
||||
go func() {
|
||||
defer p.coverWG.Done()
|
||||
p.prefetchCover(ctx, sr)
|
||||
}()
|
||||
}
|
||||
|
||||
// waitCovers blocks until every in-flight cover heal finishes. Tests call it
|
||||
// after a round before asserting on cover fetches.
|
||||
func (p *Poller) waitCovers() {
|
||||
p.coverWG.Wait()
|
||||
}
|
||||
|
||||
// fetchableSeriesURL reports whether site is a Site the registry knows and
|
||||
|
||||
+1236
-108
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@ import (
|
||||
// seriesRead carries the two facts the poll and the acquirer both extract
|
||||
// from a series page. Persistence, stamps and scheduling stay with the
|
||||
// callers, so the policies that keep the two flows distinct (stamp order,
|
||||
// cooldowns) are not swallowed by the module.
|
||||
// rests) are not swallowed by the module.
|
||||
type seriesRead struct {
|
||||
Latest latestChapter
|
||||
HasLatest bool
|
||||
@@ -22,6 +22,9 @@ type seriesRead struct {
|
||||
|
||||
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
||||
// failures so each caller keeps its own distinct log line for all three.
|
||||
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
|
||||
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
|
||||
// is how a Lane tells a refusal from an ordinary failure (issue #100).
|
||||
var (
|
||||
errNotFetchable = errors.New("series url not fetchable")
|
||||
errNoFetcher = errors.New("no fetcher for site")
|
||||
@@ -49,9 +52,21 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
||||
if err != nil {
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: %w", seriesURL, err)
|
||||
}
|
||||
if status == 403 {
|
||||
// Cloudflare's challenge response for these Sites (cf-mitigated). The
|
||||
// browser fetcher returns exactly this on a held interstitial, and a
|
||||
// plain-TLS 403 means the same: the Site is refusing.
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
|
||||
}
|
||||
if status != 200 {
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
|
||||
}
|
||||
if isInterstitial(body) {
|
||||
// A 200 that is the challenge page, not the payload: the TLS route can
|
||||
// receive this where the browser would have kept re-reading. Same
|
||||
// refusal as the 403.
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: interstitial body", errChallengeHeld, seriesURL)
|
||||
}
|
||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||
|
||||
@@ -0,0 +1,494 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// Sightings (issue #103) are specified at the Poller seam, with the store as
|
||||
// the way in: a Sighting is seeded the way handlers.Put performs one, a round
|
||||
// is run against the injected fetcher and a frozen clock, and the assertions
|
||||
// are the two observable facts — whether the Series was fetched, and what the
|
||||
// stored Latest Chapter is afterwards. Nothing here asserts counter arithmetic
|
||||
// through an internal call or reads how a deferral is represented in a row.
|
||||
|
||||
const (
|
||||
sightingSlug = "chronicles-of-the-demon-faction-f886a8af"
|
||||
sightingKey = "asura:" + sightingSlug
|
||||
sightingURL = "https://asurascans.com/comics/" + sightingSlug
|
||||
)
|
||||
|
||||
// sightingFixtureLatest is the newest chapter asuraSeriesFixture publishes.
|
||||
const sightingFixtureLatest = 181.0
|
||||
|
||||
// sight performs one Sighting exactly as the JSON API does (handlers.Put):
|
||||
// RecordSighting against the row as stored, then the Upsert that stores the
|
||||
// reported value. The order is load-bearing — the raise comparison has nothing
|
||||
// to compare against once the Upsert has landed — and the bookmark's own fields
|
||||
// are carried over untouched, which is what a userscript PUT does when it
|
||||
// echoes back the row it cached.
|
||||
func sight(t *testing.T, s *store.Store, readerID int64, key string, num float64, at time.Time) {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", key)
|
||||
}
|
||||
b, found, err := s.Get(readerID, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("sight %q: get: %v found=%v", key, err, found)
|
||||
}
|
||||
if err := s.RecordSighting(readerID, site, seriesID, &num, at.UnixMilli()); err != nil {
|
||||
t.Fatalf("sight %q: %v", key, err)
|
||||
}
|
||||
b.LatestChapter = fmt.Sprintf("Chapter %v", num)
|
||||
b.LatestChapterNum = &num
|
||||
b.UpdatedAt = at.UnixMilli()
|
||||
if _, err := s.Upsert(readerID, b); err != nil {
|
||||
t.Fatalf("sight %q: upsert: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// secondReader is another Reader on the same database. The owner seed is the
|
||||
// only reader-creation path in this package, so a second Open as a different
|
||||
// owner is how a test gets one (as TestRunOnceFetchesSharedSeriesOnce does).
|
||||
func secondReader(t *testing.T, dbURL string) *store.Store {
|
||||
t.Helper()
|
||||
other, err := store.Open(dbURL,
|
||||
store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))},
|
||||
t.TempDir(), testCoverBaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Open second reader: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { other.Close() })
|
||||
return other
|
||||
}
|
||||
|
||||
func readLatestNum(t *testing.T, s *store.Store, readerID int64, key string) float64 {
|
||||
t.Helper()
|
||||
b, ok, err := s.Get(readerID, key)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get %q: %v ok=%v", key, err, ok)
|
||||
}
|
||||
if b.LatestChapterNum == nil {
|
||||
t.Fatalf("%q has no latest chapter", key)
|
||||
}
|
||||
return *b.LatestChapterNum
|
||||
}
|
||||
|
||||
// A Series only one Reader bookmarks is the case where being wrong can hurt
|
||||
// nobody but the Reader who reported it, so their Sighting stands in for the
|
||||
// Poll and the round leaves the Series alone.
|
||||
func TestSightingOnSolitarySeriesDefersPoll(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("fetched %d times after a Sighting on a solitary Series, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// On a shared Series the Sighting still writes the Latest Chapter for everyone,
|
||||
// but the Poll happens on schedule anyway — which is what corrects a wrong
|
||||
// value within the hour instead of letting it persist.
|
||||
func TestSightingOnSharedSeriesDoesNotDeferPoll(t *testing.T) {
|
||||
s, dbURL := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
other := secondReader(t, dbURL)
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
sight(t, s, s.OwnerID(), sightingKey, 200, now.Add(-10*time.Minute))
|
||||
|
||||
// The Sighting updated the shared row immediately, before any Poll.
|
||||
if got := readLatestNum(t, s, s.OwnerID(), sightingKey); got != 200 {
|
||||
t.Fatalf("latest after the Sighting = %v, want 200", got)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times after a Sighting on a shared Series, want 1", got)
|
||||
}
|
||||
if got := readLatestNum(t, s, s.OwnerID(), sightingKey); got != sightingFixtureLatest {
|
||||
t.Fatalf("latest after the Poll = %v, want the Site's own %v", got, sightingFixtureLatest)
|
||||
}
|
||||
}
|
||||
|
||||
// Reporting a chapter is not reading one: a Sighting may move the Latest
|
||||
// Chapter and nothing else. Both the solitary and the shared case, because the
|
||||
// deferral branch must not be where this guarantee lives.
|
||||
func TestSightingLeavesProgressAndOrderingUntouched(t *testing.T) {
|
||||
for _, shared := range []bool{false, true} {
|
||||
name := "solitary"
|
||||
if shared {
|
||||
name = "shared"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, dbURL := newTestStore(t)
|
||||
read := 5.0
|
||||
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, SeriesURL: sightingURL,
|
||||
LastChapter: "Chapter 5", LastChapterNum: read,
|
||||
LastChapterURL: sightingURL + "/chapter/5", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if shared {
|
||||
other := secondReader(t, dbURL)
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
sight(t, s, s.OwnerID(), sightingKey, 200, time.UnixMilli(9_000_000))
|
||||
|
||||
b, ok, err := s.Get(s.OwnerID(), sightingKey)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get: %v ok=%v", err, ok)
|
||||
}
|
||||
if b.LatestChapterNum == nil || *b.LatestChapterNum != 200 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 200", b.LatestChapterNum)
|
||||
}
|
||||
if b.LastChapterNum != read {
|
||||
t.Fatalf("LastChapterNum = %v, want %v: a Sighting is not Progress", b.LastChapterNum, read)
|
||||
}
|
||||
if b.UpdatedAt != 1000 {
|
||||
t.Fatalf("updated_at moved to %d: a Sighting must not reorder the list", b.UpdatedAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The ceiling is what makes trusting a client report safe: however recently a
|
||||
// Series was sighted, one that has not been Polled in six hours is Polled.
|
||||
func TestSightingCeilingForcesPoll(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-7*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).runOnce(context.Background())
|
||||
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times past the %s ceiling, want 1", got, sightingCeilingRests*defaultRest)
|
||||
}
|
||||
}
|
||||
|
||||
// Deferral is decided from live facts every round, so a Series that gains a
|
||||
// second Bookmark stops deferring at once — and one that loses it defers again.
|
||||
func TestDeferralFollowsTheBookmarkCount(t *testing.T) {
|
||||
s, dbURL := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("solitary Series fetched %d times, want 0", got)
|
||||
}
|
||||
|
||||
other := secondReader(t, dbURL)
|
||||
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
||||
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed second reader: %v", err)
|
||||
}
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("shared Series fetched %d times, want 1", got)
|
||||
}
|
||||
|
||||
// The Poll above consumed the rest, so move past it before asking again.
|
||||
if err := other.Delete(other.OwnerID(), sightingKey); err != nil {
|
||||
t.Fatalf("delete second bookmark: %v", err)
|
||||
}
|
||||
later := now.Add(2 * time.Hour)
|
||||
p.Now = func() time.Time { return later }
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, later.Add(-time.Minute))
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("Series fetched %d times after returning to one Bookmark, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A Series nobody reports any more returns to the normal schedule on its own:
|
||||
// the Sighting's standing lasts one rest, not forever.
|
||||
func TestDeferralExpiresWithoutFurtherSightings(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 0 {
|
||||
t.Fatalf("fetched %d times while the Sighting stood, want 0", got)
|
||||
}
|
||||
|
||||
p.Now = func() time.Time { return now.Add(90 * time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times once the Sighting aged out, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// demonicFixture publishes one chapter in demonicscans' live page shape, so a
|
||||
// test can make a Site publish an arbitrary number rather than the one the
|
||||
// captured fixture froze.
|
||||
func demonicFixture(num float64) string {
|
||||
return fmt.Sprintf(
|
||||
`<a href="/chaptered.php?manga=11799&chapter=%v" class="chplinks" title="Catastrophic Necromancer %v">Chapter %v</a>`,
|
||||
num, num, num)
|
||||
}
|
||||
|
||||
const (
|
||||
demonicKey = "demonic:Catastrophic-Necromancer"
|
||||
demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
||||
)
|
||||
|
||||
// contradictOnce reports a chapter that does not exist and then runs the round
|
||||
// that catches it, returning when that round ran so a caller can chain the
|
||||
// next one. The wait is one rest and a minute: a Sighting stands in for exactly
|
||||
// one rest, so that is the first moment this solitary Series is Polled again.
|
||||
func contradictOnce(t *testing.T, s *store.Store, p *Poller, sightAt time.Time, real float64) time.Time {
|
||||
t.Helper()
|
||||
sight(t, s, s.OwnerID(), demonicKey, real+500, sightAt)
|
||||
at := sightAt.Add(defaultRest + time.Minute)
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != real {
|
||||
t.Fatalf("latest after the Poll = %v, want the Site's own %v", got, real)
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
func seedDemonic(t *testing.T, s *store.Store, checkedAt int64) {
|
||||
t.Helper()
|
||||
seedForCheck(t, s, demonicKey, demonicURL, checkedAt)
|
||||
}
|
||||
|
||||
// A Poll finding a lower number than stored means the Sighting that raised it
|
||||
// was false. The Reader is named — not the Series flagged — and both numbers are
|
||||
// logged, because that is what tells a broken adapter from a deliberate lie.
|
||||
func TestPollContradictingASightingNamesTheReaderAndBothNumbers(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
contradictOnce(t, s, p, now, 296)
|
||||
|
||||
got := logs.String()
|
||||
for _, want := range []string{
|
||||
fmt.Sprintf("reader %d", s.OwnerID()), "796", "296", demonicKey,
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("contradiction log = %q, want it to name %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Three contradictions cost the Reader the right to defer. Nothing here writes
|
||||
// a counter: the marks are earned through Polls, which is the only way
|
||||
// production produces them.
|
||||
func TestThreeContradictionsStopDeferral(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
start := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, start)
|
||||
at := start
|
||||
for range store.SightingDisagreementLimit {
|
||||
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
|
||||
}
|
||||
fetchesSoFar := f.callCount()
|
||||
|
||||
// The marked Reader sights the same solitary Series again. It still writes
|
||||
// the Latest Chapter — the penalty removes a privilege, it does not silence
|
||||
// anyone — but the Poll is no longer postponed: the round below runs while a
|
||||
// trusted Reader's Sighting would still be standing, and fetches anyway.
|
||||
sight(t, s, s.OwnerID(), demonicKey, 900, at.Add(31*time.Minute))
|
||||
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != 900 {
|
||||
t.Fatalf("latest after a marked Reader's Sighting = %v, want 900", got)
|
||||
}
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != fetchesSoFar+1 {
|
||||
t.Fatalf("marked Reader's Sighting still deferred the Poll (fetches %d, want %d)",
|
||||
got, fetchesSoFar+1)
|
||||
}
|
||||
}
|
||||
|
||||
// The owner's remedy for a mark a broken Site adapter produced restores the
|
||||
// privilege without a wait and without SQL.
|
||||
func TestClearingMarksRestoresDeferral(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
start := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, start)
|
||||
at := start
|
||||
for range store.SightingDisagreementLimit {
|
||||
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
|
||||
}
|
||||
if err := s.ClearReaderMarks(s.OwnerID()); err != nil {
|
||||
t.Fatalf("ClearReaderMarks: %v", err)
|
||||
}
|
||||
|
||||
fetchesSoFar := f.callCount()
|
||||
sight(t, s, s.OwnerID(), demonicKey, 900, at.Add(31*time.Minute))
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != fetchesSoFar {
|
||||
t.Fatalf("fetched %d times after the marks were cleared, want %d: deferral must resume",
|
||||
got, fetchesSoFar)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovery is automatic but expensive: twenty Polls that each confirm a
|
||||
// Sighting of this Reader's clear the marks. Each round needs a new chapter,
|
||||
// because only a report that raises the stored number is attributed and so only
|
||||
// that one can be confirmed.
|
||||
func TestTwentyAgreementsClearTheMarks(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
start := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, start)
|
||||
at := start
|
||||
for range store.SightingDisagreementLimit {
|
||||
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
|
||||
}
|
||||
|
||||
chapter := 296.0
|
||||
for range store.SightingAgreementsToClear {
|
||||
chapter++
|
||||
sight(t, s, s.OwnerID(), demonicKey, chapter, at.Add(time.Minute))
|
||||
f.body = demonicFixture(chapter) // the Site publishes what was reported
|
||||
at = at.Add(defaultRest + time.Minute)
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
}
|
||||
|
||||
fetchesSoFar := f.callCount()
|
||||
chapter++
|
||||
sight(t, s, s.OwnerID(), demonicKey, chapter, at.Add(31*time.Minute))
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != fetchesSoFar {
|
||||
t.Fatalf("fetched %d times after %d confirmations, want %d: the marks must be forgiven",
|
||||
got, store.SightingAgreementsToClear, fetchesSoFar)
|
||||
}
|
||||
}
|
||||
|
||||
// A Poll finding a higher number is the Site publishing since the Sighting and
|
||||
// means nothing about the Reader — no mark, and no credit either.
|
||||
func TestPollFindingHigherNumberIsNotAContradiction(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
// Reported truthfully, then the Site published one more.
|
||||
sight(t, s, s.OwnerID(), demonicKey, 295, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
// One rest on, the Sighting has lapsed and the Poll happens.
|
||||
p.Now = func() time.Time { return now.Add(7 * time.Hour) }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times past the ceiling, want 1", got)
|
||||
}
|
||||
|
||||
// Unmarked, so a fresh Sighting still defers.
|
||||
at := now.Add(9 * time.Hour)
|
||||
sight(t, s, s.OwnerID(), demonicKey, 296, at.Add(-time.Minute))
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("a Reader whose report the Site overtook lost the right to defer (fetches %d, want 1)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A Poll that overtakes a Sighting takes ownership of the row: the value stored
|
||||
// afterwards is the Poll's own, so a later retraction is not the Reader's fault
|
||||
// and must not be charged to them.
|
||||
func TestAttributionDoesNotSurviveAPollThatOvertookIt(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
sight(t, s, s.OwnerID(), demonicKey, 295, now.Add(-10*time.Minute))
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
at := now.Add(defaultRest + time.Minute)
|
||||
p.Now = func() time.Time { return at }
|
||||
p.runOnce(context.Background())
|
||||
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != 296 {
|
||||
t.Fatalf("latest after the Poll = %v, want the Site's own 296", got)
|
||||
}
|
||||
|
||||
var logs strings.Builder
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
t.Cleanup(func() { log.SetOutput(prev) })
|
||||
|
||||
f.body = demonicFixture(290) // the Site retracts what only the Poll wrote
|
||||
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
|
||||
p.runOnce(context.Background())
|
||||
if strings.Contains(logs.String(), "sighting contradicted") {
|
||||
t.Fatalf("a retraction of the Poll's own value was charged to a Reader: %s", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A PUT with no Latest Chapter in it — a favourite toggle, progress written
|
||||
// from a chapter page — is nobody looking at the Series page, so it buys no
|
||||
// deferral. Otherwise a client could suppress a Series' Polls while reporting
|
||||
// nothing, and with nothing reported there would be nothing to judge.
|
||||
func TestPutWithoutALatestChapterDoesNotDefer(t *testing.T) {
|
||||
s, _ := newTestStore(t)
|
||||
now := time.UnixMilli(20 * time.Hour.Milliseconds())
|
||||
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
|
||||
// The handler's own call, with the field the client omitted.
|
||||
if err := s.RecordSighting(s.OwnerID(), "demonic", "Catastrophic-Necromancer",
|
||||
nil, now.Add(-time.Minute).UnixMilli()); err != nil {
|
||||
t.Fatalf("RecordSighting: %v", err)
|
||||
}
|
||||
|
||||
f := &fakeFetcher{body: demonicFixture(296), status: 200}
|
||||
p := newTestPoller(t, s, f, now)
|
||||
p.runOnce(context.Background())
|
||||
if got := f.callCount(); got != 1 {
|
||||
t.Fatalf("fetched %d times after a PUT carrying no chapter, want 1", got)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
@@ -32,6 +33,11 @@ type site struct {
|
||||
LatestChapter func(seriesURL, body string) (latestChapter, bool)
|
||||
// Cover finds the Cover address in a fetched body.
|
||||
Cover func(seriesURL, body string) (string, bool)
|
||||
// Rest is how long a Series of this Site rests between Polls.
|
||||
Rest time.Duration
|
||||
// Gap is the Lane's strictest pace: at least one second must pass between
|
||||
// two consecutive Series-page Polls of this Site (issue #100).
|
||||
Gap time.Duration
|
||||
// Browser reads this Site's payload from a cleared browser tab; nil
|
||||
// means the page is fetched over plain TLS.
|
||||
Browser *browserRead
|
||||
@@ -47,9 +53,9 @@ type browserRead struct {
|
||||
// Done reports whether the payload arrived.
|
||||
Done func(body string) bool
|
||||
// Fallback allows the plain-TLS fetcher when no browser is configured.
|
||||
// False skips the Site instead. kagane is false — a plain fetch would
|
||||
// only ever retrieve a challenge page — and novelfull is true, because
|
||||
// its challenge is a live time-varying fact (AGENTS.md).
|
||||
// False skips the Site instead. kagane and comix are false — a plain fetch
|
||||
// would only ever retrieve a challenge page — and novelfull is true,
|
||||
// because its challenge is a live time-varying fact (AGENTS.md).
|
||||
Fallback bool
|
||||
}
|
||||
|
||||
@@ -248,14 +254,25 @@ var comixInitialDataRe = regexp.MustCompile(`(?is)<script\b[^>]*\bid\s*=\s*["']i
|
||||
// supplied, and a headless browser is a strong SSRF primitive.
|
||||
var kaganeImageURLRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`)
|
||||
|
||||
// comixImageURLRe matches comix's cover host and path shape. Pinned in full
|
||||
// (scheme, host, path characters, image extension) for the same reason
|
||||
// kaganeImageURLRe is: the address reaches a headless browser, and it can
|
||||
// originate in a client-supplied PUT body. No dot is allowed inside the path,
|
||||
// so no traversal or second extension can hide in it. Shape from a live page,
|
||||
// 2026-08-10: /039d/i/1/34/6a6742bf15736@280.jpg.
|
||||
var comixImageURLRe = regexp.MustCompile(`^https://static\.comix\.to/[A-Za-z0-9@/_-]+\.(?:jpg|jpeg|png|webp)$`)
|
||||
|
||||
// browserOnlyCoverURL reports whether the browser sidecar is the only fetcher
|
||||
// for cover bytes at imageURL. kagane's image route answers a plain fetch with
|
||||
// a challenge and `cross-origin-resource-policy: same-origin`, so a TLS fetch
|
||||
// would only ever retrieve a challenge page and must not be attempted
|
||||
// (ADR-0007). This is the byte-fetch router's per-Site knowledge; it lives in
|
||||
// the extraction module, which owns kagane's URL shapes.
|
||||
// a challenge and `cross-origin-resource-policy: same-origin`, and
|
||||
// static.comix.to answers one with the same Cloudflare challenge its pages
|
||||
// serve (measured 2026-08-12, issue #98), so a TLS fetch would only ever
|
||||
// retrieve a challenge page and must not be attempted (ADR-0007). This is the
|
||||
// byte-fetch router's per-Site knowledge; it lives in the extraction module,
|
||||
// which owns those URL shapes.
|
||||
func browserOnlyCoverURL(imageURL string) bool {
|
||||
return kaganeImageURLRe.MatchString(imageURL)
|
||||
return kaganeImageURLRe.MatchString(imageURL) ||
|
||||
comixImageURLRe.MatchString(imageURL)
|
||||
}
|
||||
|
||||
// kagane's browser-fetched series response publishes cover image IDs under
|
||||
@@ -369,6 +386,59 @@ func publishedCoverURL(value string) string {
|
||||
return strings.ReplaceAll(value, " ", "%20")
|
||||
}
|
||||
|
||||
// Poll Lane constants (issue #100). The per-Site structure is deliberately
|
||||
// uniform at first — every Site rests an hour and gaps ten seconds — but it
|
||||
// exists so a single Site can be slowed if it turns hostile, and the numbers
|
||||
// stay in the registry so the structure has a place to differ.
|
||||
const (
|
||||
// defaultRest is how long every Series rests between Polls.
|
||||
defaultRest = time.Hour
|
||||
// defaultGap is the strictest pace of every Lane unless the eligible
|
||||
// Series count forces it tighter.
|
||||
defaultGap = 10 * time.Second
|
||||
// minGap floors the effective gap. One request per second is already an
|
||||
// order of magnitude past the strictest rate rule a free-plan Site can
|
||||
// express (docs/research/cloudflare-bot-scoring-and-poll-cadence.md);
|
||||
// below it the Lane is outrunning its own plan and says so loudly.
|
||||
minGap = time.Second
|
||||
// RefuseBackoff is how long a Lane waits after its Site refused twice in
|
||||
// one run before attempting it again. Exported so the web layer can derive
|
||||
// browser reachability from the pass log over the same window (issue #145).
|
||||
RefuseBackoff = 15 * time.Minute
|
||||
// browserWakeCount and browserWakeAge gate a browser Lane's run: five or
|
||||
// more due Series, or any one of them waiting this long, or Chrome stays
|
||||
// asleep (ADR-0005 on-demand browser).
|
||||
browserWakeCount = 5
|
||||
browserWakeAge = 15 * time.Minute
|
||||
// sightingCeilingRests caps Sighting deferral (issue #103): however many
|
||||
// Sightings arrive, a Series unpolled for this many of its Site's rests is
|
||||
// Polled. It is what makes a client report safe to trust — a wrong Latest
|
||||
// Chapter dies within the ceiling deterministically, rather than in
|
||||
// expectation the way a randomised audit would have it. Six, so a Series a
|
||||
// Reader visits constantly still gets one authoritative check per working
|
||||
// day-part.
|
||||
sightingCeilingRests = 6
|
||||
)
|
||||
|
||||
// effectiveGap is a Site's pace: the registry gap, or one rest divided by the
|
||||
// eligible Series count when that is smaller, never below one second. The
|
||||
// denominator follows defaultRest rather than a literal hour so a Site whose
|
||||
// rest is ever changed keeps its per-Series pace in step. The second return is
|
||||
// true when the one-second floor engaged (and the Lane logs a warning naming
|
||||
// the Site, every round it does).
|
||||
func effectiveGap(s site, eligible int) (time.Duration, bool) {
|
||||
gap := s.Gap
|
||||
if eligible > 0 {
|
||||
if perSeries := defaultRest / time.Duration(eligible); perSeries < gap {
|
||||
gap = perSeries
|
||||
}
|
||||
}
|
||||
if gap < minGap {
|
||||
return minGap, true
|
||||
}
|
||||
return gap, false
|
||||
}
|
||||
|
||||
// sites is the registry: one entry per Site, keyed by the stored site string.
|
||||
// Adding a Site means adding an entry here and nowhere else — the dispatch
|
||||
// functions above and the poller's route list are lookups into this map. An
|
||||
@@ -379,21 +449,38 @@ var sites = map[string]site{
|
||||
Host: "asurascans.com",
|
||||
LatestChapter: asuraLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
"demonic": {
|
||||
Host: "demonicscans.org",
|
||||
LatestChapter: demonicLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
"comix": {
|
||||
Host: "comix.to",
|
||||
LatestChapter: comixLatestChapter,
|
||||
Cover: comixCoverEntry,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
Read: comixRead,
|
||||
// The interstitial is served in place of the page, so "arrived"
|
||||
// has to exclude it explicitly, as novelfull's does.
|
||||
Done: func(body string) bool { return body != "" && !isInterstitial(body) },
|
||||
// Never falls back: a plain fetch of a comix page or cover
|
||||
// retrieves only a challenge page (measured 2026-08-12).
|
||||
Fallback: false,
|
||||
},
|
||||
},
|
||||
"kagane": {
|
||||
Host: "kagane.to",
|
||||
LatestChapter: kaganeLatestChapter,
|
||||
Cover: kaganeCoverEntry,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
Read: kaganeRead,
|
||||
Done: func(body string) bool { return body != "" },
|
||||
@@ -406,6 +493,8 @@ var sites = map[string]site{
|
||||
Host: "novelfull.com",
|
||||
LatestChapter: novelfullLatestChapter,
|
||||
Cover: novelfullCoverEntry,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
Read: novelfullRead,
|
||||
// The interstitial has a DOM too, so "the payload arrived" has to
|
||||
@@ -418,13 +507,27 @@ var sites = map[string]site{
|
||||
Host: "lightnovelworld.net",
|
||||
LatestChapter: lnwLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
}
|
||||
|
||||
// SiteNames returns every registry Site, sorted. The admin Series list's Site
|
||||
// select needs the full registry, not just the Sites that have rows, and
|
||||
// laneNames() is the poller's copy of the same list — both read this.
|
||||
func SiteNames() []string {
|
||||
names := make([]string, 0, len(sites))
|
||||
for name := range sites {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// browserBackedSites is derived from the registry: the Sites whose pages are
|
||||
// read through the browser sidecar, which are also the ones granted the longer
|
||||
// cooldown. Sorted so callers that range it (the due query, the browser
|
||||
// fetcher's dispatch) see a stable order instead of map-iteration noise.
|
||||
// read through the browser sidecar. Sorted so callers that range it (the
|
||||
// browser fetcher's dispatch) see a stable order instead of map-iteration
|
||||
// noise.
|
||||
func browserBackedSites() []string {
|
||||
out := make([]string, 0, len(sites))
|
||||
for name, s := range sites {
|
||||
|
||||
@@ -41,6 +41,13 @@ const challengeFixture = `<!DOCTYPE html><html><head><title>Just a moment...</ti
|
||||
// https://comix.to/title/n8we-dungeons-and-crayons fetched 2026-08-03. comix is
|
||||
// an SPA: the page ships a JSON state blob rather than a list of chapter
|
||||
// anchors, and latestChapterUrl is where the newest chapter actually lives.
|
||||
//
|
||||
// Still the right fixture after comix moved behind the challenge (#98): the
|
||||
// browser read is an in-tab fetch of the Series URL, so the body a poll parses
|
||||
// is this same server-rendered HTML, not a rendered DOM. Confirmed against a
|
||||
// live cleared tab 2026-08-16 (TestSmokeComix): the in-tab fetch returned
|
||||
// 24793 bytes of server-rendered HTML that these same parses read a chapter
|
||||
// and a cover out of.
|
||||
const comixSeriesFixture = `
|
||||
{"firstChapterUrl":"/title/n8we-dungeons-and-crayons/5038739-chapter-1","latestChapterUrl":"/title/n8we-dungeons-and-crayons/11139891-chapter-80"},
|
||||
{""manga","recommended","n8we",1]":{"items":[{"latestChapterUrl":"/title/qqwrm-full-time-awakening/99999999-chapter-999"}]}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// TestSmokeComix answers "is comix's challenge clearing from this browser right
|
||||
// now" — a live, time-varying fact, so a red run is something to re-check
|
||||
// before it is a defect. Needs the real browser unit with outbound network:
|
||||
//
|
||||
// cd chrome && BROWSER_BIND_ADDR=127.0.0.1 docker compose up -d --build
|
||||
// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:9222 go test -run TestSmokeComix ./internal/latest
|
||||
//
|
||||
// It walks the whole read: the in-tab page fetch, both parses, and the Cover
|
||||
// bytes by direct navigation to static.comix.to. The Cover address comes out of
|
||||
// the page rather than being pinned in the test, because a stored one rots.
|
||||
func TestSmokeComix(t *testing.T) {
|
||||
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
const seriesURL = "https://comix.to/title/m12d-classmate"
|
||||
|
||||
f, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
t.Fatalf("NewBrowserFetcher: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
body, status, err := f.Get(ctx, seriesURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
t.Logf("status=%d bytes=%d", status, len(body))
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status)
|
||||
}
|
||||
chapter, ok := latestChapterFrom("comix", seriesURL, body)
|
||||
if !ok {
|
||||
t.Fatalf("no latest chapter in %d bytes — page shape changed", len(body))
|
||||
}
|
||||
t.Logf("latest chapter: %v %q", chapter.Num, chapter.Label)
|
||||
|
||||
cover, ok := coverFrom("comix", seriesURL, body)
|
||||
if !ok {
|
||||
t.Fatalf("no cover address in %d bytes — page shape changed", len(body))
|
||||
}
|
||||
t.Logf("cover: %s", cover)
|
||||
if !browserOnlyCoverURL(cover) {
|
||||
t.Fatalf("cover %q is not claimed by the browser gate: the pin and the live URL shape disagree", cover)
|
||||
}
|
||||
|
||||
bytes, contentType, err := f.Image(ctx, cover)
|
||||
if err != nil {
|
||||
t.Fatalf("Image: %v", err)
|
||||
}
|
||||
if len(bytes) < 1000 {
|
||||
t.Fatalf("cover is %d bytes, want a real image", len(bytes))
|
||||
}
|
||||
t.Logf("fetched %d bytes of %s", len(bytes), contentType)
|
||||
if _, ok := store.CoverContentType(contentType); !ok {
|
||||
t.Fatalf("content type %q is not storable", contentType)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Series filter names (issue #140), ordered permanent-then-fixable — the
|
||||
// repairs nothing will ever undo first, the ones a Poll can make right after.
|
||||
// A name is the repair a row needs, not the SQL that finds it; the values are
|
||||
// the wire form the Series list URL carries (#142). "all" is the absent and
|
||||
// unknown case: every Series.
|
||||
const (
|
||||
SeriesFilterAll = "all"
|
||||
SeriesFilterNoURL = "no_series_url"
|
||||
SeriesFilterNoChapter = "never_read_a_chapter"
|
||||
SeriesFilterNoReaders = "no_readers"
|
||||
SeriesFilterNeverChecked = "never_checked"
|
||||
SeriesFilterStale = "stale"
|
||||
SeriesFilterNoCover = "no_cover"
|
||||
SeriesFilterReaderReport = "reader_report"
|
||||
)
|
||||
|
||||
// SeriesFilter is one named hygiene predicate over the whole library. Site
|
||||
// and Kind narrow the row read; Name picks the predicate; Cutoff is the
|
||||
// staleness boundary the "stale" filter compares against, supplied by the
|
||||
// caller's clock — the store has no clock; Page is 1-based.
|
||||
type SeriesFilter struct {
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both library buckets' series
|
||||
Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll
|
||||
Cutoff int64 // unix ms; "stale" reads it, the store never does
|
||||
Page int // 1-based page of the row read; default 1
|
||||
}
|
||||
|
||||
// adminSeriesColumns is the owner's library-wide Series projection in
|
||||
// scanAdminSeries order. It is the privacy boundary: a Series' row carries
|
||||
// the Reader id that raised its Latest Chapter (latest_raised_by), and that id
|
||||
// must never leave the store package — so the projection does not select it,
|
||||
// and only the anonymous boolean in raisedByReaderAnswer crosses it.
|
||||
const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at`
|
||||
|
||||
// raisedByReaderAnswer answers "did a Reader's report set this number" without
|
||||
// naming which Reader. Kept apart from adminSeriesColumns so the column list —
|
||||
// the shape scanAdminSeries is fed — stays free of the Sighting-raiser
|
||||
// identity, and the owner learns which rows to act on and nothing about the
|
||||
// Reader behind them.
|
||||
const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader`
|
||||
|
||||
// seriesPageSize is the row read's page length. The tie-break in the query's
|
||||
// ORDER BY is what makes this a stable page boundary — see SeriesPage.
|
||||
const seriesPageSize = 50
|
||||
|
||||
// AdminSeries is one Series as the owner's library-wide view sees it: a
|
||||
// Series-level fact plus an anonymous Reader count. ReaderCount being zero is
|
||||
// the orphan marker. RaisedByReader is the only trace of the Sighting
|
||||
// mechanism here; the Reader id behind it never reaches this type.
|
||||
type AdminSeries struct {
|
||||
Site string
|
||||
SeriesID string
|
||||
Title string
|
||||
SeriesURL string
|
||||
CoverAddress string // "" = no Cover yet
|
||||
Kind string
|
||||
LatestChapter string
|
||||
LatestChapterNum *float64 // nil until first captured
|
||||
LatestCheckedAt int64
|
||||
// ForcePollAt is the owner's "check now" request stamp (issue #146), zero
|
||||
// meaning never asked. Pending is derived, never stored: a request is
|
||||
// pending while ForcePollAt is newer than LatestCheckedAt.
|
||||
ForcePollAt int64
|
||||
ReaderCount int
|
||||
RaisedByReader bool // a Reader's report set LatestChapterNum
|
||||
}
|
||||
|
||||
// SeriesPage is one page of the owner's filtered Series list plus the count
|
||||
// of every Series matching the same filter — a window number, not the page's
|
||||
// len, so the landing page's figure and the list heading come from one query.
|
||||
type SeriesPage struct {
|
||||
Rows []AdminSeries
|
||||
Total int
|
||||
}
|
||||
|
||||
// SiteSeriesShape is one Site's share of the Series matching a filter: how
|
||||
// many, and the manga/novel split. One grouped pass, then library-wide totals
|
||||
// are summed in Go over the rows — the landing page's per-Site table reads
|
||||
// this and never pays for the rows the list discards.
|
||||
type SiteSeriesShape struct {
|
||||
Site string
|
||||
Total int
|
||||
Manga int
|
||||
Novel int
|
||||
}
|
||||
|
||||
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>").
|
||||
func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
|
||||
|
||||
// adminFilter maps a filter's named predicate to its compile-time WHERE and
|
||||
// HAVING clauses and their bound parameters — the name never reaches query
|
||||
// text, and Site and Kind bind as parameters. Shared by the row read and the
|
||||
// per-Site aggregate so the two cannot disagree on what a filter means.
|
||||
//
|
||||
// The WHERE set is: no URL (an empty URL only — the host-failing-the-fetch-
|
||||
// gate case is invisible to SQL, needs the Site registry in Go, and belongs to
|
||||
// a later repair), never-read-a-chapter and never-checked as disjoint halves
|
||||
// (non-zero versus zero check stamp), stale, no cover, and Reader-report.
|
||||
// no_readers is the one HAVING predicate: it is the orphan test, an aggregate
|
||||
// over the LEFT JOIN, where a bare WHERE has no row to test.
|
||||
//
|
||||
// stale is the checked-but-old half of the stamp partition — because the
|
||||
// verdict line wants "not checked in twelve hours" as one figure, and a never
|
||||
// checked Series is already counted on its own "waiting"/never-checked
|
||||
// filter, folding it in would double-report it. The landing page computes the
|
||||
// inclusive number as stale + never-checked.
|
||||
func adminFilter(f SeriesFilter) (where, having string, args []any, err error) {
|
||||
var clauses []string
|
||||
if f.Kind != "" {
|
||||
args = append(args, f.Kind)
|
||||
clauses = append(clauses, "s.kind = $"+strconv.Itoa(len(args)))
|
||||
}
|
||||
switch f.Name {
|
||||
case "", SeriesFilterAll:
|
||||
case SeriesFilterNoURL:
|
||||
clauses = append(clauses, `s.series_url = ''`)
|
||||
case SeriesFilterNoChapter:
|
||||
clauses = append(clauses, `s.latest_checked_at <> 0 AND s.latest_chapter_num IS NULL`)
|
||||
case SeriesFilterNeverChecked:
|
||||
clauses = append(clauses, `s.latest_checked_at = 0`)
|
||||
case SeriesFilterStale:
|
||||
clauses = append(clauses, `s.latest_checked_at > 0 AND s.latest_checked_at < $`+strconv.Itoa(len(args)+1))
|
||||
args = append(args, f.Cutoff)
|
||||
case SeriesFilterNoCover:
|
||||
clauses = append(clauses, `s.cover_address = ''`)
|
||||
case SeriesFilterReaderReport:
|
||||
clauses = append(clauses, `s.latest_raised_by IS NOT NULL`)
|
||||
case SeriesFilterNoReaders:
|
||||
having = `HAVING COUNT(b.reader_id) = 0`
|
||||
default:
|
||||
return "", "", nil, fmt.Errorf("unknown series filter %q", f.Name)
|
||||
}
|
||||
if len(clauses) > 0 {
|
||||
where = "WHERE " + strings.Join(clauses, " AND ")
|
||||
}
|
||||
return where, having, args, nil
|
||||
}
|
||||
|
||||
// SeriesPage returns one page of the Series matching the filter, least
|
||||
// recently checked first. The LEFT JOIN to Bookmarks is what surfaces the
|
||||
// orphans that hygiene has to find — an inner join would hide them, exactly
|
||||
// as the Lane's join does. ReaderCount is a plain count of every Bookmark on
|
||||
// the Series, which knowingly disagrees with the two Lane queries for as long
|
||||
// as the finished lifecycle bucket exists (#140).
|
||||
//
|
||||
// The tie-break is mandatory, not decorative: every unpollable Series shares a
|
||||
// zero check stamp, so ordering on that column alone gives no stable page
|
||||
// boundary and rows would repeat or vanish across pages. (site, series_id) is
|
||||
// the primary key, hence total. The filtered total is a window count in the
|
||||
// same query — window functions run after grouping and before the limit, so
|
||||
// one where-clause cannot disagree with a second copy of itself.
|
||||
func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return SeriesPage{}, err
|
||||
}
|
||||
if f.Page < 1 {
|
||||
f.Page = 1
|
||||
}
|
||||
// Site narrowing is the row read's own; the aggregate must see every Site.
|
||||
if f.Site != "" {
|
||||
args = append(args, f.Site)
|
||||
clause := "s.site = $" + strconv.Itoa(len(args))
|
||||
if where == "" {
|
||||
where = "WHERE " + clause
|
||||
} else {
|
||||
where += " AND " + clause
|
||||
}
|
||||
}
|
||||
base := len(args)
|
||||
args = append(args, seriesPageSize, seriesPageSize*(f.Page-1))
|
||||
rows, err := s.db.Query(`
|
||||
SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`,
|
||||
COUNT(b.reader_id) AS reader_count,
|
||||
COUNT(*) OVER () AS filtered_total
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at, s.latest_raised_by
|
||||
`+having+`
|
||||
ORDER BY s.latest_checked_at, s.site, s.series_id
|
||||
LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("query series page: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := SeriesPage{}
|
||||
for rows.Next() {
|
||||
a, total, err := scanAdminSeries(rows.Scan)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("scan series page: %w", err)
|
||||
}
|
||||
out.Rows = append(out.Rows, a)
|
||||
out.Total = total
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SeriesShapes returns each Site's share of the Series matching the filter,
|
||||
// one grouped pass. Site and Page are row-read concerns and are ignored; the
|
||||
// Landing page reads this per Site and sums the totals in Go for the
|
||||
// library-wide figure.
|
||||
func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE kind = 'manga') AS manga,
|
||||
COUNT(*) FILTER (WHERE kind = 'novel') AS novel
|
||||
FROM (
|
||||
SELECT s.site, s.kind
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.kind
|
||||
`+having+`
|
||||
) shape
|
||||
GROUP BY site
|
||||
ORDER BY site`, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query series shapes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []SiteSeriesShape{}
|
||||
for rows.Next() {
|
||||
var sh SiteSeriesShape
|
||||
if err := rows.Scan(&sh.Site, &sh.Total, &sh.Manga, &sh.Novel); err != nil {
|
||||
return nil, fmt.Errorf("scan series shape: %w", err)
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer
|
||||
// order, plus the query's reader_count and filtered_total columns, and returns
|
||||
// the window total alongside the row. latest_chapter_num is NULL until first
|
||||
// captured — the "never read a chapter" state. The Sighting-raiser column is
|
||||
// never among the scanned columns.
|
||||
func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
|
||||
var (
|
||||
a AdminSeries
|
||||
latestChapterNum sql.NullFloat64
|
||||
total int
|
||||
)
|
||||
if err := scan(
|
||||
&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
|
||||
&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
|
||||
&a.ForcePollAt,
|
||||
&a.RaisedByReader, &a.ReaderCount, &total,
|
||||
); err != nil {
|
||||
return AdminSeries{}, 0, err
|
||||
}
|
||||
if latestChapterNum.Valid {
|
||||
a.LatestChapterNum = &latestChapterNum.Float64
|
||||
}
|
||||
return a, total, nil
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seriesSeed describes one Series (and optionally its bookmarks) to stand up
|
||||
// for an admin filter test. Direct SQL, because the filters separate rows the
|
||||
// Upsert path could not produce together: an orphan has no bookmark, and a
|
||||
// Reader-raised Latest Chapter needs a Sighting the store does not create.
|
||||
type seriesSeed struct {
|
||||
key string
|
||||
kind string
|
||||
url string
|
||||
cover string // cover_address
|
||||
checkedAt int64
|
||||
latestNum *float64
|
||||
bookmarks int // readers that hold it; 0 = orphan
|
||||
raisedBy bool // a Reader's report is attributed as the raiser
|
||||
}
|
||||
|
||||
// seedAdminSeries inserts one series row and its bookmarks (owner first, then
|
||||
// fresh readers), with the exact admin-relevant facts a test needs.
|
||||
func seedAdminSeries(t *testing.T, s *Store, seed seriesSeed) {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(seed.key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", seed.key)
|
||||
}
|
||||
if seed.kind == "" {
|
||||
seed.kind = "manga"
|
||||
}
|
||||
var latestChapter any = ""
|
||||
if seed.latestNum != nil {
|
||||
latestChapter = "Chapter " + strconv.FormatFloat(*seed.latestNum, 'f', -1, 64)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO series (site, series_id, title, kind, series_url, cover_address,
|
||||
latest_checked_at, latest_chapter, latest_chapter_num)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||
site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover,
|
||||
seed.checkedAt, latestChapter, seed.latestNum); err != nil {
|
||||
t.Fatalf("seed series %q: %v", seed.key, err)
|
||||
}
|
||||
for i := range seed.bookmarks {
|
||||
var readerID int64 = s.OwnerID()
|
||||
if i > 0 {
|
||||
readerID = secondReader(t, s)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO bookmarks (reader_id, site, series_id,
|
||||
last_chapter, last_chapter_num, last_chapter_url,
|
||||
favorite, status, updated_at)
|
||||
VALUES ($1, $2, $3, '', 0, '', false, 'reading', $4)`,
|
||||
readerID, site, seriesID, seed.checkedAt); err != nil {
|
||||
t.Fatalf("seed bookmark %q: %v", seed.key, err)
|
||||
}
|
||||
}
|
||||
if seed.raisedBy {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET latest_raised_by = $1 WHERE site = $2 AND series_id = $3`,
|
||||
s.OwnerID(), site, seriesID); err != nil {
|
||||
t.Fatalf("seed raised-by %q: %v", seed.key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pageKeys(t *testing.T, s *Store, f SeriesFilter) map[string]bool {
|
||||
t.Helper()
|
||||
page, err := s.SeriesPage(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(%+v): %v", f, err)
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, a := range page.Rows {
|
||||
keys[a.Key()] = true
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Each filter must return the rows it names and no others, over one shared
|
||||
// seeded mix where every healthy neighbour is present to be wrongly returned.
|
||||
// The stale cutoff is 5000: a Series checked at 9000 is current, at 2000 stale.
|
||||
func TestAdminSeriesFilters(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "https://asurascans.com/comics/healthy", cover: "aaa", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nourl", url: "", cover: "bbb", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "https://asurascans.com/comics/nochapter", cover: "ccc", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "https://asurascans.com/comics/neverchecked", cover: "ddd", checkedAt: 0, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:orphan", url: "https://asurascans.com/comics/orphan", cover: "eee", checkedAt: 9000, latestNum: new(7.0), bookmarks: 0})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:stale", url: "https://asurascans.com/comics/stale", cover: "fff", checkedAt: 2000, latestNum: new(4.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nocover", url: "https://asurascans.com/comics/nocover", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:report", url: "https://asurascans.com/comics/report", cover: "ggg", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
f SeriesFilter
|
||||
want []string
|
||||
}{
|
||||
{"all", SeriesFilter{}, []string{"asura:healthy", "asura:nourl", "asura:nochapter", "asura:neverchecked", "asura:orphan", "asura:stale", "asura:nocover", "asura:report"}},
|
||||
{"no series url", SeriesFilter{Name: SeriesFilterNoURL}, []string{"asura:nourl"}},
|
||||
{"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}},
|
||||
{"never checked", SeriesFilter{Name: SeriesFilterNeverChecked}, []string{"asura:neverchecked"}},
|
||||
{"no readers", SeriesFilter{Name: SeriesFilterNoReaders}, []string{"asura:orphan"}},
|
||||
{"stale", SeriesFilter{Name: SeriesFilterStale, Cutoff: 5000}, []string{"asura:stale"}},
|
||||
{"no cover", SeriesFilter{Name: SeriesFilterNoCover}, []string{"asura:nocover"}},
|
||||
{"reader report", SeriesFilter{Name: SeriesFilterReaderReport}, []string{"asura:report"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := pageKeys(t, s, tc.f)
|
||||
want := map[string]bool{}
|
||||
for _, k := range tc.want {
|
||||
want[k] = true
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want)
|
||||
}
|
||||
for k := range want {
|
||||
if !got[k] {
|
||||
t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// "Never read a chapter" and "never checked" are disjoint by construction:
|
||||
// the first requires a non-zero check stamp, the second a zero one. Over a
|
||||
// mix that should satisfy both, no row may be counted twice.
|
||||
func TestAdminNeverChapterAndNeverCheckedAreDisjoint(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
// A zero-stamp, no-chapter row is never-checked only: if never-read-a-
|
||||
// chapter ever lost its non-zero-stamp guard, it would claim this row too
|
||||
// and the two counts would double-report it.
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:both", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter})
|
||||
neverChecked := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNeverChecked})
|
||||
for k := range noChapter {
|
||||
if neverChecked[k] {
|
||||
t.Fatalf("row %q matches both never-read-a-chapter and never-checked", k)
|
||||
}
|
||||
}
|
||||
if !noChapter["asura:nochapter"] || !neverChecked["asura:neverchecked"] {
|
||||
t.Fatalf("disjoint split lost its own rows: no-chapter=%v never-checked=%v", noChapter, neverChecked)
|
||||
}
|
||||
}
|
||||
|
||||
// Several rows share a zero check stamp, so ordering on latest_checked_at
|
||||
// alone gives no stable page boundary. The (site, series_id) tie-break must
|
||||
// make page 2 a strict continuation of page 1: no repeat, no vanishing row.
|
||||
func TestAdminSeriesPageTieBreakIsStable(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const total = 53 // > one page, < two (page size 50)
|
||||
for i := range total {
|
||||
id := "tie" + strconv.Itoa(i)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:" + id, url: "u", checkedAt: 0, bookmarks: 1})
|
||||
}
|
||||
// A second Site's zero-stamp row is part of the same all-filter list, and
|
||||
// must land on a valid page boundary rather than duplicating or dropping
|
||||
// one of asura's rows: the tie-break is global (site, series_id).
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:z", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
wantTotal := total + 1
|
||||
|
||||
p1, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage page 1: %v", err)
|
||||
}
|
||||
p2, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage page 2: %v", err)
|
||||
}
|
||||
if len(p1.Rows) != seriesPageSize {
|
||||
t.Fatalf("page 1 has %d rows, want %d", len(p1.Rows), seriesPageSize)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range append(append([]AdminSeries{}, p1.Rows...), p2.Rows...) {
|
||||
if seen[a.Key()] {
|
||||
t.Fatalf("row %q repeats across pages", a.Key())
|
||||
}
|
||||
seen[a.Key()] = true
|
||||
}
|
||||
if len(seen) != wantTotal {
|
||||
t.Fatalf("%d distinct rows across pages, want %d (a row vanished)", len(seen), wantTotal)
|
||||
}
|
||||
if p1.Total != wantTotal {
|
||||
t.Fatalf("page total = %d, want %d (the window count must span pages)", p1.Total, wantTotal)
|
||||
}
|
||||
// A page beyond the end is empty, not an error (the list re-reads page 1).
|
||||
// The window count runs over the rows present in the result set, so an
|
||||
// overflow page has no rows and therefore no total — the caller must not
|
||||
// render it, which is exactly why the list re-reads page 1.
|
||||
pFinal, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 99})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage beyond end: %v", err)
|
||||
}
|
||||
if len(pFinal.Rows) != 0 {
|
||||
t.Fatalf("page beyond end = %d rows, want 0", len(pFinal.Rows))
|
||||
}
|
||||
}
|
||||
|
||||
// The filtered total is the window number over the same filter the rows use,
|
||||
// and the per-Site aggregate sums to the same figure — so the landing page's
|
||||
// count and the list's heading can never disagree, whoever computes them.
|
||||
func TestAdminTotalAgreesWithRowsAndShapes(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", cover: "a", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:b", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:c", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(2.0), raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:d", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
filters := []SeriesFilter{
|
||||
{},
|
||||
{Name: SeriesFilterNoCover},
|
||||
{Name: SeriesFilterReaderReport},
|
||||
{Name: SeriesFilterNoChapter},
|
||||
}
|
||||
for _, f := range filters {
|
||||
page, err := s.SeriesPage(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(%+v): %v", f, err)
|
||||
}
|
||||
want := len(page.Rows)
|
||||
if f.Page == 0 && want == seriesPageSize {
|
||||
t.Fatalf("seed produced a full page; bump the seed or drop page size in the test")
|
||||
}
|
||||
if page.Total != want {
|
||||
t.Fatalf("%+v total = %d, want %d (window count disagrees with row count)", f, page.Total, want)
|
||||
}
|
||||
shapes, err := s.SeriesShapes(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes(%+v): %v", f, err)
|
||||
}
|
||||
sum := 0
|
||||
for _, sh := range shapes {
|
||||
sum += sh.Total
|
||||
}
|
||||
if sum != want {
|
||||
t.Fatalf("%+v aggregate sum = %d, want %d (aggregate disagrees with row query)", f, sum, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The default filter's aggregate carries the library shape: per-Site
|
||||
// totals and the manga/novel split, summed in Go for library wide.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes default: %v", err)
|
||||
}
|
||||
if len(shapes) != 2 || shapes[0].Site != "asura" || shapes[1].Site != "demonic" {
|
||||
t.Fatalf("shapes = %+v, want asura then demonic", shapes)
|
||||
}
|
||||
if shapes[0].Total != 3 || shapes[0].Manga != 3 || shapes[0].Novel != 0 {
|
||||
t.Fatalf("asura shape = %+v, want 3 manga, 0 novel", shapes[0])
|
||||
}
|
||||
if shapes[1].Total != 1 || shapes[1].Manga != 0 || shapes[1].Novel != 1 {
|
||||
t.Fatalf("demonic shape = %+v, want 1 novel", shapes[1])
|
||||
}
|
||||
}
|
||||
|
||||
// Site and Library narrowing stack on a named filter without changing what
|
||||
// the filter means.
|
||||
func TestAdminFilterSiteAndKindNarrow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:aa", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(1.0)})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:ab", url: "", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:aa", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:ab", url: "", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
got := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Site: "asura"})
|
||||
if len(got) != 1 || !got["asura:ab"] {
|
||||
t.Fatalf("site+nourl = %v, want only asura:ab", got)
|
||||
}
|
||||
got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"})
|
||||
if len(got) != 1 || !got["demonic:ab"] {
|
||||
t.Fatalf("kind+nourl = %v, want only demonic:ab", got)
|
||||
}
|
||||
got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterAll, Site: "demonic", Kind: "novel"})
|
||||
if len(got) != 2 || !got["demonic:aa"] || !got["demonic:ab"] {
|
||||
t.Fatalf("site+kind+all = %v, want both demonic rows", got)
|
||||
}
|
||||
|
||||
// The aggregate ignores the Site narrowing (it is per-Site by shape), but
|
||||
// honours the Library narrowing: asura's missing-URL row is manga, so the
|
||||
// novel no-URL list is demonic alone.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes: %v", err)
|
||||
}
|
||||
if len(shapes) != 1 || shapes[0].Site != "demonic" ||
|
||||
shapes[0].Total != 1 || shapes[0].Novel != 1 {
|
||||
t.Fatalf("novel no-URL aggregate = %+v, want demonic {Total:1 Novel:1}", shapes)
|
||||
}
|
||||
}
|
||||
|
||||
// The projection is the privacy boundary: a Series whose Latest Chapter was
|
||||
// raised by a Reader's report reads back with the anonymous boolean set, not
|
||||
// with the Reader's id, and no Reader id travels in any returned row.
|
||||
func TestAdminSeriesReportsAnonymously(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:raised", url: "u", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1, raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1})
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
byKey := map[string]AdminSeries{}
|
||||
for _, a := range page.Rows {
|
||||
byKey[a.Key()] = a
|
||||
}
|
||||
if !byKey["asura:raised"].RaisedByReader {
|
||||
t.Fatal("Reader-raised Series read back RaisedByReader=false")
|
||||
}
|
||||
if byKey["asura:polled"].RaisedByReader {
|
||||
t.Fatal("Poll-raised Series read back RaisedByReader=true")
|
||||
}
|
||||
}
|
||||
|
||||
// The privacy test that cannot rot into a template-only guarantee: assert the
|
||||
// admin column constant does not carry the Sighting-raiser column and that the
|
||||
// admin row type has no field for it, modelled on the guard on the Bookmark
|
||||
// column list.
|
||||
func TestAdminProjectionHidesSightingRaiser(t *testing.T) {
|
||||
if strings.Contains(adminSeriesColumns, "latest_raised_by") {
|
||||
t.Fatal("admin column list carries latest_raised_by: the Sighting-raiser id would reach the owner")
|
||||
}
|
||||
if _, ok := reflect.TypeOf(AdminSeries{}).FieldByName("LatestRaisedBy"); ok {
|
||||
t.Fatal("AdminSeries carries a field for the Sighting-raiser id")
|
||||
}
|
||||
}
|
||||
|
||||
// An unknown filter name is rejected rather than silently meaning "all" —
|
||||
// otherwise a mistyped URL would present an empty page as the whole library.
|
||||
func TestAdminFilterUnknownNameRejected(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
for name, call := range map[string]func() error{
|
||||
"page": func() error { _, err := s.SeriesPage(SeriesFilter{Name: "bogus"}); return err },
|
||||
"shape": func() error { _, err := s.SeriesShapes(SeriesFilter{Name: "bogus"}); return err },
|
||||
} {
|
||||
if err := call(); err == nil || !strings.Contains(err.Error(), "unknown series filter") {
|
||||
t.Fatalf("%s with bogus filter = %v, want unknown-filter error", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ForceSeriesPoll is the idempotent stamp write: a second press overwrites
|
||||
// the request time, and touching a missing series is not an error.
|
||||
func TestForceSeriesPollStampsIdempotently(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
if err := s.ForceSeriesPoll("asura", "x", 42); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
if err := s.ForceSeriesPoll("asura", "x", 99); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll re-stamp: %v", err)
|
||||
}
|
||||
// Touching a missing series is not an error: the row may have been
|
||||
// orphaned, and the caller's read decides what exists.
|
||||
if err := s.ForceSeriesPoll("asura", "ghost", 99); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll missing: %v", err)
|
||||
}
|
||||
|
||||
var got int64
|
||||
if err := s.db.QueryRow(
|
||||
`SELECT force_poll_at FROM series WHERE site = 'asura' AND series_id = 'x'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read force_poll_at: %v", err)
|
||||
}
|
||||
if got != 99 {
|
||||
t.Fatalf("force_poll_at = %d, want 99 (the later press wins)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The admin projection carries the force stamp so the web layer can derive
|
||||
// the pending flag without a second read.
|
||||
func TestAdminSeriesCarriesForcePollAt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 1000, bookmarks: 1})
|
||||
if err := s.ForceSeriesPoll("asura", "x", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
if len(page.Rows) != 1 || page.Rows[0].ForcePollAt != 5000 {
|
||||
t.Fatalf("row = %+v, want ForcePollAt 5000", page.Rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
-- The owner's administrative page (issue #102) renders these counters and
|
||||
-- offers a control to clear them, deliberately shipped before the Sighting
|
||||
-- feature (issue #103) that fills them, so a false mark never needs SQL
|
||||
-- against production. Zero counters mean a trusted Reader.
|
||||
ALTER TABLE readers ADD COLUMN sighting_agreements integer NOT NULL DEFAULT 0;
|
||||
ALTER TABLE readers ADD COLUMN sighting_disagreements integer NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Sighting deferral (issue #103). latest_sighted_at is when a Reader's report
|
||||
-- last stood in for a Poll; it is separate from latest_checked_at because the
|
||||
-- six-hour ceiling has to know when the Series was last really fetched, and a
|
||||
-- Sighting writing the Poll's own column would erase that.
|
||||
-- latest_raised_by is attribution: whoever last raised this Series' Latest
|
||||
-- Chapter by Sighting, so a Poll that contradicts the value downwards names a
|
||||
-- Reader rather than flagging a row. Cleared by the Poll that judges it, NULL
|
||||
-- whenever the stored value is the Poll's own.
|
||||
ALTER TABLE series ADD COLUMN latest_sighted_at bigint NOT NULL DEFAULT 0;
|
||||
ALTER TABLE series ADD COLUMN latest_raised_by bigint REFERENCES readers(id) ON DELETE SET NULL;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- One durable state row per Poll Lane. Zero means no pause or refusal is set.
|
||||
CREATE TABLE poll_lanes (
|
||||
site text NOT NULL PRIMARY KEY,
|
||||
paused_until bigint NOT NULL DEFAULT 0,
|
||||
refuse_until bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Append-only Lane Pass log. Timestamps are unix milliseconds from the poller's clock.
|
||||
CREATE TABLE poll_passes (
|
||||
site text NOT NULL,
|
||||
ran_at bigint NOT NULL,
|
||||
skip text NOT NULL,
|
||||
due integer NOT NULL,
|
||||
checked integer NOT NULL,
|
||||
gap_ms bigint NOT NULL,
|
||||
clamped boolean NOT NULL,
|
||||
refused integer NOT NULL,
|
||||
unreachable integer NOT NULL,
|
||||
no_chapter integer NOT NULL,
|
||||
unfetchable integer NOT NULL,
|
||||
errors integer NOT NULL,
|
||||
PRIMARY KEY (site, ran_at)
|
||||
);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Admin read-model foundation (#140). The Series list's default order is
|
||||
-- least-recently-checked first, so the table — which has only its primary key
|
||||
-- today — gets an index that can serve it. A grouped query over a join may
|
||||
-- ignore the index, so this is a judgement, not a measurement: re-time on real
|
||||
-- data before adding a second.
|
||||
CREATE INDEX series_latest_checked_at_idx ON series (latest_checked_at);
|
||||
|
||||
-- force_poll_at is the "ask for one Series to be checked now" stamp (#146).
|
||||
-- Zero means never forced; nothing reads the column before that ticket wires
|
||||
-- it, so it lands here unused.
|
||||
ALTER TABLE series ADD COLUMN force_poll_at bigint NOT NULL DEFAULT 0;
|
||||
+479
-33
@@ -16,7 +16,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
@@ -39,7 +38,7 @@ type Bookmark struct {
|
||||
// origin once the bytes exist, and "" until they do — never a third-party
|
||||
// address and never an address that 404s (ADR-0007). A client may still
|
||||
// send this field and it is discarded on the way in; see Upsert.
|
||||
Cover string `json:"cover"`
|
||||
Cover string `json:"cover"`
|
||||
LastChapter string `json:"last_chapter"`
|
||||
LastChapterNum float64 `json:"last_chapter_num"`
|
||||
LastChapterURL string `json:"last_chapter_url"`
|
||||
@@ -80,10 +79,48 @@ type Series struct {
|
||||
LatestChapter string
|
||||
LatestChapterNum *float64 // nil until first captured
|
||||
LatestCheckedAt int64 // unix ms; see MarkLatestChecked
|
||||
// LatestRaisedBy is the Reader whose Sighting last raised LatestChapter,
|
||||
// and nil when the stored value is a Poll's own finding. It is what lets a
|
||||
// Poll that contradicts the value downwards name a Reader instead of
|
||||
// merely flagging the row (issue #103); the Poll that judges it clears it.
|
||||
LatestRaisedBy *int64
|
||||
|
||||
// readerCount is the number of bookmarks referencing this series, filled
|
||||
// only by the due-queue query that orders on it.
|
||||
readerCount int
|
||||
// Forced is whether the owner asked for a check now (issue #146): the
|
||||
// request stamp is newer than the check stamp. Derived in the due query,
|
||||
// never stored, and the flag that jumps the queue and opens the browser
|
||||
// wake gate.
|
||||
Forced bool
|
||||
}
|
||||
|
||||
// LanePass is one Poll Lane's durable pass snapshot. Pause and refusal stamps
|
||||
// are joined from poll_lanes on read; they are not pass facts.
|
||||
type LanePass struct {
|
||||
Site string
|
||||
RanAt int64
|
||||
Skip string
|
||||
Due, Checked int
|
||||
GapMS int64
|
||||
Clamped bool
|
||||
Refused, Unreachable, NoChapter int
|
||||
Unfetchable, Errors int
|
||||
PausedUntil, RefuseUntil int64
|
||||
}
|
||||
|
||||
// SiteOutcomes is one Site's summed Lane Pass outcomes over a caller-supplied
|
||||
// window.
|
||||
type SiteOutcomes struct {
|
||||
Site string
|
||||
Refused, Unreachable, NoChapter int
|
||||
Unfetchable, Errors int
|
||||
}
|
||||
|
||||
// LanePause is one persisted Lane pause stamp.
|
||||
type LanePause struct {
|
||||
Site string
|
||||
PausedUntil int64
|
||||
}
|
||||
|
||||
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>"),
|
||||
@@ -184,9 +221,9 @@ const (
|
||||
//go:embed migrations/*.sql
|
||||
var migrations embed.FS
|
||||
|
||||
// bookmarkColumns is the only value ever concatenated into query text. It is a
|
||||
// compile-time constant; every request value is bound as a parameter. The
|
||||
// series-owned fields are joined in from the series table, in scanBookmark
|
||||
// These column lists are the only values ever concatenated into query text.
|
||||
// They are compile-time constants; every request value is bound as a parameter.
|
||||
// The series-owned fields are joined in from the series table, in scanBookmark
|
||||
// order, so the flat Bookmark reads back whole despite the split (ADR-0004).
|
||||
const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_address,
|
||||
b.last_chapter, b.last_chapter_num, b.last_chapter_url,
|
||||
@@ -196,7 +233,11 @@ const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_add
|
||||
// due query. latest_checked_at lives only on series — see MarkLatestChecked
|
||||
// for why it stays off every client-visible write.
|
||||
const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at`
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by`
|
||||
|
||||
const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped,
|
||||
p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors,
|
||||
COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)`
|
||||
|
||||
// Owner is the person running the service: the first Reader, seeded at startup
|
||||
// so a fresh deployment has a library before anyone logs in. The seed makes
|
||||
@@ -316,25 +357,42 @@ func (s *Store) EnsureReader(discordID string, epochZeroHash [32]byte) (int64, e
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// SightingDisagreementLimit is the number of contradictions that stop that
|
||||
// Reader's Sightings from deferring a Poll (issue #103). The counters exist
|
||||
// before the mechanism that moves them, so the admin page (issue #102) can
|
||||
// clear a false mark without waiting for the Sighting feature.
|
||||
const SightingDisagreementLimit = 3
|
||||
|
||||
// Blocked reports whether this Reader's Sighting marks have reached the
|
||||
// disagreement limit, which stops their Sightings from deferring a Poll.
|
||||
func (r ReaderSummary) Blocked() bool {
|
||||
return r.Disagreements >= SightingDisagreementLimit
|
||||
}
|
||||
|
||||
// ReaderSummary is one Reader as the owner's administration panel sees them:
|
||||
// who they are and how many live sessions they hold. No credential material,
|
||||
// hashed or otherwise, is exposed.
|
||||
// who they are, how many live sessions they hold, and their Sighting marks.
|
||||
// No credential material, hashed or otherwise, is exposed.
|
||||
type ReaderSummary struct {
|
||||
ID int64
|
||||
DiscordID string
|
||||
// Sessions counts unexpired session rows — what the owner revokes.
|
||||
Sessions int
|
||||
// Agreements and Disagreements are the Sighting counters (issue #102);
|
||||
// zero means a trusted Reader.
|
||||
Agreements int
|
||||
Disagreements int
|
||||
}
|
||||
|
||||
// Readers lists every Reader with their live session count, oldest first, so
|
||||
// the owner row (always the oldest) heads the list.
|
||||
// Readers lists every Reader with their live session count and Sighting
|
||||
// marks, oldest first, so the owner row (always the oldest) heads the list.
|
||||
func (s *Store) Readers() ([]ReaderSummary, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT r.id, r.discord_id,
|
||||
r.sighting_agreements, r.sighting_disagreements,
|
||||
count(sess.id) FILTER (WHERE sess.expires_at > now()) AS sessions
|
||||
FROM readers r
|
||||
LEFT JOIN sessions sess ON sess.reader_id = r.id
|
||||
GROUP BY r.id, r.discord_id
|
||||
GROUP BY r.id, r.discord_id, r.sighting_agreements, r.sighting_disagreements
|
||||
ORDER BY r.id`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query readers: %w", err)
|
||||
@@ -344,7 +402,7 @@ func (s *Store) Readers() ([]ReaderSummary, error) {
|
||||
out := []ReaderSummary{}
|
||||
for rows.Next() {
|
||||
var r ReaderSummary
|
||||
if err := rows.Scan(&r.ID, &r.DiscordID, &r.Sessions); err != nil {
|
||||
if err := rows.Scan(&r.ID, &r.DiscordID, &r.Agreements, &r.Disagreements, &r.Sessions); err != nil {
|
||||
return nil, fmt.Errorf("scan reader: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
@@ -352,6 +410,18 @@ func (s *Store) Readers() ([]ReaderSummary, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ClearReaderMarks zeroes a Reader's Sighting counters. It is the owner's
|
||||
// remedy for a mark produced by a broken Site adapter rather than a dishonest
|
||||
// Reader: it restores a privilege, it is not destruction.
|
||||
func (s *Store) ClearReaderMarks(readerID int64) error {
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE readers SET sighting_agreements = 0, sighting_disagreements = 0
|
||||
WHERE id = $1`, readerID); err != nil {
|
||||
return fmt.Errorf("clear reader marks for reader %d: %w", readerID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readersMigration is the version that creates the readers table. The owner
|
||||
// seed runs between two migrate passes, so that the run-once migration which
|
||||
// attaches existing bookmarks (0004) finds the owner row.
|
||||
@@ -556,26 +626,43 @@ func (s *Store) scanBookmark(scan func(...any) error) (Bookmark, error) {
|
||||
}
|
||||
|
||||
// scanSeries reads one row in seriesColumns order, plus the due query's
|
||||
// reader_count column. latest_chapter_num is NULL until the first capture,
|
||||
// same as on the bookmark read path.
|
||||
// forced flag and reader_count columns. latest_chapter_num and
|
||||
// latest_raised_by are both nullable, same as latest_chapter_num on the
|
||||
// bookmark read path.
|
||||
func scanSeries(scan func(...any) error) (Series, error) {
|
||||
var (
|
||||
sr Series
|
||||
latestChapterNum sql.NullFloat64
|
||||
latestRaisedBy sql.NullInt64
|
||||
)
|
||||
if err := scan(
|
||||
&sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress,
|
||||
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt,
|
||||
&sr.readerCount,
|
||||
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy,
|
||||
&sr.Forced, &sr.readerCount,
|
||||
); err != nil {
|
||||
return Series{}, err
|
||||
}
|
||||
if latestChapterNum.Valid {
|
||||
sr.LatestChapterNum = &latestChapterNum.Float64
|
||||
}
|
||||
if latestRaisedBy.Valid {
|
||||
sr.LatestRaisedBy = &latestRaisedBy.Int64
|
||||
}
|
||||
return sr, nil
|
||||
}
|
||||
|
||||
func scanLanePass(scan func(...any) error) (LanePass, error) {
|
||||
var p LanePass
|
||||
if err := scan(
|
||||
&p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped,
|
||||
&p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors,
|
||||
&p.PausedUntil, &p.RefuseUntil,
|
||||
); err != nil {
|
||||
return LanePass{}, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying database handle.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
@@ -897,10 +984,198 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DueForLatestCheck returns series whose server-side latest-chapter check has
|
||||
// aged past the appropriate cutoff, ordered by how many bookmarks reference
|
||||
// them (descending) then least-recently-checked first, at most limit of them.
|
||||
// Browser-backed sites use browserCutoffMs; every other site uses cutoffMs.
|
||||
// RecordLanePass appends one pass and prunes every older row in the same
|
||||
// transaction. retainBefore is supplied by the poller's clock.
|
||||
func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin lane pass %s: %w", p.Site, err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO poll_passes
|
||||
(site, ran_at, skip, due, checked, gap_ms, clamped,
|
||||
refused, unreachable, no_chapter, unfetchable, errors)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
||||
p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped,
|
||||
p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil {
|
||||
return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil {
|
||||
return fmt.Errorf("prune lane passes before %d: %w", retainBefore, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit lane pass %s at %d: %w", p.Site, p.RanAt, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LatestLanePass returns the newest pass for one Site, with its current Lane
|
||||
// state joined on. A Site without a pass has no durable snapshot yet.
|
||||
func (s *Store) LatestLanePass(site string) (LanePass, bool, error) {
|
||||
p, err := scanLanePass(s.db.QueryRow(`SELECT `+lanePassColumns+`
|
||||
FROM poll_passes p
|
||||
LEFT JOIN poll_lanes l ON l.site = p.site
|
||||
WHERE p.site = $1
|
||||
ORDER BY p.ran_at DESC
|
||||
LIMIT 1`, site).Scan)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return LanePass{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return LanePass{}, false, fmt.Errorf("latest lane pass %s: %w", site, err)
|
||||
}
|
||||
return p, true, nil
|
||||
}
|
||||
|
||||
// LatestLanePasses returns the newest pass for each Site, with current Lane
|
||||
// state joined on. Sites without a pass have no row yet.
|
||||
func (s *Store) LatestLanePasses() ([]LanePass, error) {
|
||||
rows, err := s.db.Query(`SELECT ` + lanePassColumns + `
|
||||
FROM (
|
||||
SELECT DISTINCT ON (site)
|
||||
site, ran_at, skip, due, checked, gap_ms, clamped,
|
||||
refused, unreachable, no_chapter, unfetchable, errors
|
||||
FROM poll_passes
|
||||
ORDER BY site, ran_at DESC
|
||||
) p
|
||||
LEFT JOIN poll_lanes l ON l.site = p.site
|
||||
ORDER BY p.site`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query latest lane passes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []LanePass{}
|
||||
for rows.Next() {
|
||||
p, err := scanLanePass(rows.Scan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan latest lane pass: %w", err)
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LanePassOutcomes sums the named outcomes for each Site at or after since.
|
||||
// The window boundary is supplied by the caller; the store has no clock.
|
||||
func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter),
|
||||
SUM(unfetchable), SUM(errors)
|
||||
FROM poll_passes
|
||||
WHERE ran_at >= $1
|
||||
GROUP BY site
|
||||
ORDER BY site`, since)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query lane pass outcomes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []SiteOutcomes{}
|
||||
for rows.Next() {
|
||||
var outcomes SiteOutcomes
|
||||
if err := rows.Scan(
|
||||
&outcomes.Site, &outcomes.Refused, &outcomes.Unreachable,
|
||||
&outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan lane pass outcomes: %w", err)
|
||||
}
|
||||
out = append(out, outcomes)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetLaneRefusal persists a Site's refusal backoff stamp without touching its
|
||||
// pause. until is supplied by the caller's clock.
|
||||
func (s *Store) SetLaneRefusal(site string, until int64) error {
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO poll_lanes (site, refuse_until) VALUES ($1, $2)
|
||||
ON CONFLICT (site) DO UPDATE SET refuse_until = EXCLUDED.refuse_until`, site, until); err != nil {
|
||||
return fmt.Errorf("set lane refusal %s: %w", site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PauseLane persists a bounded pause. The caller must ensure until is after
|
||||
// its current timestamp; the store has no clock and rejects only the invalid
|
||||
// zero and negative sentinels.
|
||||
func (s *Store) PauseLane(site string, until int64) error {
|
||||
if until <= 0 {
|
||||
return fmt.Errorf("pause lane %s: expiry must be positive", site)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO poll_lanes (site, paused_until) VALUES ($1, $2)
|
||||
ON CONFLICT (site) DO UPDATE SET paused_until = EXCLUDED.paused_until`, site, until); err != nil {
|
||||
return fmt.Errorf("pause lane %s: %w", site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResumeLane clears only the pause stamp and keeps the Lane state row, along
|
||||
// with any refusal stamp already persisted on it.
|
||||
func (s *Store) ResumeLane(site string) error {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE poll_lanes SET paused_until = 0 WHERE site = $1`, site); err != nil {
|
||||
return fmt.Errorf("resume lane %s: %w", site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PausedLanes returns Lane rows with a nonzero pause stamp. Expiry comparison
|
||||
// stays with the caller because the store is deliberately clockless.
|
||||
func (s *Store) PausedLanes() ([]LanePause, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site, paused_until
|
||||
FROM poll_lanes
|
||||
WHERE paused_until > 0
|
||||
ORDER BY site`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query paused lanes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []LanePause{}
|
||||
for rows.Next() {
|
||||
var pause LanePause
|
||||
if err := rows.Scan(&pause.Site, &pause.PausedUntil); err != nil {
|
||||
return nil, fmt.Errorf("scan paused lane: %w", err)
|
||||
}
|
||||
out = append(out, pause)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LaneGates reads a Site's pause and refusal stamps in one row read — the
|
||||
// top-of-pass gate the poller uses (issue #141). A missing state row is the
|
||||
// default: unpaused and not refusing.
|
||||
func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err error) {
|
||||
err = s.db.QueryRow(
|
||||
`SELECT paused_until, refuse_until FROM poll_lanes WHERE site = $1`, site).
|
||||
Scan(&pausedUntil, &refuseUntil)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("lane gates %s: %w", site, err)
|
||||
}
|
||||
return pausedUntil, refuseUntil, nil
|
||||
}
|
||||
|
||||
// DueForLatestCheck returns one Site's series whose server-side
|
||||
// latest-chapter check has aged past cutoffMs, ordered by how many bookmarks
|
||||
// reference them (descending) then least-recently-checked first. One Site per
|
||||
// query, because each Poll Lane asks for its own list: the query carries one
|
||||
// Site and one cut-off instead of parallel lists (issue #100). There is no
|
||||
// limit — the Lane's own gap paces the fetches, and the batch size that used
|
||||
// to cap this query is gone with the shared pace.
|
||||
//
|
||||
// A forced Series (force_poll_at newer than latest_checked_at, issue #146)
|
||||
// overrides exactly three gates: the rest cutoff, the Sighting-deferral
|
||||
// clause and the finished-only bucket. It never overrides an empty
|
||||
// series_url or the Bookmarks join — nothing to fetch, and no consumer for
|
||||
// the result — so those stay unconditional. Forced rows sort to the front of
|
||||
// the queue; the reader-count-then-age ordering among the rest is ADR-0003.
|
||||
//
|
||||
// The reader_count ordering is the point of the split (ADR-0003): a series
|
||||
// shared by several readers is fetched once per due cycle, and the popular
|
||||
@@ -916,20 +1191,40 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
// burns requests. Archived bookmarks still count — knowing what a shelved
|
||||
// series is up to is the whole reason for archiving instead of deleting.
|
||||
// A series with no bookmarks at all never appears: the join excludes it.
|
||||
func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites []string, limit int) ([]Series, error) {
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
|
||||
//
|
||||
// ceilingMs is the Sighting deferral ceiling (issue #103): a Series whose last
|
||||
// real Poll is older than it appears however recently it was sighted. That is
|
||||
// what bounds the whole mechanism — a wrong Latest Chapter dies within the
|
||||
// ceiling deterministically rather than in expectation. Deferral itself is
|
||||
// decided here, from two facts the query already computes, so a Lane gains no
|
||||
// query per round: a Sighting younger than cutoffMs holds the Series back, but
|
||||
// only while COUNT(*) is 1. A Series a second Reader bookmarks is Polled on
|
||||
// schedule, so a wrong value the whole guild can see is corrected by a check
|
||||
// that was never postponed; on a solitary Series the only person a wrong value
|
||||
// reaches is the Reader who reported it. Whether the reporting Reader is
|
||||
// allowed to defer at all was settled when the Sighting was recorded — see
|
||||
// RecordSighting.
|
||||
func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Series, error) {
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`,
|
||||
(s.force_poll_at > s.latest_checked_at) AS forced,
|
||||
COUNT(*) AS reader_count
|
||||
FROM series s
|
||||
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
WHERE s.series_url <> ''
|
||||
AND s.latest_checked_at <= CASE
|
||||
WHEN s.site = ANY($3::text[]) THEN $2::bigint
|
||||
ELSE $1::bigint
|
||||
END
|
||||
WHERE s.site = $1
|
||||
AND s.series_url <> ''
|
||||
AND (s.latest_checked_at <= $2::bigint
|
||||
OR s.force_poll_at > s.latest_checked_at)
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
|
||||
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
ORDER BY reader_count DESC, s.latest_checked_at ASC
|
||||
LIMIT $4`, cutoffMs, browserCutoffMs, pgtype.FlatArray[string](browserSites), limit)
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at
|
||||
HAVING (COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
OR s.force_poll_at > s.latest_checked_at)
|
||||
AND (COUNT(*) > 1
|
||||
OR s.latest_sighted_at <= $2::bigint
|
||||
OR s.latest_checked_at <= $3::bigint
|
||||
OR s.force_poll_at > s.latest_checked_at)
|
||||
ORDER BY (s.force_poll_at > s.latest_checked_at) DESC,
|
||||
reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query due series: %w", err)
|
||||
}
|
||||
@@ -946,6 +1241,30 @@ func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// EligibleSeriesCount returns how many of a Site's Series still have at least
|
||||
// one bookmark outside the finished bucket. It is the denominator of the
|
||||
// Lane's pace (issue #100): the effective gap is the smaller of the registry
|
||||
// gap and one hour divided by this count, so Series that will never be Polled
|
||||
// do not make the Lane faster than it needs to be, and counting every eligible
|
||||
// Series rather than only those currently due keeps the pace steady — the
|
||||
// single worst moment to be fastest is startup, when everything is due at
|
||||
// once.
|
||||
func (s *Store) EligibleSeriesCount(site string) (int, error) {
|
||||
var n int
|
||||
err := s.db.QueryRow(`SELECT COUNT(*) FROM (
|
||||
SELECT 1
|
||||
FROM series s
|
||||
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
WHERE s.site = $1
|
||||
GROUP BY s.site, s.series_id
|
||||
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
) e`, site).Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count eligible series %s: %w", site, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// MarkLatestChecked records that the server looked at a series at ts, whatever
|
||||
// the look turned up. Marking a missing series is not an error: the row may
|
||||
// have been orphaned while a fetch was in flight.
|
||||
@@ -954,7 +1273,7 @@ func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites
|
||||
// out of the client-visible read path on purpose. PUT /bookmarks/{key} decodes
|
||||
// a whole Bookmark from the client and Upsert writes every series column it
|
||||
// knows about, so a userscript PUT — which has no idea this field exists —
|
||||
// would write a zero and reset the cooldown, making the poller re-fetch that
|
||||
// would write a zero and reset the rest, making the poller re-fetch that
|
||||
// series every tick for as long as the user kept reading it.
|
||||
func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
|
||||
if _, err := s.db.Exec(
|
||||
@@ -965,8 +1284,27 @@ func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ForceSeriesPoll stamps a Series with the owner's "check now" request
|
||||
// (issue #146): a fact about the Series the Lane's next pass reads through
|
||||
// DueForLatestCheck, never a command to the poller — so the request survives
|
||||
// a restart. Writing again overwrites the request time; the write is
|
||||
// idempotent. Touching a missing series is not an error: the row may have
|
||||
// been orphaned, and the caller's read decides what exists. The stamp never
|
||||
// expires by itself — an unanswered request keeps ageing — and pending is
|
||||
// derived as force_poll_at > latest_checked_at, which is why the poller's
|
||||
// check stamp is written before the fetch: the first attempt ends the
|
||||
// pending state whatever it returns.
|
||||
func (s *Store) ForceSeriesPoll(site, seriesID string, at int64) error {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET force_poll_at = $1 WHERE site = $2 AND series_id = $3`,
|
||||
at, site, seriesID); err != nil {
|
||||
return fmt.Errorf("force poll %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LatestCheckedAt reads the column MarkLatestChecked writes. It exists for
|
||||
// tests outside this package (the poller's own tests assert on cooldown
|
||||
// tests outside this package (the poller's own tests assert on rest
|
||||
// bookkeeping) — see MarkLatestChecked for why the field stays off the
|
||||
// client-visible row.
|
||||
func (s *Store) LatestCheckedAt(site, seriesID string) (int64, error) {
|
||||
@@ -992,3 +1330,111 @@ func (s *Store) SetLatestChapter(site, seriesID, label string, num float64) erro
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordSighting notes that a Reader's browser reported this Series' Latest
|
||||
// Chapter, which is the half of a Sighting the client body cannot express
|
||||
// (issue #103). It must be called *before* the Upsert that stores the reported
|
||||
// value: the raise test compares against what is still on the row, and after
|
||||
// the Upsert there is nothing left to compare with. A Series that does not
|
||||
// exist yet — the first Bookmark of it — is not a Sighting at all: nothing has
|
||||
// ever been Polled, so there is nothing to defer and nobody to attribute.
|
||||
//
|
||||
// Two independent effects, hence the two CASE arms. The deferral stamp is only
|
||||
// written for a Reader below the disagreement limit, so a marked Reader's
|
||||
// reports keep updating the Latest Chapter but stop postponing anything, and
|
||||
// clearing their marks restores the privilege on their next Sighting. The
|
||||
// attribution is written whenever the report raises the stored number,
|
||||
// including for a marked Reader — their Sightings are still judged, which is
|
||||
// how they earn the privilege back.
|
||||
//
|
||||
// num is the reported chapter number. A PUT that carries none — a favourite
|
||||
// toggle, or progress written from a chapter page — is no Sighting at all:
|
||||
// nobody read the Series page, so there is nothing to stand in for a Poll and
|
||||
// nothing that could later be judged.
|
||||
func (s *Store) RecordSighting(readerID int64, site, seriesID string, num *float64, ts int64) error {
|
||||
if num == nil {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE series SET
|
||||
latest_sighted_at = CASE
|
||||
WHEN (SELECT sighting_disagreements FROM readers WHERE id = $3) < $6
|
||||
THEN $4::bigint ELSE latest_sighted_at END,
|
||||
latest_raised_by = CASE
|
||||
WHEN latest_chapter_num IS NULL OR $5::double precision > latest_chapter_num
|
||||
THEN $3::bigint ELSE latest_raised_by END
|
||||
WHERE site = $1 AND series_id = $2`,
|
||||
site, seriesID, readerID, ts, *num, SightingDisagreementLimit); err != nil {
|
||||
return fmt.Errorf("record sighting %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SightingAgreementsToClear is how many Polls must confirm a Reader's
|
||||
// Sightings in a row before their disagreements are forgiven. An agreement is
|
||||
// only recorded when a Poll later confirms a Sighting, so this is twenty Polls
|
||||
// of Series that Reader bookmarks — hours to days, not twenty page views. That
|
||||
// is the intended price: recovery is automatic but cannot be outwaited, and a
|
||||
// disagreement resets the run to zero, so credit cannot be banked in advance.
|
||||
const SightingAgreementsToClear = 20
|
||||
|
||||
// RecordSightingOutcome settles what a Poll decided about the Reader whose
|
||||
// Sighting last raised this Series' Latest Chapter, and clears the attribution
|
||||
// in the same transaction so one Sighting is judged exactly once. agreed is
|
||||
// the Poll confirming the stored value; its opposite is the Poll finding a
|
||||
// lower number, which means the raise was false.
|
||||
//
|
||||
// A Poll finding a *higher* number is neither — the Site published — and takes
|
||||
// ClearSightingAttribution instead.
|
||||
func (s *Store) RecordSightingOutcome(site, seriesID string, readerID int64, agreed bool) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin sighting outcome %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// The run length is what "consecutive" means: a disagreement zeroes the
|
||||
// agreements, and completing a run zeroes both, so the next run starts
|
||||
// from nothing rather than forgiving every later disagreement instantly.
|
||||
q := `UPDATE readers SET sighting_disagreements = sighting_disagreements + 1,
|
||||
sighting_agreements = 0
|
||||
WHERE id = $1`
|
||||
args := []any{readerID}
|
||||
if agreed {
|
||||
q = `UPDATE readers SET
|
||||
sighting_agreements = CASE WHEN sighting_agreements + 1 >= $2 THEN 0
|
||||
ELSE sighting_agreements + 1 END,
|
||||
sighting_disagreements = CASE WHEN sighting_agreements + 1 >= $2 THEN 0
|
||||
ELSE sighting_disagreements END
|
||||
WHERE id = $1`
|
||||
args = append(args, SightingAgreementsToClear)
|
||||
}
|
||||
if _, err := tx.Exec(q, args...); err != nil {
|
||||
return fmt.Errorf("record sighting outcome for reader %d: %w", readerID, err)
|
||||
}
|
||||
if _, err := tx.Exec(clearAttributionSQL, site, seriesID, readerID); err != nil {
|
||||
return fmt.Errorf("clear sighting attribution %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit sighting outcome %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearSightingAttribution answers a Sighting without judging it: the Poll
|
||||
// found a higher number, so the value about to be stored is its own and this
|
||||
// Reader is no longer answerable for the row. Without it the next Poll's
|
||||
// agreement would be credited to a Reader who did not earn it.
|
||||
func (s *Store) ClearSightingAttribution(site, seriesID string, readerID int64) error {
|
||||
if _, err := s.db.Exec(clearAttributionSQL, site, seriesID, readerID); err != nil {
|
||||
return fmt.Errorf("clear sighting attribution %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clearAttributionSQL drops the attribution only while it still names the
|
||||
// Reader being judged: a Sighting landing between the due query's snapshot and
|
||||
// this write is a fresh, unjudged one and must not be erased by the previous
|
||||
// one's verdict.
|
||||
const clearAttributionSQL = `UPDATE series SET latest_raised_by = NULL
|
||||
WHERE site = $1 AND series_id = $2 AND latest_raised_by = $3`
|
||||
|
||||
@@ -277,6 +277,10 @@ func seedForCheck(t *testing.T, s *Store, key, seriesURL string, checkedAt int64
|
||||
}
|
||||
}
|
||||
|
||||
// noCeiling is a Sighting deferral ceiling no Series can reach, for the tests
|
||||
// that predate the ceiling and are about rest, ordering or buckets instead.
|
||||
const noCeiling = int64(-1)
|
||||
|
||||
func TestDueForLatestCheck(t *testing.T) {
|
||||
const hour = int64(3600_000)
|
||||
now := 10 * hour
|
||||
@@ -298,7 +302,7 @@ func TestDueForLatestCheck(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
|
||||
|
||||
due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", now-hour, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -309,22 +313,25 @@ func TestDueForLatestCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
|
||||
func TestDueForLatestCheckOldestFirstAndScopedToSite(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
// Insert newest-checked first so a correct ORDER BY has to reverse it.
|
||||
seedForCheck(t, s, "asura:c", "https://asurascans.com/comics/c", 300)
|
||||
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
|
||||
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
|
||||
// A second Site's due series must not appear in asura's list: each Lane
|
||||
// asks for one Site, and no Lane may see another's queue.
|
||||
seedForCheck(t, s, "demonic:z", "https://demonicscans.org/manga/z", 0)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 2 {
|
||||
t.Fatalf("got %d rows, want 2 (limit)", len(due))
|
||||
if len(due) != 3 {
|
||||
t.Fatalf("got %d rows, want 3 (all of asura's, none of demonic's)", len(due))
|
||||
}
|
||||
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" {
|
||||
t.Fatalf("got %q,%q; want asura:a,asura:b (oldest first)", due[0].Key(), due[1].Key())
|
||||
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" || due[2].Key() != "asura:c" {
|
||||
t.Fatalf("got %q,%q,%q; want asura:a,asura:b,asura:c (oldest first)", due[0].Key(), due[1].Key(), due[2].Key())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,7 +503,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
due, err := store.DueForLatestCheck("asura", time.Now().UnixMilli(), noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -512,6 +519,38 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The gap's denominator counts every Series the Lane will ever Poll: a
|
||||
// finished Series must not make the Lane faster than it needs to be, and
|
||||
// another Site's Series must not leak into this Site's count.
|
||||
func TestEligibleSeriesCount(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
seedForCheck(t, store, "asura:reading", "https://asurascans.com/comics/reading", 0)
|
||||
seedForCheck(t, store, "asura:archived", "https://asurascans.com/comics/archived", 0)
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:finished", Site: "asura", SeriesID: "finished",
|
||||
SeriesURL: "https://asurascans.com/comics/finished",
|
||||
Status: StatusFinished, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed finished: %v", err)
|
||||
}
|
||||
seedForCheck(t, store, "demonic:z", "https://demonicscans.org/manga/z", 0)
|
||||
|
||||
n, err := store.EligibleSeriesCount("asura")
|
||||
if err != nil {
|
||||
t.Fatalf("EligibleSeriesCount: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("eligible = %d, want 2 (finished excluded, demonic excluded)", n)
|
||||
}
|
||||
n, err = store.EligibleSeriesCount("demonic")
|
||||
if err != nil {
|
||||
t.Fatalf("EligibleSeriesCount(demonic): %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("eligible(demonic) = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisplayChapter(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -719,7 +758,7 @@ func TestMigration0008DropsLegacyCoverRows(t *testing.T) {
|
||||
func readSeries(t *testing.T, s *Store, site, seriesID string) Series {
|
||||
t.Helper()
|
||||
sr, err := scanSeries(s.db.QueryRow(
|
||||
`SELECT `+seriesColumns+`, 0 AS reader_count FROM series s
|
||||
`SELECT `+seriesColumns+`, false AS forced, 0 AS reader_count FROM series s
|
||||
WHERE s.site = $1 AND s.series_id = $2`, site, seriesID).Scan)
|
||||
if err != nil {
|
||||
t.Fatalf("read series %s:%s: %v", site, seriesID, err)
|
||||
@@ -970,7 +1009,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
|
||||
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -996,7 +1035,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
|
||||
t.Fatalf("seed orphan series: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -1299,6 +1338,86 @@ func TestReadersAndSessionRevocation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The Sighting counters ship before the mechanism that fills them (issue
|
||||
// #102 before #103), so the admin page depends on their default: a fresh
|
||||
// Reader reads back trusted. Marking one directly proves Readers() reports
|
||||
// the counters and Blocked() flips at the limit, and that ClearReaderMarks —
|
||||
// the owner's remedy for a false mark — zeroes them again.
|
||||
func TestReaderSightingMarks(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
other := secondReader(t, s)
|
||||
|
||||
readers, err := s.Readers()
|
||||
if err != nil {
|
||||
t.Fatalf("Readers: %v", err)
|
||||
}
|
||||
if len(readers) != 2 {
|
||||
t.Fatalf("readers = %d, want the owner and the second Reader", len(readers))
|
||||
}
|
||||
for _, r := range readers {
|
||||
if r.Agreements != 0 || r.Disagreements != 0 || r.Blocked() {
|
||||
t.Fatalf("fresh reader %d has marks: %+v", r.ID, r)
|
||||
}
|
||||
}
|
||||
|
||||
// The counters' default is the trusted state; writing them directly is
|
||||
// the only way to exercise the read path until issue #103 moves them.
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE readers SET sighting_agreements = 5, sighting_disagreements = 2
|
||||
WHERE id = $1`, other); err != nil {
|
||||
t.Fatalf("mark reader: %v", err)
|
||||
}
|
||||
readers, err = s.Readers()
|
||||
if err != nil {
|
||||
t.Fatalf("Readers: %v", err)
|
||||
}
|
||||
var marked *ReaderSummary
|
||||
for i := range readers {
|
||||
if readers[i].ID == other {
|
||||
marked = &readers[i]
|
||||
}
|
||||
}
|
||||
if marked == nil || marked.Agreements != 5 || marked.Disagreements != 2 {
|
||||
t.Fatalf("marked reader = %+v, want agreements 5, disagreements 2", marked)
|
||||
}
|
||||
if marked.Blocked() {
|
||||
t.Fatalf("reader with 2 disagreements is blocked; limit is %d", SightingDisagreementLimit)
|
||||
}
|
||||
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE readers SET sighting_disagreements = 3 WHERE id = $1`, other); err != nil {
|
||||
t.Fatalf("block reader: %v", err)
|
||||
}
|
||||
readers, err = s.Readers()
|
||||
if err != nil {
|
||||
t.Fatalf("Readers: %v", err)
|
||||
}
|
||||
for _, r := range readers {
|
||||
if r.ID == other && !r.Blocked() {
|
||||
t.Fatalf("reader at the disagreement limit is not blocked: %+v", r)
|
||||
}
|
||||
if r.ID == s.OwnerID() && r.Blocked() {
|
||||
t.Fatalf("untouched owner became blocked: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.ClearReaderMarks(other); err != nil {
|
||||
t.Fatalf("ClearReaderMarks: %v", err)
|
||||
}
|
||||
if err := s.ClearReaderMarks(other + 9999); err != nil {
|
||||
t.Fatalf("ClearReaderMarks(unknown id): %v", err)
|
||||
}
|
||||
readers, err = s.Readers()
|
||||
if err != nil {
|
||||
t.Fatalf("Readers: %v", err)
|
||||
}
|
||||
for _, r := range readers {
|
||||
if r.Agreements != 0 || r.Disagreements != 0 || r.Blocked() {
|
||||
t.Fatalf("reader %d not cleared: %+v", r.ID, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two Readers on one Series: one series row, two independent progresses. The
|
||||
// second Reader starts at zero however far the first has read, and the shared
|
||||
// row is still due exactly once.
|
||||
@@ -1334,7 +1453,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", time.Now().UnixMilli(), noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -1350,7 +1469,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
|
||||
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
due, err = s.DueForLatestCheck("asura", time.Now().UnixMilli(), noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck after delete: %v", err)
|
||||
}
|
||||
@@ -1465,3 +1584,318 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
|
||||
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
|
||||
}
|
||||
}
|
||||
func TestRecordLanePassPrunesBeforeInsertCutoff(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, pass := range []LanePass{
|
||||
{Site: "asura", RanAt: 99},
|
||||
{Site: "asura", RanAt: 100},
|
||||
} {
|
||||
if err := s.RecordLanePass(pass, 100); err != nil {
|
||||
t.Fatalf("RecordLanePass(%d): %v", pass.RanAt, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 200}, 100); err != nil {
|
||||
t.Fatalf("RecordLanePass(200): %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&count); err != nil {
|
||||
t.Fatalf("count passes: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Fatalf("retained passes = %d, want 2", count)
|
||||
}
|
||||
if _, ok, err := s.LatestLanePass("asura"); err != nil || !ok {
|
||||
t.Fatalf("LatestLanePass = ok %v, err %v; want latest row", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestLanePassesKeepsNewestPerSiteAndJoinsState(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, pass := range []LanePass{
|
||||
{Site: "asura", RanAt: 100, Due: 1},
|
||||
{Site: "asura", RanAt: 200, Skip: "due-query", Due: 2, Checked: 3, GapMS: 4000, Clamped: true},
|
||||
{Site: "demonic", RanAt: 150, Due: 4},
|
||||
} {
|
||||
if err := s.RecordLanePass(pass, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass(%s/%d): %v", pass.Site, pass.RanAt, err)
|
||||
}
|
||||
}
|
||||
if err := s.PauseLane("asura", 1234); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
if err := s.SetLaneRefusal("asura", 5678); err != nil {
|
||||
t.Fatalf("SetLaneRefusal: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.LatestLanePasses()
|
||||
if err != nil {
|
||||
t.Fatalf("LatestLanePasses: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("latest passes = %d, want one per Site", len(got))
|
||||
}
|
||||
bySite := map[string]LanePass{}
|
||||
for _, pass := range got {
|
||||
bySite[pass.Site] = pass
|
||||
}
|
||||
asura := bySite["asura"]
|
||||
if asura.RanAt != 200 || asura.Skip != "due-query" || asura.Due != 2 || asura.Checked != 3 || asura.GapMS != 4000 || !asura.Clamped ||
|
||||
asura.PausedUntil != 1234 || asura.RefuseUntil != 5678 {
|
||||
t.Fatalf("asura latest pass = %+v, want newest pass and joined state", asura)
|
||||
}
|
||||
if demonic := bySite["demonic"]; demonic.RanAt != 150 || demonic.Due != 4 {
|
||||
t.Fatalf("demonic latest pass = %+v, want its only pass", demonic)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLanePassOutcomesSumsWindow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, pass := range []LanePass{
|
||||
{Site: "asura", RanAt: 99, Refused: 1, Unreachable: 2, NoChapter: 3, Unfetchable: 4, Errors: 5},
|
||||
{Site: "asura", RanAt: 100, Refused: 2, Unreachable: 3, NoChapter: 4, Unfetchable: 5, Errors: 6},
|
||||
{Site: "asura", RanAt: 200, Refused: 3, Unreachable: 4, NoChapter: 5, Unfetchable: 6, Errors: 7},
|
||||
{Site: "demonic", RanAt: 150, Refused: 8, Unreachable: 9, NoChapter: 10, Unfetchable: 11, Errors: 12},
|
||||
} {
|
||||
if err := s.RecordLanePass(pass, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass(%s/%d): %v", pass.Site, pass.RanAt, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := s.LanePassOutcomes(100)
|
||||
if err != nil {
|
||||
t.Fatalf("LanePassOutcomes: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("outcome Sites = %d, want 2", len(got))
|
||||
}
|
||||
bySite := map[string]SiteOutcomes{}
|
||||
for _, outcomes := range got {
|
||||
bySite[outcomes.Site] = outcomes
|
||||
}
|
||||
if want := (SiteOutcomes{Site: "asura", Refused: 5, Unreachable: 7, NoChapter: 9, Unfetchable: 11, Errors: 13}); bySite["asura"] != want {
|
||||
t.Fatalf("asura outcomes = %+v, want %+v", bySite["asura"], want)
|
||||
}
|
||||
if want := (SiteOutcomes{Site: "demonic", Refused: 8, Unreachable: 9, NoChapter: 10, Unfetchable: 11, Errors: 12}); bySite["demonic"] != want {
|
||||
t.Fatalf("demonic outcomes = %+v, want %+v", bySite["demonic"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLaneGatesReadsOneRow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
paused, refused, err := s.LaneGates("asura")
|
||||
if err != nil || paused != 0 || refused != 0 {
|
||||
t.Fatalf("LaneGates on a missing state row = (%d, %d, %v), want (0, 0, nil)", paused, refused, err)
|
||||
}
|
||||
if err := s.PauseLane("asura", 2000); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
if err := s.SetLaneRefusal("asura", 3000); err != nil {
|
||||
t.Fatalf("SetLaneRefusal: %v", err)
|
||||
}
|
||||
paused, refused, err = s.LaneGates("asura")
|
||||
if err != nil || paused != 2000 || refused != 3000 {
|
||||
t.Fatalf("LaneGates = (%d, %d, %v), want (2000, 3000, nil)", paused, refused, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLaneStatePauseResumeAndRefusal(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, until := range []int64{0, -1} {
|
||||
if err := s.PauseLane("asura", until); err == nil {
|
||||
t.Fatalf("PauseLane(%d) accepted a non-future expiry", until)
|
||||
}
|
||||
}
|
||||
if err := s.PauseLane("asura", 2000); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
if err := s.SetLaneRefusal("asura", 3000); err != nil {
|
||||
t.Fatalf("SetLaneRefusal: %v", err)
|
||||
}
|
||||
if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 1}, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass: %v", err)
|
||||
}
|
||||
pausedUntil, refuseUntil, err := s.LaneGates("asura")
|
||||
if err != nil || pausedUntil != 2000 || refuseUntil != 3000 {
|
||||
t.Fatalf("LaneGates = %d/%d, %v; want 2000/3000", pausedUntil, refuseUntil, err)
|
||||
}
|
||||
paused, err := s.PausedLanes()
|
||||
if err != nil {
|
||||
t.Fatalf("PausedLanes: %v", err)
|
||||
}
|
||||
if len(paused) != 1 || paused[0] != (LanePause{Site: "asura", PausedUntil: 2000}) {
|
||||
t.Fatalf("PausedLanes = %+v, want asura/2000", paused)
|
||||
}
|
||||
|
||||
if err := s.ResumeLane("asura"); err != nil {
|
||||
t.Fatalf("ResumeLane: %v", err)
|
||||
}
|
||||
latest, ok, err := s.LatestLanePass("asura")
|
||||
if err != nil || !ok || latest.RefuseUntil != 3000 {
|
||||
t.Fatalf("latest refusal after resume = %+v, ok=%v, err=%v; want 3000 preserved", latest, ok, err)
|
||||
}
|
||||
if got, got2, err := s.LaneGates("asura"); err != nil || got != 0 || got2 != 3000 {
|
||||
t.Fatalf("LaneGates after resume = %d/%d, %v; want 0/3000", got, got2, err)
|
||||
}
|
||||
if paused, err := s.PausedLanes(); err != nil || len(paused) != 0 {
|
||||
t.Fatalf("PausedLanes after resume = %+v, %v; want empty", paused, err)
|
||||
}
|
||||
var rows int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM poll_lanes WHERE site = $1`, "asura").Scan(&rows); err != nil {
|
||||
t.Fatalf("count lane state: %v", err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("lane state rows after resume = %d, want 1", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A forced Series is due ahead of the rest cutoff: the request overrides the
|
||||
// rest gate so the Lane's next pass picks it up however recently it was
|
||||
// checked. An unforced series under the rest stays out.
|
||||
func TestDueForLatestCheckForcedOverridesRestCutoff(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const now = int64(10 * 3600_000)
|
||||
seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", now-30*60_000)
|
||||
seedForCheck(t, s, "asura:fresh", "https://asurascans.com/comics/fresh", now-30*60_000)
|
||||
if err := s.ForceSeriesPoll("asura", "forced", now); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", now-3600_000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 || due[0].Key() != "asura:forced" {
|
||||
t.Fatalf("due = %v, want only the forced series", due)
|
||||
}
|
||||
}
|
||||
|
||||
// The Sighting-deferral clause holds a solitary series back after a recent
|
||||
// Reader report; a forced request overrides it and puts the series back on
|
||||
// the Lane's list.
|
||||
func TestDueForLatestCheckForcedOverridesSightingDeferral(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const now = int64(10 * 3600_000)
|
||||
// One bookmark (so deferral can apply), sighted and checked 10m ago:
|
||||
// inside the deferral window and under the ceiling.
|
||||
seedForCheck(t, s, "asura:deferred", "https://asurascans.com/comics/deferred", now-10*60_000)
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET latest_sighted_at = $1 WHERE site = 'asura' AND series_id = 'deferred'`,
|
||||
now-10*60_000); err != nil {
|
||||
t.Fatalf("seed sighting: %v", err)
|
||||
}
|
||||
|
||||
// Unforced: deferred, and under the rest anyway.
|
||||
due, err := s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 0 {
|
||||
t.Fatalf("unforced deferred series is due: %v", due)
|
||||
}
|
||||
|
||||
// Forced: the request overrides the deferral.
|
||||
if err := s.ForceSeriesPoll("asura", "deferred", now); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 || due[0].Key() != "asura:deferred" {
|
||||
t.Fatalf("forced deferred series not due: %v", due)
|
||||
}
|
||||
}
|
||||
|
||||
// The finished-only bucket excludes a series whose only bookmarks are
|
||||
// finished; a forced request overrides it — the owner asked, so the Lane
|
||||
// looks.
|
||||
func TestDueForLatestCheckForcedOverridesFinishedBucket(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:reading", "https://asurascans.com/comics/reading", 0)
|
||||
if _, err := s.Upsert(s.OwnerID(), Bookmark{
|
||||
Key: "asura:finished", Site: "asura", SeriesID: "finished",
|
||||
SeriesURL: "https://asurascans.com/comics/finished",
|
||||
Status: StatusFinished, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed finished: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
for _, sr := range due {
|
||||
if sr.Key() == "asura:finished" {
|
||||
t.Fatalf("unforced finished series is due: %v", due)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.ForceSeriesPoll("asura", "finished", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, sr := range due {
|
||||
got[sr.Key()] = true
|
||||
}
|
||||
if !got["asura:reading"] || !got["asura:finished"] {
|
||||
t.Fatalf("forced finished series not due: %v", due)
|
||||
}
|
||||
}
|
||||
|
||||
// A forced Series jumps the queue: it sorts ahead of a more-read series that
|
||||
// is due by rest, without disturbing the reader-count-then-age tie-break
|
||||
// among the unforced rows (ADR-0003).
|
||||
func TestDueForLatestCheckForcedSortsFirst(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
// "popular" has two readers and is long overdue; "forced" has one reader
|
||||
// and a fresh check stamp. The forced row must come first.
|
||||
seedForCheck(t, s, "asura:popular", "https://asurascans.com/comics/popular", 100)
|
||||
seedSecondReader(t, s, "asura:popular:2", "asura", "popular", 1001)
|
||||
seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", 900)
|
||||
if err := s.ForceSeriesPoll("asura", "forced", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 2 {
|
||||
t.Fatalf("due = %d rows, want 2", len(due))
|
||||
}
|
||||
if due[0].Key() != "asura:forced" || due[1].Key() != "asura:popular" {
|
||||
t.Fatalf("due order = %q, %q; want forced first, then popular", due[0].Key(), due[1].Key())
|
||||
}
|
||||
}
|
||||
|
||||
// A forced Series with no series URL is still not fetched — nothing to fetch —
|
||||
// and one with no Bookmarks is still excluded by the join. The force flag
|
||||
// opens the three gates it is allowed to, not the whole query.
|
||||
func TestDueForLatestCheckForcedDoesNotOverrideURLOrJoin(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:nourl", "", 0)
|
||||
if err := s.ForceSeriesPoll("asura", "nourl", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO series (site, series_id, title, series_url, cover, kind,
|
||||
latest_chapter, latest_chapter_num, latest_checked_at,
|
||||
force_poll_at)
|
||||
VALUES ('asura', 'orphan', 'Orphan', 'https://asurascans.com/comics/orphan',
|
||||
'', 'manga', '', NULL, 0, 5000)`); err != nil {
|
||||
t.Fatalf("seed orphan: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 0 {
|
||||
t.Fatalf("due = %v, want neither the URL-less nor the orphan series", due)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// ownerWindow is the staleness boundary the Series list's "not checked in
|
||||
// 12h" filter compares against. Declared once; later admin tickets read it.
|
||||
const ownerWindow = 12 * time.Hour
|
||||
|
||||
// adminView is the shared shell data for an administrative page and the roster
|
||||
// fragment returned after a Reader action.
|
||||
type adminView struct {
|
||||
Page string
|
||||
Readers []store.ReaderSummary
|
||||
// OwnerID travels with the roster so it can tell the owner's own row from
|
||||
// the Readers they may act on.
|
||||
OwnerID int64
|
||||
Lanes lanesView
|
||||
SeriesList seriesListView
|
||||
// Detail is the per-Series page data; zero on every other page.
|
||||
Detail seriesDetailView
|
||||
// Overview is the landing page data; zero on every other page.
|
||||
Overview overviewView
|
||||
}
|
||||
|
||||
// adminRoute pairs a route pattern with its handler so the route list and the
|
||||
// gate cannot drift apart.
|
||||
type adminRoute struct {
|
||||
pattern string
|
||||
handler http.HandlerFunc
|
||||
}
|
||||
|
||||
// adminRoutes is every route that reaches past the acting Reader. Register
|
||||
// wraps each one in requireOwner, so a new administrative route is gated by
|
||||
// being listed here rather than by remembering to write a check inside it.
|
||||
func (h *Handler) adminRoutes() []adminRoute {
|
||||
return []adminRoute{
|
||||
{"GET /admin", h.admin},
|
||||
{"GET /admin/lanes", h.adminLanes},
|
||||
{"GET /admin/readers", h.adminReaders},
|
||||
{"GET /admin/series", h.adminSeries},
|
||||
{"GET /admin/series/{key}", h.adminSeriesDetail},
|
||||
{"POST /admin/series/{key}/poll", h.adminSeriesPoll},
|
||||
{"POST /admin/lanes/{site}/pause", h.adminLanePause},
|
||||
{"POST /admin/lanes/{site}/resume", h.adminLaneResume},
|
||||
{"GET /ui/admin/lanes", h.uiLanes},
|
||||
{"POST /readers/{id}/revoke", h.revokeReaderSessions},
|
||||
{"POST /readers/{id}/clear-marks", h.clearReaderMarks},
|
||||
}
|
||||
}
|
||||
|
||||
// AdminPatterns names every administrative route, so one test can prove the
|
||||
// owner gate covers all of them rather than one test per route. The receiver is
|
||||
// nil because only the patterns are read; the bound handlers are never called.
|
||||
func AdminPatterns() []string {
|
||||
routes := (*Handler)(nil).adminRoutes()
|
||||
out := make([]string, 0, len(routes))
|
||||
for _, rt := range routes {
|
||||
out = append(out, rt.pattern)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// requireOwner is the owner test, in one place, layered on the session gate: no
|
||||
// session is still 401, and a signed-in Reader who is not the owner gets 404
|
||||
// rather than 403 — a refusal that confirms the address exists is a refusal
|
||||
// that helps whoever is probing for it.
|
||||
func (h *Handler) requireOwner(next http.HandlerFunc) http.HandlerFunc {
|
||||
return h.requireSession(func(w http.ResponseWriter, r *http.Request) {
|
||||
if readerOf(r) != h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// admin renders the Overview landing page: a verdict line, a stats block
|
||||
// where every figure is a door into the list it counts, and the per-Site
|
||||
// library shape table — all read from the database, never from a poller.
|
||||
func (h *Handler) admin(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.overviewView()
|
||||
if err != nil {
|
||||
log.Printf("admin overview: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "overview", Overview: view})
|
||||
}
|
||||
|
||||
// adminReaders renders the Reader roster on its own bookmarkable page.
|
||||
func (h *Handler) adminReaders(w http.ResponseWriter, r *http.Request) {
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
log.Printf("admin readers: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "readers", Readers: readers, OwnerID: h.store.OwnerID()})
|
||||
}
|
||||
|
||||
func (h *Handler) renderAdmin(w http.ResponseWriter, view adminView) {
|
||||
h.render(w, http.StatusOK, "admin", view)
|
||||
}
|
||||
|
||||
// revokeReaderSessions logs one Reader out of every browser they are signed in
|
||||
// on. The owner gate is the route's, not this handler's.
|
||||
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
|
||||
target, ok := readerPathID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// The owner is not one of the Readers this endpoint reaches: revoking
|
||||
// themselves would sign out the browser making the request, which is what
|
||||
// logout is for. The roster hides the button; this refuses the hand-rolled
|
||||
// POST behind it.
|
||||
if target == h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.DeleteReaderSessions(target); err != nil {
|
||||
log.Printf("revoke sessions: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderRoster(w, "revoke sessions")
|
||||
}
|
||||
|
||||
// clearReaderMarks zeroes one Reader's Sighting counters. The guard those
|
||||
// counters feed has one known false positive — a Site changing its page shape
|
||||
// makes a correct adapter read a wrong high number and marks every honest
|
||||
// Reader of that Site at once (issue #103) — and this is its remedy. It
|
||||
// restores a privilege rather than destroying anything, so the control is
|
||||
// confirmed but never wears the destruction accent.
|
||||
func (h *Handler) clearReaderMarks(w http.ResponseWriter, r *http.Request) {
|
||||
target, ok := readerPathID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := h.store.ClearReaderMarks(target); err != nil {
|
||||
log.Printf("clear marks: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderRoster(w, "clear marks")
|
||||
}
|
||||
|
||||
// readerPathID reads the Reader a route names, answering the request itself
|
||||
// when there is nobody to act on.
|
||||
func readerPathID(w http.ResponseWriter, r *http.Request) (int64, bool) {
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "bad reader id", http.StatusBadRequest)
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
// renderRoster answers an action with the whole roster, so the counts and marks
|
||||
// it shows cannot describe the state before the tap.
|
||||
func (h *Handler) renderRoster(w http.ResponseWriter, what string) {
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
log.Printf("%s: %v", what, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "readers", adminView{Readers: readers, OwnerID: h.store.OwnerID()})
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// lanesView is the Lane status block: one row per Site's latest durable pass,
|
||||
// plus the browser fact derived from that same log. No poller is consulted —
|
||||
// the page answers from the database, so it is complete thirty seconds after
|
||||
// a deploy (issue #145).
|
||||
type lanesView struct {
|
||||
Rows []laneRow
|
||||
// PollerOff means latest-chapter polling is switched off in this
|
||||
// deployment (LATEST_CHAPTER_POLL_ENABLED). It is a config fact, not a
|
||||
// poller answering "absent": the browser line must not blame the sidecar
|
||||
// when nothing polls.
|
||||
PollerOff bool
|
||||
BrowserConfigured bool
|
||||
BrowserReachable bool
|
||||
}
|
||||
|
||||
// laneRow is one Lane formatted for reading rather than for arithmetic: the
|
||||
// template renders strings and flags, and every judgement about what they
|
||||
// mean is made here.
|
||||
type laneRow struct {
|
||||
Site string
|
||||
Due int
|
||||
Checked int
|
||||
// Gap is the last pass's pace, or "—" when no pass has reached one yet —
|
||||
// a refused Lane still reports the pace its last real pass chose, so a
|
||||
// zero here would be a figure the row never measured.
|
||||
Gap string
|
||||
Ran string
|
||||
// Chips are the named outcome counts over the owner's window, in the
|
||||
// taxonomy's fixed order. Empty writes "none observed".
|
||||
Chips []chip
|
||||
HasChips bool
|
||||
// StatePhrase is the reason this Lane declined to work: a skipped pass's
|
||||
// own sentence, or the one true stall. Empty means the pass reached its
|
||||
// loop and read normally. StateGood marks a healthy way to do nothing
|
||||
// (paused, browser asleep, nothing eligible) rather than a fault.
|
||||
StatePhrase string
|
||||
StateGood bool
|
||||
// Attention is the one flag the template colours on, so a Lane that
|
||||
// needs the owner is found at a glance rather than read for.
|
||||
Attention bool
|
||||
// Paused is the live pause state — the poll_lanes stamp the pass row
|
||||
// joins on, still in the future — not the pass's skip: the control must
|
||||
// offer Resume from the moment the owner presses Pause, with no pass
|
||||
// having run to record it (issue #147).
|
||||
Paused bool
|
||||
}
|
||||
|
||||
// chip is one named outcome count over the owner's window.
|
||||
type chip struct {
|
||||
Name string
|
||||
Count int
|
||||
}
|
||||
|
||||
// adminLanes renders the page that hosts the live Lane fragment.
|
||||
func (h *Handler) adminLanes(w http.ResponseWriter, r *http.Request) {
|
||||
h.renderAdmin(w, adminView{Page: "lanes", Lanes: h.lanesView()})
|
||||
}
|
||||
|
||||
// uiLanes answers the status block's own refresh. Only the block refreshes on
|
||||
// a timer; the roster re-renders after an action, as it always has.
|
||||
func (h *Handler) uiLanes(w http.ResponseWriter, r *http.Request) {
|
||||
h.render(w, http.StatusOK, "lanes", h.lanesView())
|
||||
}
|
||||
|
||||
// pauseDurations are the offered pause lengths, by their wire value. A fixed
|
||||
// allow-list rather than time.ParseDuration: the unoffered value must be
|
||||
// refused, and a permissive parser turns the offered set into "anything Go
|
||||
// can read" (issue #147).
|
||||
var pauseDurations = map[string]time.Duration{
|
||||
"1h": time.Hour,
|
||||
"6h": 6 * time.Hour,
|
||||
"24h": 24 * time.Hour,
|
||||
}
|
||||
|
||||
// laneSite reads the Site a lane route names, answering the request itself
|
||||
// when it is not a registry Site. The path value is client-supplied, so it
|
||||
// is checked against the registry before it reaches the store.
|
||||
func laneSite(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
site := r.PathValue("site")
|
||||
if !slices.Contains(latest.SiteNames(), site) {
|
||||
http.Error(w, "unknown site", http.StatusBadRequest)
|
||||
return "", false
|
||||
}
|
||||
return site, true
|
||||
}
|
||||
|
||||
// adminLanePause writes a bounded pause for one Site and answers with the
|
||||
// freshly rendered Lanes block, so the figures describe the state after the
|
||||
// press. The pause is a fact about the Site — the Lane's next pass reads it
|
||||
// from the durable row, never from this process — so it survives a restart.
|
||||
// The owner gate is the route's, not this handler's; the body is capped like
|
||||
// the API path caps its bodies; the Site and the duration are validated
|
||||
// here, before the store sees them (issue #147).
|
||||
func (h *Handler) adminLanePause(w http.ResponseWriter, r *http.Request) {
|
||||
site, ok := laneSite(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
d, ok := pauseDurations[r.PostFormValue("duration")]
|
||||
if !ok {
|
||||
http.Error(w, "unknown pause duration", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.PauseLane(site, time.Now().Add(d).UnixMilli()); err != nil {
|
||||
log.Printf("pause lane %s: %v", site, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "lanes", h.lanesView())
|
||||
}
|
||||
|
||||
// adminLaneResume zeroes one Site's pause and answers with the freshly
|
||||
// rendered Lanes block. Resume is the reversal of a bounded pause, so it
|
||||
// fires instantly with no confirm row (issue #147).
|
||||
func (h *Handler) adminLaneResume(w http.ResponseWriter, r *http.Request) {
|
||||
site, ok := laneSite(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := h.store.ResumeLane(site); err != nil {
|
||||
log.Printf("resume lane %s: %v", site, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "lanes", h.lanesView())
|
||||
}
|
||||
|
||||
// lanesView builds the Lane status block from the durable pass log. Both
|
||||
// reads are the store's latest-per-Site projection, so the page's seam is a
|
||||
// seeded row rather than a fake poller; errors degrade to the empty state and
|
||||
// are logged, never shown to the owner in detail.
|
||||
func (h *Handler) lanesView() lanesView {
|
||||
v := lanesView{
|
||||
PollerOff: !h.pollerEnabled,
|
||||
BrowserConfigured: h.browserConfigured,
|
||||
}
|
||||
passes, err := h.store.LatestLanePasses()
|
||||
if err != nil {
|
||||
log.Printf("admin lanes: latest passes: %v", err)
|
||||
// No evidence of a lost sidecar reads as reachable, per the same rule
|
||||
// browserReachable applies: a store failure must not condemn the
|
||||
// browser. The empty table already says no Lane has recorded a pass.
|
||||
v.BrowserReachable = true
|
||||
return v
|
||||
}
|
||||
now := time.Now()
|
||||
outcomes, err := h.store.LanePassOutcomes(now.Add(-ownerWindow).UnixMilli())
|
||||
if err != nil {
|
||||
// The rows are complete without the chips, so a failed outcome sum
|
||||
// must not blank the table into "no data yet" — that is the confident
|
||||
// wrong statement the page exists to avoid. Every row renders "none
|
||||
// observed" instead, which is honest.
|
||||
log.Printf("admin lanes: outcomes: %v", err)
|
||||
outcomes = nil
|
||||
}
|
||||
bySite := make(map[string]store.SiteOutcomes, len(outcomes))
|
||||
for _, o := range outcomes {
|
||||
bySite[o.Site] = o
|
||||
}
|
||||
v.Rows = make([]laneRow, 0, len(passes))
|
||||
v.BrowserReachable = browserReachable(passes, now)
|
||||
for _, p := range passes {
|
||||
v.Rows = append(v.Rows, buildLaneRow(p, bySite[p.Site], now))
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// browserReachable derives the sidecar's reachability from the pass log: a
|
||||
// browser Site is down when its latest pass inside the refusal backoff is a
|
||||
// sidecar loss, a missing fetcher, or an interrupted read. Only browser Sites
|
||||
// ever produce those signals, so no Site registry leaks into the web layer.
|
||||
// A configured browser with no such evidence reads as reachable; an unset
|
||||
// BROWSER_WS_URL degrades identically to a browser that is down.
|
||||
func browserReachable(passes []store.LanePass, now time.Time) bool {
|
||||
backoff := latest.RefuseBackoff
|
||||
for _, p := range passes {
|
||||
ran := time.UnixMilli(p.RanAt)
|
||||
if now.Sub(ran) >= backoff || ran.After(now) {
|
||||
continue
|
||||
}
|
||||
if p.Skip == latest.SkipSidecarDown || p.Skip == latest.SkipNoFetcher || p.Unreachable > 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// buildLaneRow turns one Site's latest pass and window outcome sums into the
|
||||
// row the template prints. The skip column is the authority on why a pass did
|
||||
// nothing; the outcomes render named and unlinked, because the pass row holds
|
||||
// counts and never identities.
|
||||
func buildLaneRow(p store.LanePass, o store.SiteOutcomes, now time.Time) laneRow {
|
||||
row := laneRow{
|
||||
Site: p.Site,
|
||||
Due: p.Due,
|
||||
Checked: p.Checked,
|
||||
Gap: "—",
|
||||
Ran: since(now, time.UnixMilli(p.RanAt)),
|
||||
}
|
||||
if p.GapMS > 0 {
|
||||
row.Gap = (time.Duration(p.GapMS) * time.Millisecond).Truncate(time.Second).String()
|
||||
}
|
||||
row.Chips = outcomeChips(o)
|
||||
row.HasChips = len(row.Chips) > 0
|
||||
row.StatePhrase, row.StateGood, row.Attention = laneState(p, now)
|
||||
row.Paused = time.UnixMilli(p.PausedUntil).After(now)
|
||||
return row
|
||||
}
|
||||
|
||||
// outcomeChips lists a Site's nonzero window sums in the taxonomy's fixed
|
||||
// order, so the chips never reorder as the window changes. None observed is
|
||||
// written by the template, not drawn as a confident zero count.
|
||||
func outcomeChips(o store.SiteOutcomes) []chip {
|
||||
fixed := []struct {
|
||||
name string
|
||||
count int
|
||||
}{
|
||||
{"refused", o.Refused},
|
||||
{"unreachable", o.Unreachable},
|
||||
{"no chapter", o.NoChapter},
|
||||
{"unfetchable", o.Unfetchable},
|
||||
{"errors", o.Errors},
|
||||
}
|
||||
var out []chip
|
||||
for _, f := range fixed {
|
||||
if f.count > 0 {
|
||||
out = append(out, chip{Name: f.name, Count: f.count})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// laneState renders the reason a Lane's last pass did nothing, in one sentence
|
||||
// per skip value with the one true stall kept apart from every Lane that
|
||||
// declined and said why. Good states — a pause, a sleeping browser, nothing
|
||||
// eligible — carry no Attention: the mark must stay spendable on the faults
|
||||
// that actually need the owner.
|
||||
func laneState(p store.LanePass, now time.Time) (phrase string, good, attention bool) {
|
||||
// The pause phrase reads the live poll_lanes stamp the pass row joins
|
||||
// on, not the pass's skip: the owner's press must render as paused on
|
||||
// the very answer it gets, with no pass having run to record it. The
|
||||
// pause is a fact about the Site, and the join delivers it (issue #147).
|
||||
if pausedUntil := time.UnixMilli(p.PausedUntil); pausedUntil.After(now) {
|
||||
phrase = "paused · resumes in " + humanDuration(pausedUntil.Sub(now))
|
||||
good = true
|
||||
return phrase, good, attention
|
||||
}
|
||||
switch p.Skip {
|
||||
case latest.SkipPaused:
|
||||
// A paused pass whose stamp has since lapsed: the Lane still
|
||||
// declined with a reason, so it is never the one true stall.
|
||||
phrase = "paused · resumes in " + humanDuration(time.UnixMilli(p.PausedUntil).Sub(now))
|
||||
good = true
|
||||
case latest.SkipRefusing:
|
||||
phrase = "refusing"
|
||||
if until := time.UnixMilli(p.RefuseUntil); until.After(now) {
|
||||
phrase += " · backs off until " + until.Format("15:04")
|
||||
}
|
||||
attention = true
|
||||
case latest.SkipSidecarDown, latest.SkipNoFetcher:
|
||||
// Known false positive shipped per spec: a sibling Lane's Chrome loss
|
||||
// stamps this Site too, and the enum deliberately has no tenth value
|
||||
// to separate it (issue #141). Render it as written.
|
||||
phrase = "no browser"
|
||||
attention = true
|
||||
case latest.SkipAsleep:
|
||||
phrase = "browser asleep"
|
||||
good = true
|
||||
case latest.SkipDueQuery:
|
||||
phrase = "due query failed"
|
||||
attention = true
|
||||
case latest.SkipEligibleCount:
|
||||
phrase = "eligible count failed"
|
||||
attention = true
|
||||
case latest.SkipNothingEligible:
|
||||
phrase = "nothing eligible"
|
||||
good = true
|
||||
}
|
||||
if phrase == "" && p.Due > 0 && p.Checked == 0 {
|
||||
// The one true stall: the pass reached its loop, Series were waiting,
|
||||
// and none were read. Every skip above is a Lane that said why.
|
||||
phrase = "not checking"
|
||||
attention = true
|
||||
}
|
||||
return phrase, good, attention
|
||||
}
|
||||
|
||||
// humanDuration renders a positive duration compactly for a "resumes in" clue
|
||||
// at the pause and refusal scales — minutes under an hour, then h and h+m.
|
||||
func humanDuration(d time.Duration) string {
|
||||
d = d.Round(time.Minute)
|
||||
if d <= 0 {
|
||||
return "soon"
|
||||
}
|
||||
if d < time.Hour {
|
||||
return fmt.Sprintf("%dm", int(d/time.Minute))
|
||||
}
|
||||
h := int(d / time.Hour)
|
||||
if m := int(d%time.Hour) / int(time.Minute); m == 0 {
|
||||
return fmt.Sprintf("%dh", h)
|
||||
} else {
|
||||
return fmt.Sprintf("%dh%dm", h, m)
|
||||
}
|
||||
}
|
||||
|
||||
// since formats how long ago a Lane last ran, at second resolution: the block
|
||||
// refreshes every thirty seconds, so anything finer is noise the owner would
|
||||
// have to ignore.
|
||||
func since(now, then time.Time) string {
|
||||
d := now.Sub(then).Truncate(time.Second)
|
||||
if d < time.Second {
|
||||
return "just now"
|
||||
}
|
||||
return d.String() + " ago"
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// overviewView is the Overview landing page's data: one verdict line, the
|
||||
// hygiene and library stats blocks, and the per-Site library shape table.
|
||||
// Every judgement — the verdict state, which figures link, what a Lane's
|
||||
// state means — is made here; the template only prints.
|
||||
type overviewView struct {
|
||||
// Verdict is the attention phrase that leads the page.
|
||||
Verdict string
|
||||
// HasCounts is false on a virgin pass log: the waiting figure would be a
|
||||
// confident zero, and "nothing has happened" must not render as health.
|
||||
HasCounts bool
|
||||
// Waiting is the sum of Due over the latest pass per Site.
|
||||
Waiting int
|
||||
// Unchecked is the number of Series not checked in the window, computed
|
||||
// as stale + never_checked: a never-checked Series is already counted on
|
||||
// its own filter, and the verdict wants the inclusive number.
|
||||
Unchecked int
|
||||
// Hygiene is the seven problem filters in the Series list's own render
|
||||
// order; Library is the library split plus the roster. Every figure is a
|
||||
// door into the list that counts it, except a zero.
|
||||
Hygiene []fig
|
||||
Library []fig
|
||||
// Sites is the per-Site library shape table, one row per Site with any
|
||||
// Series, in the store's Site order.
|
||||
Sites []siteRow
|
||||
}
|
||||
|
||||
// fig is one stats figure: its label, the list it counts, and the count
|
||||
// itself. Href empty means the count is zero: a measured zero is a real
|
||||
// figure that stays on the page, but it is not a door, because following it
|
||||
// lands on an empty list.
|
||||
type fig struct {
|
||||
Label string
|
||||
Href string
|
||||
Count int
|
||||
}
|
||||
|
||||
// siteRow is one Site's share of the library: the Series total and the three
|
||||
// hygiene counts the per-Site table carries, each a door to the list narrowed
|
||||
// to that Site, plus the Lane state phrase derived from its latest pass. The
|
||||
// table is library shape only — the Poll outcome sums live on the Lanes page.
|
||||
type siteRow struct {
|
||||
Site string
|
||||
SiteHref string
|
||||
Figs []fig
|
||||
// State is the Lane's own sentence; "" means the last pass read normally.
|
||||
// StateGood / StateBad pick the ok / bad second class.
|
||||
State string
|
||||
StateGood bool
|
||||
StateBad bool
|
||||
}
|
||||
|
||||
// overviewView assembles the landing page from the store's read model: one
|
||||
// SeriesShapes pass per filter summed in Go (the shipped surface offers eight
|
||||
// grouped passes, not a stats query — #140), the pass log's latest pass per
|
||||
// Site, and the roster. A failure in any read is a 500 with a logged reason,
|
||||
// never a page of silent zeroes.
|
||||
func (h *Handler) overviewView() (overviewView, error) {
|
||||
now := time.Now()
|
||||
cutoff := now.Add(-ownerWindow).UnixMilli()
|
||||
|
||||
shapes := make(map[string][]store.SiteSeriesShape, len(seriesFilterOrder))
|
||||
totals := make(map[string]int, len(seriesFilterOrder))
|
||||
for _, name := range seriesFilterOrder {
|
||||
rows, err := h.store.SeriesShapes(store.SeriesFilter{Name: name, Cutoff: cutoff})
|
||||
if err != nil {
|
||||
return overviewView{}, err
|
||||
}
|
||||
shapes[name] = rows
|
||||
for _, sh := range rows {
|
||||
totals[name] += sh.Total
|
||||
}
|
||||
}
|
||||
|
||||
passes, err := h.store.LatestLanePasses()
|
||||
if err != nil {
|
||||
return overviewView{}, err
|
||||
}
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
return overviewView{}, err
|
||||
}
|
||||
|
||||
view := overviewView{Waiting: waiting(passes)}
|
||||
view.Unchecked = totals[store.SeriesFilterStale] + totals[store.SeriesFilterNeverChecked]
|
||||
view.Verdict, view.HasCounts = overviewVerdict(passes, now)
|
||||
|
||||
// The seven problem filters, in seriesFilterOrder's permanent-then-fixable
|
||||
// order; the All filter's count belongs to the Library block, not to a
|
||||
// "hygiene" figure.
|
||||
hygiene := make([]fig, 0, len(seriesFilterOrder)-1)
|
||||
for _, name := range seriesFilterOrder[1:] {
|
||||
hygiene = append(hygiene, door(seriesFilterLabels[name], totals[name], seriesListHref(name, "", "", 0)))
|
||||
}
|
||||
view.Hygiene = hygiene
|
||||
|
||||
var manga, novel int
|
||||
for _, sh := range shapes[store.SeriesFilterAll] {
|
||||
manga += sh.Manga
|
||||
novel += sh.Novel
|
||||
}
|
||||
view.Library = []fig{
|
||||
door("Series", totals[store.SeriesFilterAll], seriesListHref("", "", "", 0)),
|
||||
door("Manga", manga, seriesListHref("", "", store.KindManga, 0)),
|
||||
door("Novels", novel, seriesListHref("", "", store.KindNovel, 0)),
|
||||
door("Readers", len(readers), "/admin/readers"),
|
||||
}
|
||||
|
||||
// One row per Site with any Series, from the All shapes; the hygiene
|
||||
// counts come from the same per-Site projection so the table cannot
|
||||
// disagree with the library-wide figures above it.
|
||||
siteCounts := make(map[string]map[string]int, len(shapes))
|
||||
for name, rows := range shapes {
|
||||
m := make(map[string]int, len(rows))
|
||||
for _, sh := range rows {
|
||||
m[sh.Site] = sh.Total
|
||||
}
|
||||
siteCounts[name] = m
|
||||
}
|
||||
passBySite := make(map[string]store.LanePass, len(passes))
|
||||
for _, p := range passes {
|
||||
passBySite[p.Site] = p
|
||||
}
|
||||
view.Sites = make([]siteRow, 0, len(shapes[store.SeriesFilterAll]))
|
||||
for _, sh := range shapes[store.SeriesFilterAll] {
|
||||
row := siteRow{
|
||||
Site: sh.Site,
|
||||
SiteHref: seriesListHref("", sh.Site, "", 0),
|
||||
Figs: []fig{
|
||||
door("", sh.Total, seriesListHref("", sh.Site, "", 0)),
|
||||
door("", siteCounts[store.SeriesFilterNoCover][sh.Site], seriesListHref(store.SeriesFilterNoCover, sh.Site, "", 0)),
|
||||
door("", siteCounts[store.SeriesFilterNeverChecked][sh.Site], seriesListHref(store.SeriesFilterNeverChecked, sh.Site, "", 0)),
|
||||
door("", siteCounts[store.SeriesFilterStale][sh.Site], seriesListHref(store.SeriesFilterStale, sh.Site, "", 0)),
|
||||
},
|
||||
}
|
||||
if p, ok := passBySite[sh.Site]; ok {
|
||||
row.State, row.StateGood, row.StateBad = laneState(p, now)
|
||||
} else {
|
||||
row.State = "no pass yet"
|
||||
}
|
||||
view.Sites = append(view.Sites, row)
|
||||
}
|
||||
return view, nil
|
||||
}
|
||||
|
||||
// door is one figure with its door: the list that counts it. A measured zero
|
||||
// is still a real figure, but the door closes — following it would land on an
|
||||
// empty list. The count is written once so the figure and what it links to
|
||||
// cannot drift apart.
|
||||
func door(label string, count int, href string) fig {
|
||||
if count == 0 {
|
||||
href = ""
|
||||
}
|
||||
return fig{Label: label, Href: href, Count: count}
|
||||
}
|
||||
|
||||
// overviewVerdict decides the landing page's one line from the latest pass
|
||||
// per Site: no passes at all is "no Lane has reported yet" — never confident
|
||||
// zeroes; otherwise the count of Lanes whose last pass needs the owner, or
|
||||
// "all lanes healthy". The count comes from the same laneState judgement the
|
||||
// Lanes page colours on, so the two pages cannot disagree on what a fault is.
|
||||
func overviewVerdict(passes []store.LanePass, now time.Time) (phrase string, counts bool) {
|
||||
if len(passes) == 0 {
|
||||
return "no Lane has reported yet", false
|
||||
}
|
||||
attention := 0
|
||||
for _, p := range passes {
|
||||
if _, _, attn := laneState(p, now); attn {
|
||||
attention++
|
||||
}
|
||||
}
|
||||
if attention == 0 {
|
||||
return "all lanes healthy", true
|
||||
}
|
||||
if attention == 1 {
|
||||
return "1 lane needs a look", true
|
||||
}
|
||||
return fmt.Sprintf("%d lanes need a look", attention), true
|
||||
}
|
||||
|
||||
// waiting sums Due over the latest pass per Site: how many Series the Lanes
|
||||
// found waiting, from the durable log rather than a running poller.
|
||||
func waiting(passes []store.LanePass) int {
|
||||
n := 0
|
||||
for _, p := range passes {
|
||||
n += p.Due
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// seriesPageSize matches the store's row-read page length: the pager's range
|
||||
// must agree with the LIMIT the store applies or the "of N" figure describes
|
||||
// the wrong page. The store does not export it (#140).
|
||||
const seriesPageSize = 50
|
||||
|
||||
// seriesFilterLabels names every hygiene filter for the Series list select,
|
||||
// keyed by the wire constant the URL carries. The render order is
|
||||
// seriesFilterOrder; the labels are read by later admin tickets too, so the
|
||||
// map and the constants cannot drift apart.
|
||||
var seriesFilterLabels = map[string]string{
|
||||
store.SeriesFilterAll: "All series",
|
||||
store.SeriesFilterNoURL: "No series URL",
|
||||
store.SeriesFilterNoChapter: "Never read a chapter",
|
||||
store.SeriesFilterNoReaders: "No Readers",
|
||||
store.SeriesFilterNeverChecked: "Never checked",
|
||||
store.SeriesFilterStale: "Not checked in 12h",
|
||||
store.SeriesFilterNoCover: "No cover",
|
||||
store.SeriesFilterReaderReport: "Latest from a Reader",
|
||||
}
|
||||
|
||||
// seriesFilterOrder is the select's render order: All first, then the
|
||||
// permanent repairs, then the fixable ones (issue #140).
|
||||
var seriesFilterOrder = []string{
|
||||
store.SeriesFilterAll,
|
||||
store.SeriesFilterNoURL,
|
||||
store.SeriesFilterNoChapter,
|
||||
store.SeriesFilterNoReaders,
|
||||
store.SeriesFilterNeverChecked,
|
||||
store.SeriesFilterStale,
|
||||
store.SeriesFilterNoCover,
|
||||
store.SeriesFilterReaderReport,
|
||||
}
|
||||
|
||||
// seriesListView is the Series list page's data. The template renders strings
|
||||
// and flags, and every judgement about what a value means is made here.
|
||||
type seriesListView struct {
|
||||
Filters []seriesFilterOption
|
||||
Sites []string
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both libraries
|
||||
FilterLabel string
|
||||
Rows []seriesRowView
|
||||
Total int
|
||||
// KindBoth / KindManga / KindNovel are the Library segment links, and
|
||||
// PrevHref / NextHref the pager's, all carrying the active filter, Site
|
||||
// and Kind so narrowing never drops state.
|
||||
KindBoth string
|
||||
KindManga string
|
||||
KindNovel string
|
||||
PrevHref string
|
||||
NextHref string
|
||||
Range string
|
||||
}
|
||||
|
||||
// seriesFilterOption is one entry of the Show select: its wire value, its
|
||||
// rendered label with the library-wide count, and whether it is the active
|
||||
// filter.
|
||||
type seriesFilterOption struct {
|
||||
Name string
|
||||
Label string
|
||||
Count int
|
||||
Selected bool
|
||||
}
|
||||
|
||||
// seriesRowView is one Series row formatted for the template. Band carries
|
||||
// the alternating row tint by class rather than nth-of-type, so the confirm
|
||||
// rows later tickets add are row siblings without breaking the alternation.
|
||||
// Attention tints the title patina: a row with any hygiene chip needs one.
|
||||
//
|
||||
// CanPoll is the Check now control's visibility: absent on a Series with no
|
||||
// page to fetch and on an orphan, so the owner is never offered a button that
|
||||
// can never do anything. Pending is derived — the request stamp is newer than
|
||||
// the check stamp — and Requested is its ageing label.
|
||||
type seriesRowView struct {
|
||||
Key string
|
||||
Title string
|
||||
Site string
|
||||
Ch string // chapter number; "—" until first captured
|
||||
Age string // checked age; "never" until first check
|
||||
Readers int
|
||||
Notes []string // chips, capped at two
|
||||
More int // chips past the cap, rendered as a +N tail
|
||||
Band bool
|
||||
Attention bool
|
||||
CanPoll bool
|
||||
Pending bool
|
||||
Requested string // "requested 3m ago", rendered only while pending
|
||||
}
|
||||
|
||||
// adminSeries renders the filterable, bookmarkable Series list: filter, Site,
|
||||
// Library and page all live in the query string, so the list's state is an
|
||||
// address rather than a click path.
|
||||
func (h *Handler) adminSeries(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := h.seriesListView(r)
|
||||
if err != nil {
|
||||
log.Printf("admin series: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "series", SeriesList: view})
|
||||
}
|
||||
|
||||
// adminSeriesPoll is the Check now action: it stamps the Series' force_poll_at
|
||||
// and answers with the freshly rendered row, so the figures describe the
|
||||
// state after the press. The control never commands the poller — the request
|
||||
// is a fact about the Series, and the Lane's next pass reads it through
|
||||
// DueForLatestCheck (ADR-0013). The owner gate is the route's, not this
|
||||
// handler's; the body is capped like the API path caps its bodies; the key is
|
||||
// validated here — a malformed key is a 400 and an unknown one a 404.
|
||||
func (h *Handler) adminSeriesPoll(w http.ResponseWriter, r *http.Request) {
|
||||
site, seriesID, ok := strings.Cut(r.PathValue("key"), ":")
|
||||
if !ok || site == "" || seriesID == "" {
|
||||
http.Error(w, "bad series key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "invalid form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if _, found, err := h.adminSeriesByKey(site, seriesID); err != nil {
|
||||
log.Printf("series poll %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
} else if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.ForceSeriesPoll(site, seriesID, time.Now().UnixMilli()); err != nil {
|
||||
log.Printf("series poll %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Re-read after the stamp: the answer must describe the state after the
|
||||
// press. The detail page's control swaps its meta in place and the list
|
||||
// row's swaps the row; htmx names an id target in HX-Target, so the
|
||||
// response matches the surface it came from. The row's band parity travels
|
||||
// with the press (hx-vals), so the swap keeps the zebra alternation.
|
||||
a, found, err := h.adminSeriesByKey(site, seriesID)
|
||||
if err != nil {
|
||||
log.Printf("series poll %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("HX-Target") == "detail-meta" {
|
||||
h.render(w, http.StatusOK, "series-detail-meta", h.seriesDetailView(a))
|
||||
return
|
||||
}
|
||||
band := 0
|
||||
if r.PostFormValue("band") == "1" {
|
||||
band = 1
|
||||
}
|
||||
h.render(w, http.StatusOK, "series-row", seriesRow(a, band, time.Now()))
|
||||
}
|
||||
|
||||
// seriesListView assembles one Series list view from the request's query
|
||||
// string. An unknown filter value is the absent All case, never an error: the
|
||||
// select's options are not the only way this URL can be reached.
|
||||
func (h *Handler) seriesListView(r *http.Request) (seriesListView, error) {
|
||||
q := r.URL.Query()
|
||||
filter := q.Get("filter")
|
||||
if _, ok := seriesFilterLabels[filter]; !ok {
|
||||
filter = store.SeriesFilterAll
|
||||
}
|
||||
site := q.Get("site")
|
||||
kind := q.Get("kind")
|
||||
if kind != store.KindManga && kind != store.KindNovel {
|
||||
kind = ""
|
||||
}
|
||||
page := 1
|
||||
if p, err := strconv.Atoi(q.Get("page")); err == nil && p > 1 {
|
||||
page = p
|
||||
}
|
||||
|
||||
sf := store.SeriesFilter{
|
||||
Site: site,
|
||||
Kind: kind,
|
||||
Name: filter,
|
||||
Cutoff: time.Now().Add(-ownerWindow).UnixMilli(),
|
||||
Page: page,
|
||||
}
|
||||
data, err := h.store.SeriesPage(sf)
|
||||
if err != nil {
|
||||
return seriesListView{}, err
|
||||
}
|
||||
// A page past the end is not an empty list: the store's window count runs
|
||||
// over the rows the result set carries, so an overflow page reports zero
|
||||
// rows and zero total, and the list re-reads at page 1 to know the truth.
|
||||
if len(data.Rows) == 0 && page > 1 {
|
||||
page = 1
|
||||
sf.Page = 1
|
||||
data, err = h.store.SeriesPage(sf)
|
||||
if err != nil {
|
||||
return seriesListView{}, err
|
||||
}
|
||||
}
|
||||
|
||||
view := seriesListView{
|
||||
Site: site,
|
||||
Kind: kind,
|
||||
FilterLabel: seriesFilterLabels[filter],
|
||||
Rows: make([]seriesRowView, 0, len(data.Rows)),
|
||||
Total: data.Total,
|
||||
Sites: latest.SiteNames(),
|
||||
}
|
||||
now := time.Now()
|
||||
for i, a := range data.Rows {
|
||||
view.Rows = append(view.Rows, seriesRow(a, i, now))
|
||||
}
|
||||
view.Filters, err = h.seriesFilterOptions(filter, sf.Cutoff)
|
||||
if err != nil {
|
||||
return seriesListView{}, err
|
||||
}
|
||||
view.KindBoth = seriesListHref(filter, site, "", 0)
|
||||
view.KindManga = seriesListHref(filter, site, store.KindManga, 0)
|
||||
view.KindNovel = seriesListHref(filter, site, store.KindNovel, 0)
|
||||
if page > 1 {
|
||||
view.PrevHref = seriesListHref(filter, site, kind, page-1)
|
||||
}
|
||||
if last := (data.Total + seriesPageSize - 1) / seriesPageSize; page < last {
|
||||
view.NextHref = seriesListHref(filter, site, kind, page+1)
|
||||
}
|
||||
view.Range = pagerRange(data.Total, len(data.Rows), page)
|
||||
return view, nil
|
||||
}
|
||||
|
||||
// seriesFilterOptions renders every hygiene filter with its library-wide
|
||||
// count, one SeriesShapes pass per filter summed in Go — the shipped surface
|
||||
// offers eight grouped passes, not a single stats query (#140). The counts
|
||||
// are library-wide because the select sits next to the Site narrowing and
|
||||
// must not shift as the owner narrows the list itself. Cutoff travels with
|
||||
// the stale filter, or its count would always be zero.
|
||||
func (h *Handler) seriesFilterOptions(selected string, cutoff int64) ([]seriesFilterOption, error) {
|
||||
out := make([]seriesFilterOption, 0, len(seriesFilterOrder))
|
||||
for _, name := range seriesFilterOrder {
|
||||
shapes, err := h.store.SeriesShapes(store.SeriesFilter{Name: name, Cutoff: cutoff})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count := 0
|
||||
for _, sh := range shapes {
|
||||
count += sh.Total
|
||||
}
|
||||
out = append(out, seriesFilterOption{
|
||||
Name: name,
|
||||
Label: seriesFilterLabels[name],
|
||||
Count: count,
|
||||
Selected: name == selected,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seriesRow shapes one store row for the template, capping its chips at two
|
||||
// plus a +N tail; attention marks a row that carries any.
|
||||
// pollState derives the Check now control and the pending marker (issue
|
||||
// #146), shared by the list row and the detail page: CanPoll is false on a
|
||||
// Series with no page to fetch and on an orphan, so the owner is never
|
||||
// offered a button that can never do anything. Pending is derived — the
|
||||
// request stamp is newer than the check stamp — and requested is its ageing
|
||||
// label, which never expires.
|
||||
func pollState(a store.AdminSeries, now time.Time) (canPoll, pending bool, requested string) {
|
||||
canPoll = a.SeriesURL != "" && a.ReaderCount > 0
|
||||
if a.ForcePollAt > a.LatestCheckedAt {
|
||||
pending = true
|
||||
requested = requestedAge(now, a.ForcePollAt)
|
||||
}
|
||||
return canPoll, pending, requested
|
||||
}
|
||||
|
||||
func seriesRow(a store.AdminSeries, i int, now time.Time) seriesRowView {
|
||||
canPoll, pending, requested := pollState(a, now)
|
||||
row := seriesRowView{
|
||||
Key: a.Key(),
|
||||
Site: a.Site,
|
||||
Title: a.Title,
|
||||
Readers: a.ReaderCount,
|
||||
Band: i%2 == 1,
|
||||
CanPoll: canPoll,
|
||||
Pending: pending,
|
||||
Requested: requested,
|
||||
}
|
||||
if a.LatestChapterNum != nil {
|
||||
row.Ch = strconv.FormatFloat(*a.LatestChapterNum, 'f', -1, 64)
|
||||
} else {
|
||||
row.Ch = "—"
|
||||
}
|
||||
row.Age = checkedAge(now, a.LatestCheckedAt)
|
||||
notes := seriesNotes(a, now)
|
||||
if n := len(notes); n > 2 {
|
||||
row.Notes, row.More = notes[:2], n-2
|
||||
} else {
|
||||
row.Notes = notes
|
||||
}
|
||||
row.Attention = len(notes) > 0
|
||||
return row
|
||||
}
|
||||
|
||||
// seriesNotes are a row's hygiene chips in the design's order: no URL, no
|
||||
// cover, orphan, stale, reader sighting.
|
||||
func seriesNotes(a store.AdminSeries, now time.Time) []string {
|
||||
notes := []string{}
|
||||
if a.SeriesURL == "" {
|
||||
notes = append(notes, "no URL")
|
||||
}
|
||||
if a.CoverAddress == "" {
|
||||
notes = append(notes, "no cover")
|
||||
}
|
||||
if a.ReaderCount == 0 {
|
||||
notes = append(notes, "orphan")
|
||||
}
|
||||
if a.LatestCheckedAt > 0 && a.LatestCheckedAt < now.Add(-ownerWindow).UnixMilli() {
|
||||
notes = append(notes, "stale")
|
||||
}
|
||||
if a.RaisedByReader {
|
||||
notes = append(notes, "reader sighting")
|
||||
}
|
||||
return notes
|
||||
}
|
||||
|
||||
// checkedAge formats how long ago a Series was last checked, at the
|
||||
// granularity the list reads at — minutes, hours, days. Zero means never.
|
||||
func checkedAge(now time.Time, ts int64) string {
|
||||
if ts == 0 {
|
||||
return "never"
|
||||
}
|
||||
d := now.Sub(time.UnixMilli(ts))
|
||||
switch {
|
||||
case d < time.Hour:
|
||||
m := int(d / time.Minute)
|
||||
if m < 1 {
|
||||
m = 1
|
||||
}
|
||||
return fmt.Sprintf("%dm ago", m)
|
||||
case d < 24*time.Hour:
|
||||
return fmt.Sprintf("%dh ago", int(d/time.Hour))
|
||||
default:
|
||||
return fmt.Sprintf("%dd ago", int(d/(24*time.Hour)))
|
||||
}
|
||||
}
|
||||
|
||||
// requestedAge is the pending marker's text: how long ago the owner asked,
|
||||
// and nothing about when the request will run — the page does not know when a
|
||||
// sleeping browser will wake (issue #146). An unanswered request ages forever;
|
||||
// there is no expiry.
|
||||
func requestedAge(now time.Time, ts int64) string {
|
||||
return "requested " + checkedAge(now, ts)
|
||||
}
|
||||
|
||||
// pagerRange is the pager's "1–50 of 120" line. The template renders the
|
||||
// pager only over rows (the empty state replaces it), so it is never asked
|
||||
// to describe an empty list.
|
||||
func pagerRange(total, rows, page int) string {
|
||||
from := (page-1)*seriesPageSize + 1
|
||||
return fmt.Sprintf("%d–%d of %d", from, from+rows-1, total)
|
||||
}
|
||||
|
||||
// seriesListHref is one Series list address carrying the filter, Site, Kind
|
||||
// and page. The All filter and page 1 are the absent cases and stay out of
|
||||
// the URL, so the default address is the shortest one.
|
||||
func seriesListHref(filter, site, kind string, page int) string {
|
||||
q := url.Values{}
|
||||
if filter != "" && filter != store.SeriesFilterAll {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if site != "" {
|
||||
q.Set("site", site)
|
||||
}
|
||||
if kind != "" {
|
||||
q.Set("kind", kind)
|
||||
}
|
||||
if page > 1 {
|
||||
q.Set("page", strconv.Itoa(page))
|
||||
}
|
||||
if len(q) == 0 {
|
||||
return "/admin/series"
|
||||
}
|
||||
return "/admin/series?" + q.Encode()
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// seriesDetailView is one Series' page as the owner sees it: strings and
|
||||
// flags, every judgement made here, the template left to print. ReaderCount
|
||||
// is the only figure that crosses the privacy boundary — the owner learns how
|
||||
// many Readers hold the Series, never which Reader reads what.
|
||||
type seriesDetailView struct {
|
||||
Key string // "<site>:<series_id>", the page's address and the Series' identity
|
||||
Site string
|
||||
Kind string
|
||||
// Title, Cover and Chapter come from the shared Series row; the Cover is
|
||||
// the wire URL of the stored bytes, "" before any exist.
|
||||
Title string
|
||||
Cover string
|
||||
Chapter string // Latest Chapter number, or "—" before the first capture
|
||||
Checked string // how long ago the poller last checked, or "never"
|
||||
Readers int
|
||||
|
||||
// Marks, one per hygiene fact, rendered only while it holds.
|
||||
Unpollable bool // no SeriesURL to fetch
|
||||
NoCover bool
|
||||
Orphan bool // no Reader holds the Series
|
||||
SightingRaised bool // a Reader's Sighting set the Latest Chapter
|
||||
|
||||
// Poll is the Check now control and the pending marker (issue #146): the
|
||||
// same derivation and visibility as the list row. CanPoll is false on a
|
||||
// Series with no page to fetch and on an orphan; Pending is derived —
|
||||
// the request stamp is newer than the check stamp — and Requested is its
|
||||
// ageing label.
|
||||
CanPoll bool
|
||||
Pending bool
|
||||
Requested string
|
||||
}
|
||||
|
||||
// adminSeriesDetail renders one Series' page, keyed by the composite
|
||||
// "<site>:<series_id>" the list row already shows. The row is read through
|
||||
// the list's own SeriesPage read narrowed to the key's Site: the admin
|
||||
// projection is the privacy boundary, and a dedicated single-row read would
|
||||
// be a second definition of it.
|
||||
func (h *Handler) adminSeriesDetail(w http.ResponseWriter, r *http.Request) {
|
||||
site, seriesID, ok := strings.Cut(r.PathValue("key"), ":")
|
||||
if !ok || site == "" || seriesID == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
a, found, err := h.adminSeriesByKey(site, seriesID)
|
||||
if err != nil {
|
||||
log.Printf("series detail %s: %v", site+":"+seriesID, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
h.renderAdmin(w, adminView{Page: "series-detail", Detail: h.seriesDetailView(a)})
|
||||
}
|
||||
|
||||
// adminSeriesByKey reads one Series through the list's own SeriesPage read
|
||||
// narrowed to the key's Site: the admin projection is the privacy boundary,
|
||||
// and a dedicated single-row read would be a second definition of it. Absence
|
||||
// is reported with found=false, never an error.
|
||||
// ponytail: a page scan per keyed read, one query per page of the Site's rows
|
||||
// up to the window total; a keyed read alongside SeriesPage when the library
|
||||
// outgrows the page size.
|
||||
func (h *Handler) adminSeriesByKey(site, seriesID string) (store.AdminSeries, bool, error) {
|
||||
seen := 0
|
||||
for page := 1; ; page++ {
|
||||
p, err := h.store.SeriesPage(store.SeriesFilter{Site: site, Page: page})
|
||||
if err != nil {
|
||||
return store.AdminSeries{}, false, err
|
||||
}
|
||||
seen += len(p.Rows)
|
||||
for i := range p.Rows {
|
||||
if p.Rows[i].SeriesID == seriesID {
|
||||
return p.Rows[i], true, nil
|
||||
}
|
||||
}
|
||||
if seen >= p.Total {
|
||||
break
|
||||
}
|
||||
}
|
||||
return store.AdminSeries{}, false, nil
|
||||
}
|
||||
|
||||
// seriesDetailView shapes one AdminSeries row for display: every judgement in
|
||||
// Go, the template left to print strings and flags.
|
||||
func (h *Handler) seriesDetailView(a store.AdminSeries) seriesDetailView {
|
||||
canPoll, pending, requested := pollState(a, time.Now())
|
||||
v := seriesDetailView{
|
||||
Key: a.Key(),
|
||||
Site: a.Site,
|
||||
Kind: a.Kind,
|
||||
Title: a.Title,
|
||||
Cover: h.store.CoverWireURL(a.CoverAddress),
|
||||
Readers: a.ReaderCount,
|
||||
Unpollable: a.SeriesURL == "",
|
||||
NoCover: a.CoverAddress == "",
|
||||
Orphan: a.ReaderCount == 0,
|
||||
SightingRaised: a.RaisedByReader,
|
||||
CanPoll: canPoll,
|
||||
Pending: pending,
|
||||
Requested: requested,
|
||||
}
|
||||
if a.LatestChapterNum == nil {
|
||||
v.Chapter = "—"
|
||||
} else {
|
||||
v.Chapter = strconv.FormatFloat(*a.LatestChapterNum, 'f', -1, 64)
|
||||
}
|
||||
if a.LatestCheckedAt == 0 {
|
||||
v.Checked = "never"
|
||||
} else {
|
||||
v.Checked = since(time.Now(), time.UnixMilli(a.LatestCheckedAt))
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,830 @@
|
||||
:root {
|
||||
--measure-wide: 1080px;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root {
|
||||
--measure-wide: 1080px;
|
||||
}
|
||||
}
|
||||
|
||||
.admin-sheet {
|
||||
max-width: var(--measure-wide);
|
||||
}
|
||||
|
||||
.admin-sheet .brand em {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .brand .mark > g > g:last-child {
|
||||
stroke: var(--patina);
|
||||
}
|
||||
|
||||
.topbar-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 18px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.navrow {
|
||||
display: flex;
|
||||
gap: 18px;
|
||||
padding: 2px 20px 0;
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
scrollbar-width: none;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.navrow::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.navrow a {
|
||||
flex: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-width: 44px;
|
||||
padding: 8px 0 12px;
|
||||
color: var(--mute);
|
||||
font: 400 17px var(--font-display);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.navrow a:hover {
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.navrow a.active {
|
||||
color: var(--paper);
|
||||
border-bottom: 2px solid var(--paper);
|
||||
margin-bottom: -1px;
|
||||
}
|
||||
|
||||
.admin-page {
|
||||
padding: 0 20px 40px;
|
||||
}
|
||||
|
||||
.admin-page > .sec,
|
||||
.admin-sheet .readers h2,
|
||||
.admin-sheet .lanes h2 {
|
||||
position: relative;
|
||||
margin: 0;
|
||||
padding: 26px 0 8px;
|
||||
font: 500 11px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-page > .sec::before,
|
||||
.admin-sheet .readers h2::before,
|
||||
.admin-sheet .lanes h2::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 14px;
|
||||
left: 0;
|
||||
width: 34px;
|
||||
height: 2px;
|
||||
background: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .readers,
|
||||
.admin-sheet .lanes {
|
||||
margin: 0;
|
||||
padding: 0 0 16px;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.admin-sheet .readerlist,
|
||||
.admin-sheet .lanelist {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.admin-sheet .readerlist li,
|
||||
.admin-sheet .lanelist li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 16px;
|
||||
min-height: 48px;
|
||||
padding: 10px 0;
|
||||
border-top: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-actions {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
gap: 18px;
|
||||
margin-left: auto;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .reader-actions .ghost,
|
||||
.admin-sheet .c-act .ghost {
|
||||
font-size: 12px;
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-actions .ghost.danger,
|
||||
.admin-sheet .c-act .ghost.danger {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .readerlist form {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.admin-sheet .lane-browser {
|
||||
padding: 12px 0 0;
|
||||
}
|
||||
|
||||
.admin-sheet .ghost.danger {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .ghost.danger:hover {
|
||||
color: var(--danger);
|
||||
border-bottom-color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-id {
|
||||
font: 500 15px/1.5 var(--font-mono);
|
||||
letter-spacing: .01em;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-sessions,
|
||||
.admin-sheet .reader-sightings,
|
||||
.admin-sheet .reader-blocked,
|
||||
.admin-sheet .lane-fact,
|
||||
.admin-sheet .lane-mark {
|
||||
font: 500 13px/1.4 var(--font-mono);
|
||||
letter-spacing: .06em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .reader-sessions {
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-sightings,
|
||||
.admin-sheet .lane-fact {
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .reader-blocked,
|
||||
.admin-sheet .lane-mark {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .lane-site {
|
||||
font: 400 19px/1.2 var(--font-display);
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .lanelist li.attention .lane-site {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
/* A single grid keeps row rules continuous; cell padding supplies gutters. */
|
||||
.admin-sheet .sechead {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.admin-sheet .sechead .statusline {
|
||||
padding: 0 0 8px;
|
||||
font-size: 11px;
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .statusline {
|
||||
margin: 0;
|
||||
padding: 0 0 10px;
|
||||
font: 500 12px/1 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
/* The verdict line is set in the data face, not the display face: it is
|
||||
three counts, not a page title. The judgement is the only bright thing. */
|
||||
.admin-sheet .verdict {
|
||||
padding: 16px 0 12px;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
font: 500 15px/1.6 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .verdict .attn {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .verdict .counts b {
|
||||
color: var(--paper);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(240px, 1fr) 156px 92px 110px 76px minmax(150px, 220px) 140px;
|
||||
column-gap: 0;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead > * {
|
||||
padding: 10px 14px 8px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
font: 500 12px/1 var(--font-mono);
|
||||
letter-spacing: .12em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow > * {
|
||||
padding: 11px 14px 11px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
font: 500 15px/1.5 var(--font-mono);
|
||||
letter-spacing: .01em;
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead > *:last-child,
|
||||
.admin-sheet .tbl .trow > *:last-child {
|
||||
padding-right: 0;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-title,
|
||||
.admin-sheet .tbl .c-site {
|
||||
font: 400 18px/1.35 var(--font-display);
|
||||
letter-spacing: 0;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-title a:hover,
|
||||
.admin-sheet .tbl .c-act .ghost:hover {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow.attention .c-title {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-ch,
|
||||
.admin-sheet .tbl .c-rd {
|
||||
text-align: right;
|
||||
padding-right: 26px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow .c-note .mark {
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-act {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-act .ghost + .ghost {
|
||||
margin-left: 12px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow > .confirm-row {
|
||||
grid-column: 1 / -1;
|
||||
padding: 10px 12px;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.admin-sheet .stats {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(232px, 1fr));
|
||||
margin: 4px 0 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .stat {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: baseline;
|
||||
gap: 16px;
|
||||
padding: 11px 24px 11px 0;
|
||||
}
|
||||
|
||||
.admin-sheet .stat .lbl {
|
||||
font: 500 13px/1.3 var(--font-mono);
|
||||
letter-spacing: .1em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .stat .fig {
|
||||
font: 500 19px/1 var(--font-mono);
|
||||
font-variant-numeric: tabular-nums;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .stat .fig.zero {
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .stat a.fig {
|
||||
/* The hover rule below sets the border on :hover only, which shifts the
|
||||
row a pixel on hover; the transparent baseline holds the layout. */
|
||||
border-bottom: 1px solid transparent;
|
||||
}
|
||||
|
||||
.admin-sheet .stat a.fig:hover {
|
||||
color: var(--patina);
|
||||
border-bottom: 1px solid var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .mark,
|
||||
.admin-sheet .mark-faint {
|
||||
font: 500 13px/1 var(--font-mono);
|
||||
letter-spacing: .06em;
|
||||
text-transform: uppercase;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .mark {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .mark-faint {
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .mark.mark-strong {
|
||||
font-size: 13px;
|
||||
letter-spacing: .14em;
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .mark.mark-strong::before {
|
||||
content: "";
|
||||
display: inline-block;
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
background: var(--patina);
|
||||
margin-right: 8px;
|
||||
vertical-align: .08em;
|
||||
}
|
||||
|
||||
.admin-sheet .mark.bad {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites {
|
||||
grid-template-columns: 170px repeat(4, 106px) minmax(180px, 1fr);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .c-site,
|
||||
.admin-sheet .tbl.lanes .c-site {
|
||||
font: 400 18px/1.35 var(--font-display);
|
||||
letter-spacing: 0;
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .c-state {
|
||||
color: var(--patina);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .c-state.bad,
|
||||
.admin-sheet .tbl.lanes .c-skip .bad,
|
||||
.admin-sheet .tbl.lanes .trow.attention .c-site {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-skip .ok {
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.sites .thead > *:nth-child(n+2):nth-child(-n+5),
|
||||
.admin-sheet .tbl.sites .trow > *:nth-child(n+2):nth-child(-n+5) {
|
||||
padding-right: 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes {
|
||||
grid-template-columns: 150px 62px 88px 66px 118px minmax(0, 1fr) 168px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-skip {
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-skip > * {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-ctrl {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .thead > *:nth-child(n+2):nth-child(-n+4),
|
||||
.admin-sheet .tbl.lanes .trow > *:nth-child(n+2):nth-child(-n+4) {
|
||||
padding-right: 26px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series {
|
||||
grid-template-columns: 150px 84px 104px 76px minmax(120px, 1fr) 212px;
|
||||
row-gap: 4px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .thead > *:first-child {
|
||||
padding-left: 20px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .thead > *:last-child {
|
||||
padding-right: 20px;
|
||||
}
|
||||
|
||||
/* The site cell's colour is a class, never an inline style: site is
|
||||
client-supplied and unvalidated, and a hostile value reaching a CSS
|
||||
context would render ZgotmplZ, while an unknown class degrades to the
|
||||
unstyled cell. */
|
||||
.admin-sheet .tbl .c-site.site-asura {
|
||||
color: var(--asura);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-demonic {
|
||||
color: var(--demonic);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-comix {
|
||||
color: var(--comix);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-kagane {
|
||||
color: var(--kagane);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-novelfull {
|
||||
color: var(--novelfull);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-site.site-lightnovelworld {
|
||||
color: var(--lightnovelworld);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow {
|
||||
display: grid;
|
||||
grid-column: 1 / -1;
|
||||
grid-template-columns: subgrid;
|
||||
padding: 13px 20px 14px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow.band {
|
||||
background: var(--hover);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow > * {
|
||||
padding: 0 14px 0 0;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .c-title {
|
||||
display: flex;
|
||||
grid-column: 1 / -1;
|
||||
align-items: baseline;
|
||||
gap: 16px;
|
||||
padding: 0 0 4px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .c-title .mark {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.admin-sheet .filterbar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 14px 22px;
|
||||
padding: 16px 0 14px;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .fsel {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.admin-sheet .fsel > span {
|
||||
font: 500 11px/1 var(--font-mono);
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute-2);
|
||||
}
|
||||
|
||||
.admin-sheet .fsel select,
|
||||
.admin-sheet .pausebar select {
|
||||
color-scheme: dark;
|
||||
background: var(--ink);
|
||||
border: none;
|
||||
border-bottom: 1px solid var(--field-line);
|
||||
color: var(--paper);
|
||||
font: 400 16px/1.4 var(--font-display);
|
||||
padding: 4px 4px 5px 0;
|
||||
}
|
||||
|
||||
.admin-sheet .fsel select:hover,
|
||||
.admin-sheet .pausebar select:hover {
|
||||
border-bottom-color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .fsel select option,
|
||||
.admin-sheet .pausebar select option {
|
||||
background: var(--ink);
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .segrow {
|
||||
display: inline-flex;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.admin-sheet .segrow a {
|
||||
padding: 6px 11px 7px;
|
||||
border-bottom: 2px solid transparent;
|
||||
color: var(--mute);
|
||||
font: 500 14px/1 var(--font-mono);
|
||||
}
|
||||
|
||||
.admin-sheet .segrow a.active {
|
||||
border-bottom-color: var(--patina);
|
||||
color: var(--patina);
|
||||
}
|
||||
|
||||
.admin-sheet .listhead {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
padding: 16px 0 4px;
|
||||
color: var(--paper);
|
||||
font: 400 18px/1.2 var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .listhead .lbl {
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .listhead .lbl em {
|
||||
color: var(--patina);
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
.admin-sheet .pager {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
padding: 14px 0 0;
|
||||
color: var(--mute-2);
|
||||
font: 500 11px/1 var(--font-mono);
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .pager .pg {
|
||||
color: var(--paper-dim);
|
||||
}
|
||||
|
||||
.admin-sheet .pager .pg:hover {
|
||||
color: var(--paper);
|
||||
}
|
||||
|
||||
.admin-sheet .pager .pg.disabled {
|
||||
color: var(--faint);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.admin-sheet .empty {
|
||||
padding: 28px 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.admin-sheet .empty strong {
|
||||
color: var(--paper);
|
||||
font: 400 20px var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .empty p {
|
||||
margin: 6px 0 0;
|
||||
font: 14px/1.5 var(--font-body);
|
||||
color: var(--mute);
|
||||
}
|
||||
|
||||
.admin-sheet .confirm-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 10px 12px;
|
||||
background: var(--danger-wash);
|
||||
}
|
||||
|
||||
.admin-sheet .confirm-row span {
|
||||
flex: 1 1 16ch;
|
||||
color: var(--danger-soft);
|
||||
font: 400 15px/1.3 var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .confirm-row div {
|
||||
display: flex;
|
||||
flex: none;
|
||||
gap: 12px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-back {
|
||||
display: inline-block;
|
||||
margin: 18px 0 0;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-title {
|
||||
margin: 10px 0 2px;
|
||||
color: var(--paper);
|
||||
font: 400 28px/1.25 var(--font-display);
|
||||
}
|
||||
|
||||
.admin-sheet .detail-key {
|
||||
margin: 0;
|
||||
color: var(--mute-2);
|
||||
font: 500 11px/1.4 var(--font-mono);
|
||||
letter-spacing: .08em;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 14px;
|
||||
margin: 10px 0 0;
|
||||
color: var(--mute-2);
|
||||
font: 500 12px/1.5 var(--font-mono);
|
||||
letter-spacing: .08em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .cover {
|
||||
width: 160px;
|
||||
aspect-ratio: 3 / 4;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin: 18px 0 4px;
|
||||
background: var(--hatch);
|
||||
color: var(--mute-2);
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 0 28px;
|
||||
}
|
||||
|
||||
.admin-sheet .dform {
|
||||
padding: 14px 0 0;
|
||||
}
|
||||
|
||||
.admin-sheet .dform h3 {
|
||||
margin: 0 0 4px;
|
||||
color: var(--mute-2);
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .2em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.admin-sheet .dform .field {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.admin-sheet .pausebar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
@media (max-width: 719px) {
|
||||
.admin-sheet .topbar {
|
||||
flex-wrap: wrap;
|
||||
row-gap: 12px;
|
||||
}
|
||||
|
||||
.admin-sheet .brand {
|
||||
flex: 1 1 100%;
|
||||
}
|
||||
|
||||
.admin-sheet .topbar-actions {
|
||||
margin-left: auto;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 1019px) {
|
||||
.admin-sheet .tbl.lanes {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .thead {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .trow {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: baseline;
|
||||
gap: 4px 16px;
|
||||
padding: 12px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .trow > * {
|
||||
padding: 0;
|
||||
border-bottom: none;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-site {
|
||||
width: 100%;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.lanes .c-ctrl {
|
||||
margin-left: auto;
|
||||
text-align: right;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 899px) {
|
||||
.admin-sheet .tbl,
|
||||
.admin-sheet .tbl.lanes {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .thead {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 2px 10px;
|
||||
padding: 11px 0;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .trow > * {
|
||||
padding: 0;
|
||||
border-bottom: none;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-ch,
|
||||
.admin-sheet .tbl .c-rd {
|
||||
padding-right: 0;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-title {
|
||||
width: 100%;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl.series .trow > * {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.admin-sheet .detail-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.admin-sheet .tbl .c-act {
|
||||
margin-left: auto;
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,11 @@
|
||||
--moss: #7fae86; /* finished */
|
||||
--clay: #b5906f; /* set chapter */
|
||||
--trash: #977671; /* remove, resting — icons need 3:1, not 4.5:1 */
|
||||
/* A Lane needing attention: the admin page's only accent. Verdigris — cool,
|
||||
the far side of the wheel from ember's crimson, and clear of the archive
|
||||
blue. Neither ember (new chapter) nor danger (destruction) may say
|
||||
"system unhealthy". */
|
||||
--patina: #5fb3a6;
|
||||
|
||||
/* Desktop cell borders for the two coloured action states. */
|
||||
--play-hot-line: #3a1d18;
|
||||
@@ -146,6 +151,7 @@
|
||||
--moss: #3d6c46;
|
||||
--clay: #7c5533;
|
||||
--trash: #8c6558;
|
||||
--patina: #1f6f66;
|
||||
|
||||
--play-hot-line: #f0cfc6;
|
||||
--fav-line: #e3d3a4;
|
||||
@@ -290,32 +296,6 @@ button { cursor: pointer; }
|
||||
letter-spacing: .04em;
|
||||
}
|
||||
|
||||
/* ---- reader roster (owner only): same hairline panel, one row per Reader ---- */
|
||||
.readerlist { margin: 0; padding: 0; list-style: none; }
|
||||
.readerlist li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 16px;
|
||||
min-height: 44px;
|
||||
border-top: 1px solid var(--rule);
|
||||
}
|
||||
.readerlist form { margin: 0 0 0 auto; }
|
||||
.reader-id {
|
||||
font: 500 13px/1.4 var(--font-mono);
|
||||
letter-spacing: .04em;
|
||||
color: var(--paper);
|
||||
}
|
||||
.reader-sessions {
|
||||
font: 500 10px/1 var(--font-mono);
|
||||
letter-spacing: .14em;
|
||||
text-transform: uppercase;
|
||||
color: var(--mute);
|
||||
}
|
||||
/* Revocation cuts someone off, so it wears --danger. Ember stays reserved for
|
||||
the new-chapter signal. */
|
||||
.ghost.danger { color: var(--danger); }
|
||||
.ghost.danger:hover { color: var(--danger); border-bottom-color: var(--danger); }
|
||||
|
||||
.chrome { display: flex; flex-direction: column; }
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{{/* Every owner-only address shares this shell; page content stays behind its
|
||||
bookmarkable route so the active tab survives a reload. */}}
|
||||
{{define "admin"}}
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||
<meta name="color-scheme" content="dark light">
|
||||
<title>BookmarkManager — Admin</title>
|
||||
<link rel="icon" href="/static/logo.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
<link rel="stylesheet" href="/static/admin.css">
|
||||
<link rel="preload" href="/static/fonts/instrument-serif-400-latin.woff2" as="font" type="font/woff2" crossorigin>
|
||||
<script src="/static/htmx.min.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="sheet admin-sheet">
|
||||
<header class="topbar">
|
||||
<h1 class="brand">{{template "mark" .}}<span>Bookmark<em>Manager</em></span></h1>
|
||||
<span class="topbar-actions">
|
||||
<a class="ghost" href="/">Library</a>
|
||||
<form method="post" action="/logout">
|
||||
<button type="submit" class="ghost">Log out</button>
|
||||
</form>
|
||||
</span>
|
||||
</header>
|
||||
|
||||
<nav class="navrow" aria-label="Admin pages">
|
||||
<a href="/admin" class="{{if eq .Page "overview"}}active{{end}}" {{if eq .Page "overview"}}aria-current="page"{{end}}>Overview</a>
|
||||
<a href="/admin/lanes" class="{{if eq .Page "lanes"}}active{{end}}" {{if eq .Page "lanes"}}aria-current="page"{{end}}>Lanes</a>
|
||||
<a href="/admin/readers" class="{{if eq .Page "readers"}}active{{end}}" {{if eq .Page "readers"}}aria-current="page"{{end}}>Readers</a>
|
||||
<a href="/admin/series" class="{{if or (eq .Page "series") (eq .Page "series-detail")}}active{{end}}" {{if or (eq .Page "series") (eq .Page "series-detail")}}aria-current="page"{{end}}>Series</a>
|
||||
</nav>
|
||||
|
||||
<main class="page admin-page">
|
||||
{{if eq .Page "lanes"}}
|
||||
<div aria-live="polite">{{template "lanes" .Lanes}}</div>
|
||||
{{else if eq .Page "readers"}}
|
||||
{{template "readers" .}}
|
||||
{{else if eq .Page "series"}}
|
||||
{{template "series-list" .SeriesList}}
|
||||
{{else if eq .Page "series-detail"}}
|
||||
{{template "series-detail" .Detail}}
|
||||
{{else}}{{template "overview" .Overview}}{{end}}
|
||||
</main>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
{{end}}
|
||||
@@ -28,6 +28,10 @@
|
||||
<a href="/?lib=novel&tab=all" class="{{if eq .Lib "novel"}}active{{end}}"
|
||||
{{if eq .Lib "novel"}}aria-current="page"{{end}}>Novels</a>
|
||||
</nav>
|
||||
{{/* The owner's only difference on this page: a link out to the
|
||||
administrative one. It sits beside Log out rather than in the library
|
||||
switch — that switch says which library, not which page. */}}
|
||||
{{if .Owner}}<a class="ghost" href="/admin">Admin</a>{{end}}
|
||||
<form method="post" action="/logout">
|
||||
<button type="submit" class="ghost">Log out</button>
|
||||
</form>
|
||||
@@ -76,8 +80,6 @@
|
||||
|
||||
{{template "setup" .}}
|
||||
|
||||
{{if .Owner}}{{template "readers" .}}{{end}}
|
||||
|
||||
{{template "keyrow" .}}
|
||||
|
||||
{{template "recent" .}}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
{{/* Poll Lane status: one row per Site's latest durable pass, refreshing
|
||||
itself so a run can be watched rather than sampled by reloading. The
|
||||
refresh is one attribute on the fragment root and the endpoint answers
|
||||
with this same fragment, so the swap replaces the element that asked.
|
||||
|
||||
Every figure is read from poll_passes, never from a running poller: a
|
||||
restart answers from the database the moment it is up (issue #145). The
|
||||
browser fact is a deployment-config fact plus a reachability derived from
|
||||
the pass log; the cause chips and the state phrase are decided in Go,
|
||||
this template only prints them. */}}
|
||||
{{define "lanes"}}
|
||||
<section class="lanes" id="lanes"
|
||||
hx-get="/ui/admin/lanes" hx-trigger="every 30s" hx-swap="outerHTML">
|
||||
<div class="sechead">
|
||||
<h2 class="sec">Poll Lanes</h2>
|
||||
<p class="statusline">
|
||||
{{if .PollerOff}}Polling: <span class="mark-faint">off</span>
|
||||
{{else}}Browser: {{if not .BrowserConfigured}}<span class="mark-faint">not configured</span>{{else if .BrowserReachable}}<span class="mark mark-strong">reachable</span>{{else}}<span class="mark bad">unreachable</span>{{end}}{{end}}
|
||||
</p>
|
||||
</div>
|
||||
{{if .Rows}}
|
||||
<div class="tbl lanes">
|
||||
<div class="thead">
|
||||
<span>Site</span><span>Due</span><span>Checked</span><span>Gap</span>
|
||||
<span>Last pass</span><span>Outcomes · state</span><span></span>
|
||||
</div>
|
||||
{{range .Rows}}
|
||||
<div class="trow{{if .Attention}} attention{{end}}">
|
||||
<span class="c-site">{{.Site}}</span>
|
||||
<span>{{.Due}}</span>
|
||||
<span>{{.Checked}}</span>
|
||||
<span>{{.Gap}}</span>
|
||||
<span>ran {{.Ran}}</span>
|
||||
<span class="c-skip">{{if .HasChips}}{{range $i, $c := .Chips}}{{if $i}}<span class="mark-faint"> · </span>{{end}}<span class="mark">{{$c.Name}} {{$c.Count}}</span>{{end}}{{else}}<span class="mark-faint">none observed</span>{{end}}{{if .StatePhrase}} · <span class="{{if .StateGood}}ok{{else}}bad{{end}}">{{.StatePhrase}}</span>{{end}}</span>
|
||||
{{/* The pause control lives in the one slot the design leaves for it:
|
||||
a running Lane offers the three durations and Pause; a paused Lane
|
||||
offers Resume in the same place. Pause is not destruction — it
|
||||
takes nothing away and reverses in one press — so neither wears a
|
||||
confirm row or the danger accent. The form wraps the select so the
|
||||
offered duration travels with the press. */}}
|
||||
<span class="c-ctrl">{{if .Paused}}<span class="pausebar">
|
||||
<form hx-post="/admin/lanes/{{.Site}}/resume" hx-target="#lanes" hx-swap="outerHTML">
|
||||
<button type="submit" class="ghost">Resume</button>
|
||||
</form>
|
||||
</span>{{else}}<span class="pausebar">
|
||||
<form hx-post="/admin/lanes/{{.Site}}/pause" hx-target="#lanes" hx-swap="outerHTML">
|
||||
<select name="duration" aria-label="Pause duration">
|
||||
<option>1h</option><option selected>6h</option><option>24h</option>
|
||||
</select>
|
||||
<button type="submit" class="ghost">Pause</button>
|
||||
</form>
|
||||
</span>{{end}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<p class="empty">No data yet — no Lane has recorded a pass.</p>
|
||||
{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
@@ -0,0 +1,14 @@
|
||||
{{/* The Overview landing page: one verdict line leading, then a stats block
|
||||
where every figure is a door into the list that counts it, and the
|
||||
per-Site library shape table. Every judgement — the verdict state, which
|
||||
figures link, what a Lane's state means — is made in Go; this template
|
||||
only prints. */}}
|
||||
{{define "overview"}}
|
||||
<p class="verdict"><span class="attn">{{.Verdict}}</span> {{if .HasCounts}}<span class="counts">· <b>{{.Waiting}}</b> series waiting · <b>{{.Unchecked}}</b> unchecked over 12h</span>{{end}}</p>
|
||||
<h2 class="sec">Hygiene</h2>
|
||||
<div class="stats">{{range .Hygiene}}<div class="stat"><span class="lbl">{{.Label}}</span>{{if .Href}}<a class="fig" href="{{.Href}}">{{.Count}}</a>{{else}}<span class="fig zero">{{.Count}}</span>{{end}}</div>{{end}}</div>
|
||||
<h2 class="sec">Library</h2>
|
||||
<div class="stats">{{range .Library}}<div class="stat"><span class="lbl">{{.Label}}</span>{{if .Href}}<a class="fig" href="{{.Href}}">{{.Count}}</a>{{else}}<span class="fig zero">{{.Count}}</span>{{end}}</div>{{end}}</div>
|
||||
<h2 class="sec">Sites · last 12h</h2>
|
||||
<div class="tbl sites"><div class="thead"><span>Site</span><span>Series</span><span>No cover</span><span>Never chk</span><span>Stale</span><span>State</span></div>{{range .Sites}}<div class="trow"><a class="c-site site-{{.Site}}" href="{{.SiteHref}}">{{.Site}}</a>{{range .Figs}}{{if .Href}}<a class="fig" href="{{.Href}}">{{.Count}}</a>{{else}}<span class="fig zero">{{.Count}}</span>{{end}}{{end}}<span class="c-state{{if .StateGood}} ok{{end}}{{if .StateBad}} bad{{end}}">{{.State}}</span></div>{{end}}</div>
|
||||
{{end}}
|
||||
@@ -1,29 +1,48 @@
|
||||
{{/* The owner's Reader roster. Rendered only for the owner (listView.Owner),
|
||||
and re-rendered whole as the response to a revocation so the session
|
||||
counts cannot describe the state before the tap. Revocation is
|
||||
confirm-gated: it signs someone out of every device at once. */}}
|
||||
{{/* The Reader roster, on the owner's administrative page. Re-rendered whole
|
||||
as the response to an action so the counts and marks it shows cannot
|
||||
describe the state before the tap. Both actions are confirm-gated: one
|
||||
signs a Reader out of every device at once, the other wipes a record.
|
||||
|
||||
Owner-only at route registration, so nothing here re-tests who is asking. */}}
|
||||
{{define "readers"}}
|
||||
<details class="setup" id="readers">
|
||||
<summary>Readers</summary>
|
||||
<section class="readers" id="readers">
|
||||
<h2>Readers</h2>
|
||||
<p class="setup-copy">Everyone who has signed in through Discord. Revoking
|
||||
signs a Reader out of every device; their library and bookmarks are
|
||||
untouched, and they can sign in again.</p>
|
||||
untouched, and they can sign in again. The Sighting counters record how
|
||||
often a later Poll confirmed or contradicted what that Reader's browser
|
||||
reported; enough contradictions stop their reports deferring a Poll, and
|
||||
clearing the marks gives that back.</p>
|
||||
<ul class="readerlist">
|
||||
{{range .Readers}}
|
||||
<li>
|
||||
<span class="reader-id">{{.DiscordID}}</span>
|
||||
<span class="reader-sessions">{{.Sessions}} session{{if ne .Sessions 1}}s{{end}}</span>
|
||||
{{/* The owner's own row never offers Revoke: it is the one row where the
|
||||
button would sign the tapping browser out, and the endpoint refuses
|
||||
it anyway. Logout is the deliberate way to do that. */}}
|
||||
{{if and .Sessions (ne .ID $.OwnerID)}}
|
||||
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
|
||||
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
|
||||
<button type="submit" class="ghost danger">Revoke sessions</button>
|
||||
</form>
|
||||
{{end}}
|
||||
<span class="reader-sightings">{{.Agreements}} confirmed / {{.Disagreements}} contradicted</span>
|
||||
{{/* Blocked is spelled out rather than left to be worked out from two
|
||||
numbers and a threshold. */}}
|
||||
{{if .Blocked}}<span class="reader-blocked">deferral blocked</span>{{end}}
|
||||
<span class="reader-actions">
|
||||
{{/* Clearing restores a privilege, so it is a plain ghost button —
|
||||
the destruction accent belongs to revocation alone. It is offered
|
||||
on every row, including one reading zero: the remedy must be
|
||||
findable before the counters climb, not after. */}}
|
||||
<form hx-post="/readers/{{.ID}}/clear-marks" hx-target="#readers" hx-swap="outerHTML"
|
||||
hx-confirm="Clearing wipes this Reader's whole Sighting record, confirmations included. Clear?">
|
||||
<button type="submit" class="ghost">Clear marks</button>
|
||||
</form>
|
||||
{{/* The owner's own row never offers Revoke: it is the one row where the
|
||||
button would sign the tapping browser out, and the endpoint refuses
|
||||
it anyway. Logout is the deliberate way to do that. */}}
|
||||
{{if and .Sessions (ne .ID $.OwnerID)}}
|
||||
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
|
||||
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
|
||||
<button type="submit" class="ghost danger">Revoke sessions</button>
|
||||
</form>
|
||||
{{end}}
|
||||
</span>
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
</details>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{{/* Per-Series page: one address per Series, keyed "<site>:<series_id>" so the
|
||||
list row is one hop from it. Everything here is a Series-level fact plus
|
||||
the anonymous Reader count. The Check now control lands in its own .dform
|
||||
below the (empty) .detail-grid; the pending marker rides the meta line
|
||||
with the other marks. */}}
|
||||
{{define "series-detail"}}
|
||||
<a class="ghost detail-back" href="/admin/series">← Series</a>
|
||||
<h1 class="detail-title">{{.Title}}</h1>
|
||||
<p class="detail-key">{{.Key}} · {{.Site}} · {{.Kind}}</p>
|
||||
{{if .Cover}}<div class="cover"><img src="{{.Cover}}" alt="" loading="lazy"></div>
|
||||
{{else}}<div class="cover"></div>{{end}}
|
||||
{{template "series-detail-meta" .}}
|
||||
<div class="detail-grid"></div>
|
||||
{{if .CanPoll}}
|
||||
<div class="dform">
|
||||
<div class="field"><a class="ghost act" hx-post="/admin/series/{{.Key}}/poll" hx-target="#detail-meta" hx-swap="outerHTML" href="#">Check now</a></div>
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{/* series-detail-meta is the meta line, and the answer a Check now press on
|
||||
the detail page swaps into its place: the same marks, re-rendered after
|
||||
the stamp so the pending marker shows. */}}
|
||||
{{define "series-detail-meta"}}
|
||||
<div class="detail-meta" id="detail-meta">
|
||||
<span>ch {{.Chapter}}</span>
|
||||
<span>checked {{.Checked}}</span>
|
||||
<span>{{.Readers}} readers</span>
|
||||
{{if .Pending}}<span class="mark">{{.Requested}}</span>{{end}}
|
||||
{{if .Unpollable}}<span class="mark">unpollable</span>{{end}}
|
||||
{{if .NoCover}}<span class="mark">no cover</span>{{end}}
|
||||
{{if .Orphan}}<span class="mark">orphan</span>{{end}}
|
||||
{{if .SightingRaised}}<span class="mark">sighting-raised</span>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -0,0 +1,53 @@
|
||||
{{/* The Series list: every Series across every Reader's library, filtered by
|
||||
one hygiene rule and narrowed by Site and Library. Filter, Site, Library
|
||||
and page all live in the query string, so the list's state is an address
|
||||
that can be bookmarked: the two selects submit the GET form, and the
|
||||
Library segment links and the pager preserve the filter and Site. */}}
|
||||
{{define "series-list"}}
|
||||
<form class="filterbar" method="get" action="/admin/series">
|
||||
<input type="hidden" name="kind" value="{{.Kind}}">
|
||||
<label class="fsel"><span>Show</span><select name="filter" onchange="this.form.submit()">
|
||||
{{range .Filters}}<option value="{{.Name}}"{{if .Selected}} selected{{end}}>{{.Label}} ({{.Count}})</option>{{end}}
|
||||
</select></label>
|
||||
<label class="fsel"><span>Site</span><select name="site" onchange="this.form.submit()">
|
||||
<option value=""{{if not .Site}} selected{{end}}>All sites</option>
|
||||
{{range .Sites}}<option value="{{.}}"{{if eq $.Site .}} selected{{end}}>{{.}}</option>{{end}}
|
||||
</select></label>
|
||||
<span class="segrow">
|
||||
<a href="{{.KindBoth}}"{{if not .Kind}} class="active"{{end}}>both</a>
|
||||
<a href="{{.KindManga}}"{{if eq .Kind "manga"}} class="active"{{end}}>manga</a>
|
||||
<a href="{{.KindNovel}}"{{if eq .Kind "novel"}} class="active"{{end}}>novels</a>
|
||||
</span>
|
||||
</form>
|
||||
<div class="listhead">{{.Total}} series <span class="lbl">· <em>{{.FilterLabel}}</em></span></div>
|
||||
{{if .Rows}}
|
||||
<div class="tbl series">
|
||||
<div class="thead"><span>Site</span><span class="c-ch">Ch</span><span>Checked</span><span class="c-rd">Readers</span><span>Notes</span><span></span></div>
|
||||
{{range .Rows}}{{template "series-row" .}}{{end}}
|
||||
</div>
|
||||
<div class="pager">
|
||||
{{if .PrevHref}}<a class="pg" href="{{.PrevHref}}">‹ prev</a>{{else}}<span class="pg disabled">‹ prev</span>{{end}}
|
||||
<span>{{.Range}}</span>
|
||||
{{if .NextHref}}<a class="pg" href="{{.NextHref}}">next ›</a>{{else}}<span class="pg disabled">next ›</span>{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="empty"><strong>No series</strong><p>Nothing matches <em>{{.FilterLabel}}</em>.</p></div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{/* series-row is one Series list row, and the answer a Check now press swaps
|
||||
into the row's place (hx-target="closest .trow"): it must render the
|
||||
pending marker the press created. The control is absent on a Series with
|
||||
no page to fetch and on an orphan, so the owner is never offered a button
|
||||
that can never do anything. */}}
|
||||
{{define "series-row"}}
|
||||
<div class="trow{{if .Attention}} attention{{end}}{{if .Band}} band{{end}}">
|
||||
<span class="c-title"><a href="/admin/series/{{.Key}}">{{.Title}}</a>{{if .Pending}}<span class="mark">{{.Requested}}</span>{{end}}</span>
|
||||
<span class="c-site site-{{.Site}}">{{.Site}}</span>
|
||||
<span class="c-ch">{{.Ch}}</span>
|
||||
<span>{{.Age}}</span>
|
||||
<span class="c-rd">{{.Readers}}</span>
|
||||
<span class="c-note">{{range .Notes}}<span class="mark">{{.}}</span>{{end}}{{if .More}}<span class="mark mark-faint">+{{.More}}</span>{{end}}</span>
|
||||
<span class="c-act">{{if .CanPoll}}<a class="ghost act" hx-post="/admin/series/{{.Key}}/poll" hx-target="closest .trow" hx-swap="outerHTML" hx-vals='{"band":{{if .Band}}1{{else}}0{{end}}}' href="#">Check now</a>{{end}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
+26
-56
@@ -50,6 +50,14 @@ type Handler struct {
|
||||
// httpClient is the plain stdlib client that talks to Discord. It is not
|
||||
// an injected interface: tests point APIBase at a stub server instead.
|
||||
httpClient *http.Client
|
||||
// pollerEnabled reports whether latest-chapter polling is switched on in
|
||||
// this deployment (LATEST_CHAPTER_POLL_ENABLED) and browserConfigured
|
||||
// whether a browser sidecar is configured (BROWSER_WS_URL set). Both are
|
||||
// deployment facts resolved by the composition root; the Lanes page (issue
|
||||
// #145) reports them from config and derives reachability from the pass
|
||||
// log rather than from whether a poller goroutine happened to start.
|
||||
pollerEnabled bool
|
||||
browserConfigured bool
|
||||
}
|
||||
|
||||
// listView is what every list-rendering template receives.
|
||||
@@ -77,14 +85,9 @@ type listView struct {
|
||||
// It is not "newly registered": a Reader who deletes their last bookmark is
|
||||
// in the same position and needs the same links.
|
||||
EmptyLibrary bool
|
||||
// Owner marks the acting Reader as the deployment's owner, which unlocks
|
||||
// the Readers panel. Nothing else in the UI differs.
|
||||
// Owner marks the acting Reader as the deployment's owner, which offers
|
||||
// the link to the administrative page. Nothing else in the UI differs.
|
||||
Owner bool
|
||||
// Readers is the owner's roster, populated only for the owner's own page
|
||||
// render and the revocation fragment. OwnerID travels with it so the roster
|
||||
// can tell the owner's own row apart from the Readers they may revoke.
|
||||
Readers []store.ReaderSummary
|
||||
OwnerID int64
|
||||
}
|
||||
|
||||
// PageURL and ListURL are the two link shapes every tab needs. Building them
|
||||
@@ -111,7 +114,12 @@ type loginView struct {
|
||||
|
||||
// New parses every template up front so a broken one kills the process at
|
||||
// startup rather than the first request that touches it.
|
||||
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string) (*Handler, error) {
|
||||
//
|
||||
// pollerEnabled and browserConfigured are deployment facts the composition
|
||||
// root resolves from LATEST_CHAPTER_POLL_ENABLED and BROWSER_WS_URL: the Lanes
|
||||
// page (issue #145) reports them and derives browser reachability from the
|
||||
// pass log, so no running poller is wired through here at all.
|
||||
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string, pollerEnabled, browserConfigured bool) (*Handler, error) {
|
||||
tmpl, err := template.ParseFS(templateFS, "templates/*.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -126,6 +134,8 @@ func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, nove
|
||||
states: newOAuthStates(),
|
||||
limiter: session.NewLoginLimiter(),
|
||||
httpClient: &http.Client{Timeout: discordTimeout},
|
||||
pollerEnabled: pollerEnabled,
|
||||
browserConfigured: browserConfigured,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -149,9 +159,11 @@ func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /install/manga-bookmark.user.js", h.requireSession(h.installUserscript("manga-bookmark.user.js")))
|
||||
mux.HandleFunc("GET /install/novel-bookmark.user.js", h.requireSession(h.installUserscript("novel-bookmark.user.js")))
|
||||
mux.HandleFunc("POST /rotate-token", h.requireSession(h.rotateToken))
|
||||
|
||||
// Owner-only: the one place the UI crosses the Reader boundary.
|
||||
mux.HandleFunc("POST /readers/{id}/revoke", h.requireSession(h.revokeReaderSessions))
|
||||
// Owner-only: every route that reaches past the acting Reader is gated in
|
||||
// one place, so a missing gate is visible in the route list.
|
||||
for _, rt := range h.adminRoutes() {
|
||||
mux.HandleFunc(rt.pattern, h.requireOwner(rt.handler))
|
||||
}
|
||||
}
|
||||
|
||||
// staticHandler serves the embedded assets. An hour, not longer: assets are
|
||||
@@ -240,14 +252,9 @@ func (h *Handler) index(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if readerID == h.store.OwnerID() {
|
||||
view.Owner, view.OwnerID = true, readerID
|
||||
if view.Readers, err = h.store.Readers(); err != nil {
|
||||
log.Printf("index readers: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
// The owner's page differs only by the link to the administrative page:
|
||||
// the roster lives there now, so the page read every day is only reading.
|
||||
view.Owner = readerID == h.store.OwnerID()
|
||||
h.render(w, http.StatusOK, "app", view)
|
||||
}
|
||||
|
||||
@@ -618,40 +625,3 @@ func (h *Handler) rotateToken(w http.ResponseWriter, r *http.Request) {
|
||||
view := listView{Lib: store.KindManga, Rotated: true}
|
||||
h.render(w, http.StatusOK, "setup", view)
|
||||
}
|
||||
|
||||
// revokeReaderSessions logs one Reader out of every browser they are signed
|
||||
// in on. Owner-only: it reaches across the Reader boundary every other handler
|
||||
// respects, so the guard is a comparison against the seeded owner rather than
|
||||
// a role a Reader could acquire. A non-owner gets 404 — the panel does not
|
||||
// exist for them, so neither should the endpoint.
|
||||
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
|
||||
if readerOf(r) != h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
target, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "bad reader id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// The owner is not one of the Readers this endpoint reaches: revoking
|
||||
// themselves would sign out the browser making the request, which is what
|
||||
// logout is for. The roster hides the button; this refuses the hand-rolled
|
||||
// POST behind it.
|
||||
if target == h.store.OwnerID() {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := h.store.DeleteReaderSessions(target); err != nil {
|
||||
log.Printf("revoke sessions: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
readers, err := h.store.Readers()
|
||||
if err != nil {
|
||||
log.Printf("revoke sessions: %v", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
h.render(w, http.StatusOK, "readers", listView{Owner: true, Readers: readers, OwnerID: h.store.OwnerID()})
|
||||
}
|
||||
|
||||
+38
-94
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -55,36 +54,26 @@ type Config struct {
|
||||
// /u/{token}/novel-bookmark.user.js. Same bindmount, second script: the
|
||||
// two libraries are separate installs.
|
||||
NovelUserscriptPath string
|
||||
// BrowserWSURL is the CDP websocket the poller's browser Sites read
|
||||
// through. Set means a browser sidecar is configured in this deployment —
|
||||
// the Lanes page reports the fact and derives reachability from the pass
|
||||
// log rather than asking the poller (issue #145).
|
||||
BrowserWSURL string
|
||||
// LatestPoll configures the background latest-chapter fetcher.
|
||||
LatestPoll LatestPoll
|
||||
}
|
||||
|
||||
// LatestPoll configures the background latest-chapter poller.
|
||||
//
|
||||
// Sizing: batch x (cooldown / interval) is how many series hold a true cooldown
|
||||
// cadence — 14 x (1h / 10m) = 84 with these defaults, which covers this
|
||||
// deployment. Past that nothing breaks; the effective cadence stretches to
|
||||
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
|
||||
// Only the kill switch lives here. Pace is per Site — rest time and gap are
|
||||
// registry properties (internal/latest/sites.go, issue #100), because each
|
||||
// Lane has to be able to differ from the others. The five environment
|
||||
// settings that used to size a shared pace (cooldown, browser cooldown,
|
||||
// interval, stagger, batch) are gone with it: no deployed .env may carry them.
|
||||
type LatestPoll struct {
|
||||
Enabled bool
|
||||
Cooldown time.Duration
|
||||
BrowserCooldown time.Duration
|
||||
Interval time.Duration
|
||||
Stagger time.Duration
|
||||
Batch int
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
const (
|
||||
defaultPollCooldown = time.Hour
|
||||
defaultBrowserPollCooldown = 6 * time.Hour
|
||||
defaultPollInterval = 10 * time.Minute
|
||||
defaultPollStagger = 20 * time.Second
|
||||
defaultPollBatch = 14
|
||||
// minPollCooldown keeps a typo from turning a polite background check into
|
||||
// a hammer against sites that are already bot-scoring us.
|
||||
minPollCooldown = 15 * time.Minute
|
||||
)
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
@@ -107,66 +96,11 @@ func envBool(key string, def bool) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// envDuration reads a duration env var. An unparseable or non-positive value
|
||||
// falls back to def and logs rather than failing startup: the poller is an
|
||||
// enhancement, and a typo in one of its knobs must not stop bookmark sync.
|
||||
func envDuration(key string, def time.Duration) time.Duration {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
d, err := time.ParseDuration(raw)
|
||||
if err != nil || d <= 0 {
|
||||
log.Printf("config: %s=%q is not a positive duration, using %s", key, raw, def)
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// envInt reads a positive integer env var, with the same fallback policy.
|
||||
func envInt(key string, def int) int {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
log.Printf("config: %s=%q is not a positive integer, using %d", key, raw, def)
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func clampPollCooldown(name string, d time.Duration) time.Duration {
|
||||
if d < minPollCooldown {
|
||||
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
|
||||
return minPollCooldown
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// loadLatestPoll reads the poller's settings, clamping anything that would make
|
||||
// it antisocial.
|
||||
// loadLatestPoll reads the poller's settings. The pace knobs that used to be
|
||||
// clamped here are registry properties now (issue #100), so there is nothing
|
||||
// left to clamp.
|
||||
func loadLatestPoll() LatestPoll {
|
||||
p := LatestPoll{
|
||||
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
|
||||
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
|
||||
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
|
||||
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
|
||||
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
|
||||
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
|
||||
}
|
||||
p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
|
||||
p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
|
||||
// batch x stagger has to fit inside one tick or a batch is still running
|
||||
// when the next one is due. Run() serialises them, so this degrades to a
|
||||
// slower cadence rather than to overlapping fetches — worth a warning, not
|
||||
// a failure.
|
||||
if span := time.Duration(p.Batch) * p.Stagger; span > p.Interval {
|
||||
log.Printf("config: batch(%d) x stagger(%s) = %s exceeds interval %s; batches will overrun their tick",
|
||||
p.Batch, p.Stagger, span, p.Interval)
|
||||
}
|
||||
return p
|
||||
return LatestPoll{Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true)}
|
||||
}
|
||||
|
||||
func loadConfig() Config {
|
||||
@@ -179,6 +113,7 @@ func loadConfig() Config {
|
||||
OwnerDiscordID: os.Getenv("OWNER_DISCORD_ID"),
|
||||
UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"),
|
||||
NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"),
|
||||
BrowserWSURL: os.Getenv("BROWSER_WS_URL"),
|
||||
LatestPoll: loadLatestPoll(),
|
||||
}
|
||||
c.Discord = web.DiscordConfig{
|
||||
@@ -200,6 +135,10 @@ func loadConfig() Config {
|
||||
// newRouter wires routes and middleware. CORS is the outermost layer so
|
||||
// preflight OPTIONS short-circuits before auth; /bookmarks* is auth-protected,
|
||||
// /healthz is public.
|
||||
//
|
||||
// The web layer learns the deployment's poller and browser config from cfg —
|
||||
// nothing of the running poller is wired through here; the Lanes page reads
|
||||
// the database (issue #145).
|
||||
func newRouter(s *store.Store, cfg Config) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
h := &api.Handler{Store: s}
|
||||
@@ -229,9 +168,12 @@ func newRouter(s *store.Store, cfg Config) http.Handler {
|
||||
mux.Handle("/bookmarks/", auth)
|
||||
|
||||
// The browser UI is always registered; signing in is Discord OAuth, so
|
||||
// there is no password to forget and no gate to leave unset.
|
||||
// there is no password to forget and no gate to leave unset. The poller
|
||||
// and browser facts are config, not the poller's: the Lanes page reads
|
||||
// the pass log and reports the deployment as configured.
|
||||
wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey),
|
||||
cfg.UserscriptPath, cfg.NovelUserscriptPath)
|
||||
cfg.UserscriptPath, cfg.NovelUserscriptPath,
|
||||
cfg.LatestPoll.Enabled, strings.TrimSpace(cfg.BrowserWSURL) != "")
|
||||
if err != nil {
|
||||
log.Fatalf("web handler: %v", err)
|
||||
}
|
||||
@@ -309,7 +251,7 @@ func main() {
|
||||
var browser latest.Fetcher
|
||||
pollCtx, stopPoll := context.WithCancel(context.Background())
|
||||
defer stopPoll()
|
||||
if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" {
|
||||
if ws := strings.TrimSpace(cfg.BrowserWSURL); ws != "" {
|
||||
bf, err := latest.NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
log.Printf("browser fetcher disabled: %v", err)
|
||||
@@ -346,6 +288,9 @@ func main() {
|
||||
}
|
||||
s.OnSeriesCreated = acq.Acquire
|
||||
}
|
||||
// The poller's only connection to the web layer is the database now: it is
|
||||
// started for its own sake, and the Lanes page reads the pass rows it
|
||||
// records (issue #145).
|
||||
startLatestPoller(pollCtx, s, cfg.LatestPoll, browser)
|
||||
|
||||
srv := &http.Server{
|
||||
@@ -377,7 +322,8 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
|
||||
// newLatestPoller wires the fetcher seams into the poller. Pace is registry
|
||||
// property, not config (issue #100), so there are no knobs to pass through.
|
||||
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
|
||||
var covers latest.BrowserCoverFetcher
|
||||
if f, ok := browser.(latest.BrowserCoverFetcher); ok {
|
||||
@@ -390,27 +336,24 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch
|
||||
CoverFetch: covers,
|
||||
CoverBytesFetch: latest.NewCoverFetcher(),
|
||||
Now: time.Now,
|
||||
Cooldown: cfg.Cooldown,
|
||||
BrowserCooldown: cfg.BrowserCooldown,
|
||||
Interval: cfg.Interval,
|
||||
Stagger: cfg.Stagger,
|
||||
Batch: cfg.Batch,
|
||||
}
|
||||
}
|
||||
|
||||
// startLatestPoller launches the background poller unless it is disabled or its
|
||||
// HTTP client cannot be built. Any problem here is logged and skipped: this
|
||||
// feature going missing degrades the service to userscript-only latest-chapter
|
||||
// tracking, which is exactly how it behaved before.
|
||||
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) {
|
||||
// tracking, which is exactly how it behaved before. It returns the running
|
||||
// Poller, or nil when there is none; the caller starts it for its own sake —
|
||||
// the Lanes page reads the pass log, so no return value is wired anywhere.
|
||||
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) *latest.Poller {
|
||||
if !cfg.Enabled {
|
||||
log.Println("latest-chapter poller: disabled by config")
|
||||
return
|
||||
return nil
|
||||
}
|
||||
f, err := latest.NewTLSFetcher()
|
||||
if err != nil {
|
||||
log.Printf("latest-chapter poller: disabled, cannot build client: %v", err)
|
||||
return
|
||||
return nil
|
||||
}
|
||||
// Nil browser: sites behind a JavaScript challenge are simply not polled,
|
||||
// and their latest_chapter comes from the userscript alone — which is how
|
||||
@@ -418,4 +361,5 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, brow
|
||||
p := newLatestPoller(s, cfg, f, browser)
|
||||
|
||||
go p.Run(ctx)
|
||||
return p
|
||||
}
|
||||
|
||||
+21
-117
@@ -9,31 +9,15 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/latest"
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
func TestLoadLatestPollDefaults(t *testing.T) {
|
||||
for _, k := range []string{
|
||||
"LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN",
|
||||
"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", "LATEST_CHAPTER_POLL_INTERVAL",
|
||||
"LATEST_CHAPTER_POLL_STAGGER", "LATEST_CHAPTER_POLL_BATCH",
|
||||
} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
|
||||
got := loadLatestPoll()
|
||||
want := LatestPoll{
|
||||
Enabled: true,
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
Interval: 10 * time.Minute,
|
||||
Stagger: 20 * time.Second,
|
||||
Batch: 14,
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, want)
|
||||
t.Setenv("LATEST_CHAPTER_POLL_ENABLED", "")
|
||||
if got := loadLatestPoll(); got != (LatestPoll{Enabled: true}) {
|
||||
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, LatestPoll{Enabled: true})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,105 +47,25 @@ func TestLoadLatestPollEnabledParsing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
wantFrom func(LatestPoll) any
|
||||
want any
|
||||
}{
|
||||
{
|
||||
name: "cooldown below the floor is clamped up",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "1m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Cooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "cooldown at the floor is kept",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "15m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Cooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown below the floor is clamped up",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "1m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown at the floor is kept",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "15m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown override is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "8h"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 8 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "browser cooldown unparseable value falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "six hours"},
|
||||
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
|
||||
want: 6 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "a valid override is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Interval },
|
||||
want: 5 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "an unparseable duration falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "ten minutes"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Interval },
|
||||
want: 10 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "a zero duration falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_STAGGER": "0s"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Stagger },
|
||||
want: 20 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "a valid batch is honoured",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "30"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Batch },
|
||||
want: 30,
|
||||
},
|
||||
{
|
||||
name: "a negative batch falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "-5"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Batch },
|
||||
want: 14,
|
||||
},
|
||||
{
|
||||
name: "a non-numeric batch falls back",
|
||||
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "lots"},
|
||||
wantFrom: func(p LatestPoll) any { return p.Batch },
|
||||
want: 14,
|
||||
},
|
||||
// newLatestPoller wires the fetcher seams; pace lives in the registry, so
|
||||
// nothing here sizes a cooldown any more.
|
||||
func TestNewLatestPollerWiresFetchers(t *testing.T) {
|
||||
tls := &latest.TLSFetcher{}
|
||||
p := newLatestPoller(nil, LatestPoll{Enabled: true}, tls, nil)
|
||||
if p.Fetch != tls {
|
||||
t.Fatalf("Fetch not wired")
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for k, v := range tt.env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
if got := tt.wantFrom(loadLatestPoll()); got != tt.want {
|
||||
t.Fatalf("got %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
if p.BrowserFetch != nil {
|
||||
t.Fatalf("BrowserFetch = %v, want nil for a browser-less deployment", p.BrowserFetch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLatestPollerWiresCooldowns(t *testing.T) {
|
||||
p := newLatestPoller(nil, LatestPoll{
|
||||
Cooldown: time.Hour,
|
||||
BrowserCooldown: 6 * time.Hour,
|
||||
}, nil, nil)
|
||||
if p.Cooldown != time.Hour || p.BrowserCooldown != 6*time.Hour {
|
||||
t.Fatalf("poller cooldowns = %s/%s, want 1h/6h", p.Cooldown, p.BrowserCooldown)
|
||||
if p.CoverFetch != nil {
|
||||
t.Fatalf("CoverFetch = %v, want nil when the browser is absent", p.CoverFetch)
|
||||
}
|
||||
if p.CoverBytesFetch == nil {
|
||||
t.Fatalf("CoverBytesFetch = nil, want the TLS cover fetcher")
|
||||
}
|
||||
if p.Now == nil {
|
||||
t.Fatalf("Now = nil, want the live clock")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1705
-14
File diff suppressed because it is too large
Load Diff
+4
-7
@@ -58,14 +58,11 @@ services:
|
||||
# Second script from the same bindmount; the novel library is a separate
|
||||
# Violentmonkey install.
|
||||
NOVEL_USERSCRIPT_PATH: ${NOVEL_USERSCRIPT_PATH:-/userscript/novel-bookmark.user.js}
|
||||
# Latest-chapter poller. LATEST_CHAPTER_POLL_ENABLED=0 in .env is the kill
|
||||
# switch; it only takes effect because these are listed here.
|
||||
# Latest-chapter poller. LATEST_CHAPTER_POLL_ENABLED=0 in .env is the
|
||||
# kill switch; it only takes effect because it is listed here. Pace is
|
||||
# per Site in the registry (one Poll Lane per Site, issue #100) — the
|
||||
# cooldown/interval/stagger/batch knobs are gone with the shared pace.
|
||||
LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1}
|
||||
LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h}
|
||||
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN: ${LATEST_CHAPTER_POLL_BROWSER_COOLDOWN:-6h}
|
||||
LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m}
|
||||
LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14}
|
||||
LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s}
|
||||
# CDP endpoint for sites behind a JavaScript challenge (kagane,
|
||||
# novelfull). The browser is not part of this stack — it runs on the home
|
||||
# machine as its own unit (chrome/docker-compose.yml) and is reached over
|
||||
|
||||
@@ -19,8 +19,15 @@ and because the Series row now knows how many Readers hold it, the poll queue is
|
||||
absorbs the shortfall. That ordering is only expressible because the split happened.
|
||||
|
||||
Raising throughput instead was rejected: sweeping 400 Series hourly needs the stagger
|
||||
cut from 20s to ~9s, doubling request rate against sites that already bot-score the
|
||||
single VPS IP.
|
||||
cut from 20s to ~9s, doubling request rate against sites already fronted by Cloudflare
|
||||
from the single VPS IP.
|
||||
|
||||
Corrected 2026-08-12: the original wording said those sites "bot-score" the VPS IP.
|
||||
They do not — the 1-99 bot score is Enterprise Bot Management only, and no per-IP
|
||||
request rate is documented as an input to challenge issuance
|
||||
(`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`). The decision stands on
|
||||
its first argument, sweep depth versus the 1-hour cooldown; the rate-limit fear was
|
||||
never evidenced.
|
||||
|
||||
## Only the Poll writes Series fields
|
||||
|
||||
|
||||
@@ -23,8 +23,15 @@ requests: the poller's due query joins bookmarks, production held four kagane
|
||||
series and no bookmarks on any of them, and with no kagane bookmark the web UI
|
||||
never rendered a kagane cover either.
|
||||
|
||||
The home machine has 5.9 GiB of swap and a residential egress, which Cloudflare
|
||||
scores better than a datacenter IP. Both machines were already on the tailnet.
|
||||
The home machine has 5.9 GiB of swap and a residential egress, which avoids the
|
||||
cloud-hosting-IP signature Cloudflare's Bot Fight Mode documentedly challenges. Both
|
||||
machines were already on the tailnet.
|
||||
|
||||
Corrected 2026-08-12: the original wording said Cloudflare "scores" a residential
|
||||
egress better than a datacenter IP. There is no score on a free-plan zone; what is
|
||||
documented is signature matching, and hosting-provider IP space is one of the
|
||||
signatures (`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`). Memory was
|
||||
the load-bearing reason regardless.
|
||||
|
||||
This move is only safe because covers are persisted (ADR-0005's sibling work,
|
||||
issue #43/#45) and the browser is on-demand (ADR-0005). Without stored covers a
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# ADR-0010: Poll Lanes — one independent Poll stream per Site
|
||||
|
||||
Date: 2026-08-16
|
||||
Status: accepted
|
||||
|
||||
## Decision
|
||||
|
||||
Replace the single shared polling pace with one **Poll Lane** per Site: an
|
||||
independent goroutine that polls only that Site's Series, paced by that Site's
|
||||
registry entry. Pace moves out of config and into the Site registry
|
||||
(`internal/latest/sites.go`): every entry carries a `Rest` (how long a Series
|
||||
rests between Polls) and a `Gap` (how long the Lane waits between fetches).
|
||||
|
||||
Rest is enforced by the due query's WHERE clause (`latest_checked_at <= now -
|
||||
Rest`), never by a timer — the same mechanism that enforced the old cooldown.
|
||||
The Lane enforces its own gap by sleeping between fetches. `effectiveGap` is
|
||||
the registry gap, or one hour divided by the Site's eligible Series count when
|
||||
that is smaller, never below one second.
|
||||
|
||||
The five environment settings that used to size the shared pace —
|
||||
`LATEST_CHAPTER_POLL_COOLDOWN`, `_BROWSER_COOLDOWN`, `_INTERVAL`, `_BATCH`,
|
||||
`_STAGGER` — are deleted. Only the kill switch `LATEST_CHAPTER_POLL_ENABLED`
|
||||
remains. No deployed `.env` may carry the deleted knobs.
|
||||
|
||||
## Why
|
||||
|
||||
The shared pace capped the whole backend at roughly 180 Polls an hour (one
|
||||
20-second stagger across one queue). ~60 Series today, scaling to hundreds or
|
||||
thousands, would stretch the hour beyond what the New Chapter signal can
|
||||
tolerate. Worse, the queue mixed Sites with very different costs: kagane and
|
||||
comix pay seconds of a serialized single-tab Chrome per Poll (a challenged
|
||||
page, ADR-0005), and one hostile Site burning its challenge timeout made every
|
||||
other Site's Series wait — "one hostile Site can eat most of an hour".
|
||||
|
||||
Lanes fix both at once:
|
||||
|
||||
- **Throughput scales per Site.** The six Lanes fetch concurrently; a Lane's
|
||||
own gap paces it. The browser Lanes' combined ceiling stays about 360 Polls
|
||||
an hour (one tab), and when they cannot keep up the wait past Rest grows and
|
||||
is logged every pass — the "behind by X" measurement, so the decision to
|
||||
give browser Sites more pages is made from data.
|
||||
- **Hostility is contained.** A refusal (two challenge-held reads in one
|
||||
pass) stops only that Site's Lane for `refuseBackoff` (15m); the rest of
|
||||
that Lane's Series stay unstamped and due. A lost browser gates the other
|
||||
browser Lanes' passes for the same window — the flag is shared Poller
|
||||
state, so the loss is noticed once instead of once per Lane per pass, and
|
||||
decays after 15m so the Lanes probe again. One Site can no longer tax the
|
||||
others.
|
||||
|
||||
## Tradeoffs and rejections
|
||||
|
||||
- **Per-Site env knobs** (e.g. `KAGANE_POLL_GAP`) rejected: the registry is
|
||||
the single place pace lives, testable and reviewable; config knobs would
|
||||
recreate the shared-pace sprawl with six times the surface. All six entries
|
||||
are deliberately uniform at first — rest an hour, gap ten seconds — so the
|
||||
structure exists to differ without inventing numbers for Sites that have
|
||||
not earned them.
|
||||
- **Dynamic gap** (`rest / eligible`) is the one knob that stays automatic:
|
||||
a Site with more Series than one per ten seconds would otherwise back up
|
||||
behind its own gap, and the per-Series share of the hour is the natural
|
||||
pace. The ten-second default is not arbitrary: one request per ten seconds
|
||||
is the strictest rate rule a free-plan Site can even express (per-zone
|
||||
rate limiting, as documented in
|
||||
`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`), so the
|
||||
default pace is exactly what the most restrictive Site would demand of us.
|
||||
The computed gap never goes below one second and logs loudly when the
|
||||
floor engages.
|
||||
- **Timer-based pacing** rejected: the old ticker made the poller's rate a
|
||||
function of wall clock rather than of what was due. The due-query cutoff is
|
||||
the only rate authority; the Lane sleep just prevents hammering.
|
||||
- **Batch size** (the old `_BATCH` cap) is gone with the shared pace: a Lane
|
||||
processes everything due, paced by its gap. There is no global queue left
|
||||
to bound.
|
||||
|
||||
## Constraints preserved
|
||||
|
||||
- Stamp-before-fetch ("attempted" semantics): an untried Series stays due, so
|
||||
a browser that appears after a restart finds its full queue waiting.
|
||||
- Browser wake gate (ADR-0005): a browser Lane leaves Chrome asleep below
|
||||
five due Series and 15 minutes of wait, per Lane.
|
||||
- The browser is not in the API stack (ADR-0006): an unreachable browser
|
||||
degrades a Lane exactly as an unset `BROWSER_WS_URL` — browser-only Sites
|
||||
skipped, plain-TLS unaffected, stored covers still served.
|
||||
- Cover heals moved to background goroutines (joined by the test suite via
|
||||
`waitCovers`) so a slow cover CDN cannot consume a Lane's gap.
|
||||
|
||||
Supersedes the pace mechanics of ADR-0003's "raise throughput instead" note
|
||||
(the stagger cut it rejected is what the per-Lane gap replaces) and the
|
||||
6-hour browser cooldown introduced with the browser-backed Sites; the
|
||||
1-hour browser rest was already cleared as safe by
|
||||
`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`.
|
||||
@@ -0,0 +1,139 @@
|
||||
# ADR-0011: Sightings — a Reader report defers a Poll where being wrong hurts only them
|
||||
|
||||
Date: 2026-08-16
|
||||
Status: accepted
|
||||
|
||||
## Decision
|
||||
|
||||
A **Sighting** is the Latest Chapter the Reader's own browser read off the
|
||||
Series page and PUT to the backend. It is now allowed to stand in for a Poll,
|
||||
under one restriction and one ceiling:
|
||||
|
||||
- **Solitary Series only.** A Sighting defers the Poll of a Series exactly one
|
||||
Bookmark points at. A Series two Readers share is Polled on schedule no matter
|
||||
how recently it was sighted.
|
||||
- **One rest of standing.** A Sighting postpones Polls for one Rest
|
||||
(`defaultRest`, an hour), not forever: a Series nobody visits again returns to
|
||||
the normal schedule by itself.
|
||||
- **Six-rest ceiling.** `sightingCeilingRests = 6`, counted in the Site's own
|
||||
Rest — six hours everywhere today. However many Sightings arrive, a Series
|
||||
unpolled that long is Polled.
|
||||
|
||||
Both live in the due query's HAVING clause (`store.DueForLatestCheck`), beside
|
||||
the Rest cutoff — the same place the schedule has always been decided, so no
|
||||
timer and no second code path can disagree with it.
|
||||
|
||||
Attribution and judgement:
|
||||
|
||||
- `Store.RecordSighting` runs *before* the Upsert that stores the reported
|
||||
value, because the raise test needs the row as it stands. A report that raises
|
||||
the stored Latest Chapter names its Reader in `series.latest_raised_by`.
|
||||
- The Poll is the oracle. `Poller.checkOne` already compares what the Site
|
||||
publishes against what is stored, so judgement costs no extra request: a lower
|
||||
number contradicts the Sighting (`sighting_disagreements + 1`, both numbers and
|
||||
the Reader logged), the same number confirms it (`sighting_agreements + 1`), a
|
||||
**higher** number is the Site publishing and means nothing either way — but it
|
||||
does clear the attribution (`Store.ClearSightingAttribution`), because the
|
||||
value stored afterwards is the Poll's own and nobody must answer for it.
|
||||
- At `SightingDisagreementLimit` (3) that Reader's Sightings stop deferring
|
||||
anything. They still write the Latest Chapter — the penalty removes a
|
||||
privilege, it does not silence anyone.
|
||||
- `SightingAgreementsToClear` (20) consecutive confirmations forgive the
|
||||
disagreements. A disagreement resets the run to zero.
|
||||
- The owner clears marks from the administration page (issue #102, shipped
|
||||
first precisely so a false mark has a remedy the day the mechanism lands).
|
||||
|
||||
One client change was required, and only one. Both userscripts stopped short of
|
||||
PUTting a read whose number had not moved (`applyLatestChapterIfChanged`), so
|
||||
the case this whole mechanism exists for — visiting a Series with nothing new —
|
||||
never reached the backend. `reportLatestChapter` now sends it, skipping only the
|
||||
local write and the re-render. A numberless PUT (favourite toggle, progress from
|
||||
a chapter page) is not a Sighting and defers nothing: nobody read the Series
|
||||
page, so there would be nothing to judge later.
|
||||
|
||||
## Why
|
||||
|
||||
Most of the backend's work was redundant. The userscript reads the Latest
|
||||
Chapter on every Series page visit; minutes later the Poll Lane fetches the same
|
||||
page for the same number. Deferring on a report converts a visit into a Poll
|
||||
saved, which is Lane capacity handed back to Series nobody is
|
||||
reading.
|
||||
|
||||
The restriction is the whole safety argument, and it is about **blast radius**,
|
||||
not about trust arithmetic:
|
||||
|
||||
- On a solitary Series, a wrong report can only mislead the Reader who made it.
|
||||
There is nobody else's ember to falsify.
|
||||
- On a shared Series it could mislead someone else, so a report never postpones
|
||||
anything there.
|
||||
|
||||
The ceiling bounds the damage in time: a false value dies within six hours
|
||||
whatever happens, because the Poll that finds it is guaranteed. That is also
|
||||
what makes lying pointless — the six-hour audit is certain, not sampled, so a
|
||||
determined attacker buys at most three ceilings' worth of a wrong number on
|
||||
their own Series and then loses deferral entirely.
|
||||
|
||||
The cost of recovery is deliberate. An agreement is only recorded when a later
|
||||
Poll confirms a Sighting, so twenty agreements are twenty Polls of Series that
|
||||
Reader bookmarks — hours to days of real time, not twenty page views. Waiting is
|
||||
therefore not a strategy, and credit cannot be banked in advance.
|
||||
|
||||
## Tradeoffs and rejections
|
||||
|
||||
- **Trusting a Sighting on a shared Series** rejected: it is the only case where
|
||||
one Reader's mistake reaches another Reader's list, and no amount of
|
||||
reputation makes that recoverable within the six-hour window.
|
||||
- **Cross-Reader agreement, voting, weighting, consensus scoring** rejected on
|
||||
evidence: every truth-discovery method estimates source reliability by
|
||||
comparing sources on the same object, and the standard survey states outright
|
||||
that an object provided by very few sources cannot have its confidence
|
||||
evaluated — Li, Gao, Meng, Li, Su, Zhao, Fan, Han, *A Survey on Truth
|
||||
Discovery*, SIGMOD Record 45(1), 2016 (arXiv:1505.02463), §"Challenges" on
|
||||
sparse sources. With the two Readers this backend actually has, a
|
||||
disagreement is a coin flip. The Poll is an authoritative oracle, so it is
|
||||
the only judge.
|
||||
- **A randomised audit** (Poll a fraction of deferred Series) rejected in favour
|
||||
of the fixed ceiling. Sampling an oracle against untrusted reports is the
|
||||
gold-question technique from crowdsourcing quality control — Le, Edmonds,
|
||||
Hester, Biewald, *Ensuring quality in crowdsourced search relevance
|
||||
evaluation: the effects of training question distribution*, SIGIR 2010
|
||||
Workshop on Crowdsourcing for Search Evaluation, which inserts known answers
|
||||
sporadically and adjusts each worker's trust from them. The ceiling is the
|
||||
same idea made deterministic: sampling prices an attack in expectation, a
|
||||
guaranteed six-hour audit prices it as a certainty, which is what makes the
|
||||
solitary-Series rule defensible in one sentence.
|
||||
- **A trust *ratio*** (agreements over judgements, as that same gold-question
|
||||
scheme uses) rejected for two thresholds: a ratio lets an attacker bank
|
||||
credit first and spend it on lies later, and it needs the owner watching a
|
||||
score to act. Three-and-twenty is a threshold both ways — a disagreement
|
||||
resets the run to zero, so credit cannot be pre-bought, and recovery happens
|
||||
without the owner in the loop.
|
||||
- **Blocking a marked Reader's writes** rejected: the Latest Chapter they report
|
||||
is still the best available value, and their Sightings must keep being judged
|
||||
or they could never earn the privilege back.
|
||||
- **Per-Series flagging** rejected in favour of per-Reader marks: a Series is
|
||||
not the thing that can be wrong. Naming the Reader and logging both numbers is
|
||||
also what distinguishes a broken Site adapter (every Reader of that Site
|
||||
contradicted at once) from one bad actor.
|
||||
- **Timers or a background reputation job** rejected: deferral is recomputed
|
||||
from live facts every round — Bookmark count and sighting timestamp — so a
|
||||
Series that gains a second Bookmark stops deferring at once, with nothing to
|
||||
invalidate. The Reader's marks are the one input read earlier, when the
|
||||
Sighting is recorded rather than when the round runs: a Reader who crosses
|
||||
the threshold, or has their marks cleared, changes behaviour from their next
|
||||
Sighting on, and the standing they already bought lasts out its rest. That is
|
||||
bounded by one rest and costs one subselect instead of joining `readers` into
|
||||
the due query on every round.
|
||||
|
||||
## Constraints preserved
|
||||
|
||||
- A Sighting is not Progress: it may move the Latest Chapter and nothing else.
|
||||
`updated_at` never moves, so a report cannot reorder the list (ADR-0004).
|
||||
- The Latest Chapter is a Series-level fact (ADR-0003): a Sighting writes the
|
||||
shared row, so every Reader of a shared Series sees it immediately — deferral
|
||||
is the only thing the solitary rule withholds.
|
||||
- Ember means new chapter only (`docs/design-system.md`): a marked Reader
|
||||
renders no differently in their own list, and nothing about the trust model
|
||||
reaches the Series list's colour.
|
||||
- The Poll remains authoritative. Where a Sighting and a Poll disagree, the
|
||||
Poll's value is what gets stored.
|
||||
@@ -0,0 +1,44 @@
|
||||
# ADR-0012: Persisted lane state
|
||||
|
||||
Date: 2026-08-21
|
||||
Status: accepted
|
||||
|
||||
Supersedes the in-memory lane snapshot carried by `latest`'s `LaneState`/`Status`
|
||||
and the `web.LaneReporter` seam (ADR-0010 wrote the durable rows this page now
|
||||
reads).
|
||||
|
||||
## Decision
|
||||
|
||||
The admin Lanes page stops reading the poller's in-memory Lane state and
|
||||
becomes a read of `poll_passes`/`poll_lanes` in Postgres. There is no
|
||||
`LaneReporter` interface: `web/admin_lanes.go` walks `store.LatestLanePasses()`
|
||||
into one row per Site and adds the window's outcome sums from
|
||||
`store.LanePassOutcomes()`. The `latest` package's `LaneState`/`Status` snapshot
|
||||
and its `web.LaneReporter` seam are deleted.
|
||||
|
||||
The browser is a deployment configuration fact plus a reachability derived
|
||||
from the pass log: `BROWSER_WS_URL` set means "configured", and the browser is
|
||||
"reachable" unless a recent browser-Site pass inside `latest.RefuseBackoff` is
|
||||
a sidecar loss, a missing fetcher, or an interrupted read. A skip reason is
|
||||
the whole difference between a Lane resting and a Lane stuck: a skipped pass
|
||||
prints its sentence, and only an empty skip with Series due and none read
|
||||
draws the true-stall fault. Sleep skips never count toward `Attention`.
|
||||
|
||||
## Why
|
||||
|
||||
The old page lived on a poller snapshot. Because that state was in memory, a
|
||||
deploy erased it: the page read zeroes until a fresh pass ran, and browser
|
||||
reachability came through a reporter interface only a live poller could
|
||||
serve. Making the page answer from the database means a restart is complete
|
||||
the instant the store is up, the browser fact survives a poller restart, and
|
||||
a Lane that has not yet gathered figures shows a placeholder rather than a
|
||||
confident zero.
|
||||
|
||||
## Constraints
|
||||
|
||||
The poller still owns the writes: each pass exit records one row (ADR-0010),
|
||||
and a pass that returns before gathering figures carries the previous pass's
|
||||
numbers forward instead of recording zeroes. A skip is a stable wire string;
|
||||
`asleep` never counts toward `Attention`. When polling is switched off
|
||||
(`LATEST_CHAPTER_POLL_ENABLED` unset) the page must say so, and the browser
|
||||
statusline appears only when polling is switched on.
|
||||
@@ -0,0 +1,60 @@
|
||||
# ADR-0013: Commands through the database
|
||||
|
||||
Date: 2026-08-22
|
||||
Status: accepted
|
||||
|
||||
## Decision
|
||||
|
||||
Owner interventions are **facts about rows, never commands to the poller**.
|
||||
*Check now* (`POST /admin/series/{key}/poll`) writes one stamp —
|
||||
`series.force_poll_at`, unix ms, zero meaning never asked (the column landed
|
||||
in migration 0014) — and the poller's next pass reads it through
|
||||
`Store.DueForLatestCheck`. The control never signals the running process, so
|
||||
a request survives a restart, and the whole surface is testable with no
|
||||
poller running at all.
|
||||
|
||||
**Pending is derived, never stored**: a Series is pending while
|
||||
`force_poll_at > latest_checked_at`. It self-clears with no second write and
|
||||
no sweeper because the check stamp is written *before* the fetch (the same
|
||||
"attempted" discipline as ADR-0010) — the first attempt ends the pending
|
||||
state whatever the attempt returns. There is no expiry: a request the Lane
|
||||
never reaches keeps ageing in the UI, and an old pending marker is itself the
|
||||
evidence that a Lane is stuck. Writing again re-stamps the request time; the
|
||||
write is idempotent.
|
||||
|
||||
**Queue-jump rules.** A forced Series overrides exactly three gates in the
|
||||
due query: the rest cutoff, the Sighting-deferral clause and the finished-only
|
||||
bucket, and it sorts to the front of the queue
|
||||
(`ORDER BY forced DESC, reader_count DESC, latest_checked_at ASC`). It never
|
||||
overrides an empty `series_url` (nothing to fetch), the Bookmarks join (a
|
||||
Series no Reader holds has no consumer for the result), the Lane's refusal
|
||||
backoff, the sidecar-down skip, or the Lane's gap — the last three are
|
||||
poller-side gates the query cannot see and must not. The one pass-level gate
|
||||
a forced Series does open is the browser wake threshold: a human asking wakes
|
||||
a sleeping Chrome, where the thresholds exist to stop the machine waking
|
||||
itself for one unattended check. If the home machine is off, nothing happens
|
||||
and the request ages visibly, which is correct.
|
||||
|
||||
Rejected: zeroing the check stamp as the force signal. It would corrupt the
|
||||
never-checked and stale counts the landing page exists to show, and make a
|
||||
pending marker impossible.
|
||||
|
||||
## Why
|
||||
|
||||
A stuck-looking Series previously waited for its turn in the Lane's hour, and
|
||||
there was no way to ask for one check sooner. A direct poller command would
|
||||
have been lost on every restart and untestable without a running poller; a
|
||||
row the poller already reads is neither. Deriving pending from the two stamps
|
||||
keeps the flag honest across restarts and makes the mechanism two column
|
||||
writes and three query clauses instead of a state machine.
|
||||
|
||||
## Constraints
|
||||
|
||||
- The finished-status clause the force flag overrides is today's Lifecycle
|
||||
test; a later spec in this series deletes it wholesale rather than amending
|
||||
it, so the clause stays as it stands.
|
||||
- The control is unconfirmed (it takes nothing away) and renders no
|
||||
`.confirm-row`; it is hidden on a Series with no `series_url` and on an
|
||||
orphan — the same pair the due query refuses to override.
|
||||
- The answer to a press is the freshly rendered row, so the figures describe
|
||||
the state after the press.
|
||||
+17
-4
@@ -10,7 +10,7 @@ Implemented in:
|
||||
|
||||
| Surface | Files |
|
||||
| --- | --- |
|
||||
| Web UI (login, list, card, empty, errors) | `backend/internal/web/static/style.css`, `backend/internal/web/templates/{app,card,list,login,chrome,icons}.html`, `backend/internal/web/static/filter.js` |
|
||||
| Web UI (login, list, card, empty, errors, admin) | `backend/internal/web/static/style.css`, `backend/internal/web/templates/{app,admin,lanes,readers,card,list,login,chrome,icons}.html`, `backend/internal/web/static/filter.js` |
|
||||
| Userscript panel (Shadow DOM) | `userscript/manga-bookmark.user.js` — `TEMPLATE` and `CSS` at the bottom of the IIFE |
|
||||
|
||||
## 1. The one idea
|
||||
@@ -73,6 +73,7 @@ Defined once in `backend/internal/web/static/style.css` `:root`, mirrored in the
|
||||
| `--moss` | `#7fae86` | `#3d6c46` | finished accent |
|
||||
| `--clay` | `#b5906f` | `#7c5533` | set-chapter accent |
|
||||
| `--trash` | `#977671` | `#8c6558` | remove, at rest — icons need 3:1, not 4.5:1 |
|
||||
| `--patina` | `#5fb3a6` | `#1f6f66` | admin page only — a Poll Lane needing attention, a Reader whose reports are blocked |
|
||||
| `--play-hot-line` | `#3a1d18` | `#f0cfc6` | desktop cell border, play when `.is-new` |
|
||||
| `--fav-line` | `#332b14` | `#e3d3a4` | desktop cell border, favourite when on |
|
||||
| `--asura` | `#7d93a5` | `#4f6b80` | site tag |
|
||||
@@ -81,9 +82,13 @@ Defined once in `backend/internal/web/static/style.css` `:root`, mirrored in the
|
||||
| `--kagane` | `#9a8aa5` | `#6f5f7d` | site tag |
|
||||
| `--hatch` / `--hatch-dim` | 135° 5px stripe | paper stripe | missing-cover slot |
|
||||
|
||||
`--slate`/`--moss`/`--clay`/`--brass` are held at the same weight deliberately:
|
||||
one accent per action, so a press says which lane it belongs to, with none of
|
||||
them competing with ember. Dark is the default (`color-scheme: dark light`);
|
||||
`--slate`/`--moss`/`--clay`/`--brass`/`--patina` are held at the same weight
|
||||
deliberately: one accent per meaning, so a press says which lane it belongs to,
|
||||
with none of them competing with ember. `--patina` is the admin page's only
|
||||
colour — a cool verdigris, the far side of the wheel from ember's crimson and
|
||||
clear of the archive blue: system health is neither a new chapter nor
|
||||
destruction, so it borrows neither `--ember` nor `--danger`.
|
||||
Dark is the default (`color-scheme: dark light`);
|
||||
light is a `@media (prefers-color-scheme: light)` override of the same names.
|
||||
**Any new colour must be added in both branches** — light is not a filter over
|
||||
dark, the hues are re-tuned.
|
||||
@@ -140,6 +145,14 @@ Recurring specs (copy these rather than inventing sizes):
|
||||
main#list article.card … | .empty
|
||||
```
|
||||
|
||||
The owner's admin page (`admin.html`) is the same sheet with two sections in
|
||||
place of the list — `.lanes` (Poll Lane rows) and `.readers` (the roster) —
|
||||
and no library switch: it belongs to neither library, so its topbar carries a
|
||||
plain `.ghost.back` link home. Both sections are eyebrow + hairline-separated
|
||||
rows, the shape the roster already had as a fold-out. `.lanes` refreshes itself
|
||||
every 30s via `hx-get="/ui/admin/lanes"` with `hx-swap="outerHTML"`; the roster
|
||||
re-renders only in answer to an action.
|
||||
|
||||
**Brand mark**: an inline `<svg class="mark">` (`viewBox="0 0 200 172"`),
|
||||
defined once in `chrome.html`'s `mark` template and reused by `app.html` and
|
||||
`login.html` so it takes the page's `--ink`/`currentColor`/`--ember` rather
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
# Cloudflare bot scoring and poll cadence — what is actually documented
|
||||
|
||||
Research note for the browser-backed poller cadence decision (kagane.to, novelfull.com, comix.to). All pages were fetched live from **developers.cloudflare.com / blog.cloudflare.com on 2026-08-12**. Primary sources only: Cloudflare's own documentation, Cloudflare blog posts, and RFCs/standards where noted. Where Cloudflare does not publicly document something, this note says **`Not publicly documented`** instead of guessing. Repo-measured facts from the existing poller work are reused without re-derivation and marked as such.
|
||||
|
||||
The site configuration of the three challenged sites (which plan, which bot product, Challenge Passage setting, whether Precursor is enabled) is **not observable from outside** — Cloudflare does not expose a zone's security configuration to anonymous clients. Anything in this note that depends on those unknowns is flagged `[INFERENCE]`.
|
||||
|
||||
---
|
||||
|
||||
## Short answer
|
||||
|
||||
**No — polling once per hour per series, from one residential IP through one real Chrome holding a valid `cf_clearance`, carries no documented challenge risk beyond polling every six hours.** Challenge issuance on Free/Pro-grade protection (Bot Fight Mode, WAF rules) is signature- and fingerprint-driven (headless browsers, cloud-hosted IPs, browser signals); the only rate-aware detector — the per-request bot score — exists solely on Enterprise Bot Management, and free-plan Rate Limiting Rules count per-IP over 10-second windows, which 20–60 requests/hour cannot trip. Both cadences re-solve the challenge every visit anyway: `cf_clearance` expires after **30 minutes by default** (site-configurable), so a 1-hour gap always finds it expired. The documented lever that matters — already verified in this repo — is **fingerprint quality**: real Chrome + real timezone clears in ~4 s; headless variants never do. Residual, undocumented risk is site-specific: Challenge Passage, Precursor (behavior-bound re-challenge), and custom WAF rules are zone settings not observable from outside.
|
||||
|
||||
---
|
||||
|
||||
## Summary answer table
|
||||
|
||||
| Question | Answer | Section |
|
||||
|---|---|---|
|
||||
| What is a bot score? | Integer 1–99 = Cloudflare's certainty a request is automated; **Enterprise Bot Management only**; everyone else gets coarse "bot groupings" (Pro+ analytics) or nothing. | §1 |
|
||||
| Is request frequency documented as a bot-score input? | Partially: ML inputs are "headers, session characteristics, and browser signals"; the `__cf_bm` cookie "measures a single user's request pattern". **No numeric rate threshold is documented.** Volume policing is Rate Limiting, a separate product. | §1, §5 |
|
||||
| What can a free-plan site deploy? | Bot Fight Mode only: challenges *signatures* (headless browsers, cloud-hosting IPs) with a computational challenge; JavaScript Detections forced on; no scores, no analytics, cannot be skipped/customized. | §2 |
|
||||
| Does the free tier score continuously? | **No.** No score exists on Free at all — granular scores need Enterprise Bot Management, groupings need Pro+. BFM just challenges signature matches. | §2 |
|
||||
| What does `cf-mitigated: challenge` mean? | The response was a Cloudflare Challenge Page (any type); `challenge` is the only value; body is always `text/html`. | §3 |
|
||||
| How is a successful solve remembered? | `cf_clearance` cookie, issued with `SameSite=None; Secure; Partitioned`; suppresses challenges while valid. | §3, §4 |
|
||||
| `cf_clearance` lifetime? | **30 minutes by default**, configurable by the site via Challenge Passage (15–45 min recommended); +skew minutes; +1 h for XHR. | §4 |
|
||||
| Is `cf_clearance` bound to IP / device? | Documented: "securely tied to the specific visitor and device it was issued to"; the *solve request* must come from the same IP that received the challenge (different IP → invalid solve → challenge loop). Replay from another machine/IP is therefore **not** valid. | §4 |
|
||||
| What invalidates clearance early? | Precursor (if enabled): suspicious session → clearance reduced/invalidated, re-challenge even before expiry. Zone-level toggle; unobservable from outside. | §4 |
|
||||
| Does polling more often raise challenge risk? | **No documented mechanism at 20–60 req/hour.** Free-plan rate limiting is 10 s/IP-only; DDoS thresholds are ~1,000 errors/sec. Scores (the only rate-aware thing) are Enterprise-only. | §5 |
|
||||
| Is there a documented "legitimate poller" path? | Yes, but it requires **self-identification** (Web Bot Auth signature or published IP list + stable UA) via the verified-bots application — not anonymity. robots.txt is voluntary; nothing exempts anonymous scrapers. | §6 |
|
||||
|
||||
---
|
||||
|
||||
## 1. What a bot score is and what feeds it
|
||||
|
||||
### 1.1 The score itself
|
||||
|
||||
Cloudflare documents the bot score as "a score from _1_ to _99_ that indicates how likely that request came from a bot" — 1 = quite certain automated, 99 = quite certain human. Source: [Bot scores — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot-score/), read 2026-08-12.
|
||||
|
||||
Two access tiers, both gated:
|
||||
|
||||
- **Granular 1–99 scores are only available to Enterprise customers who purchased Bot Management.** "All other customers can only access this information through bot groupings in Bot Analytics" (categories: `Not computed` = 0, `Automated` = 1, `Likely automated` = 2–29, `Likely human` = 30–99, `Verified bot`). Bot groupings themselves require "a Pro plan or higher". Source: [Bot scores — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot-score/), read 2026-08-12.
|
||||
- A score of 0 means "Bot Management did not evaluate the request" (redirected, handled by another feature) — "does not indicate the request is safe or human". Same source.
|
||||
|
||||
So: **on a Free-plan site there is no bot score at all, for anyone.** `[INFERENCE]` the three challenged sites are almost certainly not Enterprise Bot Management customers, but this is not externally verifiable.
|
||||
|
||||
### 1.2 The detection engines (Enterprise Bot Management)
|
||||
|
||||
Cloudflare documents four engines, all stated to apply to Enterprise Bot Management (the bot-score page: "The following detection engines only apply to Enterprise Bot Management"). Sources: [Bot scores — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot-score/) and [Bot detection engines — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot-detection-engines/), both read 2026-08-12.
|
||||
|
||||
| Engine | Documented behavior | Score it produces |
|
||||
|---|---|---|
|
||||
| **Heuristics** | "Processes all requests"; pattern matching against "a growing database of malicious fingerprints". | 1 for high-confidence deterministic detections; occasionally 29 "where Cloudflare has identified automated traffic and is still assessing traffic overlap" |
|
||||
| **Machine learning** | Supervised model, trained on "billions of daily requests". Input variables: "headers, session characteristics, and browser signals". Output: "predicted probability that a client is human (such as the probability of successfully solving a Challenge)". | Most scores 2–99 |
|
||||
| **Anomaly detection** | Unsupervised; learns a per-domain baseline, flags outlier requests; **deprecated, not onboarding new customers**. | 1 |
|
||||
| **JavaScript detections** | "Identifies headless browsers and other automation tools" via "a lightweight, invisible JavaScript injection"; runs client-side; "blocks, challenges, or passes requests to other engines". Enabled by default (but optional) in Bot Management. | Pass/fail (`cf.bot_management.js_detection.passed`), not a score |
|
||||
|
||||
Crucially, the ML engine's documented inputs are *headers, session characteristics, and browser signals* — **no rate or per-IP volume parameter is listed.** The only place request patterns appear is the `__cf_bm` cookie note: "Cloudflare uses the `__cf_bm` cookie to smooth out the bot score and reduce false positives… The Bot Management cookie measures a single user's request pattern and applies it to the machine learning data to generate a reliable bot score for all of that user's requests." ([Bot scores — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot-score/), read 2026-08-12). So frequency is *a* signal inside Enterprise Bot Management via the session cookie — **but no numeric threshold, window, or per-IP rate is published anywhere.** `Not publicly documented`: any specific requests-per-hour / requests-per-IP value that raises or lowers a bot score.
|
||||
|
||||
### 1.3 Rate limiting is a separate product
|
||||
|
||||
Volume enforcement is not part of bot scoring at all. Rate Limiting Rules are a distinct WAF product with their own evaluation phase (`http_ratelimit`, running after custom rules and before SBFM). Sources: [Rate limiting rules — Cloudflare docs](https://developers.cloudflare.com/waf/rate-limiting-rules/) and [Security features interoperability — Cloudflare docs](https://developers.cloudflare.com/waf/feature-interoperability/), read 2026-08-12. See §5 for what the Free plan's version of that product can actually do.
|
||||
|
||||
---
|
||||
|
||||
## 2. The free-plan reality
|
||||
|
||||
### 2.1 What each plan gets
|
||||
|
||||
Cloudflare's plan table ([Plans — Cloudflare docs](https://developers.cloudflare.com/bots/plans/), read 2026-08-12):
|
||||
|
||||
| Plan | Bot product | Documented detections | Action | Control |
|
||||
|---|---|---|---|---|
|
||||
| **Free** | **Bot Fight Mode** (BFM) | "Simple bots from cloud hosting providers and headless browsers" | "Cloudflare issues a computationally expensive challenge" | Applied to all traffic across the domain; no exceptions possible |
|
||||
| **Pro / Business / Enterprise (no BM)** | **Super Bot Fight Mode** (SBFM) | Configurable actions per bot category (Definitely automated / Likely automated / Verified bots) | Challenge or block | Runs on Ruleset Engine; **can** be skipped via custom rules |
|
||||
| **Enterprise + Bot Management** | Bot Management | "Simple and sophisticated bots, headless browsers, and domain-specific anomalies" | Customer-chosen (block, challenges) | Per-path / per-IP rules; access to bot score, JA3/JA4, bot tags, detection IDs |
|
||||
|
||||
Sources: [Plans — Free](https://developers.cloudflare.com/bots/plans/free/), [Plans — Bot Management for Enterprise](https://developers.cloudflare.com/bots/plans/bm-subscription/), [Bot Fight Mode — Cloudflare docs](https://developers.cloudflare.com/bots/get-started/bot-fight-mode/), [Super Bot Fight Mode — Cloudflare docs](https://developers.cloudflare.com/bots/get-started/super-bot-fight-mode/) — all read 2026-08-12.
|
||||
|
||||
### 2.2 Bot Fight Mode specifics (the Free-plan product)
|
||||
|
||||
- Identifies "traffic matching patterns of known bots" and "issues computationally expensive challenges that force the requesting client to perform CPU-intensive calculations". ([Bot Fight Mode — Cloudflare docs](https://developers.cloudflare.com/bots/get-started/bot-fight-mode/), read 2026-08-12.)
|
||||
- It "does not run on the Ruleset Engine — it operates in a separate evaluation pipeline where _Skip_, _Bypass_, and _Allow_ actions have no effect"; **you cannot bypass or skip BFM** with custom rules or Page Rules. ([Bot Fight Mode — Cloudflare docs](https://developers.cloudflare.com/bots/get-started/bot-fight-mode/) and [Security features interoperability — Cloudflare docs](https://developers.cloudflare.com/waf/feature-interoperability/), read 2026-08-12.)
|
||||
- **JavaScript Detections is automatically enabled for BFM customers and cannot be disabled.** ([Bot Fight Mode — Cloudflare docs](https://developers.cloudflare.com/bots/get-started/bot-fight-mode/), read 2026-08-12.) This is the documented hook that explains the repo's `headless-shell` / `HeadlessChrome` failures: JSD "identifies headless browsers" ([JavaScript detections — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/), read 2026-08-12).
|
||||
- False positives on *legitimate automated* traffic are acknowledged as expected behavior: "false positives can occur where legitimate human or automated traffic is incorrectly challenged or blocked", and the only remedies are disabling BFM or upgrading to Bot Management. ([Handle False Positives — Cloudflare docs](https://developers.cloudflare.com/bots/troubleshooting/false-positives/), read 2026-08-12.)
|
||||
|
||||
### 2.3 Does the free tier "score" continuously?
|
||||
|
||||
**No.** The Free plan exposes no score, no bot analytics (groupings need Pro+), and no per-request decision data. BFM is a static on/off toggle that challenges signature matches; there is no continuous per-request score on Free. ([Plans — Free](https://developers.cloudflare.com/bots/plans/free/) and [Bot scores — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot-score/), read 2026-08-12.) JSD *runs* on every HTML request even on Free (forced by BFM), but the only documented way to act on its result — the `cf.bot_management.js_detection.passed` field — is gated behind an Enterprise Bot Management subscription ("Prerequisites: You must have an Enterprise Bot Management subscription"). ([JavaScript detections — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/), read 2026-08-12.) On Free, JSD output feeds BFM's internal challenge decision; `Not publicly documented` exactly how.
|
||||
|
||||
`[INFERENCE]` The observed behavior on the three sites (real Chrome + real timezone passes in ~4 s; headless variants never do) is consistent with either BFM or a WAF custom rule using a challenge action — Cloudflare does not expose which product a site runs, and the failure signature (HeadlessChrome UA / headless-shell never passing) matches JSD's documented headless-browser detection either way.
|
||||
|
||||
---
|
||||
|
||||
## 3. Managed Challenge / JS challenge mechanics and `cf-mitigated`
|
||||
|
||||
### 3.1 What the observed response is
|
||||
|
||||
A `403` with `cf-mitigated: challenge`, `server: cloudflare`, and a "Just a moment…" body is a **Cloudflare Challenge Page**. Cloudflare documents: "the Challenge Page response (regardless of the Challenge Page type) will have the `cf-mitigated` header present and set to `challenge`… `challenge` is the only valid value. The header is set for all Challenge Page types", and "the content-type of a challenge will be `text/html`". ([Detect a Challenge Page response — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/detect-response/), read 2026-08-12.)
|
||||
|
||||
### 3.2 What a Challenge Page does
|
||||
|
||||
"An interstitial Challenge Page… acts as a gate between the visitor and your website… The Challenge Page intercepts the visitor… by holding the request and evaluating the browser environment for automated signals, and serving a challenge. The visitor cannot reach their destination without passing the challenge." ([Interstitial Challenge Pages — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/), read 2026-08-12.)
|
||||
|
||||
Three variants, in increasing severity ([Interstitial Challenge Pages — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/), read 2026-08-12):
|
||||
|
||||
- **Non-Interactive**: Cloudflare judges automation from browser signals gathered by injected JS; the page needs no human interaction, typically < 5 s of JS processing.
|
||||
- **Managed Challenge**: "Cloudflare dynamically chooses the appropriate type of challenge… based on the characteristics of a request from the signals indicated by their browser. Most human visitors are automatically verified and the Challenge Page will display **Successful**. However, if Cloudflare detects non-human attributes… they may be required to interact." Cloudflare's stated recommendation for WAF rules.
|
||||
- **Interactive**: requires explicit human interaction (CAPTCHA-style). Cloudflare's "End the CAPTCHA era" position is that Managed Challenges should make this rare.
|
||||
|
||||
Cloudflare's own framing matches the repo's ~4 s real-Chrome solve: a normal browser passes with no interaction (Managed Challenge auto-verify or Non-Interactive JS processing).
|
||||
|
||||
### 3.3 Which product issues which challenge
|
||||
|
||||
Documented mapping ([How Challenges work — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/concepts/how-challenges-work/), read 2026-08-12):
|
||||
|
||||
| Trigger | Challenge type |
|
||||
|---|---|
|
||||
| WAF custom rules, **rate limiting rules**, IP Access rules | Interstitial Challenge Page |
|
||||
| Bot Management | JavaScript Detections (invisible, per-request) |
|
||||
| **Bot Fight Mode / Super Bot Fight Mode** | Interstitial Challenge Page |
|
||||
| Under Attack Mode | Managed Challenge |
|
||||
|
||||
### 3.4 How a successful solve is remembered
|
||||
|
||||
Solving issues the **`cf_clearance`** cookie: "Clearance Cookie stores the proof of challenge passed. It is used to no longer issue a challenge if present. It is required to reach an origin server." ([Cloudflare Cookies — Cloudflare docs](https://developers.cloudflare.com/fundamentals/reference/policies-compliances/cloudflare-cookies/), read 2026-08-12.) "When that visitor tries to access other parts of your website, Cloudflare evaluates the cookie before presenting another challenge. If the cookie is still valid, no challenges will be shown." ([Challenge Passage — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/challenge-passage/), read 2026-08-12.)
|
||||
|
||||
`cf_clearance` is set with `SameSite=None; Secure; Partitioned`; because of the `Partitioned` (CHIPS) attribute, "a clearance obtained in one top-level context is not reused in a different top-level context" — so a clearance from kagane.to does not carry to comix.to even on the same browser. ([SameSite cookie interaction — Cloudflare docs](https://developers.cloudflare.com/waf/troubleshooting/samesite-cookie-interaction/), read 2026-08-12.)
|
||||
|
||||
---
|
||||
|
||||
## 4. `cf_clearance` — lifetime, binding, invalidation (the key question)
|
||||
|
||||
### 4.1 Lifetime
|
||||
|
||||
- **Default: 30 minutes.** "By default, the `cf_clearance` cookie has a lifetime of 30 minutes. Cloudflare recommends a setting between 15 and 45 minutes." The site owner can change it via the **Challenge Passage** setting. ([Challenge Passage — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/challenge-passage/), read 2026-08-12; also [SameSite cookie interaction — Cloudflare docs](https://developers.cloudflare.com/waf/troubleshooting/samesite-cookie-interaction/), read 2026-08-12.)
|
||||
- Validation grace: "a few extra minutes are included to account for clock skew. For XmlHTTP requests, an extra hour is added to the validation time." ([Challenge Passage — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/challenge-passage/), read 2026-08-12.)
|
||||
- "The Challenge Passage does not apply to rate limiting rules." Same source.
|
||||
- `Not publicly documented`: whether the three target sites have changed the Challenge Passage from the 30-minute default, and whether there is any maximum value Cloudflare enforces.
|
||||
|
||||
**Consequence for cadence:** with the default 30-minute TTL, *any* poll cadence ≥ 1 hour finds the cookie expired and re-solves the challenge on every visit. A 1-hour and a 6-hour cadence therefore differ only in *how many times per day* the browser re-solves (~4× for the same series), not in whether a re-solve happens. This repo already measured the re-solve cost: ~4 s with real Chrome + real timezone. `[INFERENCE]` a site could raise the Challenge Passage to hours/days, which would make a 1-hour cadence *cheaper* (cookie still valid, no re-solve) — but that setting is unobservable and unlikely to be long on free manga sites.
|
||||
|
||||
### 4.2 What it is bound to — can clearance be replayed from another IP?
|
||||
|
||||
Documented statements, both from Cloudflare's own docs:
|
||||
|
||||
1. **Device/visitor binding:** "The cookie is securely tied to the specific visitor and device it was issued to, preventing reuse across machines." ([Clearance — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/concepts/clearance/), read 2026-08-12.)
|
||||
2. **IP binding of the solve:** under Challenge limitations, Cloudflare lists "Client software where the solve request of a Managed Challenge comes from a different IP than the original IP a Challenge request was issued to. For example, if you receive the Challenge from one IP and solve it using another IP, the solve is not valid and you may encounter a Challenge loop." ([How Challenges work — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/concepts/how-challenges-work/), read 2026-08-12.)
|
||||
3. **Top-level-site binding** via CHIPS partitioning: clearance is not reused across embedding contexts. ([SameSite cookie interaction — Cloudflare docs](https://developers.cloudflare.com/waf/troubleshooting/samesite-cookie-interaction/), read 2026-08-12.)
|
||||
|
||||
So the repo's belief is **verified by primary sources**: a `cf_clearance` obtained on one machine cannot be replayed from a different IP — the solve is IP-bound and the cookie is device-bound. `Not publicly documented`: whether the cookie value is also cryptographically bound to the User-Agent or TLS/JA3 fingerprint. The only UA-adjacent documented statement is the reverse direction: challenge *solving* breaks when a browser extension modifies the User-Agent or Canvas/WebGL APIs ("Cloudflare Challenges cannot support… Browser extensions that modify the browser's User-Agent value or Web APIs such as Canvas and WebGL") — i.e., tampering with browser signals is documented to *fail* challenges ([How Challenges work — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/concepts/how-challenges-work/), read 2026-08-12).
|
||||
|
||||
### 4.3 Two-tier clearance, and the behavior-bound invalidation (Precursor)
|
||||
|
||||
`cf_clearance` now carries two kinds of clearance ([Clearance — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/concepts/clearance/), read 2026-08-12):
|
||||
|
||||
- **Challenge clearance** — granted by solving a challenge; level-gated (Interactive > Managed > Non-Interactive; higher clears bypass lower challenges); "remains valid for the duration configured by the customer (Challenge Passage), **unless Precursor determines the session is suspicious**".
|
||||
- **Precursor clearance** — "continuously updated based on session behavior"; an ongoing client-side process that periodically reassesses behavior. "If Precursor determines that a session is suspicious: the visitor's effective Challenge clearance may be **reduced or invalidated**; the visitor may be **re-challenged, even if the cookie has not expired**."
|
||||
|
||||
Precursor is documented as "client-side, session-based verification that continuously evaluates visitor behavior to identify automation… to detect automation that appears legitimate in individual requests but exhibits non-human patterns across a session", writing session state back into `cf_clearance`. It is a **zone-level toggle** (Security → Settings → Precursor; modes: Minimize Friction default, Maximize Security recommended), and "Precursor supersedes JavaScript Detections (JSD)". ([Precursor — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/precursor/), read 2026-08-12.)
|
||||
|
||||
**Implication:** the only documented mechanism by which *behavior over time* (as opposed to a single request's fingerprint) can revoke a valid clearance is Precursor — and it is opt-in per zone. `[INFERENCE]` It is unlikely to be enabled on free manga sites, but this is not observable from outside. With Precursor off (the default posture `[INFERENCE]`), a valid `cf_clearance` is honored purely on TTL + device/IP binding.
|
||||
|
||||
---
|
||||
|
||||
## 5. Does polling more often raise challenge risk?
|
||||
|
||||
### 5.1 Is per-IP request rate an input to challenge issuance? (documented answer: no such lever on non-Enterprise protection)
|
||||
|
||||
- **Bot scores** (the only per-request automated-detection output) are Enterprise-Bot-Management-only (§1.1); the ML engine's documented inputs are headers/session/browser signals, with request *pattern* entering only via `__cf_bm` — and no numeric rate is published (§1.2). `Not publicly documented`: any requests-per-hour value that changes a bot score or challenge probability.
|
||||
- **BFM/SBFM** match "patterns of known bots" — signatures, not volumes ([Bot Fight Mode — Cloudflare docs](https://developers.cloudflare.com/bots/get-started/bot-fight-mode/), read 2026-08-12).
|
||||
- **Rate limiting** is the product that polices volume, and it is opt-in per zone with explicit per-plan constraints (§5.2). Cloudflare's only documented link between "one valid clearance + high volume" is a *recommendation to site owners*: "Cloudflare recommends that customers add a rate limiting rule based on the `cf_clearance` cookie value. This helps ensure that a single, valid cookie cannot be abused by one machine to send an excessive volume of requests." ([Clearance — Cloudflare docs](https://developers.cloudflare.com/cloudflare-challenges/concepts/clearance/), read 2026-08-12.) Note: counting by cookie value is only available on Enterprise (see table below); a Free/Pro site cannot even build that rule.
|
||||
|
||||
### 5.2 What Rate Limiting Rules would do to 20–60 requests/hour
|
||||
|
||||
Rate limiting rules are opt-in; nothing runs them unless the site creates a rule. The documented per-plan capabilities ([Rate limiting rules — Cloudflare docs](https://developers.cloudflare.com/waf/rate-limiting-rules/), read 2026-08-12):
|
||||
|
||||
| Capability | Free | Pro | Business |
|
||||
|---|---|---|---|
|
||||
| Number of rules | **1** | 2 | 5 |
|
||||
| Counting characteristics | **IP only** | IP only | IP, IP w/ NAT |
|
||||
| Counting period | **10 s only** | ≤ 1 min | ≤ 10 min |
|
||||
| Mitigation timeout | **10 s** | ≤ 1 h | ≤ 1 day |
|
||||
| Fields in expression | **Path, Verified Bot** | + Host, URI, Full URI, Query | + Method, Source IP, User Agent |
|
||||
|
||||
Even the most aggressive free-plan rule (1 request per 10 s = 360/hour) is 6–18× above our 20–60/hour volume; the counting window is 10 s, so a per-hour burst is invisible to it. At 1 request per minute worst-case, **20–60 requests/hour from one residential IP cannot trip any rate limiting rule the Free plan can express.** Also documented: rate limiting is approximate, not precise — "there may be a delay of up to a few seconds between detecting a request and updating rate counters… excess requests could still reach the origin", and counters are per-data-center. ([Rate limiting rules — Cloudflare docs](https://developers.cloudflare.com/waf/rate-limiting-rules/), read 2026-08-12.) `[INFERENCE]` a site could also challenge on rate via WAF custom rules, but that requires Pro+ (custom rules are not on Free) and its own configuration.
|
||||
|
||||
### 5.3 DDoS protection (always on, all plans)
|
||||
|
||||
HTTP DDoS Attack Protection "is always enabled" and can only be tuned, not disabled. The only *published numeric* thresholds are error-rate-based: origin-error floods mitigate at the default "High" sensitivity of **1,000 errors per second** (Pro+ also requires 5× normal origin traffic). Per-IP volumetric thresholds are adaptive and `Not publicly documented` in the managed ruleset docs. 20–60 requests/hour is ~9 orders of magnitude below the published figure. ([HTTP DDoS Attack Protection — Cloudflare docs](https://developers.cloudflare.com/ddos-protection/managed-rulesets/http/), read 2026-08-12.)
|
||||
|
||||
### 5.4 Execution order (which product fires first)
|
||||
|
||||
Documented phase order: `ddos_l7` → custom rules → `http_ratelimit` (rate limiting) → managed rules → `http_request_sbfm` (SBFM); BFM runs outside this pipeline and cannot be skipped; a terminating action (block/challenge) stops later phases. ([Security features interoperability — Cloudflare docs](https://developers.cloudflare.com/waf/feature-interoperability/), read 2026-08-12.) Practical reading: on the three sites, the challenge we see could come from any of these stages; none of them documents a volume input at our scale (§5.1–5.3).
|
||||
|
||||
---
|
||||
|
||||
## 6. The documented legitimate side
|
||||
|
||||
### 6.1 Verified bots — the only "treated well" path, and it requires self-identification
|
||||
|
||||
Cloudflare documents a Verified bot as one meeting two bars ([Verified bots — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot/verified-bots/), read 2026-08-12):
|
||||
|
||||
1. **Honest self-identification** — "through a cryptographic Web Bot Auth signature, a published IP list with a stable user-agent, or reverse DNS".
|
||||
2. **Non-abusive behavior** — "it obeys `robots.txt` and crawl directives, **maintains reasonable request rates**, and has not been observed evading website owner preferences or attacking sites".
|
||||
|
||||
Relevant verified-bot *behavior classes* exist for exactly this kind of client: "**Feed Fetching** — RSS readers, podcast aggregators, and news feed bots" and "**Monitoring & Operations** — Uptime monitoring, webhooks, and health checks". Becoming verified requires an application via the dashboard and validation via Web Bot Auth or IP validation; breach of the policy (e.g. "An AI Crawler that does not respect the crawl-delay directive") removes the bot from the allowlist. ([Verified bots — Cloudflare docs](https://developers.cloudflare.com/bots/concepts/bot/verified-bots/), read 2026-08-12.)
|
||||
|
||||
"Historically, Verified bots have been excluded in default bot configurations across all plans" (same source) — i.e., verified bots are *default-allowed* under SBFM/Bot Management. **But** this path is the opposite of what a scraper wants: it requires the poller to publicly identify itself (stable, published IPs or cryptographic signatures) and to have its identity vetted by Cloudflare — and the *site* still decides via verified-bot policy whether to allow the category. There is **no documented mechanism for an anonymous low-volume automated client to be treated well.** `[INFERENCE]` a manga-site scraper would never qualify (it would be classified as Data Collection / scraping behavior, which is not a default-allowed class).
|
||||
|
||||
### 6.2 robots.txt and crawl control
|
||||
|
||||
- `robots.txt` **compliance is voluntary** — "The file expresses your preferences, but it does not prevent crawlers from accessing your content at a technical level." Enforcement requires Cloudflare's AI Crawl Control. ([robots.txt setting — Cloudflare docs](https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/), read 2026-08-12.)
|
||||
- The managed `robots.txt` feature (all plans) is aimed at AI crawlers; it prepends `Disallow` rules for AI bots and a Content Signals Policy. It does not create any allowance for generic scrapers. (Same source.)
|
||||
- RFC-side: the robots exclusion standard is an unauthenticated convention; nothing in it grants access rights. ([RFC 9309 "Robots Exclusion Protocol"](https://www.rfc-editor.org/rfc/rfc9309.html) — read 2026-08-12.) The standard defines crawl-delay etc. as voluntary directives; Cloudflare's docs are the operative statement for CF-protected sites.
|
||||
|
||||
### 6.3 The documented takeaway for "slowing down vs. fingerprint quality"
|
||||
|
||||
Cloudflare's own documentation repeatedly points at **browser/device signals** as the decision input on non-Enterprise protection (JSD detecting headless browsers; Managed Challenge choosing based on "signals indicated by their browser"; challenges failing when UA/Canvas/WebGL are modified — §3.2, §4.2), and at **identity** (verified bots) as the only legitimacy signal for automation (§6.1). Request *rate* appears only as: (a) an unnamed component of Enterprise-ML "session characteristics", (b) a voluntary verified-bot behavioral bar, and (c) the separate, opt-in, Free-plan-impotent Rate Limiting product (§5). Nothing documented says "slow down and you'll be challenged less" for a free-plan site — **fingerprint quality is the lever that the documentation actually describes**, which matches this repo's measurements (real Chrome + real timezone passes; every headless variant fails regardless of rate).
|
||||
|
||||
---
|
||||
|
||||
## 7. Implications for cadence design
|
||||
|
||||
### Documented facts (with sources above)
|
||||
|
||||
1. **1-hour vs 6-hour cadence is not a documented risk lever.** Challenge issuance on Free/Pro-grade protection is signature-based; the rate-aware scoring only exists on Enterprise Bot Management; free-plan rate limiting cannot express a limit our volume could trip (§1, §2, §5).
|
||||
2. **Every poll ≥ 1 hour re-solves the challenge anyway.** `cf_clearance` defaults to 30 minutes; Challenge Passage is site-configurable and unobservable. The re-solve cost is what this repo measured (~4 s, real Chrome + real timezone) (§4.1, repo measurements).
|
||||
3. **The documented failure modes are fingerprint, not rate:** headless browsers (JSD), cloud-hosting IPs (BFM heuristics), modified UA/Canvas/WebGL (challenge solve failure) (§2.2, §3, §4.2).
|
||||
4. **Clearance is not portable:** device-bound + solve-IP-bound + CHIPS-partitioned; replaying a cookie from another IP is documented invalid (§4.2).
|
||||
5. **Anonymity has no documented "good citizen" path:** the only legitimate-automation route (verified bots) requires self-identification and site-side allowance (§6).
|
||||
6. **The one behavior-bound revocation mechanism (Precursor) is opt-in per zone**, not a default documented behavior (§4.3).
|
||||
|
||||
### Inferences (not documented)
|
||||
|
||||
- `[INFERENCE]` The three sites run Free/Pro-grade protection (BFM, SBFM, or WAF challenge rules), not Enterprise Bot Management; therefore no continuous per-request bot score exists for our traffic.
|
||||
- `[INFERENCE]` The sites have not changed Challenge Passage to hours/days (free manga sites default to the 30-minute default); if they had, hourly polling would get *cheaper* (valid cookie, no re-solve).
|
||||
- `[INFERENCE]` Precursor is not enabled on these sites; if it were, hourly re-visits from an automated Chrome could accumulate session-behavior signals and trigger re-challenge even with a valid cookie — the only documented scenario in which polling *frequency* (via session behavior) could matter.
|
||||
- `[INFERENCE]` 6-hour cooldowns buy nothing documented beyond raw request-count reduction (fewer challenge solves per day, less origin load); the risk profile at 1 request/hour/series is not documented to differ from 6 request/hour/series.
|
||||
- `[INFERENCE]` If the owner wants belt-and-braces, the engineering levers that match the documentation are: keep the real-Chrome fingerprint (no UA spoofing, no headless-shell, real timezone — already done), keep a persistent user-data profile so `cf_clearance`/`__cf_bm` persist across visits, and treat any change of exit IP (e.g. home connection rebooting to a new IP) as a guaranteed re-solve, since clearance does not travel with the IP.
|
||||
|
||||
### Bottom line
|
||||
|
||||
Moving browser-backed sites from 6-hour to 1-hour per-series cooldown is **not contradicted by any documented Cloudflare mechanism** at 20–60 requests/hour from one residential IP through one real Chrome. The documented risk is carried by fingerprint quality (already solved in this repo) and by unobservable site configuration (Challenge Passage, Precursor, possible custom WAF rules). The residual, non-documented risk is that these sites sit behind Cloudflare's *proprietary* detection, and Cloudflare publishes neither its per-IP thresholds nor the ML feature set — so "no documented lever" is not "no lever".
|
||||
@@ -0,0 +1,344 @@
|
||||
# GIF — maximum byte size of a file
|
||||
|
||||
Research note for Gitea issue #71 (backend `maxBodyBytes` = 4 MiB rejects the
|
||||
8,571,192-byte animated cover GIF at
|
||||
`https://cdn.asurascans.com/asura-images/covers/a-dragonslayers-peerless-regression.gif`).
|
||||
|
||||
All facts fetched live on **2026-08-17**: the GIF89a spec at
|
||||
`https://www.w3.org/Graphics/GIF/spec-gif89a.txt`, Go stdlib `image/gif`
|
||||
sources at `/usr/local/go/src/image/gif/reader.go` (Go 1.26.5), Chromium
|
||||
`blink/renderer/platform/image-decoders/` sources via
|
||||
`chromium.googlesource.com`, Firefox `image/decoders/nsGIFDecoder2.cpp` via
|
||||
`hg.mozilla.org`, and cover bytes probed with plain `curl` (desktop Chrome UA;
|
||||
`HEAD`/ranged `GET`). **No Cloudflare challenge was encountered on any CDN
|
||||
probe** — every request returned real headers, consistent with the AGENTS.md
|
||||
note of 2026-07-26 that plain `curl` works against both scan sites from the
|
||||
dev machine and the VPS.
|
||||
|
||||
Every claim carries the URL it came from, or a reproducible command.
|
||||
Interpretation rather than observation is marked `[INFERENCE]`.
|
||||
|
||||
---
|
||||
|
||||
## 1. Summary answer table
|
||||
|
||||
| Question | Answer | Evidence |
|
||||
|---|---|---|
|
||||
| Does the GIF89a spec define a maximum file size? | **No.** There is no file-size field anywhere in the format; the only numeric ceilings are per-field (16-bit screen/image dimensions, 255-byte sub-blocks, 12-bit LZW codes). | §2 |
|
||||
| Maximum logical screen | 65535 × 65535 pixels (unsigned 16-bit width/height). | §2.1 |
|
||||
| Number of frames / image descriptors | Unbounded — "An unlimited number of images may be present per Data Stream." | §2.2 |
|
||||
| Formal max byte size of any single GIF | None. Single-frame worst case ≈ **6.44 GB** (12-bit LZW, max canvas); animated GIFs are **unbounded** because frames are unbounded. | §3 |
|
||||
| Does the backend's decoder (Go `image/gif`) bound size? | **No.** It reads 16-bit dimensions and allocates `width×height` bytes per frame; a 65535² frame forces a ~4 GiB allocation. No total-size or dimension guard. | §4.1 |
|
||||
| Do browsers bound on-disk GIF size? | Chromium and Firefox: no on-wire size cap in their GIF readers; Chromium caps *decoded* memory at min(4 B × pixels, platform budget). | §4.3, §4.4 |
|
||||
| Real cover sizes (asurascans, n=25) | min 190,410 B · median 1,275,082 B · p90 4,524,788 B · max 8,571,192 B · **3/25 > 4 MiB** (two JPEGs and the animated GIF) | §5 |
|
||||
| Real cover sizes (demonicscans/readermc, n=78) | min 13,298 B · median 63,061 B · max 801,200 B · 0/78 > 4 MiB | §5 |
|
||||
| Comparable service caps | GitHub: 10 MB for images/GIFs. Discord API: default 10 MiB per file. Wikimedia: 100 MiB upload / 5 GiB host. | §6 |
|
||||
| Recommended cover cap for #71 | **10 MiB** (separate from the 4 MiB series-page cap). Covers 100% of the 103 observed covers; matches GitHub/Discord calibration; ≤ 20 MiB worst-case transient per concurrent fetch+serve on a 1974 MiB swapless VPS. | §7 |
|
||||
|
||||
---
|
||||
|
||||
## 2. What the GIF89a specification actually bounds
|
||||
|
||||
Source: `https://www.w3.org/Graphics/GIF/spec-gif89a.txt` (fetched 2026-08-17).
|
||||
|
||||
### 2.1 Fixed-width fields — the only hard ceilings
|
||||
|
||||
The format is a stream of fixed-width blocks; the numeric fields that *do*
|
||||
have a ceiling are all 16-bit unsigned, little-endian ("multi-byte numeric
|
||||
fields are ordered Least Significant Byte first", §4 of the spec):
|
||||
|
||||
- **Logical Screen Width / Height** — "Unsigned" 2-byte fields (§18, Logical
|
||||
Screen Descriptor) → maximum **65535 × 65535** pixels.
|
||||
- **Image Left / Top Position, Image Width / Height** — "Unsigned" 2-byte
|
||||
fields (§20, Image Descriptor). Each image "must fit within the boundaries
|
||||
of the Logical Screen" (§20a), so an image cannot exceed the 65535² canvas
|
||||
even though its own fields would allow it.
|
||||
- **Data sub-blocks** — "A data sub-block may contain from 0 to 255 data
|
||||
bytes" (§15); each sub-block is preceded by a 1-byte size field and the
|
||||
stream is terminated by a 0x00 Block Terminator (§16). This bounds a
|
||||
*chunk*, not the stream.
|
||||
- **Global/Local Color Tables** — optional, "3 x 2^(Size of Global Color
|
||||
Table+1)" bytes with a 3-bit size field → at most 3 × 2⁸ = **768 bytes**
|
||||
each (§19, §21).
|
||||
- **LZW codes** — "The output codes are of variable length, starting at
|
||||
<code size>+1 bits per code, **up to 12 bits per code**. This defines a
|
||||
maximum code value of 4095 (0xFFF)" (Appendix F, COMPRESSION, rule 4).
|
||||
- **Trailer** — a single byte, fixed value 0x3B, "indicating the end of the
|
||||
GIF Data Stream" (§27).
|
||||
|
||||
### 2.2 What is unbounded
|
||||
|
||||
- **Number of images (frames).** §20a, verbatim: "This block is REQUIRED for
|
||||
an image. Exactly one Image Descriptor must be present per image in the
|
||||
Data Stream. **An unlimited number of images may be present per Data
|
||||
Stream.**"
|
||||
- **The Data Stream itself.** The grammar in Appendix B is
|
||||
`<GIF Data Stream> ::= Header <Logical Screen> <Data>* Trailer`, and the
|
||||
spec states "the entity Data … may be repeated any number of times,
|
||||
including 0 times." There is **no field anywhere that carries a file size,
|
||||
byte count, frame count, or total-length value**. §13 (Block Sizes) only
|
||||
defines sizes *within* blocks.
|
||||
|
||||
### 2.3 Verdict
|
||||
|
||||
**The GIF89a specification defines no maximum file size.** The only hard
|
||||
bounds are per-field: 65535×65535 pixels per screen/image, 255 bytes per
|
||||
sub-block, 12 bits per LZW code, and one trailer byte. A compliant decoder
|
||||
must process whatever stream the blocks describe. Any byte ceiling a
|
||||
particular GIF actually hits is therefore *implicit* — 16-bit dimensions,
|
||||
LZW code width, decoder memory, or an external policy — never something the
|
||||
format itself enforces. `[INFERENCE]` This is why real-world GIFs cap out at
|
||||
"a few GB at most" and every service that wants a bound has to impose one
|
||||
itself (see §6; Wikimedia explicitly documents that a 4 GiB host limit was a
|
||||
storage-representation artifact of 32-bit integers, `phab:T191805`, not a
|
||||
format limit).
|
||||
|
||||
---
|
||||
|
||||
## 3. Theoretical worst case
|
||||
|
||||
### 3.1 Single frame, maximal canvas, 8-bit pixels
|
||||
|
||||
| Quantity | Value | Derivation |
|
||||
|---|---|---|
|
||||
| Max pixels | 4,294,836,225 | 65535 × 65535 |
|
||||
| Raw 8-bit palette-index raster | 4,294,836,225 B ≈ **4.29 GB / 4.00 GiB** | 1 byte per pixel (Table Based Image Data, §22; Go's `image.Paletted` uses exactly 1 byte/pixel) |
|
||||
| LZW worst case | ≈ **6.44 GB / 6.00 GiB** | codes ≤ 12 bits each (Appendix F), at most ~1 code per pixel for incompressible data → ≤ 12 bits/px = 1.5 B/px → 4,294,836,225 × 1.5 B |
|
||||
| Sub-block overhead | ≈ +25.3 MB | every ≤255-byte chunk carries a 1-byte size field (§15): ⌈6,442,254,338 / 255⌉ ≈ 25,263,743 size bytes, + 1 block terminator |
|
||||
| Fixed overhead | ≈ +1.6 KB | header 6 B (§17) + logical screen descriptor 7 B (§18) + global color table ≤ 768 B (§19) + image descriptor 10 B (§20) + local color table ≤ 768 B (§21) + LZW minimum code size 1 B (§22) |
|
||||
|
||||
So a **single maximal-frame GIF cannot exceed ≈ 6.47 GB on the wire**
|
||||
(12-bit LZW bound), and LZW being lossless means the real byte count depends
|
||||
entirely on image content — the same canvas can be a few KB (flat color) or
|
||||
~6 GB (noise).
|
||||
|
||||
Two caveats, both marked `[INFERENCE]`:
|
||||
|
||||
- The "1.5 B/px" figure assumes ~one emitted code per pixel. An encoder is
|
||||
permitted to emit a Clear code at any point (Appendix F: "The Clear code
|
||||
can appear at any point in the image data stream"), so a
|
||||
pathological-but-compliant encoder emitting clear+pixel per pixel reaches
|
||||
~24 bits/px ≈ 12.9 GB for the max canvas. Real encoders do not do this;
|
||||
12-bit/px is the practical bound.
|
||||
- The spec's deferred-clear note (cover sheet) explicitly allows an encoder
|
||||
to keep using a full table at 12-bit codes without clearing, so the 12-bit
|
||||
cap holds for the whole stream, it cannot "grow" past 12 bits.
|
||||
|
||||
### 3.2 Animated GIFs: unbounded
|
||||
|
||||
Every frame is one Image Descriptor, each bounded by the 65535² canvas, but
|
||||
the *count* of frames is unbounded (§2.2). Total bytes = sum over frames —
|
||||
therefore **there is no finite maximum byte size for an animated GIF** in
|
||||
the format. The only thing that stops a real one is decoder memory, a
|
||||
service cap, or disk space. `[INFERENCE]` This is the category the issue #71
|
||||
cover falls into: it is an animated GIF (NETSCAPE2.0 loop extension found at
|
||||
offset 0x310 of the file, verified 2026-08-17 by a ranged GET), and its
|
||||
8,571,192 bytes are ~2.04× the current 4 MiB backend cap.
|
||||
|
||||
---
|
||||
|
||||
## 4. Decoder-side real limits
|
||||
|
||||
### 4.1 Go `image/gif` (the backend's decoder path, stdlib)
|
||||
|
||||
Source: `/usr/local/go/src/image/gif/reader.go`, Go 1.26.5.
|
||||
|
||||
- Dimensions are read as little-endian uint16 — `left/top/width/height :=
|
||||
int(d.tmp[N]) + int(d.tmp[N+1])<<8` (reader.go:490-493) — so the format
|
||||
ceiling 65535 applies, and nothing smaller is enforced.
|
||||
- The only geometric check is that each frame fits inside the logical
|
||||
screen: `if left+width > d.width || top+height > d.height` →
|
||||
`errors.New("gif: frame bounds larger than image bounds")` (reader.go:512-513).
|
||||
- **There is no file-size, byte-count, frame-count, or pixel-count guard.**
|
||||
Each frame allocates `image.NewPaletted(...)` (reader.go:515) — a
|
||||
`[]byte` of width×height — so decoding one legal 65535² frame attempts a
|
||||
**~4.29 GB allocation**. `DecodeAll` (reader.go:603-605) additionally
|
||||
retains every frame's `Pix` slice for the lifetime of the returned `*GIF`.
|
||||
- `[INFERENCE]` On the 1974 MiB swapless VPS (root AGENTS.md), decoding such
|
||||
a file would OOM rather than error cleanly; nothing in stdlib protects
|
||||
the process. This matters for §7: the backend stores cover bytes without
|
||||
decoding them (see §5.3), so the fetch path never triggers this — but any
|
||||
future "validate/re-encode server-side" scheme would.
|
||||
- Grep for `MaxInt|limit|too large|bounds` in reader.go: the only hits are
|
||||
the frame-bounds check above and the `tmp [1024]byte` scratch buffer
|
||||
(reader.go:109); no size caps exist.
|
||||
|
||||
### 4.2 giflib / libgif
|
||||
|
||||
**Not verified from source.** On 2026-08-17 the giflib sources were not
|
||||
reachable from this network: `github.com/giflib/giflib` returns 404 (repo
|
||||
gone/moved), `gitlab.com/giflib/giflib/-/raw/...` answers a Cloudflare
|
||||
"Just a moment…" challenge, and the SourceForge project download path
|
||||
404s. No limit claim about giflib is made here. `[INFERENCE]` giflib is
|
||||
widely known to be allocation-driven with no dimension cap, but that is not
|
||||
checked against source and is not needed for issue #71 (the backend uses Go
|
||||
stdlib, not giflib).
|
||||
|
||||
### 4.3 Chromium (browser behaviour, first-party source)
|
||||
|
||||
- `third_party/blink/renderer/platform/image-decoders/gif/gif_image_reader.cc`
|
||||
(via `chromium.googlesource.com/chromium/src/+/main/...`, fetched
|
||||
2026-08-17): **no GIF byte-size or dimension cap found** — grep for
|
||||
`max|limit|too large|dimension|65535|overflow` matches only license text.
|
||||
- The base `ImageDecoder` caps *decoded memory*, not transfer size:
|
||||
`CalculateMaxDecodedBytes` computes `min(4 * num_pixels, platform_max_decoded_bytes)`
|
||||
(8 bytes/pixel for high-bit-depth), and the header comment says "Ignoring
|
||||
this limit can cause excessive memory use or even crashes on low-memory
|
||||
devices"
|
||||
(`image_decoder.cc:94-117`, `image_decoder.h:545-549`). The GIF reader
|
||||
itself is untouched by this — it is a decoded-buffer budget.
|
||||
- Practical consequence `[INFERENCE]`: a browser will happily download and
|
||||
store a multi-GB GIF from its own cache perspective; Chromium only limits
|
||||
what it *decodes* into pixels.
|
||||
|
||||
### 4.4 Firefox
|
||||
|
||||
`image/decoders/nsGIFDecoder2.cpp` (via `hg.mozilla.org/mozilla-central/
|
||||
raw-file/tip/...`, fetched 2026-08-17): **no dimension or size limit**; the
|
||||
only guards are on LZW code width (`MAX_BITS` = 12, "maximum codeword size
|
||||
of 12 bits") and the decode stack. Nothing bounds the on-disk byte size.
|
||||
|
||||
### 4.5 Summary
|
||||
|
||||
No mainstream decoder enforces a byte-size ceiling; they stop at the 16-bit
|
||||
dimension ceiling (Go, by construction) or at decoded-memory budgets
|
||||
(Chromium) or nowhere (Firefox). A GIF's byte size is policed only by
|
||||
*storage* policies — which is what §6 calibrates and §7 sets.
|
||||
|
||||
---
|
||||
|
||||
## 5. Practical distribution — what real manga covers weigh
|
||||
|
||||
Probed **2026-08-17** with `curl -sI` (HEAD) and ranged GETs, desktop Chrome
|
||||
UA. No Cloudflare block on any request. Sample = covers *as the backend
|
||||
would fetch them* (the `og:image`/page-listed cover URL), not thumbnails we
|
||||
chose by hand.
|
||||
|
||||
### 5.1 Exact commands
|
||||
|
||||
```sh
|
||||
# asurascans.com — harvest cover URLs from the homepage, then HEAD each
|
||||
curl -s -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/126.0" https://asurascans.com/ -o home.html
|
||||
grep -oE 'https://cdn\.asurascans\.com/asura-images/covers/[^"&\\< ]+\.(webp|gif|jpg|jpeg|png)' home.html \
|
||||
| sort -u | grep -v '\-400\.' | head -25 > sample.txt # one full-res cover per series, no -400 thumbs
|
||||
while read -r u; do curl -s -A "…Chrome/126.0" -I "$u" | tr -d '\r' \
|
||||
| grep -iE '^content-length:'; done < sample.txt
|
||||
|
||||
# demonicscans.org — covers live on readermc.org (ADR-0007), URLs contain spaces/UTF-8
|
||||
curl -s -A "…Chrome/126.0" https://demonicscans.org/ -o demonic.html
|
||||
grep -oE 'src="https://readermc\.org/images/thumbnails/[^"]+"' demonic.html | tr -d 'src="' > demonic.txt
|
||||
# …plus og:image from 5 manga pages (Catastrophic-Necromancer, Magic-Emperor, …)
|
||||
# each URL percent-encoded per path segment (urllib.parse.quote, safe=':/') before HEAD
|
||||
```
|
||||
|
||||
### 5.2 asurascans — 25 full-res covers (mixed formats)
|
||||
|
||||
Homepage fetched 200 (664,700 B). All 25 returned `200` with a real
|
||||
`Content-Length`. Distribution:
|
||||
|
||||
| Statistic | Bytes |
|
||||
|---|---|
|
||||
| n | 25 |
|
||||
| min | 190,410 |
|
||||
| median | 1,275,082 |
|
||||
| p90 | 4,524,788 |
|
||||
| max | 8,571,192 |
|
||||
| mean | 1,943,651 |
|
||||
| **> 4 MiB (4,194,304)** | **3 (12%)** — `a-dragonslayers-peerless-regression.gif` 8,571,192 (the issue #71 cover, animated: NETSCAPE2.0 at 0x310, 550×733, 256 colors); `bad-born-blood.3008f6.webp` 4,524,788 `image/jpeg`; `ending-maker.cfbf53.webp` 4,619,303 `image/jpeg` |
|
||||
|
||||
Notes: the CDN serves `Content-Type` by stored bytes, not by URL extension
|
||||
(the `.webp` URLs return `image/png`, `image/jpeg`, or `image/webp` — the
|
||||
sample spans all four of `png/jpeg/webp/gif`). Two of the three over-cap
|
||||
files are **not GIFs**, so the current 4 MiB cap already silently drops 12%
|
||||
of asura covers of any format. p90 itself (4.52 MB) exceeds the cap.
|
||||
|
||||
### 5.3 demonicscans — 78 covers on readermc.org
|
||||
|
||||
78 unique cover URLs (73 from the homepage's `/images/thumbnails/` plus 5
|
||||
`og:image` values from manga pages — demonicscans publishes the thumbnail
|
||||
file as the full cover, so that is exactly what the backend would fetch).
|
||||
**78/78 returned 200 with a real Content-Length** (spaces and UTF-8 in the
|
||||
filenames were percent-encoded per path segment; the homepage's raw HTML
|
||||
carries `’`-style mojibake for curly quotes, which was repaired by
|
||||
latin-1→utf-8 re-encoding before probing).
|
||||
|
||||
| Statistic | Bytes |
|
||||
|---|---|
|
||||
| n | 78 |
|
||||
| min | 13,298 |
|
||||
| median | 63,061 |
|
||||
| p90 | 206,994 |
|
||||
| max | 801,200 |
|
||||
| mean | 110,038 |
|
||||
| > 4 MiB | 0 |
|
||||
|
||||
### 5.4 Reading
|
||||
|
||||
`[INFERENCE]` asurascans covers are the heavy tail (median 1.3 MB, top
|
||||
decile > 4 MiB, occasional ~5–9 MB), demonicscans covers are tiny (all
|
||||
< 0.8 MB). A cover cap must be chosen against the *asura* distribution —
|
||||
the 8.57 MB animated GIF is not a freak one-off outlier; the 90th
|
||||
percentile already crosses 4 MiB and two JPEGs sit between 4.5–4.7 MB.
|
||||
|
||||
---
|
||||
|
||||
## 6. Comparable documented byte caps (first-party docs only)
|
||||
|
||||
| Service | Cap | Source (fetched 2026-08-17) |
|
||||
|---|---|---|
|
||||
| GitHub (issues/PR comments) | **10 MB for images and gifs**; 25 MB other files; 10/100 MB video | `https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/attaching-files` — "The maximum file size is: 10MB for images and gifs … 25MB for all other files" |
|
||||
| Discord (API uploads) | default **10 MiB per file**, higher with Nitro / boost tier | `https://discord.com/developers/docs/reference#uploading-files` — "The file upload size limit applies to each file in a request. The default limit is `10 MiB` for all users" (help-center article `support.discord.com/hc/en-us/articles/115002935588` exists but answered 403 from this network on the probe date, so its figures were not verified here) |
|
||||
| Wikimedia Commons | **100 MiB** upload limit; hosting up to **5 GiB**; GIF thumbnails limited to **100 megapixels**; prior 4 GiB host cap was a 32-bit storage artifact (phab:T191805) | `https://commons.wikimedia.org/wiki/Commons:Maximum_file_size` |
|
||||
| MDN | nothing — MDN documents no byte-size limit for images; browsers impose none (see §4.3–4.4) | `[INFERENCE]` from absence in the platform docs read in §4 |
|
||||
|
||||
Calibration takeaway: two major platforms independently land on **~10 MB**
|
||||
as the ceiling for an uploadable image/GIF (GitHub exactly 10 MB, Discord
|
||||
exactly 10 MiB), with Wikimedia the outlier at 100 MiB/5 GiB because it is a
|
||||
media *archive*. A 10 MiB cover cap is therefore squarely inside industry
|
||||
normal.
|
||||
|
||||
---
|
||||
|
||||
## 7. Recommendation for issue #71
|
||||
|
||||
**Raise the cover cap to 10 MiB (10,485,760 B) — as a separate constant, not
|
||||
by moving the shared one.**
|
||||
|
||||
Why:
|
||||
|
||||
- **Fits the measured reality.** The largest observed cover is 8,571,192 B
|
||||
(the issue's animated GIF) = 82% of 10 MiB; 10 MiB covers **100% of the
|
||||
103 sampled covers** and the *entire* asura distribution, including its
|
||||
heavy tail. 4 MiB rejects 12% of asura covers (two of them plain JPEGs).
|
||||
- **Matches industry calibration** (§6): GitHub 10 MB images/GIFs, Discord
|
||||
10 MiB default. A 10 MiB cap is a number every engineer recognizes, and
|
||||
it leaves ~18% headroom over the current worst observed file.
|
||||
- **Costs little on the target hardware.** The backend buffers cover bytes
|
||||
whole during fetch (`backend/internal/latest/cover.go`: `ContentLength >
|
||||
maxBodyBytes` rejection at :155, then `io.ReadAll(io.LimitReader(…,
|
||||
maxBodyBytes+1))` at :158) and loads the full body per `GET /covers/…`
|
||||
(`backend/internal/api/handlers.go`, `Cover` → `w.Write(body)`). Worst
|
||||
case per concurrent fetch **+** serve is therefore 2 × cap = 20 MiB; even
|
||||
ten of each concurrently is ~200 MiB of a 1974 MiB swapless VPS (~10%),
|
||||
and the browser unit (471 MiB, root AGENTS.md) is no longer on that box.
|
||||
The 4 MiB series-page cap is *not* the issue — measured pages run
|
||||
100 KB–1.2 MB (`backend/internal/latest/fetch.go` comment) — so keep it.
|
||||
- **The cap is a separate knob.** Today one `const maxBodyBytes = 4 << 20`
|
||||
(`backend/internal/latest/fetch.go:17`) gates *both* series pages and
|
||||
covers (`cover.go` references it). Raising it wholesale would loosen the
|
||||
page-side memory guard for no benefit; a cover-specific constant (e.g.
|
||||
`maxCoverBytes = 10 << 20`) keeps the two policies independent. The fetch
|
||||
already double-checks `ContentLength` and the post-`LimitReader` length,
|
||||
so a larger constant changes nothing else.
|
||||
|
||||
Alternatives and their costs:
|
||||
|
||||
| Option | Cost |
|
||||
|---|---|
|
||||
| Keep 4 MiB | 12% of asura covers (incl. non-GIF JPEGs) never stored — current bug, silent missing covers. |
|
||||
| 16 MiB cap | 2× headroom over the observed max for future GIFs; +60% worst-case transient memory vs 10 MiB; diverges from the GitHub/Discord 10 MB calibration. |
|
||||
| Server-side re-encode / downscale covers | Requires decoding → Go `image/gif` allocates width×height per frame with **no guard** (§4.1); a legal 65535² GIF forces a ~4.29 GB allocation on a 1974 MiB swapless box — OOM, not an error. Also mutates bytes, which the store treats as immutable/content-addressed (ADR-0007). Highest risk, no upside at this scale. |
|
||||
| No cap | Unbounded transient memory and disk; rejected outright. |
|
||||
|
||||
Decision is the user's; on the evidence, **10 MiB for covers, 4 MiB for
|
||||
pages** is the defensible middle.
|
||||
@@ -1,235 +0,0 @@
|
||||
{
|
||||
"0": "HTMX Library Internals",
|
||||
"1": "Cover Fetch Test Helpers",
|
||||
"2": "Manga Userscript Adapters",
|
||||
"3": "Novel Userscript Adapters",
|
||||
"4": "Series Acquisition Tests",
|
||||
"5": "Bookmarks API Tests",
|
||||
"6": "Storage Choice ADR",
|
||||
"7": "Cover & Acquire Internals",
|
||||
"8": "System Architecture Concepts",
|
||||
"9": "Session Middleware",
|
||||
"10": "Go Test Helpers",
|
||||
"11": "Store Tests",
|
||||
"12": "Bookmarks API Handler",
|
||||
"13": "Web UI Handlers",
|
||||
"14": "Go Error Handling",
|
||||
"15": "CDP Browser Client",
|
||||
"17": "Go Code Style Guide",
|
||||
"18": "Agent Skills",
|
||||
"20": "I/O Performance Patterns",
|
||||
"21": "CPU Optimization",
|
||||
"22": "Caching Patterns",
|
||||
"23": "Browser Entrypoint",
|
||||
"24": "Memory Allocation & GC",
|
||||
"25": "Cover Fetcher Tests",
|
||||
"26": "readSeries",
|
||||
"27": "Find Skills Guide",
|
||||
"28": "Allocation Patterns",
|
||||
"29": "Observability & Alerting",
|
||||
"31": "Memory Layout",
|
||||
"32": "Repo Hard Constraints",
|
||||
"33": "Go Testing Guide",
|
||||
"34": "Session Store",
|
||||
"35": "Web UI Filter Logic",
|
||||
"36": "Userscript Test Harness",
|
||||
"37": "Product & Security Context",
|
||||
"38": "novel-logic.test.js",
|
||||
"39": "UI Critique 2026-07-26A",
|
||||
"40": "UI Critique 2026-07-26B",
|
||||
"43": "Issue Tracker & Triage",
|
||||
"44": "Ticket Workflow",
|
||||
"45": "Go Perf Alert Rules",
|
||||
"46": "Userscript Display Logic",
|
||||
"47": "Go Perf Skill Docs",
|
||||
"48": "Login Page Art",
|
||||
"49": "BookmarkManager Logo",
|
||||
"50": "Skills CLI",
|
||||
"51": "Skills Leaderboard",
|
||||
"52": "Complex Condition Extraction",
|
||||
"53": "Sentinel Errors",
|
||||
"54": "errors.As Patterns",
|
||||
"55": "errors.Is Patterns",
|
||||
"56": "errors.Join Patterns",
|
||||
"57": "Error Wrapping",
|
||||
"58": "Single Error Handling",
|
||||
"59": "SIMD Optimizations",
|
||||
"60": "GOGC Tuning",
|
||||
"61": "GOMEMLIMIT",
|
||||
"62": "Bottleneck Decision Tree",
|
||||
"63": "pprof Profiling",
|
||||
"64": "Test Timeout Helper",
|
||||
"65": "httptest Patterns",
|
||||
"66": "testify Suite Pattern",
|
||||
"67": "go:embed Fixtures",
|
||||
"68": "clockwork Time Mocking",
|
||||
"69": "testify Mocking",
|
||||
"70": "t.ArtifactDir Helper",
|
||||
"71": "Subtests Pitfall",
|
||||
"72": "golang-benchmark Skill",
|
||||
"73": "golang-concurrency Skill",
|
||||
"74": "golang-ci Skill",
|
||||
"75": "golang-database Skill",
|
||||
"76": "golang-lint Skill",
|
||||
"77": "testify Skill",
|
||||
"78": "Build Tag Integration Tests",
|
||||
"79": "Test Naming Convention",
|
||||
"80": "UI Critique A Finding",
|
||||
"81": "UI Critique B Finding",
|
||||
"82": "P0 Overflow Bug",
|
||||
"83": "P1 hx-indicator Gap",
|
||||
"84": "golang-benchmark Skill (ext)",
|
||||
"85": "golang-concurrency Skill (ext)",
|
||||
"86": "golang-ci Skill (ext)",
|
||||
"87": "golang-data-structures Skill (ext)",
|
||||
"88": "golang-database Skill (ext)",
|
||||
"89": "golang-design-patterns Skill (ext)",
|
||||
"90": "golang-documentation Skill (ext)",
|
||||
"91": "golang-gopls Skill (ext)",
|
||||
"92": "golang-lint Skill (ext)",
|
||||
"93": "golang-naming Skill (ext)",
|
||||
"94": "golang-observability Skill (ext)",
|
||||
"95": "golang-refactoring Skill (ext)",
|
||||
"96": "golang-safety Skill (ext)",
|
||||
"97": "golang-samber-oops Skill (ext)",
|
||||
"98": "golang-samber-slog Skill (ext)",
|
||||
"99": "golang-structs-interfaces Skill (ext)",
|
||||
"100": "golang-troubleshooting Skill (ext)",
|
||||
"101": "promql-cli Skill",
|
||||
"102": "Backend Module",
|
||||
"103": "bookmark-api Service",
|
||||
"104": "AGENTS.md",
|
||||
"105": "reviewer.md",
|
||||
"106": "Redeploy runbook",
|
||||
"107": "1. Backend",
|
||||
"108": "Deployment",
|
||||
"109": "Cinder — BookmarkManager design system",
|
||||
"110": "Implement tickets",
|
||||
"111": "SQLite → Postgres cutover runbook",
|
||||
"112": "Testing the userscript",
|
||||
"113": "ADR-0007: The backend hosts every Site's Cover bytes",
|
||||
"114": "Issue tracker: Gitea (`tea` CLI)",
|
||||
"115": "ADR-0006: The browser runs on the home machine, over the tailnet",
|
||||
"116": "ADR-0008: A Series identity is discovered from the Site's links, never derived from an address",
|
||||
"117": "Domain Docs",
|
||||
"118": "ticket-implementer.md",
|
||||
"119": "implementer.md",
|
||||
"120": "Series is a shared entity, and only the Poll may update it",
|
||||
"121": "Postgres replaces SQLite as the primary datastore",
|
||||
"122": "Identity comes from Discord OAuth; we store no passwords and send no email",
|
||||
"123": "The wire format stays flat and deliberately does not mirror the schema",
|
||||
"124": "ADR-0005: On-demand browser sidecar",
|
||||
"126": "Bookmark Manager",
|
||||
"127": "triage-labels.md",
|
||||
"128": "Cross-Ticket Contract",
|
||||
"129": "Implement Tickets Skill",
|
||||
"130": "Orchestrator Role",
|
||||
"131": "resolving-merge-conflicts Skill",
|
||||
"132": "tdd Skill",
|
||||
"133": "Ticket Wave Batching",
|
||||
"134": "Four-Object Browser Stub",
|
||||
"135": "Module Export Hook",
|
||||
"136": "logic.test.js Test Harness",
|
||||
"137": "manga-bookmark.user.js",
|
||||
"138": "stripBuildHash",
|
||||
"139": "Testing the Userscript Skill",
|
||||
"140": "cr-spec Agent",
|
||||
"141": "cr-standards Agent",
|
||||
"142": "Escalate Rather Than Guess",
|
||||
"143": "Status Contract",
|
||||
"144": "Ticket Implementer Agent",
|
||||
"145": "Worktree Isolation",
|
||||
"146": "Escalate Rather Than Guess (opencode)",
|
||||
"147": "Implementer Subagent (opencode)",
|
||||
"148": "Subagent-Driven Development",
|
||||
"149": "Code Quality Review",
|
||||
"150": "Reviewer Subagent (opencode)",
|
||||
"151": "Finding Severity Rubric",
|
||||
"152": "Spec Compliance Review",
|
||||
"161": "Why Use samber/oops",
|
||||
"162": "singleflight Cache Stampede Prevention",
|
||||
"163": "Struct Field Alignment",
|
||||
"164": "testing/synctest Deterministic Goroutine Testing",
|
||||
"165": "API Package (Bookmark JSON Handlers)",
|
||||
"166": "Cover Acquisition & Serving Pipeline",
|
||||
"167": "Backend AGENTS.md Guidance",
|
||||
"168": "HTTP Middleware (Auth/Gzip/CORS)",
|
||||
"169": "Latest Package (Site Parsers & Poller)",
|
||||
"170": "Latest-Chapter Poller",
|
||||
"171": "Main Composition Root",
|
||||
"172": "Migration-Owned Schema",
|
||||
"173": "Session Package (Cookie Signing & Rate Limit)",
|
||||
"174": "updated_at List-Order Rule",
|
||||
"175": "Userscript Package (Serving Handler)",
|
||||
"176": "AGENTS.md",
|
||||
"177": "Backend CLAUDE.md Guidance",
|
||||
"178": "Graphify Knowledge Graph (graphify-out/)",
|
||||
"179": "CLAUDE.md (Symlink to AGENTS.md)",
|
||||
"192": "ADR-0001 (Drop modernc.org/sqlite)",
|
||||
"193": "ADR-0003 (Split Shared Series Facts)",
|
||||
"194": "SQLite-to-Postgres Cutover Runbook",
|
||||
"195": "Import SQL Generation Rules",
|
||||
"196": "Throwaway Import Generator",
|
||||
"206": "Real scaling limit is the poller outbound fetch budget",
|
||||
"207": "PostgreSQL (jackc/pgx/v5)",
|
||||
"208": "SQLite (modernc.org/sqlite)",
|
||||
"209": "Postgres chosen for future supportability, not concurrency",
|
||||
"210": "Per-Reader bearer token for userscripts",
|
||||
"211": "Discord OAuth2 (authorization code grant)",
|
||||
"212": "ADR-0002: Discord OAuth, no passwords, no email",
|
||||
"213": "Discord snowflake is the sole identity (lock-in)",
|
||||
"214": "Bookmark (per-Reader state: Progress, Favourite, Lifecycle)",
|
||||
"215": "Deduplicate polling per Series (reader_count DESC queue)",
|
||||
"216": "ADR-0003: Series is shared, only the Poll updates it",
|
||||
"217": "Only the Poll writes Series fields (security boundary)",
|
||||
"218": "Series (shared entity keyed site+series_id)",
|
||||
"219": "ADR-0004: Wire format stays flat, does not mirror schema",
|
||||
"220": "Flat wire shape is a contract, not an implementation detail",
|
||||
"221": "Installed userscripts must keep working (14-day grace window)",
|
||||
"222": "CDP (Chrome DevTools Protocol) endpoint",
|
||||
"223": "headless-shell service (socat-fronted CDP)",
|
||||
"224": "Start Chrome on first CDP connection, reap after 300s idle",
|
||||
"225": "BROWSER_WS_URL configuration seam",
|
||||
"226": "ADR-0006: Browser runs on the home machine over the tailnet",
|
||||
"227": "Browser moved home: VPS memory pressure, no requests served",
|
||||
"228": "Tailnet (Tailscale network)",
|
||||
"229": "Content-addressed filesystem storage (SHA-256 of source URL)",
|
||||
"230": "Cover (Series image bytes)",
|
||||
"231": "Deny-class destination control for outbound fetch",
|
||||
"232": "ADR-0007: Backend hosts every Site's Cover bytes",
|
||||
"233": "kagane CORP same-origin cover restriction",
|
||||
"234": "Backend acquires, stores, serves every Cover (uniformity)",
|
||||
"235": "a[aria-label='All Chapter'] anchor pointer",
|
||||
"236": "Series identity is discovered from the Site's links",
|
||||
"237": "ADR-0008: Series identity discovered, never derived",
|
||||
"238": "Chapter slug vs series slug divergence (~7% measured)",
|
||||
"239": "Scan truncated at first wpd-threads marker",
|
||||
"240": "Surface ADR conflicts explicitly rather than silently overriding",
|
||||
"241": "Domain docs: single-context layout guidance",
|
||||
"242": "/domain-modeling skill (lazy CONTEXT.md creation)",
|
||||
"243": "CONTEXT.md glossary (ubiquitous language)",
|
||||
"244": "Gitea (tea CLI, gitea.violetcrown.my.id)",
|
||||
"245": "wayfinder map/ticket mechanism",
|
||||
"246": "Triage labels: canonical roles to tracker labels",
|
||||
"247": "Canonical triage role labels (needs-triage ... wontfix)",
|
||||
"248": "Cinder (BookmarkManager Web UI design system)",
|
||||
"249": "Heat is typographic: ember reserved for unread chapters",
|
||||
"250": "Design tokens (dark + light branches, no hardcoded hex)",
|
||||
"251": "Three type roles: display serif / mono small-caps / sans",
|
||||
"252": "a[aria-label='All Chapter'] priority pointer",
|
||||
"253": "Research: lightnovelworld chapter slug vs series slug",
|
||||
"254": "Gitea issue #77 (chapter vs series slug)",
|
||||
"255": "Slug divergence measurements (3/41 diverge, 1 split)",
|
||||
"256": "Unscoped chapter regex is SAFE, truncated at wpd-threads",
|
||||
"257": "BookmarkManager",
|
||||
"258": "Bromite (Primary Device)",
|
||||
"259": "Dark-First Design Constraint",
|
||||
"260": "Discord Guild Membership",
|
||||
"261": "Reader Isolation Invariant",
|
||||
"268": "Browser Unit Redeploy",
|
||||
"269": "pg_dump Hot Backup",
|
||||
"270": "Redeploy Runbook",
|
||||
"271": "Rollback Strategy",
|
||||
"273": "AGENTS.md",
|
||||
"279": "Userscript CLAUDE guidance"
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
.
|
||||
@@ -1,558 +0,0 @@
|
||||
# Graph Report - mangaBookmark (2026-08-12)
|
||||
|
||||
## Corpus Check
|
||||
- 110 files · ~267,791 words
|
||||
- Verdict: corpus is large enough that graph structure adds value.
|
||||
|
||||
## Summary
|
||||
- 1621 nodes · 3242 edges · 233 communities (63 shown, 170 thin omitted)
|
||||
- Extraction: 90% EXTRACTED · 10% INFERRED · 0% AMBIGUOUS · INFERRED: 312 edges (avg confidence: 0.77)
|
||||
- Token cost: 0 input · 0 output
|
||||
|
||||
## Graph Freshness
|
||||
- Built from commit: `8ae98816`
|
||||
- Run `git rev-parse HEAD` and compare to check if the graph is stale.
|
||||
- Run `graphify update .` after code changes (no API cost).
|
||||
|
||||
## Community Hubs (Navigation)
|
||||
- [[_COMMUNITY_HTMX Library Internals|HTMX Library Internals]]
|
||||
- [[_COMMUNITY_Cover Fetch Test Helpers|Cover Fetch Test Helpers]]
|
||||
- [[_COMMUNITY_Manga Userscript Adapters|Manga Userscript Adapters]]
|
||||
- [[_COMMUNITY_Novel Userscript Adapters|Novel Userscript Adapters]]
|
||||
- [[_COMMUNITY_Series Acquisition Tests|Series Acquisition Tests]]
|
||||
- [[_COMMUNITY_Bookmarks API Tests|Bookmarks API Tests]]
|
||||
- [[_COMMUNITY_Storage Choice ADR|Storage Choice ADR]]
|
||||
- [[_COMMUNITY_Cover & Acquire Internals|Cover & Acquire Internals]]
|
||||
- [[_COMMUNITY_System Architecture Concepts|System Architecture Concepts]]
|
||||
- [[_COMMUNITY_Session Middleware|Session Middleware]]
|
||||
- [[_COMMUNITY_Go Test Helpers|Go Test Helpers]]
|
||||
- [[_COMMUNITY_Store Tests|Store Tests]]
|
||||
- [[_COMMUNITY_Bookmarks API Handler|Bookmarks API Handler]]
|
||||
- [[_COMMUNITY_Web UI Handlers|Web UI Handlers]]
|
||||
- [[_COMMUNITY_Go Error Handling|Go Error Handling]]
|
||||
- [[_COMMUNITY_CDP Browser Client|CDP Browser Client]]
|
||||
- [[_COMMUNITY_Go Code Style Guide|Go Code Style Guide]]
|
||||
- [[_COMMUNITY_Agent Skills|Agent Skills]]
|
||||
- [[_COMMUNITY_IO Performance Patterns|I/O Performance Patterns]]
|
||||
- [[_COMMUNITY_CPU Optimization|CPU Optimization]]
|
||||
- [[_COMMUNITY_Caching Patterns|Caching Patterns]]
|
||||
- [[_COMMUNITY_Browser Entrypoint|Browser Entrypoint]]
|
||||
- [[_COMMUNITY_Memory Allocation & GC|Memory Allocation & GC]]
|
||||
- [[_COMMUNITY_Cover Fetcher Tests|Cover Fetcher Tests]]
|
||||
- [[_COMMUNITY_readSeries|readSeries]]
|
||||
- [[_COMMUNITY_Find Skills Guide|Find Skills Guide]]
|
||||
- [[_COMMUNITY_Allocation Patterns|Allocation Patterns]]
|
||||
- [[_COMMUNITY_Observability & Alerting|Observability & Alerting]]
|
||||
- [[_COMMUNITY_Memory Layout|Memory Layout]]
|
||||
- [[_COMMUNITY_Repo Hard Constraints|Repo Hard Constraints]]
|
||||
- [[_COMMUNITY_Go Testing Guide|Go Testing Guide]]
|
||||
- [[_COMMUNITY_Session Store|Session Store]]
|
||||
- [[_COMMUNITY_Web UI Filter Logic|Web UI Filter Logic]]
|
||||
- [[_COMMUNITY_Userscript Test Harness|Userscript Test Harness]]
|
||||
- [[_COMMUNITY_Product & Security Context|Product & Security Context]]
|
||||
- [[_COMMUNITY_novel-logic.test.js|novel-logic.test.js]]
|
||||
- [[_COMMUNITY_UI Critique 2026-07-26A|UI Critique 2026-07-26A]]
|
||||
- [[_COMMUNITY_UI Critique 2026-07-26B|UI Critique 2026-07-26B]]
|
||||
- [[_COMMUNITY_Issue Tracker & Triage|Issue Tracker & Triage]]
|
||||
- [[_COMMUNITY_Ticket Workflow|Ticket Workflow]]
|
||||
- [[_COMMUNITY_Go Perf Alert Rules|Go Perf Alert Rules]]
|
||||
- [[_COMMUNITY_Userscript Display Logic|Userscript Display Logic]]
|
||||
- [[_COMMUNITY_Go Perf Skill Docs|Go Perf Skill Docs]]
|
||||
- [[_COMMUNITY_Login Page Art|Login Page Art]]
|
||||
- [[_COMMUNITY_BookmarkManager Logo|BookmarkManager Logo]]
|
||||
- [[_COMMUNITY_Skills CLI|Skills CLI]]
|
||||
- [[_COMMUNITY_Skills Leaderboard|Skills Leaderboard]]
|
||||
- [[_COMMUNITY_Complex Condition Extraction|Complex Condition Extraction]]
|
||||
- [[_COMMUNITY_Sentinel Errors|Sentinel Errors]]
|
||||
- [[_COMMUNITY_errors.As Patterns|errors.As Patterns]]
|
||||
- [[_COMMUNITY_errors.Is Patterns|errors.Is Patterns]]
|
||||
- [[_COMMUNITY_errors.Join Patterns|errors.Join Patterns]]
|
||||
- [[_COMMUNITY_Error Wrapping|Error Wrapping]]
|
||||
- [[_COMMUNITY_Single Error Handling|Single Error Handling]]
|
||||
- [[_COMMUNITY_SIMD Optimizations|SIMD Optimizations]]
|
||||
- [[_COMMUNITY_GOGC Tuning|GOGC Tuning]]
|
||||
- [[_COMMUNITY_GOMEMLIMIT|GOMEMLIMIT]]
|
||||
- [[_COMMUNITY_Bottleneck Decision Tree|Bottleneck Decision Tree]]
|
||||
- [[_COMMUNITY_pprof Profiling|pprof Profiling]]
|
||||
- [[_COMMUNITY_Test Timeout Helper|Test Timeout Helper]]
|
||||
- [[_COMMUNITY_httptest Patterns|httptest Patterns]]
|
||||
- [[_COMMUNITY_testify Suite Pattern|testify Suite Pattern]]
|
||||
- [[_COMMUNITY_goembed Fixtures|go:embed Fixtures]]
|
||||
- [[_COMMUNITY_clockwork Time Mocking|clockwork Time Mocking]]
|
||||
- [[_COMMUNITY_testify Mocking|testify Mocking]]
|
||||
- [[_COMMUNITY_t.ArtifactDir Helper|t.ArtifactDir Helper]]
|
||||
- [[_COMMUNITY_Subtests Pitfall|Subtests Pitfall]]
|
||||
- [[_COMMUNITY_golang-benchmark Skill|golang-benchmark Skill]]
|
||||
- [[_COMMUNITY_golang-concurrency Skill|golang-concurrency Skill]]
|
||||
- [[_COMMUNITY_golang-ci Skill|golang-ci Skill]]
|
||||
- [[_COMMUNITY_golang-database Skill|golang-database Skill]]
|
||||
- [[_COMMUNITY_golang-lint Skill|golang-lint Skill]]
|
||||
- [[_COMMUNITY_testify Skill|testify Skill]]
|
||||
- [[_COMMUNITY_Build Tag Integration Tests|Build Tag Integration Tests]]
|
||||
- [[_COMMUNITY_Test Naming Convention|Test Naming Convention]]
|
||||
- [[_COMMUNITY_UI Critique A Finding|UI Critique A Finding]]
|
||||
- [[_COMMUNITY_UI Critique B Finding|UI Critique B Finding]]
|
||||
- [[_COMMUNITY_P0 Overflow Bug|P0 Overflow Bug]]
|
||||
- [[_COMMUNITY_P1 hx-indicator Gap|P1 hx-indicator Gap]]
|
||||
- [[_COMMUNITY_golang-benchmark Skill (ext)|golang-benchmark Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-concurrency Skill (ext)|golang-concurrency Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-ci Skill (ext)|golang-ci Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-data-structures Skill (ext)|golang-data-structures Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-database Skill (ext)|golang-database Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-design-patterns Skill (ext)|golang-design-patterns Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-documentation Skill (ext)|golang-documentation Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-gopls Skill (ext)|golang-gopls Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-lint Skill (ext)|golang-lint Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-naming Skill (ext)|golang-naming Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-observability Skill (ext)|golang-observability Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-refactoring Skill (ext)|golang-refactoring Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-safety Skill (ext)|golang-safety Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-samber-oops Skill (ext)|golang-samber-oops Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-samber-slog Skill (ext)|golang-samber-slog Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-structs-interfaces Skill (ext)|golang-structs-interfaces Skill (ext)]]
|
||||
- [[_COMMUNITY_golang-troubleshooting Skill (ext)|golang-troubleshooting Skill (ext)]]
|
||||
- [[_COMMUNITY_promql-cli Skill|promql-cli Skill]]
|
||||
- [[_COMMUNITY_Backend Module|Backend Module]]
|
||||
- [[_COMMUNITY_bookmark-api Service|bookmark-api Service]]
|
||||
- [[_COMMUNITY_AGENTS|AGENTS.md]]
|
||||
- [[_COMMUNITY_reviewer|reviewer.md]]
|
||||
- [[_COMMUNITY_Redeploy runbook|Redeploy runbook]]
|
||||
- [[_COMMUNITY_1. Backend|1. Backend]]
|
||||
- [[_COMMUNITY_Deployment|Deployment]]
|
||||
- [[_COMMUNITY_Cinder — BookmarkManager design system|Cinder — BookmarkManager design system]]
|
||||
- [[_COMMUNITY_Implement tickets|Implement tickets]]
|
||||
- [[_COMMUNITY_SQLite → Postgres cutover runbook|SQLite → Postgres cutover runbook]]
|
||||
- [[_COMMUNITY_Testing the userscript|Testing the userscript]]
|
||||
- [[_COMMUNITY_ADR-0007 The backend hosts every Site's Cover bytes|ADR-0007: The backend hosts every Site's Cover bytes]]
|
||||
- [[_COMMUNITY_Issue tracker Gitea (`tea` CLI)|Issue tracker: Gitea (`tea` CLI)]]
|
||||
- [[_COMMUNITY_ADR-0006 The browser runs on the home machine, over the tailnet|ADR-0006: The browser runs on the home machine, over the tailnet]]
|
||||
- [[_COMMUNITY_ADR-0008 A Series identity is discovered from the Site's links, never derived from an address|ADR-0008: A Series identity is discovered from the Site's links, never derived from an address]]
|
||||
- [[_COMMUNITY_Domain Docs|Domain Docs]]
|
||||
- [[_COMMUNITY_ticket-implementer|ticket-implementer.md]]
|
||||
- [[_COMMUNITY_implementer|implementer.md]]
|
||||
- [[_COMMUNITY_Series is a shared entity, and only the Poll may update it|Series is a shared entity, and only the Poll may update it]]
|
||||
- [[_COMMUNITY_Postgres replaces SQLite as the primary datastore|Postgres replaces SQLite as the primary datastore]]
|
||||
- [[_COMMUNITY_Identity comes from Discord OAuth; we store no passwords and send no email|Identity comes from Discord OAuth; we store no passwords and send no email]]
|
||||
- [[_COMMUNITY_The wire format stays flat and deliberately does not mirror the schema|The wire format stays flat and deliberately does not mirror the schema]]
|
||||
- [[_COMMUNITY_ADR-0005 On-demand browser sidecar|ADR-0005: On-demand browser sidecar]]
|
||||
- [[_COMMUNITY_Bookmark Manager|Bookmark Manager]]
|
||||
- [[_COMMUNITY_triage-labels|triage-labels.md]]
|
||||
- [[_COMMUNITY_Cross-Ticket Contract|Cross-Ticket Contract]]
|
||||
- [[_COMMUNITY_Implement Tickets Skill|Implement Tickets Skill]]
|
||||
- [[_COMMUNITY_Orchestrator Role|Orchestrator Role]]
|
||||
- [[_COMMUNITY_resolving-merge-conflicts Skill|resolving-merge-conflicts Skill]]
|
||||
- [[_COMMUNITY_tdd Skill|tdd Skill]]
|
||||
- [[_COMMUNITY_Ticket Wave Batching|Ticket Wave Batching]]
|
||||
- [[_COMMUNITY_Four-Object Browser Stub|Four-Object Browser Stub]]
|
||||
- [[_COMMUNITY_Module Export Hook|Module Export Hook]]
|
||||
- [[_COMMUNITY_logic.test.js Test Harness|logic.test.js Test Harness]]
|
||||
- [[_COMMUNITY_manga-bookmark.user.js|manga-bookmark.user.js]]
|
||||
- [[_COMMUNITY_stripBuildHash|stripBuildHash]]
|
||||
- [[_COMMUNITY_Testing the Userscript Skill|Testing the Userscript Skill]]
|
||||
- [[_COMMUNITY_cr-spec Agent|cr-spec Agent]]
|
||||
- [[_COMMUNITY_cr-standards Agent|cr-standards Agent]]
|
||||
- [[_COMMUNITY_Escalate Rather Than Guess|Escalate Rather Than Guess]]
|
||||
- [[_COMMUNITY_Status Contract|Status Contract]]
|
||||
- [[_COMMUNITY_Ticket Implementer Agent|Ticket Implementer Agent]]
|
||||
- [[_COMMUNITY_Worktree Isolation|Worktree Isolation]]
|
||||
- [[_COMMUNITY_Escalate Rather Than Guess (opencode)|Escalate Rather Than Guess (opencode)]]
|
||||
- [[_COMMUNITY_Implementer Subagent (opencode)|Implementer Subagent (opencode)]]
|
||||
- [[_COMMUNITY_Subagent-Driven Development|Subagent-Driven Development]]
|
||||
- [[_COMMUNITY_Code Quality Review|Code Quality Review]]
|
||||
- [[_COMMUNITY_Reviewer Subagent (opencode)|Reviewer Subagent (opencode)]]
|
||||
- [[_COMMUNITY_Finding Severity Rubric|Finding Severity Rubric]]
|
||||
- [[_COMMUNITY_Spec Compliance Review|Spec Compliance Review]]
|
||||
- [[_COMMUNITY_Why Use samberoops|Why Use samber/oops]]
|
||||
- [[_COMMUNITY_singleflight Cache Stampede Prevention|singleflight Cache Stampede Prevention]]
|
||||
- [[_COMMUNITY_Struct Field Alignment|Struct Field Alignment]]
|
||||
- [[_COMMUNITY_testingsynctest Deterministic Goroutine Testing|testing/synctest Deterministic Goroutine Testing]]
|
||||
- [[_COMMUNITY_API Package (Bookmark JSON Handlers)|API Package (Bookmark JSON Handlers)]]
|
||||
- [[_COMMUNITY_Cover Acquisition & Serving Pipeline|Cover Acquisition & Serving Pipeline]]
|
||||
- [[_COMMUNITY_Backend AGENTS.md Guidance|Backend AGENTS.md Guidance]]
|
||||
- [[_COMMUNITY_HTTP Middleware (AuthGzipCORS)|HTTP Middleware (Auth/Gzip/CORS)]]
|
||||
- [[_COMMUNITY_Latest Package (Site Parsers & Poller)|Latest Package (Site Parsers & Poller)]]
|
||||
- [[_COMMUNITY_Latest-Chapter Poller|Latest-Chapter Poller]]
|
||||
- [[_COMMUNITY_Main Composition Root|Main Composition Root]]
|
||||
- [[_COMMUNITY_Migration-Owned Schema|Migration-Owned Schema]]
|
||||
- [[_COMMUNITY_Session Package (Cookie Signing & Rate Limit)|Session Package (Cookie Signing & Rate Limit)]]
|
||||
- [[_COMMUNITY_updated_at List-Order Rule|updated_at List-Order Rule]]
|
||||
- [[_COMMUNITY_Userscript Package (Serving Handler)|Userscript Package (Serving Handler)]]
|
||||
- [[_COMMUNITY_Backend CLAUDE.md Guidance|Backend CLAUDE.md Guidance]]
|
||||
- [[_COMMUNITY_Graphify Knowledge Graph (graphify-out)|Graphify Knowledge Graph (graphify-out/)]]
|
||||
- [[_COMMUNITY_CLAUDE.md (Symlink to AGENTS.md)|CLAUDE.md (Symlink to AGENTS.md)]]
|
||||
- [[_COMMUNITY_ADR-0001 (Drop modernc.orgsqlite)|ADR-0001 (Drop modernc.org/sqlite)]]
|
||||
- [[_COMMUNITY_ADR-0003 (Split Shared Series Facts)|ADR-0003 (Split Shared Series Facts)]]
|
||||
- [[_COMMUNITY_SQLite-to-Postgres Cutover Runbook|SQLite-to-Postgres Cutover Runbook]]
|
||||
- [[_COMMUNITY_Import SQL Generation Rules|Import SQL Generation Rules]]
|
||||
- [[_COMMUNITY_Throwaway Import Generator|Throwaway Import Generator]]
|
||||
- [[_COMMUNITY_Real scaling limit is the poller outbound fetch budget|Real scaling limit is the poller outbound fetch budget]]
|
||||
- [[_COMMUNITY_PostgreSQL (jackcpgxv5)|PostgreSQL (jackc/pgx/v5)]]
|
||||
- [[_COMMUNITY_SQLite (modernc.orgsqlite)|SQLite (modernc.org/sqlite)]]
|
||||
- [[_COMMUNITY_Postgres chosen for future supportability, not concurrency|Postgres chosen for future supportability, not concurrency]]
|
||||
- [[_COMMUNITY_Per-Reader bearer token for userscripts|Per-Reader bearer token for userscripts]]
|
||||
- [[_COMMUNITY_Discord OAuth2 (authorization code grant)|Discord OAuth2 (authorization code grant)]]
|
||||
- [[_COMMUNITY_ADR-0002 Discord OAuth, no passwords, no email|ADR-0002: Discord OAuth, no passwords, no email]]
|
||||
- [[_COMMUNITY_Discord snowflake is the sole identity (lock-in)|Discord snowflake is the sole identity (lock-in)]]
|
||||
- [[_COMMUNITY_Bookmark (per-Reader state Progress, Favourite, Lifecycle)|Bookmark (per-Reader state: Progress, Favourite, Lifecycle)]]
|
||||
- [[_COMMUNITY_Deduplicate polling per Series (reader_count DESC queue)|Deduplicate polling per Series (reader_count DESC queue)]]
|
||||
- [[_COMMUNITY_ADR-0003 Series is shared, only the Poll updates it|ADR-0003: Series is shared, only the Poll updates it]]
|
||||
- [[_COMMUNITY_Only the Poll writes Series fields (security boundary)|Only the Poll writes Series fields (security boundary)]]
|
||||
- [[_COMMUNITY_Series (shared entity keyed site+series_id)|Series (shared entity keyed site+series_id)]]
|
||||
- [[_COMMUNITY_ADR-0004 Wire format stays flat, does not mirror schema|ADR-0004: Wire format stays flat, does not mirror schema]]
|
||||
- [[_COMMUNITY_Flat wire shape is a contract, not an implementation detail|Flat wire shape is a contract, not an implementation detail]]
|
||||
- [[_COMMUNITY_Installed userscripts must keep working (14-day grace window)|Installed userscripts must keep working (14-day grace window)]]
|
||||
- [[_COMMUNITY_CDP (Chrome DevTools Protocol) endpoint|CDP (Chrome DevTools Protocol) endpoint]]
|
||||
- [[_COMMUNITY_headless-shell service (socat-fronted CDP)|headless-shell service (socat-fronted CDP)]]
|
||||
- [[_COMMUNITY_Start Chrome on first CDP connection, reap after 300s idle|Start Chrome on first CDP connection, reap after 300s idle]]
|
||||
- [[_COMMUNITY_BROWSER_WS_URL configuration seam|BROWSER_WS_URL configuration seam]]
|
||||
- [[_COMMUNITY_ADR-0006 Browser runs on the home machine over the tailnet|ADR-0006: Browser runs on the home machine over the tailnet]]
|
||||
- [[_COMMUNITY_Browser moved home VPS memory pressure, no requests served|Browser moved home: VPS memory pressure, no requests served]]
|
||||
- [[_COMMUNITY_Tailnet (Tailscale network)|Tailnet (Tailscale network)]]
|
||||
- [[_COMMUNITY_Content-addressed filesystem storage (SHA-256 of source URL)|Content-addressed filesystem storage (SHA-256 of source URL)]]
|
||||
- [[_COMMUNITY_Cover (Series image bytes)|Cover (Series image bytes)]]
|
||||
- [[_COMMUNITY_Deny-class destination control for outbound fetch|Deny-class destination control for outbound fetch]]
|
||||
- [[_COMMUNITY_ADR-0007 Backend hosts every Site's Cover bytes|ADR-0007: Backend hosts every Site's Cover bytes]]
|
||||
- [[_COMMUNITY_kagane CORP same-origin cover restriction|kagane CORP same-origin cover restriction]]
|
||||
- [[_COMMUNITY_Backend acquires, stores, serves every Cover (uniformity)|Backend acquires, stores, serves every Cover (uniformity)]]
|
||||
- [[_COMMUNITY_aaria-label='All Chapter' anchor pointer|a[aria-label='All Chapter'] anchor pointer]]
|
||||
- [[_COMMUNITY_Series identity is discovered from the Site's links|Series identity is discovered from the Site's links]]
|
||||
- [[_COMMUNITY_ADR-0008 Series identity discovered, never derived|ADR-0008: Series identity discovered, never derived]]
|
||||
- [[_COMMUNITY_Chapter slug vs series slug divergence (~7% measured)|Chapter slug vs series slug divergence (~7% measured)]]
|
||||
- [[_COMMUNITY_Scan truncated at first wpd-threads marker|Scan truncated at first wpd-threads marker]]
|
||||
- [[_COMMUNITY_Surface ADR conflicts explicitly rather than silently overriding|Surface ADR conflicts explicitly rather than silently overriding]]
|
||||
- [[_COMMUNITY_Domain docs single-context layout guidance|Domain docs: single-context layout guidance]]
|
||||
- [[_COMMUNITY_domain-modeling skill (lazy CONTEXT.md creation)|/domain-modeling skill (lazy CONTEXT.md creation)]]
|
||||
- [[_COMMUNITY_CONTEXT.md glossary (ubiquitous language)|CONTEXT.md glossary (ubiquitous language)]]
|
||||
- [[_COMMUNITY_Gitea (tea CLI, gitea.violetcrown.my.id)|Gitea (tea CLI, gitea.violetcrown.my.id)]]
|
||||
- [[_COMMUNITY_wayfinder mapticket mechanism|wayfinder map/ticket mechanism]]
|
||||
- [[_COMMUNITY_Triage labels canonical roles to tracker labels|Triage labels: canonical roles to tracker labels]]
|
||||
- [[_COMMUNITY_Canonical triage role labels (needs-triage ... wontfix)|Canonical triage role labels (needs-triage ... wontfix)]]
|
||||
- [[_COMMUNITY_Cinder (BookmarkManager Web UI design system)|Cinder (BookmarkManager Web UI design system)]]
|
||||
- [[_COMMUNITY_Heat is typographic ember reserved for unread chapters|Heat is typographic: ember reserved for unread chapters]]
|
||||
- [[_COMMUNITY_Design tokens (dark + light branches, no hardcoded hex)|Design tokens (dark + light branches, no hardcoded hex)]]
|
||||
- [[_COMMUNITY_Three type roles display serif mono small-caps sans|Three type roles: display serif / mono small-caps / sans]]
|
||||
- [[_COMMUNITY_aaria-label='All Chapter' priority pointer|a[aria-label='All Chapter'] priority pointer]]
|
||||
- [[_COMMUNITY_Research lightnovelworld chapter slug vs series slug|Research: lightnovelworld chapter slug vs series slug]]
|
||||
- [[_COMMUNITY_Gitea issue 77 (chapter vs series slug)|Gitea issue #77 (chapter vs series slug)]]
|
||||
- [[_COMMUNITY_Slug divergence measurements (341 diverge, 1 split)|Slug divergence measurements (3/41 diverge, 1 split)]]
|
||||
- [[_COMMUNITY_Unscoped chapter regex is SAFE, truncated at wpd-threads|Unscoped chapter regex is SAFE, truncated at wpd-threads]]
|
||||
- [[_COMMUNITY_BookmarkManager|BookmarkManager]]
|
||||
- [[_COMMUNITY_Bromite (Primary Device)|Bromite (Primary Device)]]
|
||||
- [[_COMMUNITY_Dark-First Design Constraint|Dark-First Design Constraint]]
|
||||
- [[_COMMUNITY_Discord Guild Membership|Discord Guild Membership]]
|
||||
- [[_COMMUNITY_Reader Isolation Invariant|Reader Isolation Invariant]]
|
||||
- [[_COMMUNITY_Browser Unit Redeploy|Browser Unit Redeploy]]
|
||||
- [[_COMMUNITY_pg_dump Hot Backup|pg_dump Hot Backup]]
|
||||
- [[_COMMUNITY_Redeploy Runbook|Redeploy Runbook]]
|
||||
- [[_COMMUNITY_Rollback Strategy|Rollback Strategy]]
|
||||
- [[_COMMUNITY_AGENTS|AGENTS.md]]
|
||||
- [[_COMMUNITY_Userscript CLAUDE guidance|Userscript CLAUDE guidance]]
|
||||
|
||||
## God Nodes (most connected - your core abstractions)
|
||||
1. `testConfig()` - 53 edges
|
||||
2. `newWebTestServer()` - 49 edges
|
||||
3. `newTestStore()` - 42 edges
|
||||
4. `newTestStore()` - 41 edges
|
||||
5. `e()` - 33 edges
|
||||
6. `Handler` - 29 edges
|
||||
7. `ne()` - 28 edges
|
||||
8. `De()` - 28 edges
|
||||
9. `Open()` - 27 edges
|
||||
10. `se()` - 27 edges
|
||||
|
||||
## Surprising Connections (you probably didn't know these)
|
||||
- `Browser Sidecar Service` --semantically_similar_to--> `Browser Sidecar (BROWSER_WS_URL)` [INFERRED] [semantically similar]
|
||||
chrome/docker-compose.yml → backend/AGENTS.md
|
||||
- `el()` --indirect_call--> `c()` [INFERRED]
|
||||
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
|
||||
- `el()` --indirect_call--> `c()` [INFERRED]
|
||||
userscript/novel-bookmark.user.js → backend/internal/web/static/htmx.min.js
|
||||
- `latestChapterFromAnchors()` --indirect_call--> `re()` [INFERRED]
|
||||
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
|
||||
- `el()` --indirect_call--> `k()` [INFERRED]
|
||||
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
|
||||
|
||||
## Import Cycles
|
||||
- None detected.
|
||||
|
||||
## Hyperedges (group relationships)
|
||||
- **Batch Ticket Implementation Pipeline** — _claude_skills_implement_tickets_skill_implement_tickets, _omp_agents_ticket_implementer_ticket_implementer, _omp_agents_ticket_implementer_cr_spec, _omp_agents_ticket_implementer_cr_standards [INFERRED 0.85]
|
||||
- **Subagent-Driven Development Pipeline** — _opencode_agent_implementer_implementer, _opencode_agent_reviewer_reviewer, _opencode_agent_implementer_subagent_driven_development [INFERRED 0.85]
|
||||
- **Go HTML Template Family** — backend_internal_web_templates_app_doc, backend_internal_web_templates_card_doc, backend_internal_web_templates_list_doc, backend_internal_web_templates_chrome_doc, backend_internal_web_templates_login_doc, backend_internal_web_templates_setup_doc, backend_internal_web_templates_readers_doc, backend_internal_web_templates_icons_doc [INFERRED 0.95]
|
||||
- **htmx Fragment Swap Flow** — backend_internal_web_templates_app_doc, backend_internal_web_templates_card_doc, backend_internal_web_templates_chrome_doc, backend_internal_web_templates_setup_doc, backend_internal_web_templates_readers_doc [INFERRED 0.95]
|
||||
- **Backend owns the truth (single-writer ownership of shared facts)** — docs_adr_0003_series_shared_and_poll_owned_poll_owned_writes, docs_adr_0004_wire_format_does_not_mirror_the_schema_flat_wire_contract, docs_adr_0007_backend_hosts_cover_bytes_server_side_covers [INFERRED 0.85]
|
||||
- **Headless browser infrastructure (sidecar, on-demand, home deployment)** — docs_adr_0005_on_demand_browser_headless_shell, docs_adr_0005_on_demand_browser_cdp, docs_adr_0005_on_demand_browser_on_demand_start, docs_adr_0006_browser_on_the_home_machine_home_machine_rationale [INFERRED 0.85]
|
||||
- **lightnovelworld series-identity investigation and fix** — docs_research_lightnovelworld_chapter_vs_series_slug_issue_77, docs_research_lightnovelworld_chapter_vs_series_slug_unscoped_regex, docs_adr_0008_series_identity_is_discovered_not_derived_discovered_identity [INFERRED 0.85]
|
||||
|
||||
## Communities (233 total, 170 thin omitted)
|
||||
|
||||
### Community 0 - "HTMX Library Internals"
|
||||
Cohesion: 0.08
|
||||
Nodes (101): A(), ae(), an(), at(), B(), be(), bn(), bt() (+93 more)
|
||||
|
||||
### Community 1 - "Cover Fetch Test Helpers"
|
||||
Cohesion: 0.10
|
||||
Nodes (84): floatPtr(), testConfig(), getCover(), Cookie, Handler, ResponseRecorder, T, TestListRendersAcquiredCover() (+76 more)
|
||||
|
||||
### Community 2 - "Manga Userscript Adapters"
|
||||
Cohesion: 0.06
|
||||
Nodes (76): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+68 more)
|
||||
|
||||
### Community 3 - "Novel Userscript Adapters"
|
||||
Cohesion: 0.06
|
||||
Nodes (79): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+71 more)
|
||||
|
||||
### Community 4 - "Series Acquisition Tests"
|
||||
Cohesion: 0.10
|
||||
Nodes (67): bookmarkNewKaganeSeries(), bookmarkNewNovelfullSeries(), bookmarkNewSeries(), Context, Store, T, newAcquirer(), readBookmark() (+59 more)
|
||||
|
||||
### Community 5 - "Bookmarks API Tests"
|
||||
Cohesion: 0.08
|
||||
Nodes (67): auth(), getBookmarks(), Handler, Request, Store, T, newTestServer(), newTestStore() (+59 more)
|
||||
|
||||
### Community 7 - "Cover & Acquire Internals"
|
||||
Cohesion: 0.09
|
||||
Nodes (31): Addr, Context, Store, defaultCoverResolver(), fetchCoverBytes(), Client, Context, NewCoverFetcher() (+23 more)
|
||||
|
||||
### Community 8 - "System Architecture Concepts"
|
||||
Cohesion: 0.10
|
||||
Nodes (26): Confirm-Gated Destructive Actions, Discord OAuth & Guild-Membership Gate, Lifecycle Buckets (reading/archived/finished), Reader-Owned Store, HMAC-Derived Reader Credentials, Web Package (Browser UI + Templates), app.html — App Shell Template, Manga/Novel Library Switch (+18 more)
|
||||
|
||||
### Community 9 - "Session Middleware"
|
||||
Cohesion: 0.07
|
||||
Nodes (35): ClearCookie(), ClientIP(), Duration, Mutex, Request, ResponseWriter, Time, isHTTPS() (+27 more)
|
||||
|
||||
### Community 10 - "Go Test Helpers"
|
||||
Cohesion: 0.05
|
||||
Nodes (39): Test Helpers, Test Timeout, Basic Handler Test, HTTP Handler Testing, Query Parameters and Headers, Docker Compose Fixture, Integration Testing, SQL Schema Fixture (+31 more)
|
||||
|
||||
### Community 11 - "Store Tests"
|
||||
Cohesion: 0.07
|
||||
Nodes (79): M, TestMain(), M, TestMain(), M, Main(), start(), URL() (+71 more)
|
||||
|
||||
### Community 12 - "Bookmarks API Handler"
|
||||
Cohesion: 0.08
|
||||
Nodes (33): Handler, Request, ResponseWriter, Store, Healthz(), writeJSON(), Auth(), compressible() (+25 more)
|
||||
|
||||
### Community 13 - "Web UI Handlers"
|
||||
Cohesion: 0.06
|
||||
Nodes (24): coverRelativePath(), coverSourceAddress(), displayChapter(), Store, currentLib(), currentTab(), filterBookmarks(), Client (+16 more)
|
||||
|
||||
### Community 14 - "Go Error Handling"
|
||||
Cohesion: 0.06
|
||||
Nodes (33): Creating Errors, Custom Error Types, Custom types that wrap other errors, Decision table: which error strategy to use, Error Creation, Error String Conventions, Errors as Values, `errors.New` — static error messages (+25 more)
|
||||
|
||||
### Community 15 - "CDP Browser Client"
|
||||
Cohesion: 0.08
|
||||
Nodes (31): awaitPromise(), browserConnectionLost(), classifyBrowserError(), Action, Context, Mutex, jsString(), kaganeAPIURL() (+23 more)
|
||||
|
||||
### Community 17 - "Go Code Style Guide"
|
||||
Cohesion: 0.08
|
||||
Nodes (23): Code Style Details, Extract Complex Conditions, Value vs Pointer Arguments, Code Organization Within Files, Complex Conditions & Init Scope, Composite Literals, Control Flow, Cross-References (+15 more)
|
||||
|
||||
### Community 20 - "I/O Performance Patterns"
|
||||
Cohesion: 0.11
|
||||
Nodes (18): Avoid io.ReadAll for large payloads, Batch Operations, Buffered I/O, Cgo Overhead, Channel: batch processing from a stream, Concurrent Multi-Stage Pipelines, Connection pooling, Database: batch inserts over row-by-row (+10 more)
|
||||
|
||||
### Community 21 - "CPU Optimization"
|
||||
Cohesion: 0.13
|
||||
Nodes (15): Cache Locality, Contiguous 2D allocation, CPU Optimization, False Sharing, Function Inlining, Handling CPU-specific instruction sets, Instruction-Level Parallelism, Monotonic Time (+7 more)
|
||||
|
||||
### Community 22 - "Caching Patterns"
|
||||
Cohesion: 0.13
|
||||
Nodes (14): Algorithmic Complexity, Avoid iterator chains, Caching Patterns, Compiled Pattern Caching, Early returns and short-circuit loops, LRU caches, Map lookups over slice scanning, Precomputed lookup tables (+6 more)
|
||||
|
||||
### Community 23 - "Browser Entrypoint"
|
||||
Cohesion: 0.35
|
||||
Nodes (14): browser_alive(), connection(), connection_signal(), finish_connection(), has_connections(), lock(), reaper(), entrypoint.sh script (+6 more)
|
||||
|
||||
### Community 24 - "Memory Allocation & GC"
|
||||
Cohesion: 0.13
|
||||
Nodes (15): Allocation Rate Reduction, Ballast pattern (pre-Go 1.19), Garbage Collector Tuning, GC pacing, GC Profiling and Diagnostics, GODEBUG=gctrace=1, GOGC (default: 100), GOMAXPROCS in Containers (+7 more)
|
||||
|
||||
### Community 25 - "Cover Fetcher Tests"
|
||||
Cohesion: 0.33
|
||||
Nodes (12): coverResponse(), Request, T, TestCoverFetcherCanonicalisesJpgAlias(), TestCoverFetcherFetchesPublicHTTPSImage(), TestCoverFetcherRefusesUnsafeDestinationsBeforeRequest(), TestCoverFetcherRejectsNonImage(), TestCoverFetcherRejectsOversizedBody() (+4 more)
|
||||
|
||||
### Community 26 - "readSeries"
|
||||
Cohesion: 0.13
|
||||
Nodes (28): asuraLatestChapter(), browserOnlyCoverURL(), comixCoverEntry(), comixCoverURL(), comixLatestChapter(), comixSeriesID(), coverFrom(), demonicLatestChapter() (+20 more)
|
||||
|
||||
### Community 27 - "Find Skills Guide"
|
||||
Cohesion: 0.14
|
||||
Nodes (13): Common Skill Categories, Find Skills, How to Help Users Find Skills, Step 1: Understand What They Need, Step 2: Check the Leaderboard First, Step 3: Search for Skills, Step 4: Verify Quality Before Recommending, Step 5: Present Options to the User (+5 more)
|
||||
|
||||
### Community 28 - "Allocation Patterns"
|
||||
Cohesion: 0.14
|
||||
Nodes (14): Allocation Patterns, Backing Array Leaks, Direct indexing vs append, Eliminate redundant map lookups, Interface boxing, Map never shrinks, Map size hints, Memory Optimization (+6 more)
|
||||
|
||||
### Community 29 - "Observability & Alerting"
|
||||
Cohesion: 0.22
|
||||
Nodes (9): Alerting rules (examples), CPU saturation, GC pressure, Goroutine leaks, Grafana Dashboards, Memory leaks, Prometheus Metrics for Go, PromQL Queries for Performance Diagnosis (+1 more)
|
||||
|
||||
### Community 31 - "Memory Layout"
|
||||
Cohesion: 0.40
|
||||
Nodes (5): Map of pointers for large, frequently updated structs, Memory Layout, Pointer receivers for large structs, Struct field alignment, Zero-size field at end of struct
|
||||
|
||||
### Community 33 - "Go Testing Guide"
|
||||
Cohesion: 0.20
|
||||
Nodes (10): CI Regression Detection, Common Mistakes, Core Philosophy, Cross-References, Decision Tree: Where Is Time Spent?, Deep Dives, Go Performance Optimization, Iterative Optimization Methodology (+2 more)
|
||||
|
||||
### Community 34 - "Session Store"
|
||||
Cohesion: 0.28
|
||||
Nodes (4): Duration, Store, Time, Session
|
||||
|
||||
### Community 35 - "Web UI Filter Logic"
|
||||
Cohesion: 0.31
|
||||
Nodes (5): closeCardPanels(), setActiveTab(), toggleChapterForm(), toggleConfirmRow(), togglePanel()
|
||||
|
||||
### Community 37 - "Product & Security Context"
|
||||
Cohesion: 0.17
|
||||
Nodes (11): Accessibility & Inclusion, Brand Commitments, Capabilities and Constraints, Evidence on Hand, Operating Context, Platform, Positioning, Product (+3 more)
|
||||
|
||||
### Community 38 - "novel-logic.test.js"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): ADR-0009: A Site answers fixed questions; an unusual Site owns its own fetch, Consequences, Considered options, Decision, Why
|
||||
|
||||
### Community 39 - "UI Critique 2026-07-26A"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): Design Health Score, Design Specificity Verdict, Minor Observations, Persona Red Flags, Priority Issues, Questions to Consider
|
||||
|
||||
### Community 40 - "UI Critique 2026-07-26B"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): Design Health Score, Design Specificity Verdict, Minor Observations, Persona Red Flags, Priority Issues, Questions to Consider
|
||||
|
||||
### Community 45 - "Go Perf Alert Rules"
|
||||
Cohesion: 0.50
|
||||
Nodes (4): Prometheus Alerting Rules (Go Performance), GoroutineLeak Alert, HighGCPauseTime Alert, MemoryNearLimit Alert
|
||||
|
||||
### Community 46 - "Userscript Display Logic"
|
||||
Cohesion: 0.07
|
||||
Nodes (27): 1. Summary answer table, 2. The two reference pages, 3. Chapter page → series URL: every in-page pointer, in priority order, 4.1 Sample method, 4.2 Divergence results, 4.3 Is there a derivable rule? **No.**, 4.4 The split case — a slug can change *mid-series*, 4. The reverse direction, and how common divergence is (+19 more)
|
||||
|
||||
### Community 47 - "Go Perf Skill Docs"
|
||||
Cohesion: 0.22
|
||||
Nodes (5): Continuous Profiling, Production Observability for Performance, Pyroscope pull mode (via Grafana Alloy), Pyroscope push mode, Real-Time Visualization (Development)
|
||||
|
||||
### Community 48 - "Login Page Art"
|
||||
Cohesion: 0.67
|
||||
Nodes (3): Fantasy Sword, Fiery Volcanic Scene, Login Art: Sword in Volcanic Rock
|
||||
|
||||
### Community 49 - "BookmarkManager Logo"
|
||||
Cohesion: 1.00
|
||||
Nodes (3): Mirrored Double Bookmark Mark, Ember Flame Accent, BookmarkManager Logo
|
||||
|
||||
### Community 103 - "bookmark-api Service"
|
||||
Cohesion: 0.24
|
||||
Nodes (10): Backend Go Service (stdlib net/http), Browser Sidecar (BROWSER_WS_URL), Browser Sidecar Service, CDP Endpoint (Tailnet-Bound :9222), Persistent Chrome Profile Volume, chrome/docker-compose.yml — Browser Deployable Unit, bookmark-api Prod Override, CDP Never on Shared Proxy Network (+2 more)
|
||||
|
||||
### Community 104 - "AGENTS.md"
|
||||
Cohesion: 0.12
|
||||
Nodes (14): Agent skills, Architecture, Commands, Comments, Design system, Domain docs, Forge: Gitea, not GitHub, graphify (+6 more)
|
||||
|
||||
### Community 105 - "reviewer.md"
|
||||
Cohesion: 0.12
|
||||
Nodes (15): Assessment, Calibration, Critical (Must Fix), Do Not Trust the Report, Important (Should Fix), Inputs, Issues, Method (+7 more)
|
||||
|
||||
### Community 106 - "Redeploy runbook"
|
||||
Cohesion: 0.12
|
||||
Nodes (15): 0. Preflight, 1. Back up the database, 2. Pull the new code, 3. Rebuild and restart, 4. Verify the deploy, 5. Smoke-test the full loop, 6. Rollback, 7. The whole thing, as one block (+7 more)
|
||||
|
||||
### Community 107 - "1. Backend"
|
||||
Cohesion: 0.13
|
||||
Nodes (14): 1. Backend, 2. Userscript, Adapter reference (verified live 2026-07-24), Config (env), Deploy behind your reverse proxy, Desktop iteration (optional), Develop / test, Endpoints (+6 more)
|
||||
|
||||
### Community 108 - "Deployment"
|
||||
Cohesion: 0.14
|
||||
Nodes (13): 0. Prerequisites, 1. Configure `.env`, 1b. Web UI, 2. Build + start, 3. Verify over HTTPS, 4. Configure the userscript, 5. Install on Bromite, 6. Smoke-test the full loop (+5 more)
|
||||
|
||||
### Community 109 - "Cinder — BookmarkManager design system"
|
||||
Cohesion: 0.20
|
||||
Nodes (9): 1. The one idea, 2. Tokens, 3. Type, 4. Components (web UI), 5. Components (userscript panel), 6. Motion, 7. Accessibility floor (not negotiable), 8. Adding something new — checklist (+1 more)
|
||||
|
||||
### Community 110 - "Implement tickets"
|
||||
Cohesion: 0.22
|
||||
Nodes (8): 1. Collect the tickets, 2. Plan the batch, 3. Get the plan approved, 4. Run a wave, 5. Land the wave, 6. Close the batch, Implement tickets, Ticket #<n> — <title>
|
||||
|
||||
### Community 111 - "SQLite → Postgres cutover runbook"
|
||||
Cohesion: 0.22
|
||||
Nodes (8): 0. The generator is throwaway, 1. Stop the old API and take a fresh export, 2. Bring up Postgres with the schema and the owner Reader, 3. Generate the import SQL, 4. Review it by eye, 5. Apply it, 6. Afterwards, SQLite → Postgres cutover runbook
|
||||
|
||||
### Community 112 - "Testing the userscript"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): Adding a test, Commands, Gotchas, How the harness works, Testing the userscript, What is NOT testable here
|
||||
|
||||
### Community 113 - "ADR-0007: The backend hosts every Site's Cover bytes"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): ADR-0007: The backend hosts every Site's Cover bytes, Consequences, Considered options, Decision, Two deliberate relaxations, Why a future reader will find this surprising
|
||||
|
||||
### Community 114 - "Issue tracker: Gitea (`tea` CLI)"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): Conventions, Issue tracker: Gitea (`tea` CLI), Pull requests as a triage surface, Wayfinding operations, When a skill says "fetch the relevant ticket", When a skill says "publish to the issue tracker"
|
||||
|
||||
### Community 115 - "ADR-0006: The browser runs on the home machine, over the tailnet"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): ADR-0006: The browser runs on the home machine, over the tailnet, Consequences, Constraints, Decision, Why
|
||||
|
||||
### Community 116 - "ADR-0008: A Series identity is discovered from the Site's links, never derived from an address"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): ADR-0008: A Series identity is discovered from the Site's links, never derived from an address, Consequences, Considered options, Decision, Why
|
||||
|
||||
### Community 117 - "Domain Docs"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): Before exploring, read these, Domain Docs, File structure, Flag ADR conflicts, Use the glossary's vocabulary
|
||||
|
||||
### Community 118 - "ticket-implementer.md"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): Escalate rather than guess, Order of work, Report, Review, The worktree is your whole world
|
||||
|
||||
### Community 119 - "implementer.md"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): Before You Begin, Report Format, Self-Review Before Reporting, When You're in Over Your Head, Your Job
|
||||
|
||||
### Community 120 - "Series is a shared entity, and only the Poll may update it"
|
||||
Cohesion: 0.40
|
||||
Nodes (4): Consequences, Only the Poll writes Series fields, Series is a shared entity, and only the Poll may update it, Why
|
||||
|
||||
### Community 121 - "Postgres replaces SQLite as the primary datastore"
|
||||
Cohesion: 0.50
|
||||
Nodes (3): Consequences, Considered options, Postgres replaces SQLite as the primary datastore
|
||||
|
||||
### Community 122 - "Identity comes from Discord OAuth; we store no passwords and send no email"
|
||||
Cohesion: 0.50
|
||||
Nodes (3): Consequences, Considered options, Identity comes from Discord OAuth; we store no passwords and send no email
|
||||
|
||||
### Community 123 - "The wire format stays flat and deliberately does not mirror the schema"
|
||||
Cohesion: 0.50
|
||||
Nodes (3): Consequence, The wire format stays flat and deliberately does not mirror the schema, Why a future reader will find this surprising
|
||||
|
||||
### Community 124 - "ADR-0005: On-demand browser sidecar"
|
||||
Cohesion: 0.50
|
||||
Nodes (3): ADR-0005: On-demand browser sidecar, Constraints, Decision
|
||||
|
||||
### Community 273 - "AGENTS.md"
|
||||
Cohesion: 0.50
|
||||
Nodes (3): Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trust), Second script: `novel-bookmark.user.js`, Userscript structure (single IIFE, `manga-bookmark.user.js`)
|
||||
|
||||
## Knowledge Gaps
|
||||
- **508 isolated node(s):** `bookmarkmanager/backend`, `ctxKey`, `loginView`, `ctxKey`, `test` (+503 more)
|
||||
These have ≤1 connection - possible missing edges or undocumented components.
|
||||
- **170 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes.
|
||||
|
||||
## Suggested Questions
|
||||
_Questions this graph is uniquely positioned to answer:_
|
||||
|
||||
- **Why does `New()` connect `Series Acquisition Tests` to `Bookmarks API Tests`, `Cover & Acquire Internals`, `Session Middleware`, `Store Tests`, `Web UI Handlers`?**
|
||||
_High betweenness centrality (0.052) - this node is a cross-community bridge._
|
||||
- **Why does `Open()` connect `Store Tests` to `Cover Fetch Test Helpers`, `Web UI Handlers`, `Series Acquisition Tests`, `Bookmarks API Tests`?**
|
||||
_High betweenness centrality (0.034) - this node is a cross-community bridge._
|
||||
- **Why does `newRouter()` connect `Bookmarks API Tests` to `Cover Fetch Test Helpers`, `Bookmarks API Handler`, `Series Acquisition Tests`?**
|
||||
_High betweenness centrality (0.027) - this node is a cross-community bridge._
|
||||
- **Are the 47 inferred relationships involving `testConfig()` (e.g. with `TestListRendersAcquiredCover()` and `TestPublicCoverNeverEchoesNonImage()`) actually correct?**
|
||||
_`testConfig()` has 47 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **Are the 8 inferred relationships involving `newWebTestServer()` (e.g. with `TestListRendersAcquiredCover()` and `TestPublicCoverRejectsUnknownAddress()`) actually correct?**
|
||||
_`newWebTestServer()` has 8 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **Are the 6 inferred relationships involving `newTestStore()` (e.g. with `TestCreateAndGetSession()` and `TestDeleteSessionIsPerReader()`) actually correct?**
|
||||
_`newTestStore()` has 6 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **What connects `bookmarkmanager/backend`, `ctxKey`, `loginView` to the rest of the system?**
|
||||
_548 weakly-connected nodes found - possible documentation gaps or missing edges._
|
||||
File diff suppressed because one or more lines are too long
-51547
File diff suppressed because it is too large
Load Diff
@@ -1,622 +0,0 @@
|
||||
{
|
||||
".agents/skills/golang-code-style/evals/evals.json": {
|
||||
"mtime": 1784884678.6627614,
|
||||
"ast_hash": "bec0e12446e7af3cd05de9b6d42badd8",
|
||||
"semantic_hash": "bec0e12446e7af3cd05de9b6d42badd8"
|
||||
},
|
||||
".agents/skills/golang-error-handling/evals/evals.json": {
|
||||
"mtime": 1784884678.6655047,
|
||||
"ast_hash": "275d710b774fba1e1d0bc098d3646c6d",
|
||||
"semantic_hash": "275d710b774fba1e1d0bc098d3646c6d"
|
||||
},
|
||||
".agents/skills/golang-performance/evals/evals.json": {
|
||||
"mtime": 1784884678.6688662,
|
||||
"ast_hash": "4f06df87f90aa0e4f6deaa318b47689f",
|
||||
"semantic_hash": "4f06df87f90aa0e4f6deaa318b47689f"
|
||||
},
|
||||
".agents/skills/golang-testing/evals/evals.json": {
|
||||
"mtime": 1784884678.6721346,
|
||||
"ast_hash": "60a821bbfd20c6fe8bba996b8b540dd4",
|
||||
"semantic_hash": "60a821bbfd20c6fe8bba996b8b540dd4"
|
||||
},
|
||||
"backend/go.mod": {
|
||||
"mtime": 1786216141.668644,
|
||||
"ast_hash": "dac242903b0e98c3e4395159d609e08e",
|
||||
"semantic_hash": "dac242903b0e98c3e4395159d609e08e"
|
||||
},
|
||||
"backend/main.go": {
|
||||
"mtime": 1786363889.5678573,
|
||||
"ast_hash": "6e98a3ae91aaa132df251e43c4dfca6d",
|
||||
"semantic_hash": "6e98a3ae91aaa132df251e43c4dfca6d"
|
||||
},
|
||||
"skills-lock.json": {
|
||||
"mtime": 1784884678.6842625,
|
||||
"ast_hash": "4a94ac85bad6bce330d085bcc0ae3ffd",
|
||||
"semantic_hash": "4a94ac85bad6bce330d085bcc0ae3ffd"
|
||||
},
|
||||
"userscript/manga-bookmark.user.js": {
|
||||
"mtime": 1786488438.9080842,
|
||||
"ast_hash": "1f8bcddd3632d709f058a8401af8f127",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
".agents/skills/find-skills/SKILL.md": {
|
||||
"mtime": 1784884338.760326,
|
||||
"ast_hash": "62b297abdee9aea84c577ab2e04e1974",
|
||||
"semantic_hash": "62b297abdee9aea84c577ab2e04e1974"
|
||||
},
|
||||
".agents/skills/golang-code-style/SKILL.md": {
|
||||
"mtime": 1784884678.6623824,
|
||||
"ast_hash": "d6a01e6f64550a5c8d59dac2e948000e",
|
||||
"semantic_hash": "d6a01e6f64550a5c8d59dac2e948000e"
|
||||
},
|
||||
".agents/skills/golang-code-style/references/details.md": {
|
||||
"mtime": 1784884678.6627865,
|
||||
"ast_hash": "19891e396a986f1b24bf34b7a2854fcb",
|
||||
"semantic_hash": "19891e396a986f1b24bf34b7a2854fcb"
|
||||
},
|
||||
".agents/skills/golang-error-handling/SKILL.md": {
|
||||
"mtime": 1784884678.665052,
|
||||
"ast_hash": "8b7970f472adb240e5bc4bde863d43a6",
|
||||
"semantic_hash": "8b7970f472adb240e5bc4bde863d43a6"
|
||||
},
|
||||
".agents/skills/golang-error-handling/references/error-creation.md": {
|
||||
"mtime": 1784884678.665524,
|
||||
"ast_hash": "248dbf75492c68faef2334b8d83bd080",
|
||||
"semantic_hash": "248dbf75492c68faef2334b8d83bd080"
|
||||
},
|
||||
".agents/skills/golang-error-handling/references/error-handling.md": {
|
||||
"mtime": 1784884678.6655412,
|
||||
"ast_hash": "c2424ee3999b05963b199e0100727aa3",
|
||||
"semantic_hash": "c2424ee3999b05963b199e0100727aa3"
|
||||
},
|
||||
".agents/skills/golang-error-handling/references/error-wrapping.md": {
|
||||
"mtime": 1784884678.6655717,
|
||||
"ast_hash": "4a15d9266a1ea0c8bb1951e6b9c0f586",
|
||||
"semantic_hash": "4a15d9266a1ea0c8bb1951e6b9c0f586"
|
||||
},
|
||||
".agents/skills/golang-performance/SKILL.md": {
|
||||
"mtime": 1784884678.667833,
|
||||
"ast_hash": "35a15fd129c5bedaada3fd4df0b6ba8c",
|
||||
"semantic_hash": "35a15fd129c5bedaada3fd4df0b6ba8c"
|
||||
},
|
||||
".agents/skills/golang-performance/assets/prometheus-alerts.yml": {
|
||||
"mtime": 1784884678.668527,
|
||||
"ast_hash": "fa9357ffa87c4f894fc21afa9707c4db",
|
||||
"semantic_hash": "fa9357ffa87c4f894fc21afa9707c4db"
|
||||
},
|
||||
".agents/skills/golang-performance/references/caching.md": {
|
||||
"mtime": 1784884678.6690567,
|
||||
"ast_hash": "807c42a82994e5548dbf7eb6e30c71e0",
|
||||
"semantic_hash": "807c42a82994e5548dbf7eb6e30c71e0"
|
||||
},
|
||||
".agents/skills/golang-performance/references/cpu.md": {
|
||||
"mtime": 1784884678.669087,
|
||||
"ast_hash": "6d52e532ef51a35cc4134694c944517d",
|
||||
"semantic_hash": "6d52e532ef51a35cc4134694c944517d"
|
||||
},
|
||||
".agents/skills/golang-performance/references/io-networking.md": {
|
||||
"mtime": 1784884678.6691036,
|
||||
"ast_hash": "95c5dd51f728fd69c945a92ff021d766",
|
||||
"semantic_hash": "95c5dd51f728fd69c945a92ff021d766"
|
||||
},
|
||||
".agents/skills/golang-performance/references/memory.md": {
|
||||
"mtime": 1784884678.6691158,
|
||||
"ast_hash": "3b2108df06b4cfb3980fa80bbd9ebcff",
|
||||
"semantic_hash": "3b2108df06b4cfb3980fa80bbd9ebcff"
|
||||
},
|
||||
".agents/skills/golang-performance/references/observability.md": {
|
||||
"mtime": 1784884678.6691446,
|
||||
"ast_hash": "0aa8a498e8d55ccdd4990ad187bae828",
|
||||
"semantic_hash": "0aa8a498e8d55ccdd4990ad187bae828"
|
||||
},
|
||||
".agents/skills/golang-performance/references/runtime.md": {
|
||||
"mtime": 1784884678.669426,
|
||||
"ast_hash": "26386c33b3a3794aaef0556713bf8f3a",
|
||||
"semantic_hash": "26386c33b3a3794aaef0556713bf8f3a"
|
||||
},
|
||||
".agents/skills/golang-testing/SKILL.md": {
|
||||
"mtime": 1784884678.6716368,
|
||||
"ast_hash": "0a9b9793bba2a239db94e980272a393e",
|
||||
"semantic_hash": "0a9b9793bba2a239db94e980272a393e"
|
||||
},
|
||||
".agents/skills/golang-testing/references/helpers.md": {
|
||||
"mtime": 1784884678.6722167,
|
||||
"ast_hash": "0aecb7cfbeb9b374bf61c52e324d9d3f",
|
||||
"semantic_hash": "0aecb7cfbeb9b374bf61c52e324d9d3f"
|
||||
},
|
||||
".agents/skills/golang-testing/references/http-testing.md": {
|
||||
"mtime": 1784884678.67225,
|
||||
"ast_hash": "9111110c28a7fbbffc3537aad786b390",
|
||||
"semantic_hash": "9111110c28a7fbbffc3537aad786b390"
|
||||
},
|
||||
".agents/skills/golang-testing/references/integration-testing.md": {
|
||||
"mtime": 1784884678.67225,
|
||||
"ast_hash": "fcf9861bc36ae56e3fb7a1c18aa0e77f",
|
||||
"semantic_hash": "fcf9861bc36ae56e3fb7a1c18aa0e77f"
|
||||
},
|
||||
".agents/skills/golang-testing/references/mocking.md": {
|
||||
"mtime": 1784884678.6722653,
|
||||
"ast_hash": "3a08979e4603aae5c32a58d5b6c39765",
|
||||
"semantic_hash": "3a08979e4603aae5c32a58d5b6c39765"
|
||||
},
|
||||
"CLAUDE.md": {
|
||||
"mtime": 1786488493.5922732,
|
||||
"ast_hash": "7362a25f37333a2a6a55a5aeccf9b0cb",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"DEPLOY.md": {
|
||||
"mtime": 1786488464.5532806,
|
||||
"ast_hash": "2b7b537aa1c0954400c19acc4b634029",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"README.md": {
|
||||
"mtime": 1786488487.8305292,
|
||||
"ast_hash": "9d6be8aa8a2946c23ad48d8f2864b5ca",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"docker-compose.prod.yml": {
|
||||
"mtime": 1786292465.8305523,
|
||||
"ast_hash": "0751998a532297b8ac507a01ec48dc31",
|
||||
"semantic_hash": "0751998a532297b8ac507a01ec48dc31"
|
||||
},
|
||||
"docker-compose.yml": {
|
||||
"mtime": 1786488450.3508112,
|
||||
"ast_hash": "124fd581bf0a662ff15012abfdb40a92",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
".claude/settings.json": {
|
||||
"mtime": 1784951973.1869545,
|
||||
"ast_hash": "e51077b6a7f1f67afc748f1a32a1557d",
|
||||
"semantic_hash": "e51077b6a7f1f67afc748f1a32a1557d"
|
||||
},
|
||||
"backend/web_test.go": {
|
||||
"mtime": 1786216141.700644,
|
||||
"ast_hash": "8f1b093b59eb1ed81bc7fc0c22495c50",
|
||||
"semantic_hash": "8f1b093b59eb1ed81bc7fc0c22495c50"
|
||||
},
|
||||
"backend/main_test.go": {
|
||||
"mtime": 1786363889.5678573,
|
||||
"ast_hash": "8a165955cf28ad47481fec5ea7afb3d6",
|
||||
"semantic_hash": "8a165955cf28ad47481fec5ea7afb3d6"
|
||||
},
|
||||
".claude/settings.local.json": {
|
||||
"mtime": 1785697645.350201,
|
||||
"ast_hash": "9a1ac6369f968e8df4be9dcff0948f70",
|
||||
"semantic_hash": "9a1ac6369f968e8df4be9dcff0948f70"
|
||||
},
|
||||
"PRODUCT.md": {
|
||||
"mtime": 1786216141.660644,
|
||||
"ast_hash": "c52072d1978286060087fa0686f9c7f9",
|
||||
"semantic_hash": "c52072d1978286060087fa0686f9c7f9"
|
||||
},
|
||||
"backend/.impeccable/critique/2026-07-26T15-50-42Z__backend-templates-app-html.md": {
|
||||
"mtime": 1785128576.2412457,
|
||||
"ast_hash": "d08627c27f22d453125db6c1ab4b71ec",
|
||||
"semantic_hash": "d08627c27f22d453125db6c1ab4b71ec"
|
||||
},
|
||||
"backend/.impeccable/critique/2026-07-26T17-08-41Z__backend-templates-app-html.md": {
|
||||
"mtime": 1785128576.2452524,
|
||||
"ast_hash": "e69a8340a371579ca3ea689660f7d7bd",
|
||||
"semantic_hash": "e69a8340a371579ca3ea689660f7d7bd"
|
||||
},
|
||||
"AGENTS.md": {
|
||||
"mtime": 1786488493.5922732,
|
||||
"ast_hash": "7362a25f37333a2a6a55a5aeccf9b0cb",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"userscript/test/logic.test.js": {
|
||||
"mtime": 1786488563.581765,
|
||||
"ast_hash": "80d512bfe6fa8b847f3ba6169c321a74",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
".claude/skills/testing-the-userscript/SKILL.md": {
|
||||
"mtime": 1786363889.5489495,
|
||||
"ast_hash": "8f3c0132eb4787a2c8736eb99f7689af",
|
||||
"semantic_hash": "8f3c0132eb4787a2c8736eb99f7689af"
|
||||
},
|
||||
"REDEPLOY.md": {
|
||||
"mtime": 1786363889.552731,
|
||||
"ast_hash": "d0baf08b95e7b5986234a9f36759c12e",
|
||||
"semantic_hash": "d0baf08b95e7b5986234a9f36759c12e"
|
||||
},
|
||||
"docs/design-system.md": {
|
||||
"mtime": 1786022513.9623306,
|
||||
"ast_hash": "421cd7e57f02d4b467f120ca6ddd7b6a",
|
||||
"semantic_hash": "421cd7e57f02d4b467f120ca6ddd7b6a"
|
||||
},
|
||||
"backend/api_test.go": {
|
||||
"mtime": 1786488480.637822,
|
||||
"ast_hash": "8e4b9293bc2e45ee3f42027315594fd5",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/cover_test.go": {
|
||||
"mtime": 1786363889.552731,
|
||||
"ast_hash": "c7e313d6c92eb28e6d370e5e89035984",
|
||||
"semantic_hash": "c7e313d6c92eb28e6d370e5e89035984"
|
||||
},
|
||||
"backend/internal/api/handlers.go": {
|
||||
"mtime": 1786363889.552731,
|
||||
"ast_hash": "59e6b8767ab19839bb8f82891a7e4616",
|
||||
"semantic_hash": "59e6b8767ab19839bb8f82891a7e4616"
|
||||
},
|
||||
"backend/internal/httpmw/middleware.go": {
|
||||
"mtime": 1786216141.672644,
|
||||
"ast_hash": "385b36f58488b7e6d93eb6d6034e9ee3",
|
||||
"semantic_hash": "385b36f58488b7e6d93eb6d6034e9ee3"
|
||||
},
|
||||
"backend/internal/latest/browser.go": {
|
||||
"mtime": 1786469354.923288,
|
||||
"ast_hash": "ed129a7f00601ea90c877ff29fa21220",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/browser_test.go": {
|
||||
"mtime": 1786262323.6964688,
|
||||
"ast_hash": "e900f92971486f47d7ef76e9a95217fe",
|
||||
"semantic_hash": "e900f92971486f47d7ef76e9a95217fe"
|
||||
},
|
||||
"backend/internal/latest/fetch.go": {
|
||||
"mtime": 1786446006.9219902,
|
||||
"ast_hash": "3e20ad86aa46783e9aa95c2b746551ee",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/poller.go": {
|
||||
"mtime": 1786469641.8342345,
|
||||
"ast_hash": "44fef6074ac2eaffc8233f46aad5236b",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/poller_test.go": {
|
||||
"mtime": 1786469644.9306462,
|
||||
"ast_hash": "64bc838c822f1bf33bbf9e291215454b",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/sites.go": {
|
||||
"mtime": 1786469348.5394242,
|
||||
"ast_hash": "b744cc685363317a526cc3bebceea39e",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/sites_test.go": {
|
||||
"mtime": 1786446006.9219902,
|
||||
"ast_hash": "eabca9014a306e3c71d238b0ae499f61",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/smoke_image_test.go": {
|
||||
"mtime": 1786363889.5602942,
|
||||
"ast_hash": "db068cb59575f8c82669acbaf84bcaed",
|
||||
"semantic_hash": "db068cb59575f8c82669acbaf84bcaed"
|
||||
},
|
||||
"backend/internal/pgtest/pgtest.go": {
|
||||
"mtime": 1786216141.6766438,
|
||||
"ast_hash": "f60372d41516e66f7aaeb272da227d6e",
|
||||
"semantic_hash": "f60372d41516e66f7aaeb272da227d6e"
|
||||
},
|
||||
"backend/internal/session/session.go": {
|
||||
"mtime": 1786216141.6766438,
|
||||
"ast_hash": "9952474ffb22c825d6f075b866ee26f4",
|
||||
"semantic_hash": "9952474ffb22c825d6f075b866ee26f4"
|
||||
},
|
||||
"backend/internal/session/session_test.go": {
|
||||
"mtime": 1786216141.680644,
|
||||
"ast_hash": "37ffd00964e7a67350c68ed50c6503c5",
|
||||
"semantic_hash": "37ffd00964e7a67350c68ed50c6503c5"
|
||||
},
|
||||
"backend/internal/store/migrations/0001_bookmarks.sql": {
|
||||
"mtime": 1786216141.680644,
|
||||
"ast_hash": "f87ccfb2c25c43f93021177ced0bfae4",
|
||||
"semantic_hash": "f87ccfb2c25c43f93021177ced0bfae4"
|
||||
},
|
||||
"backend/internal/store/migrations/0002_series.sql": {
|
||||
"mtime": 1786216141.6820722,
|
||||
"ast_hash": "5dc98771e0c0f6e8416b434c280b0efb",
|
||||
"semantic_hash": "5dc98771e0c0f6e8416b434c280b0efb"
|
||||
},
|
||||
"backend/internal/store/migrations/0003_reader.sql": {
|
||||
"mtime": 1786216141.6820722,
|
||||
"ast_hash": "444a97799f38f6222f87d4e9fb2d6258",
|
||||
"semantic_hash": "444a97799f38f6222f87d4e9fb2d6258"
|
||||
},
|
||||
"backend/internal/store/migrations/0004_owner_bookmarks.sql": {
|
||||
"mtime": 1786216141.684644,
|
||||
"ast_hash": "e4fa900cc223865d3ecd4c60c5707a65",
|
||||
"semantic_hash": "e4fa900cc223865d3ecd4c60c5707a65"
|
||||
},
|
||||
"backend/internal/store/migrations/0005_sessions.sql": {
|
||||
"mtime": 1786216141.684644,
|
||||
"ast_hash": "5158887ebc57cf8c16a7b821b61cd760",
|
||||
"semantic_hash": "5158887ebc57cf8c16a7b821b61cd760"
|
||||
},
|
||||
"backend/internal/store/migrations/0006_reader_token_epoch.sql": {
|
||||
"mtime": 1786216141.684644,
|
||||
"ast_hash": "3093cc1c3aae0cd9643d04105d045402",
|
||||
"semantic_hash": "3093cc1c3aae0cd9643d04105d045402"
|
||||
},
|
||||
"backend/internal/store/sessions.go": {
|
||||
"mtime": 1786216141.684644,
|
||||
"ast_hash": "eee3510cc6172ef4b1da820474c26b01",
|
||||
"semantic_hash": "eee3510cc6172ef4b1da820474c26b01"
|
||||
},
|
||||
"backend/internal/store/sessions_test.go": {
|
||||
"mtime": 1786216141.684644,
|
||||
"ast_hash": "0b6764a0ee20f5cb7748eecd31a1d220",
|
||||
"semantic_hash": "0b6764a0ee20f5cb7748eecd31a1d220"
|
||||
},
|
||||
"backend/internal/store/store.go": {
|
||||
"mtime": 1786363889.5602942,
|
||||
"ast_hash": "54367a8ab043983e2491b2eb2650961c",
|
||||
"semantic_hash": "54367a8ab043983e2491b2eb2650961c"
|
||||
},
|
||||
"backend/internal/store/store_test.go": {
|
||||
"mtime": 1786363889.5602942,
|
||||
"ast_hash": "dc823fd77bcce2268114e31d759b20a5",
|
||||
"semantic_hash": "dc823fd77bcce2268114e31d759b20a5"
|
||||
},
|
||||
"backend/internal/userscript/userscript.go": {
|
||||
"mtime": 1786216141.692644,
|
||||
"ast_hash": "aa13a71b1c9eefe4930fd31f27722328",
|
||||
"semantic_hash": "aa13a71b1c9eefe4930fd31f27722328"
|
||||
},
|
||||
"backend/internal/userscript/userscript_test.go": {
|
||||
"mtime": 1786216141.692644,
|
||||
"ast_hash": "6c050968d7b8b67956da1a3136d2c3c7",
|
||||
"semantic_hash": "6c050968d7b8b67956da1a3136d2c3c7"
|
||||
},
|
||||
"backend/internal/web/discord.go": {
|
||||
"mtime": 1786216141.692644,
|
||||
"ast_hash": "69e4959c65fa67d7491aedf6a71bb575",
|
||||
"semantic_hash": "69e4959c65fa67d7491aedf6a71bb575"
|
||||
},
|
||||
"backend/internal/web/oauth_test.go": {
|
||||
"mtime": 1786216141.692644,
|
||||
"ast_hash": "a3bddeb70dd8d7eb14139da808723ea8",
|
||||
"semantic_hash": "a3bddeb70dd8d7eb14139da808723ea8"
|
||||
},
|
||||
"backend/internal/web/static/filter.js": {
|
||||
"mtime": 1786022513.9473197,
|
||||
"ast_hash": "b4ee3306201bfd88b148b96801972617",
|
||||
"semantic_hash": "b4ee3306201bfd88b148b96801972617"
|
||||
},
|
||||
"backend/internal/web/static/htmx.min.js": {
|
||||
"mtime": 1785873769.5512016,
|
||||
"ast_hash": "19a573773be4ca22570ca2f8543120c5",
|
||||
"semantic_hash": "19a573773be4ca22570ca2f8543120c5"
|
||||
},
|
||||
"backend/internal/web/web.go": {
|
||||
"mtime": 1786363889.5678573,
|
||||
"ast_hash": "8308949c658d3a08ce1c3cdbea6a907c",
|
||||
"semantic_hash": "8308949c658d3a08ce1c3cdbea6a907c"
|
||||
},
|
||||
"backend/reader_credential_test.go": {
|
||||
"mtime": 1786216141.700644,
|
||||
"ast_hash": "2a751ea6d9c06635aa3179db0aef1b2b",
|
||||
"semantic_hash": "2a751ea6d9c06635aa3179db0aef1b2b"
|
||||
},
|
||||
"chrome/entrypoint.sh": {
|
||||
"mtime": 1786363889.5678573,
|
||||
"ast_hash": "8008a187690764436540fab47ba0cfcc",
|
||||
"semantic_hash": "8008a187690764436540fab47ba0cfcc"
|
||||
},
|
||||
"userscript/novel-bookmark.user.js": {
|
||||
"mtime": 1786454823.798836,
|
||||
"ast_hash": "834effb0821f8d6c9f57f6554a5db462",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"userscript/test/novel-logic.test.js": {
|
||||
"mtime": 1786454566.070801,
|
||||
"ast_hash": "b25a7377af210dd0aff8284501fd0252",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
".opencode/agent/implementer.md": {
|
||||
"mtime": 1785873769.5402634,
|
||||
"ast_hash": "000de469c18d68352027e10c8ce8acfb",
|
||||
"semantic_hash": "000de469c18d68352027e10c8ce8acfb"
|
||||
},
|
||||
".opencode/agent/reviewer.md": {
|
||||
"mtime": 1785873769.5416775,
|
||||
"ast_hash": "e44a2f6f624db044e19508bc5ab05592",
|
||||
"semantic_hash": "e44a2f6f624db044e19508bc5ab05592"
|
||||
},
|
||||
"CONTEXT.md": {
|
||||
"mtime": 1786466512.6719532,
|
||||
"ast_hash": "544b7d93f1d5cb9d347cb0b92fc3a709",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"CUTOVER.md": {
|
||||
"mtime": 1786216141.660644,
|
||||
"ast_hash": "6c6f3e4c4c2f57867894280bce728c50",
|
||||
"semantic_hash": "6c6f3e4c4c2f57867894280bce728c50"
|
||||
},
|
||||
"backend/AGENTS.md": {
|
||||
"mtime": 1786363889.552731,
|
||||
"ast_hash": "6356ee58447f299e5fa7aa83486bbbe0",
|
||||
"semantic_hash": "6356ee58447f299e5fa7aa83486bbbe0"
|
||||
},
|
||||
"backend/CLAUDE.md": {
|
||||
"mtime": 1786363889.552731,
|
||||
"ast_hash": "6356ee58447f299e5fa7aa83486bbbe0",
|
||||
"semantic_hash": "6356ee58447f299e5fa7aa83486bbbe0"
|
||||
},
|
||||
"backend/internal/web/templates/app.html": {
|
||||
"mtime": 1786216141.692644,
|
||||
"ast_hash": "3965e20e204afb71ba2a3aa86cb7c61c",
|
||||
"semantic_hash": "3965e20e204afb71ba2a3aa86cb7c61c"
|
||||
},
|
||||
"backend/internal/web/templates/card.html": {
|
||||
"mtime": 1786363889.5678573,
|
||||
"ast_hash": "ab83ae0dbb34fd40c146a7cc1263173e",
|
||||
"semantic_hash": "ab83ae0dbb34fd40c146a7cc1263173e"
|
||||
},
|
||||
"backend/internal/web/templates/chrome.html": {
|
||||
"mtime": 1786363889.5678573,
|
||||
"ast_hash": "d80b27cf3bd9d131075c485dd169dfcc",
|
||||
"semantic_hash": "d80b27cf3bd9d131075c485dd169dfcc"
|
||||
},
|
||||
"backend/internal/web/templates/icons.html": {
|
||||
"mtime": 1785873769.553139,
|
||||
"ast_hash": "8e10c507c32934a92463b4bca9e34fe6",
|
||||
"semantic_hash": "8e10c507c32934a92463b4bca9e34fe6"
|
||||
},
|
||||
"backend/internal/web/templates/list.html": {
|
||||
"mtime": 1786216141.696644,
|
||||
"ast_hash": "365548aace8c06559a1f66db0ae47256",
|
||||
"semantic_hash": "365548aace8c06559a1f66db0ae47256"
|
||||
},
|
||||
"backend/internal/web/templates/login.html": {
|
||||
"mtime": 1786216141.696644,
|
||||
"ast_hash": "bcc3101498a66cf8b79f9d97c6c9cd6b",
|
||||
"semantic_hash": "bcc3101498a66cf8b79f9d97c6c9cd6b"
|
||||
},
|
||||
"backend/internal/web/templates/readers.html": {
|
||||
"mtime": 1786216141.696644,
|
||||
"ast_hash": "c5034e76bd20a705d2799cb5ecb328f0",
|
||||
"semantic_hash": "c5034e76bd20a705d2799cb5ecb328f0"
|
||||
},
|
||||
"backend/internal/web/templates/setup.html": {
|
||||
"mtime": 1786216141.696644,
|
||||
"ast_hash": "72e93c0b827414063596f7338987d879",
|
||||
"semantic_hash": "72e93c0b827414063596f7338987d879"
|
||||
},
|
||||
"docs/adr/0001-postgresql-over-sqlite.md": {
|
||||
"mtime": 1786216141.704644,
|
||||
"ast_hash": "abfb08754cee58be67311377904f8ca4",
|
||||
"semantic_hash": "abfb08754cee58be67311377904f8ca4"
|
||||
},
|
||||
"docs/adr/0002-discord-oauth-no-passwords-no-email.md": {
|
||||
"mtime": 1786216141.7071996,
|
||||
"ast_hash": "852a04d86659385085da6ffc8b489933",
|
||||
"semantic_hash": "852a04d86659385085da6ffc8b489933"
|
||||
},
|
||||
"docs/adr/0003-series-shared-and-poll-owned.md": {
|
||||
"mtime": 1786216141.7071996,
|
||||
"ast_hash": "58bb4d24e20f6f7adc1a8b9cf7967c3f",
|
||||
"semantic_hash": "58bb4d24e20f6f7adc1a8b9cf7967c3f"
|
||||
},
|
||||
"docs/adr/0004-wire-format-does-not-mirror-the-schema.md": {
|
||||
"mtime": 1786216141.7071996,
|
||||
"ast_hash": "a6ea2770dec2156f78a65b35ba06902a",
|
||||
"semantic_hash": "a6ea2770dec2156f78a65b35ba06902a"
|
||||
},
|
||||
"docs/agents/domain.md": {
|
||||
"mtime": 1786216141.7071996,
|
||||
"ast_hash": "6f99318ac6cb9825b613bfde55d76091",
|
||||
"semantic_hash": "6f99318ac6cb9825b613bfde55d76091"
|
||||
},
|
||||
"docs/agents/issue-tracker.md": {
|
||||
"mtime": 1786216141.7087462,
|
||||
"ast_hash": "1342e66ccb84a84fd579fb6dc0b8243a",
|
||||
"semantic_hash": "1342e66ccb84a84fd579fb6dc0b8243a"
|
||||
},
|
||||
"docs/agents/triage-labels.md": {
|
||||
"mtime": 1786216141.7087462,
|
||||
"ast_hash": "69114d07ed792d6bb1d13758ba5435e1",
|
||||
"semantic_hash": "69114d07ed792d6bb1d13758ba5435e1"
|
||||
},
|
||||
"userscript/AGENTS.md": {
|
||||
"mtime": 1786454605.6727684,
|
||||
"ast_hash": "e276ffe9a6e7b55fd3235466a1995c22",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"userscript/CLAUDE.md": {
|
||||
"mtime": 1786454605.6727684,
|
||||
"ast_hash": "e276ffe9a6e7b55fd3235466a1995c22",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/web/static/login-art.png": {
|
||||
"mtime": 1786022513.9585779,
|
||||
"ast_hash": "05d7863cba344a946256719a0c9ef959",
|
||||
"semantic_hash": "05d7863cba344a946256719a0c9ef959"
|
||||
},
|
||||
"backend/internal/web/static/logo.svg": {
|
||||
"mtime": 1786022513.9585779,
|
||||
"ast_hash": "d0d34d0f08a25b53176cc55989b7babe",
|
||||
"semantic_hash": "d0d34d0f08a25b53176cc55989b7babe"
|
||||
},
|
||||
"backend/internal/store/migrations/0007_covers.sql": {
|
||||
"mtime": 1786262323.6964688,
|
||||
"ast_hash": "6033ce0701be1236ed175362363bd96c",
|
||||
"semantic_hash": "6033ce0701be1236ed175362363bd96c"
|
||||
},
|
||||
"docs/adr/0005-on-demand-browser.md": {
|
||||
"mtime": 1786292465.8305523,
|
||||
"ast_hash": "8cf2fb8c0a66c2c7d82ae433b99ca42b",
|
||||
"semantic_hash": "8cf2fb8c0a66c2c7d82ae433b99ca42b"
|
||||
},
|
||||
"chrome/docker-compose.yml": {
|
||||
"mtime": 1786292465.8305523,
|
||||
"ast_hash": "5605599395a3f085904e78a2bfec1e58",
|
||||
"semantic_hash": "5605599395a3f085904e78a2bfec1e58"
|
||||
},
|
||||
"docs/adr/0006-browser-on-the-home-machine.md": {
|
||||
"mtime": 1786292465.8305523,
|
||||
"ast_hash": "dfd6bbc045d23315f2942ee8d98eb7db",
|
||||
"semantic_hash": "dfd6bbc045d23315f2942ee8d98eb7db"
|
||||
},
|
||||
"docs/adr/0007-backend-hosts-cover-bytes.md": {
|
||||
"mtime": 1786292465.8305523,
|
||||
"ast_hash": "b19e38045b3dcda7dd59634ed9227a68",
|
||||
"semantic_hash": "b19e38045b3dcda7dd59634ed9227a68"
|
||||
},
|
||||
"backend/internal/store/migrations/0008_filesystem_covers.sql": {
|
||||
"mtime": 1786363889.5602942,
|
||||
"ast_hash": "46cf7822d4f667e3cab36b547abe5e97",
|
||||
"semantic_hash": "46cf7822d4f667e3cab36b547abe5e97"
|
||||
},
|
||||
"backend/internal/latest/cover.go": {
|
||||
"mtime": 1786363889.5565126,
|
||||
"ast_hash": "e6749cfe3cd7c2e71d4392dde84f55f9",
|
||||
"semantic_hash": "e6749cfe3cd7c2e71d4392dde84f55f9"
|
||||
},
|
||||
"backend/internal/latest/cover_fetch_test.go": {
|
||||
"mtime": 1786363889.5565126,
|
||||
"ast_hash": "60d9eb7c59a3751baf4f31c7655217e7",
|
||||
"semantic_hash": "60d9eb7c59a3751baf4f31c7655217e7"
|
||||
},
|
||||
"backend/internal/latest/acquire.go": {
|
||||
"mtime": 1786468950.9432797,
|
||||
"ast_hash": "6c1ad34bbe9f5b49d0fd1eae9093f55d",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/acquire_test.go": {
|
||||
"mtime": 1786363889.5565126,
|
||||
"ast_hash": "7bd9f41814f6bf59d8998dfb81cf990a",
|
||||
"semantic_hash": "7bd9f41814f6bf59d8998dfb81cf990a"
|
||||
},
|
||||
"backend/internal/store/migrations/0009_series_cover_address.sql": {
|
||||
"mtime": 1786363889.5602942,
|
||||
"ast_hash": "4c1f6328b2e1a95828fad6d88d474c2d",
|
||||
"semantic_hash": "4c1f6328b2e1a95828fad6d88d474c2d"
|
||||
},
|
||||
".claude/skills/implement-tickets/SKILL.md": {
|
||||
"mtime": 1786417841.7117643,
|
||||
"ast_hash": "3060e32e19cc571d91871a98f18afe53",
|
||||
"semantic_hash": "3060e32e19cc571d91871a98f18afe53"
|
||||
},
|
||||
".omp/agents/ticket-implementer.md": {
|
||||
"mtime": 1786417841.7163916,
|
||||
"ast_hash": "0150a46c0d21c572b71b3d87d21ac925",
|
||||
"semantic_hash": "0150a46c0d21c572b71b3d87d21ac925"
|
||||
},
|
||||
"docs/adr/0008-series-identity-is-discovered-not-derived.md": {
|
||||
"mtime": 1786417841.7163916,
|
||||
"ast_hash": "3ce6d64ef6a8a39f27c257b39065389e",
|
||||
"semantic_hash": "3ce6d64ef6a8a39f27c257b39065389e"
|
||||
},
|
||||
"docs/research/lightnovelworld-chapter-vs-series-slug.md": {
|
||||
"mtime": 1786417841.7163916,
|
||||
"ast_hash": "74a4e538875f0a7e8ca3d5dc48c0bb53",
|
||||
"semantic_hash": "74a4e538875f0a7e8ca3d5dc48c0bb53"
|
||||
},
|
||||
"backend/internal/latest/smoke_lnw_test.go": {
|
||||
"mtime": 1786446006.9219902,
|
||||
"ast_hash": "2d65da8a081759172918fdf159760f45",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"docs/adr/0009-a-site-answers-questions-its-own-way.md": {
|
||||
"mtime": 1786469367.6875648,
|
||||
"ast_hash": "8039012a5b6de2359ff1a47079f51b66",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/internal/latest/read.go": {
|
||||
"mtime": 1786469329.7445614,
|
||||
"ast_hash": "3cf29046ddaef39fafb1df70b9f9ae8c",
|
||||
"semantic_hash": ""
|
||||
}
|
||||
}
|
||||
+94
-84
@@ -1,93 +1,103 @@
|
||||
Guidance for OpenCode (and Claude Code) working under `userscript/`. See root `AGENTS.md` for the project-wide architecture diagram, hard constraints, and design system.
|
||||
Scope: `userscript/`.
|
||||
|
||||
### Userscript structure (single IIFE, `manga-bookmark.user.js`)
|
||||
Each entry names the code that holds the truth. The prose is only what the code
|
||||
cannot tell you: rationale, invariants a refactor would break, and dated
|
||||
observations about sites we don't control.
|
||||
|
||||
1. **Site adapters** — one per host, `detect(location, document)` return page `type` + IDs. Identify type/IDs from **URL regex** (most stable); pull `title` from **`og:title`** (or the page heading where a site ships no og: tags), not CSS classes. **No adapter reads a cover**: the backend acquires, stores and serves every Cover from its own origin (ADR-0007), the wire's `cover` is already an address on our origin, and `apiPut` strips any `cover` off an outgoing body.
|
||||
2. **API client** — `apiGet/apiPut/apiDelete` with bearer header; `localStorage` key `bmgr:manga:cache` for instant render + offline fallback.
|
||||
3. **Progress logic** — auto-upsert `last_chapter` only when `chapterNum >= stored last_chapter_num` (re-reading old chapters must not regress progress; unparseable -> set current). Manual panel override forces any value.
|
||||
4. **Retry queue** — every write go through `pushBookmark`/`pushDelete`, so
|
||||
failed mutation park in `localStorage` (`bmgr:manga:queue`) and replayed on
|
||||
next navigation, reconnect, or `refresh()`. Entries are markers
|
||||
(`{key, op, sendStatus, attempts}`), never payloads — body read from
|
||||
cache at send time, so one entry per key give ordering and coalescing for
|
||||
free. `sendStatus` is **sticky**: while archive pending, later writes to
|
||||
that key keep carrying bucket, which stop successful
|
||||
in-between write from silently un-archiving series. `refresh()` drains
|
||||
before it fetches and overlays anything still pending, so list never
|
||||
flaps. 400 drops entry, 401 abort pass and keep queue, and
|
||||
transient failures retry to cap of 10. Latest-chapter writes deliberately
|
||||
stay out of queue. See
|
||||
`docs/superpowers/specs/2026-07-27-offline-retry-queue-design.md`.
|
||||
5. **UI** — rendered inside **Shadow DOM** root to isolate from site CSS
|
||||
(critical on mobile). Three tabs (All / Favourites / Archived) and row of
|
||||
link chips to web UI and both manga sites; `WEB_BASE` sits in CONFIG
|
||||
block next to `API_BASE`. FAB is `7 × 44` edge tab whose *hit* area
|
||||
widened to `28 × 72` by invisible `#hit` child; `#fab` must keep
|
||||
`touch-action: none` and must **not** regain `overflow: hidden`. Since
|
||||
`touch-action` resolved at gesture start, strip can't be both
|
||||
browser-scrolled and script-dragged, so `makeDraggable` splits by intent: swipe
|
||||
from `#hit` scrolls via `window.scrollBy`, hold of `ARM_MS` arms
|
||||
reposition drag, visible sliver drags with no hold. See
|
||||
`docs/superpowers/specs/2026-07-28-edge-tab-hitbox-design.md`.
|
||||
6. **SPA navigation** — Asura is Astro, client-routed on comic/chapter pages: patch `history.pushState`/`replaceState` + listen `popstate`, re-run `detect()` on URL change so auto-update fire without reload. Demonic uses classic reloads (initial `document-idle` run suffice).
|
||||
### Structure — single IIFE, `manga-bookmark.user.js`
|
||||
|
||||
### Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trust)
|
||||
Six parts, in file order: site adapters, API client, progress logic, retry
|
||||
queue, UI, SPA navigation.
|
||||
|
||||
- **asurascans.com**: series `/comics/<slug>` (slug carries trailing
|
||||
site-wide build-hash suffix, e.g. `-059befe1`, that **rotates on every
|
||||
redeploy**), chapter `/comics/<slug>/chapter/<n>`. `seriesId` must strip
|
||||
hash (`/-[0-9a-f]{8}$/`, `stripBuildHash` in userscript,
|
||||
`asuraBuildHash` in backend); URLs keep full slug — stale-hash
|
||||
URLs 302 to current ones. Astro-rendered; chapter links present in raw
|
||||
server HTML.
|
||||
- **demonicscans.org**: series `/manga/<slug>` (slug may URL-encode punctuation, e.g. `%2527` for `'`), chapter `/title/<slug>/chapter/<n>/<page>` (older `chaptered.php?manga=<id>&chapter=<n>` form still exists as redirect, what series-page chapter-list anchors link through).
|
||||
Encodings (incl. triple-encoded punctuation like `%25252D`) identical
|
||||
on /manga/ and /title/ pages, so decode-once seriesIds match — verified
|
||||
2026-07-28.
|
||||
- **comix.to**: series `/title/<id>-<slug>`, chapter
|
||||
`/title/<id>-<slug>/<uploadId>-chapter-<n>`. Only the leading `<id>` is
|
||||
identity — the slug re-renders when a series is renamed (`comixSeriesId`).
|
||||
An SPA that **never rewrites `og:title`**: the server-rendered head keeps
|
||||
whatever document loaded first, so on a cold load `og:title` is the homepage's
|
||||
"Comix — Read Comics online for free" and after an in-page hop it is the
|
||||
*previous* series' name. `document.title` is the one thing client routing does
|
||||
update, so titles come from there, with the chapter page's `" · Ch.<n>"` tail
|
||||
stripped. It publishes no `og:image` either, which is one of the reasons cover
|
||||
**Site adapters** — one per host, `detect(location, document)` returning page
|
||||
`type` + IDs.
|
||||
|
||||
- Identify type and IDs from **URL regex**, which is the most stable surface a
|
||||
site exposes; take `title` from **`og:title`** (or the page heading where a
|
||||
site ships no og: tags), never CSS classes.
|
||||
- **No adapter reads a cover.** The backend acquires, stores and serves every
|
||||
Cover from its own origin, the wire `cover` is already an address there, and
|
||||
`apiPut` strips any `cover` off an outgoing body.
|
||||
|
||||
**Progress logic** — auto-upsert `last_chapter` only when
|
||||
`chapterNum >= stored last_chapter_num`; unparseable sets the current value.
|
||||
Re-reading an old chapter must not regress progress. A manual panel override
|
||||
forces any value.
|
||||
|
||||
**Retry queue** — every write goes through `pushBookmark`/`pushDelete`.
|
||||
|
||||
- Entries are markers (`{key, op, sendStatus, attempts}`), **never payloads**:
|
||||
the body is read from cache at send time, so one entry per key gives ordering
|
||||
and coalescing for free.
|
||||
- `sendStatus` is **sticky** — while an archive is pending, later writes to that
|
||||
key keep carrying the bucket. Without it a successful in-between write
|
||||
silently un-archives the series.
|
||||
- `refresh()` drains before it fetches and overlays anything still pending, so
|
||||
the list never flaps.
|
||||
- 400 drops the entry, 401 aborts the pass and keeps the queue, transient
|
||||
failures retry to a cap. Latest-chapter writes deliberately stay out of the
|
||||
queue.
|
||||
|
||||
**UI** — rendered inside a **Shadow DOM** root to isolate it from site CSS,
|
||||
which is critical on mobile.
|
||||
|
||||
- The FAB's *hit* area is widened by an invisible `#hit` child. `#fab` must keep
|
||||
`touch-action: none` and must **not** regain `overflow: hidden`.
|
||||
- `touch-action` is resolved at gesture start, so the strip cannot be both
|
||||
browser-scrolled and script-dragged. `makeDraggable` therefore splits by
|
||||
intent: a swipe from `#hit` scrolls via `window.scrollBy`, a hold of `ARM_MS`
|
||||
arms a reposition drag, and the visible sliver drags with no hold.
|
||||
|
||||
**SPA navigation** — Asura is Astro and client-routes on comic/chapter pages, so
|
||||
`history.pushState`/`replaceState` are patched and `popstate` listened to, and
|
||||
`detect()` re-runs on URL change. Demonic uses classic reloads, where the
|
||||
initial `document-idle` run suffices.
|
||||
|
||||
### Live URL shapes
|
||||
|
||||
Encoded in the adapters; the notes below are the parts a reader of the regex
|
||||
would get wrong. **Verified 2026-07-26 unless dated otherwise — sites drift, so
|
||||
re-check against a live page before trusting any of it.**
|
||||
|
||||
- **asurascans.com** — the series slug carries a site-wide build-hash suffix
|
||||
(e.g. `-059befe1`) that **rotates on every redeploy**, so `seriesId` must
|
||||
strip it (`stripBuildHash` here, `asuraBuildHash` in the backend) while URLs
|
||||
keep the full slug — stale-hash URLs 302 to current ones.
|
||||
- **demonicscans.org** — slugs may URL-encode punctuation, and the older
|
||||
`chaptered.php?manga=<id>&chapter=<n>` form still exists as a redirect, which
|
||||
is what series-page chapter-list anchors link through. Encodings (including
|
||||
triple-encoded punctuation like `%25252D`) are identical on `/manga/` and
|
||||
`/title/` pages, so decode-once seriesIds match (verified 2026-07-28).
|
||||
- **comix.to** — only the leading `<id>` is identity; the slug re-renders when a
|
||||
series is renamed (`comixSeriesId`). It is an SPA that **never rewrites
|
||||
`og:title`**: the server-rendered head keeps whatever document loaded first,
|
||||
so on a cold load `og:title` is the homepage's name and after an in-page hop
|
||||
it is the *previous* series'. `document.title` is the one thing client routing
|
||||
updates, hence titles come from there with the chapter page's `" · Ch.<n>"`
|
||||
tail stripped. It publishes no `og:image` either, one of the reasons cover
|
||||
acquisition moved to the backend.
|
||||
- **kagane.to**: series `/series/<uuid>`, reader
|
||||
`/series/<uuid>/reader/<bookUuid>`. Reader URLs carry no chapter number, so
|
||||
the number comes out of `og:title`. Two shapes exist: `"<Series> - Chapter
|
||||
<n>[ - Episode <n>]"` and, for volume-numbered series, `"<Series> - Volume <v>
|
||||
Chapter <n>"` with no episode name — both must yield a bare series title, or
|
||||
the volume tail lands in the bookmark's title.
|
||||
Its covers are challenge- and CORP-protected, so nothing outside kagane.to can
|
||||
load one directly; the panel renders the backend's own cover address like every
|
||||
other Site. Behind a Cloudflare JS challenge, so the backend polls it
|
||||
through the headless browser.
|
||||
- **novelfull.com** (novel script): series `/<slug>.html`, chapter
|
||||
`/<slug>/chapter-<n>[-<title-slug>].html`. No `og:*` tags at all — title from
|
||||
`h3.title` (series) or `a.truyen-title` (chapter); the script reads no cover.
|
||||
Behind a Cloudflare JS challenge no TLS fingerprint
|
||||
clears, so the backend polls it through the headless browser.
|
||||
- **lightnovelworld.net** (novel script): series `/novel/<slug>/`, chapter
|
||||
`/<slug>-chapter-<n>/` — flat, at the site root. The chapter path's slug is a
|
||||
Chapter Slug, not an identity: the Series address is read off the page's
|
||||
- **kagane.to** — reader URLs carry no chapter number, so the number comes out
|
||||
of `og:title`. Two shapes exist, `"<Series> - Chapter <n>[ - Episode <n>]"`
|
||||
and `"<Series> - Volume <v> Chapter <n>"`; both must yield a bare series
|
||||
title, or the volume tail lands in the bookmark's title. Its covers are
|
||||
challenge- and CORP-protected, so nothing outside kagane.to can load one —
|
||||
the panel renders the backend's cover address like every other Site.
|
||||
- **novelfull.com** (novel script) — no `og:*` tags at all, so the title comes
|
||||
from `h3.title` (series) or `a.truyen-title` (chapter).
|
||||
- **lightnovelworld.net** (novel script) — chapter paths are flat at the site
|
||||
root and their slug is a **Chapter Slug, not an identity**: a Series may
|
||||
publish under several. The Series address is read off the page's
|
||||
`a[aria-label='All Chapter']` (fallback: the BreadcrumbList's second crumb),
|
||||
and a Series may publish under several Chapter Slugs. A chapter page with no
|
||||
pointer resolves to `other`, so no Bookmark is offered. `h1.entry-title` is
|
||||
the clean title on a series page and `<Title> Chapter <n>` on a chapter page.
|
||||
Its series page lists every chapter with an
|
||||
absolute href, so the backend polls it with the plain TLS client.
|
||||
The client performs no latest-chapter scan for this Site: the Poll's
|
||||
one-hour cooldown dominates the client's four-hour throttle, so a scan
|
||||
would add no freshness, and the page's wpdiscuz thread is a public write
|
||||
surface a scan would have to truncate at. `computeLatestChapter` yields
|
||||
null here and `backgroundRefreshLatest` skips the Site before any fetch.
|
||||
and a chapter page with no pointer resolves to `other` so no Bookmark is
|
||||
offered. The client runs **no latest-chapter scan** for this Site —
|
||||
`computeLatestChapter` yields null and `backgroundRefreshLatest` skips it
|
||||
before any fetch — because the backend Poll's one-hour cooldown dominates the
|
||||
client's four-hour throttle, so a scan would add no freshness while having to
|
||||
truncate at the page's wpdiscuz thread, a public write surface.
|
||||
|
||||
### Second script: `novel-bookmark.user.js`
|
||||
### Second script — `novel-bookmark.user.js`
|
||||
|
||||
A copy of the manga script with two adapters, `LIBRARY = "novel"` and
|
||||
`STORE_PREFIX = "bmgr:novel:"`. No migration loop (this script has no previous
|
||||
installation to carry keys over from). Installed alongside the manga script;
|
||||
both write to the same backend with the same `LIBRARY` column discriminating
|
||||
them.
|
||||
`STORE_PREFIX = "bmgr:novel:"`. No migration loop, because this script has no
|
||||
previous installation to carry keys over from. Installed alongside the manga
|
||||
script; both write to the same backend, discriminated by `LIBRARY`.
|
||||
|
||||
@@ -905,27 +905,37 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Records the newest chapter a site has published. Silent: this fires from
|
||||
// Reports the newest chapter a site has published. Silent: this fires from
|
||||
// page visits and background checks the user did not ask for, and it never
|
||||
// reorders the list — updated_at is a candidate the server discards unless
|
||||
// reading progress moved.
|
||||
async function applyLatestChapterIfChanged(existing, latest) {
|
||||
//
|
||||
// An unchanged number is still sent. It is the read that lets the backend
|
||||
// skip its own poll of this series (a Sighting, issue #103), so the common
|
||||
// case — visiting a series with nothing new — is exactly the one worth
|
||||
// reporting. Only the local write and the re-render are skipped.
|
||||
async function reportLatestChapter(existing, latest) {
|
||||
if (!existing || !latest) return;
|
||||
if (existing.latest_chapter_num === latest.num) return;
|
||||
const bm = Object.assign({}, existing, {
|
||||
latest_chapter: latest.label,
|
||||
latest_chapter_num: latest.num,
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
upsertLocal(bm);
|
||||
render();
|
||||
const changed = existing.latest_chapter_num !== latest.num;
|
||||
let bm = existing;
|
||||
if (changed) {
|
||||
bm = Object.assign({}, existing, {
|
||||
latest_chapter: latest.label,
|
||||
latest_chapter_num: latest.num,
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
upsertLocal(bm);
|
||||
render();
|
||||
}
|
||||
// A queued write owns this row; the drain sends latest_chapter
|
||||
// with it, carrying the correct bucket.
|
||||
if (queueGet(bm.key)) return;
|
||||
try {
|
||||
const saved = await apiPut(bm.key, bm);
|
||||
upsertLocal(saved);
|
||||
render();
|
||||
if (changed) {
|
||||
upsertLocal(saved);
|
||||
render();
|
||||
}
|
||||
} catch (e) {
|
||||
/* offline — the local cache still shows it, retried on a later visit */
|
||||
}
|
||||
@@ -937,7 +947,7 @@
|
||||
if (p.type !== "series") return;
|
||||
const existing = state.byKey[keyOf(p)];
|
||||
if (!existing) return;
|
||||
applyLatestChapterIfChanged(
|
||||
reportLatestChapter(
|
||||
existing,
|
||||
computeLatestChapter(p.site, anchorsFromDocument(document), p.seriesId)
|
||||
);
|
||||
@@ -977,7 +987,7 @@
|
||||
const html = await res.text();
|
||||
latest = computeLatestChapter(bm.site, anchorsFromHTML(html), bm.series_id);
|
||||
}
|
||||
await applyLatestChapterIfChanged(state.byKey[bm.key] || bm, latest);
|
||||
await reportLatestChapter(state.byKey[bm.key] || bm, latest);
|
||||
} catch (e) {
|
||||
/* offline or blocked — try again after the throttle window */
|
||||
}
|
||||
|
||||
@@ -809,27 +809,37 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Records the newest chapter a site has published. Silent: this fires from
|
||||
// Reports the newest chapter a site has published. Silent: this fires from
|
||||
// page visits and background checks the user did not ask for, and it never
|
||||
// reorders the list — updated_at is a candidate the server discards unless
|
||||
// reading progress moved.
|
||||
async function applyLatestChapterIfChanged(existing, latest) {
|
||||
//
|
||||
// An unchanged number is still sent. It is the read that lets the backend
|
||||
// skip its own poll of this series (a Sighting, issue #103), so the common
|
||||
// case — visiting a series with nothing new — is exactly the one worth
|
||||
// reporting. Only the local write and the re-render are skipped.
|
||||
async function reportLatestChapter(existing, latest) {
|
||||
if (!existing || !latest) return;
|
||||
if (existing.latest_chapter_num === latest.num) return;
|
||||
const bm = Object.assign({}, existing, {
|
||||
latest_chapter: latest.label,
|
||||
latest_chapter_num: latest.num,
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
upsertLocal(bm);
|
||||
render();
|
||||
const changed = existing.latest_chapter_num !== latest.num;
|
||||
let bm = existing;
|
||||
if (changed) {
|
||||
bm = Object.assign({}, existing, {
|
||||
latest_chapter: latest.label,
|
||||
latest_chapter_num: latest.num,
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
upsertLocal(bm);
|
||||
render();
|
||||
}
|
||||
// A queued write owns this row; the drain sends latest_chapter
|
||||
// with it, carrying the correct bucket.
|
||||
if (queueGet(bm.key)) return;
|
||||
try {
|
||||
const saved = await apiPut(bm.key, bm);
|
||||
upsertLocal(saved);
|
||||
render();
|
||||
if (changed) {
|
||||
upsertLocal(saved);
|
||||
render();
|
||||
}
|
||||
} catch (e) {
|
||||
/* offline — the local cache still shows it, retried on a later visit */
|
||||
}
|
||||
@@ -841,7 +851,7 @@
|
||||
if (p.type !== "series") return;
|
||||
const existing = state.byKey[keyOf(p)];
|
||||
if (!existing) return;
|
||||
applyLatestChapterIfChanged(
|
||||
reportLatestChapter(
|
||||
existing,
|
||||
computeLatestChapter(p.site, anchorsFromDocument(document), p.seriesId)
|
||||
);
|
||||
@@ -883,7 +893,7 @@
|
||||
if (!res.ok) continue;
|
||||
const html = await res.text();
|
||||
const latest = computeLatestChapter(bm.site, anchorsFromHTML(html), bm.series_id);
|
||||
await applyLatestChapterIfChanged(state.byKey[bm.key] || bm, latest);
|
||||
await reportLatestChapter(state.byKey[bm.key] || bm, latest);
|
||||
} catch (e) {
|
||||
/* offline or blocked — try again after the throttle window */
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user