Compare commits

..

4 Commits

Author SHA1 Message Date
sulthan 55ecb29b81 docs: make the comix DNS-hijack note resolver-conditional, not machine-local
The note read as a repo-wide fact ('this dev machine', 'here'), which is
meaningless in a clone elsewhere and invites adding --add-host
unconditionally. Now: symptom (ERR_CERT_COMMON_NAME_INVALID), the check
(getent hosts inside the container), the workaround, and an explicit warning
not to bake the hosts into chrome/docker-compose.yml.
2026-08-16 12:11:29 +07:00
sulthan 82a11f255b chore: refresh the code graph for the comix browser path 2026-08-16 12:08:39 +07:00
sulthan f000cc7eaa docs(latest): correct the browser-site comments the comix change made stale
Review findings from #98: BrowserFetcher's doc still described two sites and
two read shapes, Get's doc enumerated them a second time, and the Fallback
field doc omitted comix. The comix fixture comment now records the live
in-tab body it was re-checked against.
2026-08-16 12:08:28 +07:00
sulthan 86160c164a feat(latest): poll comix through the browser sidecar (#98)
comix.to began answering plain-TLS fetches with a Cloudflare JavaScript
challenge on 2026-08-12, so every poll got a 403 interstitial and its cover
host static.comix.to is gated the same way. comix joins kagane and novelfull
as a browser Site: one registry entry, no plain-TLS fallback, and cover bytes
routed through the browser's image path behind a fully pinned URL pattern.

The read is an in-tab fetch of the Series URL, not a DOM render: comix is an
SPA, so rendering costs ~65 requests for the same server-rendered HTML one
fetch returns (24.5 KB, ~480 ms). Parsers and stored Series identity are
untouched.

Verified live against the real browser unit: page 24793 bytes in one fetch,
chapter 53, cover accepted by the pin and 26862 image bytes retrieved by
direct navigation (comix's Series page sets cross-origin-embedder-policy:
require-corp, so an in-page fetch of the cover host cannot work).
2026-08-16 12:05:11 +07:00
51 changed files with 54591 additions and 3532 deletions
+20 -1
View File
@@ -1,5 +1,24 @@
{
"hooks": {
"PreToolUse": []
"PreToolUse": [
{
"matcher": "Bash",
"hooks": [
{
"type": "command",
"command": "CMD=$(python3 -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',d).get('command',''))\" 2>/dev/null || true); case \"$CMD\" in *grep*|*rg\\ *|*ripgrep*|*find\\ *|*fd\\ *|*ack\\ *|*ag\\ *) [ -f graphify-out/graph.json ] && echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"MANDATORY: graphify-out/graph.json exists. You MUST run `graphify query \\\"<question>\\\"` before grepping raw files. Only grep after graphify has oriented you, or to modify/debug specific lines.\"}}' || true ;; esac"
}
]
},
{
"matcher": "Read|Glob",
"hooks": [
{
"type": "command",
"command": "HIT=$(python3 -c \"import json,sys;d=json.load(sys.stdin);t=d.get('tool_input',d);exts=('.py','.js','.ts','.tsx','.jsx','.astro','.vue','.svelte','.go','.rs','.java','.rb','.c','.h','.cpp','.hpp','.cc','.cs','.kt','.swift','.php','.scala','.lua','.sh','.md','.rst','.txt','.mdx');vals=[str(t.get('file_path') or ''),str(t.get('pattern') or ''),str(t.get('path') or '')];j=' '.join(vals).lower().replace(chr(92),'/');tails=[('.'+x.rsplit('.',1)[-1]) for v in vals if v for x in [v.lower().replace(chr(92),'/').rsplit('/',1)[-1]] if '.' in x];sys.stdout.write('1' if 'graphify-out/' not in j and any(tl in exts for tl in tails) else '')\" 2>/dev/null || true); if [ \"$HIT\" = 1 ] && [ -f graphify-out/graph.json ]; then echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"MANDATORY: graphify-out/graph.json exists. You MUST run graphify before reading source files. Use: `graphify query \\\"<question>\\\"` (scoped subgraph), `graphify explain \\\"<concept>\\\"`, or `graphify path \\\"<A>\\\" \\\"<B>\\\"`. Only read raw files after graphify has oriented you, or to modify/debug specific lines. This rule applies to subagents too \u2014 include it in every subagent prompt involving code exploration.\"}}'; fi || true"
}
]
}
]
}
}
+2 -1
View File
@@ -11,7 +11,8 @@ to the tracker. You do not write the implementation — every line of ticket cod
is written by a `ticket-implementer` subagent in its own git worktree. Reach for
the editor yourself only for a merge conflict resolution.
Ticket source and `tea` usage: `docs/agents/issue-tracker.md`.
Ticket source and `tea` usage: `docs/agents/issue-tracker.md`. Codebase
questions: `graphify query "<question>"` before grepping.
## 1. Collect the tickets
+18 -5
View File
@@ -73,13 +73,26 @@ DISCORD_REDIRECT_URI=
# The backend re-checks each bookmarked series' newest published chapter on its
# own schedule, so latest_chapter stays fresh even when you never open the manga
# sites. This runs in parallel with the userscript's own in-browser check.
# Set to 0 to turn it off entirely. Pace is per Site (one Poll Lane per Site,
# issue #100) and lives in the backend registry, not here — there is nothing
# else to configure.
# Set to 0 to turn it off entirely.
# LATEST_CHAPTER_POLL_ENABLED=1
#
# Every Site rests an hour between checks and gaps ten seconds between fetches;
# a Site with many Series tightens its own gap. See backend/internal/latest/sites.go.
# Two independent clocks. COOLDOWN is how long a plain-TLS series rests between
# checks; BROWSER_COOLDOWN is the longer rest for kagane and novelfull. INTERVAL
# is how often the poller wakes up and looks for series past their cooldowns.
# Shortening INTERVAL cannot shorten either cooldown.
LATEST_CHAPTER_POLL_COOLDOWN=1h # plain-TLS per series, floor 15m
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN=6h # browser-backed per series, floor 15m
LATEST_CHAPTER_POLL_INTERVAL=10m # how often to wake
LATEST_CHAPTER_POLL_BATCH=14 # series per wake
LATEST_CHAPTER_POLL_STAGGER=20s # delay between fetches in a batch
#
# Uses a ticker, not an immediate first run: the first poll happens one
# INTERVAL after startup, not at startup. A container restarting more often
# than INTERVAL never polls.
#
# BATCH x (COOLDOWN / INTERVAL) series hold the cooldown cadence — 84 with these
# defaults. Beyond that the cadence stretches uniformly rather than breaking;
# raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL.
# CDP endpoint of the browser, used for the two sites behind a Cloudflare
# JavaScript challenge (kagane, novelfull) and by the web UI's kagane cover
+5
View File
@@ -5,6 +5,11 @@
backend/server
backend/backend
.playwright-mcp/
# graphify map is committed; only regenerable/local parts are ignored
graphify-out/cost.json
graphify-out/cache/
graphify-out/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/
graphify-out/.rebuild.lock
plans/
.scratch/
docs/superpowers/
+10
View File
@@ -163,3 +163,13 @@ Default five-role vocabulary, label strings unchanged (`needs-triage`, `needs-in
### Domain docs
Single-context: one root `CONTEXT.md` plus `docs/adr/`, both created lazily. See `docs/agents/domain.md`.
## graphify
Project has knowledge graph at graphify-out/ with god nodes, community structure, cross-file relationships.
Rules:
- For codebase questions and exploration, always first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. Return scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
- If graphify-out/wiki/index.md exists, use for broad navigation instead of raw source browsing.
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain don't surface enough context.
- After modifying code, run `graphify update .` to keep graph current (AST-only, no API cost).
+1 -1
View File
@@ -86,7 +86,7 @@ _Avoid_: client report, user poll, observation, claim
**Acquisition**:
The single read of a Series page made the moment the Series first exists, giving it
both its Latest Chapter and its Cover without waiting for the Lane's pace. Distinct
both its Latest Chapter and its Cover without waiting out the Poll queue. Distinct
from a Poll in the two ways that matter: a Reader is present — it is triggered by
their first Bookmark of that Series — and it is the only read that establishes a
Cover rather than refreshing facts. It happens once in a Series's life; every later
+1 -9
View File
@@ -93,14 +93,6 @@ the backend dials but not the password the database expects, and `bookmark-api`
crash-loops on `password authentication failed`. Set it before §2 and leave it
alone.
An `.env` written before issue #100 carries the old poll-pace names
(`LATEST_CHAPTER_POLL_COOLDOWN`, `_BROWSER_COOLDOWN`, `_INTERVAL`, `_BATCH`,
`_STAGGER`). All five are dead configuration now — the pace lives in the Site
registry (`backend/internal/latest/sites.go`), so **delete those lines** and
keep only the kill switch `LATEST_CHAPTER_POLL_ENABLED`. Leaving them behind
is harmless (nothing reads them) but silently misleads the next person who
edits the file.
> Match `TRAEFIK_ENTRYPOINT` / `TRAEFIK_CERTRESOLVER` to your Traefik's actual
> names (check your Traefik static config — common alternatives: `https`,
> `myresolver`, `cloudflare`). Wrong names = no certificate issued.
@@ -513,7 +505,7 @@ picks up a restarted Chrome's new debugger UUID by itself.
| kagane rows never get a `latest_chapter`; log says `browser fetcher disabled` or nothing at all | `BROWSER_WS_URL` unset. Expected before §7 is done. |
| kagane polls all fail; log shows a 500 from `/json/version` | `BROWSER_WS_URL` names a MagicDNS hostname (or any name). Chrome's DevTools handler only accepts an IP or `localhost` — use the tailnet IP. |
| kagane polls fail with a connection error | Home machine off, off the tailnet, or the unit is down. `tailscale ping <machine>`, then `docker compose ps` in its `chrome/`. Costs freshness only; stored covers keep serving. |
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series. |
| kagane cover is a placeholder for a newly bookmarked series | Its cover has never been fetched and the browser is unreachable. It fills in on the next successful poll of that series (up to `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN`, default 6h). |
| `compose` in `chrome/` errors `set BROWSER_BIND_ADDR to this machine's tailnet IP` | No `chrome/.env`, or the variable is empty. Deliberate — it has no default so an unset value cannot publish CDP to the LAN. |
| browser container restarts, or is OOM-killed | `docker inspect bookmark-browser --format '{{.RestartCount}} {{.State.OOMKilled}}'`. The 512 MiB cap is sized against a measured 645 MiB untuned peak; a real breach is a Chrome regression worth reading `docker logs` for, not a number to raise reflexively. |
+6 -1
View File
@@ -53,7 +53,12 @@ covers are stored, so the library renders in full with the browser switched off.
| `DISCORD_API_BASE` | `https://discord.com/api/v10` | Test seam — tests point it at a local stub so the real token exchange runs. |
| `USERSCRIPT_PATH` | `/userscript/manga-bookmark.user.js` | Bindmounted file served at `/u/{token}/manga-bookmark.user.js`. |
| `NOVEL_USERSCRIPT_PATH` | `/userscript/novel-bookmark.user.js` | Same, for the novel library. |
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. Pace is per Site in the registry — one Poll Lane per Site, each with its own rest and gap (issue #100) — so no other knobs exist. |
| `LATEST_CHAPTER_POLL_ENABLED` | `1` | `0` turns the poller off entirely. |
| `LATEST_CHAPTER_POLL_COOLDOWN` | `1h` | Rest between checks of one plain-TLS series; floor `15m`. |
| `LATEST_CHAPTER_POLL_BROWSER_COOLDOWN` | `6h` | Rest between checks of one browser-backed series; floor `15m`. |
| `LATEST_CHAPTER_POLL_INTERVAL` | `10m` | How often the poller wakes. Cannot shorten either cooldown. |
| `LATEST_CHAPTER_POLL_BATCH` | `14` | Series per wake. Keep `BATCH × STAGGER` under `INTERVAL`. |
| `LATEST_CHAPTER_POLL_STAGGER` | `20s` | Delay between fetches in a batch — this is the outbound request rate. |
Compose reads a few more from the same `.env` that the backend never sees:
`POSTGRES_PASSWORD` (required — `DATABASE_URL` is built from it, and Postgres
+21 -94
View File
@@ -27,8 +27,7 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
`series` keyed `(site, series_id)`
(`asura`|`demonic`|`comix`|`kagane`|`novelfull`|`lightnovelworld`) owns the
shared facts — title, cover, canonical URL, `kind` (`manga`|`novel`),
Latest Chapter, `latest_checked_at`, and the Sighting pair
`latest_sighted_at`/`latest_raised_by` (issue #103) — and `bookmarks` holds only what
Latest Chapter, `latest_checked_at` — and `bookmarks` holds only what
differs between readers: progress, favourite, lifecycle bucket,
`updated_at`. A bookmark is keyed `(reader_id, site, series_id)` — no
surrogate id; the wire `key` is derived as `site:series_id` on read — and
@@ -75,65 +74,20 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
Remove's row wear ember wash, two reversible ones wear `.calm` grey.
`--ember` stay reserved for new-chapter signal: busy bar and inline
error use `--mute`.
- **Latest-chapter poller:** one goroutine per Site (a Poll Lane, issue #100),
each re-checking that Site's bookmarked series' newest published chapter from
backend's own network access, so `latest_chapter` stays fresh when the user
isn't browsing. Second, parallel signal — the userscript keeps its own
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` schedule, and its
`reportLatestChapter` PUTs every read, unchanged numbers included, because an
unchanged read is exactly the Sighting worth deferring a Poll on (#103).
Two independent clocks: per-series rest (`series.latest_checked_at`,
enforced by `Store.DueForLatestCheck`'s WHERE clause — `now - Rest`) and
per-Lane gap (the Lane sleeping between fetches, `effectiveGap`). Both live
in the Site registry (`internal/latest/sites.go`), not config: the five env
knobs that used to size a shared pace are gone.
- **Latest-chapter poller:** ticker goroutine in same binary re-check
each bookmarked series' newest published chapter from backend's own
network access, so `latest_chapter` stay fresh when user not
browsing. Second, parallel signal — userscript keep own
`maybeCaptureLatestOnSeriesPage`/`backgroundRefreshLatest` logic unchanged.
Two independent clocks: per-series cooldown (`series.latest_checked_at`,
enforced by `Store.DueForLatestCheck`'s WHERE clause) and wake interval.
The poller walks **Series, not Bookmarks** — a series referenced by several
bookmarks is fetched once per cycle, and the due queue orders
`reader_count DESC, latest_checked_at ASC` (ADR-0003). Series row stamped
*before* fetch so broken series wait out the rest instead of retrying
*before* fetch so broken series wait out full cooldown instead of retrying
every tick; found chapter written straight to the series row via
`Store.SetLatestChapter`, so a bookmark's `updated_at` — and the list
order — is never touched.
**Sightings** (issue #103, ADR-0011) let a Reader's own page read defer a
Poll: `Store.RecordSighting` — called by the PUT handler *before* the Upsert,
because the raise test needs the row as it stands — stamps
`series.latest_sighted_at` and, when the report raises the stored number,
names its Reader in `series.latest_raised_by`. The due query's HAVING clause
is where deferral lives: a Series is skipped only while it has exactly one
Bookmark, was sighted within one Rest, and is under the ceiling
(`sightingCeilingRests`, six of that Site's rests) since its last Poll. So a
shared Series is never deferred, and no Series goes six hours unpolled
whatever arrives. `checkOne` judges the named Reader off the comparison it
already makes: a lower number is a contradiction (logged with the Reader and
both numbers), the same number an agreement, a higher number the Site
publishing and neither — that last one clears the attribution instead, since
the value the Poll then stores is its own and a later retraction is not the
Reader's fault. Three contradictions
(`store.SightingDisagreementLimit`) stop that Reader deferring — their
reports still write the Latest Chapter — and twenty consecutive agreements
(`store.SightingAgreementsToClear`) forgive them, as does the owner's
clear-marks control. Deferral is recomputed from live facts every round, so
nothing needs invalidating when a Series gains a second Bookmark; the one
input read earlier is the Reader's marks, checked when the Sighting is
recorded, so crossing the threshold or being cleared takes effect from that
Reader's next Sighting and the standing already bought lasts out its rest.
Refusals and browser loss are Lane-local: two `errChallengeHeld` in one pass
stop that Site for `refuseBackoff` (15m) while other Lanes continue; an
`errBrowserInterrupted` (remote Chrome restart) sets a shared Poller flag
that makes the other browser Lanes skip their passes for the same 15m, so a
restarting Chrome doesn't stamp one Series per Lane per pass — after the
window the flag decays and they probe again. Browser Lanes wake Chrome only
when 5+ Series are due or one has waited 15m (ADR-0005 on-demand browser),
and cover work (both healing a stored source URL and filling a blank from
the series page) runs in the background so a slow CDN can't consume a
Lane's gap.
A refusal is only ever the challenge *page*: `isInterstitial` matches the
orchestration path `/cdn-cgi/challenge-platform/h/`, never the bare prefix.
Cloudflare injects `/cdn-cgi/challenge-platform/scripts/jsd/main.js` into
ordinary 200 pages once a zone turns JS detections on, which demonic did on
2026-08-16 — the prefix match then read every real demonic page as a refusal
and parked that Lane in 15m backoff while plain TLS was returning the full
series page.
Fetches use `bogdanfinn/tls-client` with Chrome profile as defence in depth
against fingerprint-based blocking; any failure log and skip. kagane, comix
and novelfull sit behind Cloudflare JavaScript challenges the TLS client
@@ -198,17 +152,15 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
`_REDIRECT_URI` (required; Discord OAuth for the browser UI),
`DISCORD_REQUIRED_ROLE` (optional role gate, empty by default),
`DISCORD_API_BASE` (default `https://discord.com/api/v10`),
`LATEST_CHAPTER_POLL_ENABLED` (background latest-chapter poller kill
switch, default on). Pace is per Site in the registry (issue #100): every
Site rests an hour and gaps ten seconds, a Site with more eligible Series
than 360 tightens its own gap toward the 1s floor, and browser Lanes wake
Chrome only on demand (ADR-0005). The `_COOLDOWN`/`_BROWSER_COOLDOWN`/
`_INTERVAL`/`_BATCH`/`_STAGGER` knobs that used to size a shared pace are
gone. The 1h rest for browser Sites is safe on documented grounds: a
challenged page costs seconds of a serialized single-tab browser, free-plan
zones have no bot score and no published per-IP rate input, and
`cf_clearance` expires in 30 minutes so every cadence at or above 1h
re-solves anyway —
`LATEST_CHAPTER_POLL_ENABLED`/`_COOLDOWN`/`_BROWSER_COOLDOWN`/`_INTERVAL`/
`_BATCH`/`_STAGGER` (background latest-chapter poller; defaults on,
`1h` plain-TLS cooldown, `6h` browser cooldown, `10m`/`14`/`20s`; both
cooldowns have a `15m` floor). The browser cooldown is longer for cost, not
for safety: a challenged page costs seconds of a serialized single-tab
browser, while a plain read costs one request. It buys no documented
reduction in challenge risk — free-plan zones have no bot score and no
published per-IP rate input, and `cf_clearance` expires in 30 minutes so
every cadence at or above 1h re-solves anyway —
`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`.
`USERSCRIPT_PATH` and `NOVEL_USERSCRIPT_PATH` (files served at
`/u/{token}/manga-bookmark.user.js` and `/u/{token}/novel-bookmark.user.js`,
@@ -244,31 +196,6 @@ Guidance for OpenCode (and Claude Code) working under `backend/`. See root `AGEN
`POST /rotate-token` (atomic epoch bump + hash
rewrite; invalidates every installed copy, so the panel warns to reinstall
on all devices).
- **Owner-only admin page (`internal/web/admin.go`, issue #102):** `GET /admin`
carries the Reader roster (sessions, Sighting counters, `POST
/readers/{id}/revoke` and `POST /readers/{id}/clear-marks`) and Poll Lane
status (`GET /ui/admin/lanes`, self-refreshing every 30s). Every route that
reaches past the acting Reader is listed in `adminRoutes()` and wrapped in
`requireOwner` at registration — add a route there, not a check inside a
handler; `web.AdminPatterns()` is what the gate test walks. A non-owner gets
404, never 403. Lane figures come from the running poller through the
`web.LaneReporter` seam (`latest.Poller.LaneStatus`), never from a table: a
nil reporter or a Lane that has not finished a pass renders "no data yet"
rather than zeroes. `main.newRouter` takes the reporter as an interface and
converts a nil `*Poller` to a nil interface — a typed nil would make the page
claim a poller exists.
The one owner comparison left outside `requireOwner` is in `index`
(`view.Owner = readerID == h.store.OwnerID()`): it gates a link, not an
endpoint, so it is a rendering decision a registration-time wrapper cannot
express — do not "unify" it into the gate.
A Lane pass that returns before computing its figures (refusal backoff,
sidecar down) carries the previous pass's due count and gap forward rather
than recording zeroes; a Lane that has never reached a pace renders no gap at
all. `Checked` next to `Due` is what separates a stopped Lane from a quiet
one, so neither figure may be dropped from the row.
Due-without-Checked is *not* by itself a stall: a browser Lane under both
wake thresholds sets `LaneState.Asleep` at the on-demand gate and renders
"browser asleep" instead of "not checking", and never counts toward
`Attention`. That is the commonest healthy state for kagane, comix and
novelfull — one due Series, nothing checked — so spending the stall mark on
it would train the owner to ignore the mark that matters.
Owner-only `POST /readers/{id}/revoke` (drops one Reader's session rows and
re-renders the `readers` panel; 404 for any non-owner) is the only route that
reaches across Readers.
+4 -44
View File
@@ -48,7 +48,7 @@ func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
func newTestServer(t *testing.T) http.Handler {
t.Helper()
return newRouter(newTestStore(t), testConfig(), nil)
return newRouter(newTestStore(t), testConfig())
}
func newTestStore(t *testing.T) *store.Store {
@@ -599,7 +599,7 @@ func TestLoadConfigDiscord(t *testing.T) {
// cooldown and the poller would re-fetch that series on every single tick.
func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
s := newTestStore(t)
srv := newRouter(s, testConfig(), nil)
srv := newRouter(s, testConfig())
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 777)
@@ -621,46 +621,6 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) {
}
}
// Sighting deferral (issue #103) only reaches production through the PUT
// handler: the store and poller can be right and the feature still dead if the
// handler never records the report. Asserted where a client can see it - the
// series stops being due the moment the PUT lands.
func TestPutRecordsASighting(t *testing.T) {
s := newTestStore(t)
srv := newRouter(s, testConfig(), nil)
now := time.Now().UnixMilli()
hour := time.Hour.Milliseconds()
seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", now-2*hour)
due, err := s.DueForLatestCheck("asura", now-hour, now-6*hour)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 1 {
t.Fatalf("due before the PUT = %d series, want 1", len(due))
}
body := `{"key":"asura:x","site":"asura","series_id":"x",
"series_url":"https://asurascans.com/comics/x",
"last_chapter":"Chapter 5","last_chapter_num":5,
"latest_chapter":"Chapter 9","latest_chapter_num":9}`
req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, auth(req))
if rec.Code != http.StatusOK {
t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String())
}
due, err = s.DueForLatestCheck("asura", now-hour, now-6*hour)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 0 {
t.Fatalf("due after the PUT = %d series, want 0: the handler recorded no Sighting", len(due))
}
}
// The userscript route is registered outside the web UI's Discord auth, so it
// must keep working whatever the web config — see internal/userscript for the
// handler's own behaviour. The credential in the path is the owner's derived
@@ -677,7 +637,7 @@ func TestUserscriptServedWithWebUIDisabled(t *testing.T) {
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/u/"+ownerCredential()+"/manga-bookmark.user.js", nil)
newRouter(s, cfg, nil).ServeHTTP(rr, req)
newRouter(s, cfg).ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code)
}
@@ -699,7 +659,7 @@ func TestNovelUserscriptServed(t *testing.T) {
cfg := testConfig()
cfg.NovelUserscriptPath = novelPath
srv := newRouter(s, cfg, nil)
srv := newRouter(s, cfg)
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet,
+1 -1
View File
@@ -98,7 +98,7 @@ func TestPublicCoverNeverEchoesNonImage(t *testing.T) {
if _, err := db.Exec(`UPDATE covers SET content_type = 'text/html' WHERE address = $1`, address); err != nil {
t.Fatalf("poison row: %v", err)
}
rr := getCover(t, newRouter(st, testConfig(), nil), "/covers/"+address, nil)
rr := getCover(t, newRouter(st, testConfig()), "/covers/"+address, nil)
if rr.Code == http.StatusOK {
t.Fatalf("status = 200, want a refusal for a non-image row (body %q)", rr.Body.String())
}
+1 -12
View File
@@ -99,18 +99,7 @@ func (h *Handler) Put(w http.ResponseWriter, r *http.Request) {
// reading progress actually moved. Any client value is ignored.
b.UpdatedAt = time.Now().UnixMilli()
// A userscript PUT is a Sighting: the Reader's browser was on the Series
// page and read its Latest Chapter (issue #103). Recorded before the
// Upsert, which is what makes the raise comparison possible, and never
// from the web UI's own read-modify-write — a Reader toggling a favourite
// has not looked at the Site and must not postpone a Poll. A failure here
// costs a deferral, not the write, so it is logged and dropped.
readerID := httpmw.ReaderID(r)
if err := h.Store.RecordSighting(readerID, b.Site, b.SeriesID, b.LatestChapterNum, b.UpdatedAt); err != nil {
log.Printf("record sighting: %v", err)
}
stored, err := h.Store.Upsert(readerID, b)
stored, err := h.Store.Upsert(httpmw.ReaderID(r), b)
if err != nil {
log.Printf("upsert: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
+1 -1
View File
@@ -120,7 +120,7 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
// Stamped after success — the reverse of the poller, which stamps before
// the fetch: the Reader is here, watching the Series they just created, so
// a failed acquisition must leave the row due for a fast retry rather than
// consuming the rest. The stamp happens even when the page read
// consuming the cooldown. The stamp happens even when the page read
// succeeded but produced no facts to persist.
if err := a.Store.MarkLatestChecked(sr.Site, sr.SeriesID, time.Now().UnixMilli()); err != nil {
log.Printf("acquire %q: mark checked: %v", sr.Key(), err)
+5 -12
View File
@@ -18,7 +18,7 @@ import (
// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
// challenge clears in a few seconds when it clears at all; anything longer is a
// challenge that is not going to pass, and the caller's rest was already
// challenge that is not going to pass, and the caller's cooldown was already
// stamped before this ran.
const challengeTimeout = 45 * time.Second
@@ -213,7 +213,7 @@ func (f *BrowserFetcher) Image(ctx context.Context, imageURL string) ([]byte, st
// errChallengeHeld reports that the budget ran out with the interstitial still
// up. Distinct from a transport failure: it means "this site said no", which
// the poller answers with a refusal backoff for that Site's Lane (issue #100).
// the poller answers with a 403 and its ordinary cooldown.
var errChallengeHeld = errors.New("challenge held")
// errBrowserInterrupted distinguishes a remote Chrome restart from the
@@ -250,18 +250,11 @@ func browserConnectionLost(ctx context.Context) bool {
const challengePollInterval = 2 * time.Second
// isInterstitial reports whether html is Cloudflare's challenge page rather
// than the site's own. Matched on the challenge orchestration path
// (/cdn-cgi/challenge-platform/h/<b|g|x>/orchestrate/...), which is stable
// across the interstitial's wording and locale — the visible "Just a
// than the site's own. Matched on the challenge runtime's script path, which is
// stable across the interstitial's wording and locale — the visible "Just a
// moment..." title is neither.
//
// The bare "/cdn-cgi/challenge-platform/" prefix is NOT enough: Cloudflare
// injects /cdn-cgi/challenge-platform/scripts/jsd/main.js into ordinary 200
// pages when JS detections are on, so matching the prefix declared every real
// demonic page a refusal and parked that Lane in 15m backoff (observed
// 2026-08-16, demonic turned detections on).
func isInterstitial(html string) bool {
return strings.Contains(html, "/cdn-cgi/challenge-platform/h/")
return strings.Contains(html, "/cdn-cgi/challenge-platform/")
}
// run navigates to target and re-reads until done reports an answer, bounded by
-19
View File
@@ -117,25 +117,6 @@ func TestBrowserOnlyCoverURL(t *testing.T) {
})
}
}
// The jsd script is injected into ordinary 200 pages when a zone turns JS
// detections on; only the orchestration path means the page itself is the
// challenge. Conflating the two parked the demonic Lane in refusal backoff
// while every fetch was in fact the real series page (observed 2026-08-16).
func TestIsInterstitial(t *testing.T) {
if !isInterstitial(challengeFixture) {
t.Fatal("challenge page not detected as interstitial")
}
const jsdInjected = `<html><head><title>The Possessed Grappler</title>
<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head>
<body><a href="/chaptered.php?manga=13721&chapter=22">Chapter 22</a></body></html>`
if isInterstitial(jsdInjected) {
t.Fatal("real page carrying the injected jsd script misread as interstitial")
}
if got, ok := demonicLatestChapter("", jsdInjected); !ok || got.Label != "Chapter 22" {
t.Fatalf("demonicLatestChapter = %+v, ok = %v, want Chapter 22", got, ok)
}
}
func TestClassifyBrowserInterruption(t *testing.T) {
if err := classifyBrowserError(context.Background(), true, context.Canceled); !errors.Is(err, errBrowserInterrupted) {
t.Fatalf("classifyBrowserError(context.Canceled) = %v, want browser interruption", err)
+5 -14
View File
@@ -47,15 +47,6 @@ func fetchCoverBytes(ctx context.Context, cover string, browser BrowserCoverFetc
// inject it to exercise hostile DNS results without touching the live network.
type CoverResolver func(context.Context, string) ([]netip.Addr, error)
// maxCoverBytes caps one cover, separately from the series-page maxBodyBytes:
// a cover is a bounded binary asset, not a text page, and 4 MiB rejected 12%
// of asurascans covers measured 2026-08-17 (p90 4.52 MB, max 8.57 MB — two of
// the three over-cap files were JPEGs, not the animated GIF of issue #71).
// 10 MiB is ~18% headroom over that worst case and matches the GitHub and
// Discord image limits; see docs/research/gif-maximum-byte-size.md. GIF itself
// has no maximum size, so this number is policy, not format.
const maxCoverBytes = 10 << 20
// TLSCoverFetcher retrieves image bytes with the standard HTTPS client. Unlike
// TLSFetcher, it does not need a browser fingerprint: cover hosts are public
// CDNs and the response is accepted only after the destination gate passes.
@@ -161,15 +152,15 @@ func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte,
if !ok {
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", raw)
}
if resp.ContentLength > maxCoverBytes {
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxCoverBytes)
if resp.ContentLength > maxBodyBytes {
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverBytes+1))
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes+1))
if err != nil {
return nil, "", fmt.Errorf("read cover: %w", err)
}
if len(body) > maxCoverBytes {
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxCoverBytes)
if len(body) > maxBodyBytes {
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
}
return body, contentType, nil
}
+1 -24
View File
@@ -171,7 +171,7 @@ func TestCoverFetcherRejectsOversizedBody(t *testing.T) {
var calls int
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxCoverBytes+1))
response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxBodyBytes+1))
response.ContentLength = -1
return response, nil
})}
@@ -187,29 +187,6 @@ func TestCoverFetcherRejectsOversizedBody(t *testing.T) {
}
}
// Covers between the series-page cap and the cover cap must be accepted: the
// 4 MiB page cap rejected 12% of asurascans covers (issue #71).
func TestCoverFetcherAcceptsCoverOverPageCap(t *testing.T) {
body := bytes.Repeat([]byte("x"), maxBodyBytes+1)
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return coverResponse(http.StatusOK, "image/gif", "", body), nil
})}
fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil
})
got, contentType, err := fetcher.Fetch(context.Background(), "https://cdn.example/big.gif")
if err != nil {
t.Fatalf("Fetch rejected a %d-byte cover: %v", len(body), err)
}
if len(got) != len(body) {
t.Fatalf("body = %d bytes, want %d", len(got), len(body))
}
if contentType != "image/gif" {
t.Fatalf("content type = %q, want image/gif", contentType)
}
}
func TestCoverFetcherRejectsNonImage(t *testing.T) {
var calls int
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+68 -336
View File
@@ -5,8 +5,6 @@ import (
"errors"
"log"
"net/url"
"sort"
"sync"
"time"
"bookmarkmanager/backend/internal/store"
@@ -30,11 +28,15 @@ type BrowserCoverFetcher interface {
// in parallel and report the same observable fact, so whichever writes last wins
// and neither needs to know about the other.
//
// Every Site gets its own Poll Lane: one independent stream of Polls with its
// own pace, running concurrently with every other Site's (issue #100). Rest
// time and gap live in the Site registry, not here — see sites.go. Rest is
// enforced by the WHERE clause in DueForLatestCheck rather than by any timer;
// the gap is enforced by the Lane sleeping between fetches.
// Two clocks, deliberately independent:
//
// - Interval is how often this goroutine wakes up and looks.
// - Cooldowns are how long a series rests since its own last check. Browser-
// backed sites use the longer BrowserCooldown.
//
// Cooldowns are enforced by the WHERE clause in DueForLatestCheck rather than
// by any timer. Shortening Interval therefore cannot shorten anyone's cooldown;
// it only makes the poller wake up and find nothing due more often.
type Poller struct {
Store *store.Store
Fetch Fetcher
@@ -48,23 +50,11 @@ type Poller struct {
// same failure-isolated prefetch path.
CoverBytesFetch CoverBytesFetcher
Now func() time.Time // injected so tests can freeze it
// refuseUntil gates a Site's Lane after it refused twice in one run: no
// Series of that Site is attempted again before this time (issue #100).
// browserDownAt is when a browser Lane last lost the sidecar; the other
// browser Lanes skip their passes for the next refuseBackoff, so a
// restarting Chrome does not stamp one Series per pass per Lane (story 20).
mu sync.Mutex
refuseUntil map[string]time.Time
browserDownAt time.Time
// laneStates is the owner's page snapshot of each Lane's last pass
// (issue #102), keyed by Site. Guarded by mu; a Site appears only after
// its first pass, so a restart renders "no data yet" rather than zeroes.
laneStates map[string]LaneState
// coverWG tracks in-flight cover work. Covers heal in the background so a
// slow cover host cannot delay the next Series-page Poll; tests join it
// before asserting on cover fetches.
coverWG sync.WaitGroup
Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
// fillBlankCover gives a Series its Cover when it has none. The blank state is
@@ -86,14 +76,7 @@ func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, cover stri
if cover == "" {
return
}
// Like healCover, the fill runs in the background: a large import of
// blanks would otherwise pay one og:image fetch per Series against the
// Lane's gap (issue #100, story 12).
p.coverWG.Add(1)
go func() {
defer p.coverWG.Done()
p.storeCover(ctx, sr, cover)
}()
p.storeCover(ctx, sr, cover)
}
// prefetchCover heals Series that already carry a third-party source URL but
@@ -160,259 +143,72 @@ func fetcherFor(site string, browser, tls Fetcher) Fetcher {
return nil
}
// Run polls until ctx is cancelled: one goroutine per Site Lane, each pacing
// itself by the Site's effective gap. Lanes share nothing but the store and
// the browser fetcher's single tab (BrowserFetcher serializes itself), so one
// hostile Site burns only its own budget.
// laneNames returns every registry Site in the deterministic order both Run
// and runOnce iterate: sorted, so lane behaviour and its tests agree on who
// runs first.
func laneNames() []string {
names := make([]string, 0, len(sites))
for name := range sites {
names = append(names, name)
}
sort.Strings(names)
return names
}
// Run polls until ctx is cancelled.
//
// runOnce is called synchronously, so a batch that overruns the tick delays the
// next one instead of stacking a second batch on top of it. That is the intended
// failure mode for a misconfigured batch x stagger: a slower cadence, never
// concurrent fetch storms.
func (p *Poller) Run(ctx context.Context) {
names := laneNames()
log.Printf("latest-chapter poller: %d lanes, rest=%s gap=%s", len(names), defaultRest, defaultGap)
for _, name := range names {
go p.lane(ctx, name)
}
<-ctx.Done()
log.Println("latest-chapter poller: stopped")
}
// lane is one Site's Poll Lane: one pass, then sleep the pace the pass
// reported, then another pass, until ctx is cancelled. The sleep is the whole
// pace discipline — a pass that fetched nothing still reports its gap so the
// Lane wakes often enough to notice Series as they become due. The pass shares
// the Poller's browser-down state, so a sidecar loss is noticed once and the
// other browser Lanes skip passes until the backoff window decays.
func (p *Poller) lane(ctx context.Context, name string) {
log.Printf("latest-chapter poller: interval=%s cooldown=%s browser-cooldown=%s batch=%d stagger=%s",
p.Interval, p.Cooldown, p.BrowserCooldown, p.Batch, p.Stagger)
t := time.NewTicker(p.Interval)
defer t.Stop()
for {
pace := p.runLanePass(ctx, name, true)
if ctx.Err() != nil {
return
}
select {
case <-ctx.Done():
log.Println("latest-chapter poller: stopped")
return
case <-time.After(pace):
case <-t.C:
p.runOnce(ctx)
}
}
}
// runOnce processes one round: one pass of every Lane, back to back, no real
// time passing. This is the deterministic entry point the test suite drives a
// round at a time. The production Run loop does the same work paced by its own
// sleeps; pacing is the only difference.
// runOnce processes one batch of due series.
func (p *Poller) runOnce(ctx context.Context) {
for _, name := range laneNames() {
p.runLanePass(ctx, name, false)
}
}
// runLanePass processes one pass of one Site's Lane: select the due Series,
// pace through them, and report how long the Lane should wait before its next
// pass. paced spaces consecutive fetches by the Site's effective gap — the
// production Lane's rate limit; the deterministic test entry runs back to back.
func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.Duration {
now := p.Now()
// Snapshot this pass for the owner's page (issue #102). Recorded on every
// return path, with the figures filled in where the pass computes them.
st := LaneState{Site: name, LastRun: now, Browser: isBrowserSite(name)}
defer func() { p.recordLaneState(st) }()
if until := p.refusalBackoff(name); now.Before(until) {
// Cooling down after a refusal: do not attempt this Site at all.
return until.Sub(now)
}
if isBrowserSite(name) {
if downFor, down := p.browserDownFor(now); down && downFor < refuseBackoff {
// A sibling browser Lane lost the sidecar within the backoff
// window: skip this pass, so a restarting Chrome does not stamp
// this Site's Series one pass at a time. After refuseBackoff the
// flag decays and the Lane probes again (issue #100, story 20).
log.Printf("latest poll %s: browser lane skipping pass (sidecar down %s ago)", name, downFor)
return refuseBackoff - downFor
}
}
s := sites[name]
f := fetcherFor(name, p.BrowserFetch, p.Fetch)
if f == nil {
// No fetcher at all right now (browser absent, no fallback): every
// Series stays unstamped and due, so a browser that appears after a
// restart finds its full queue waiting (issue #100).
st.Gap = defaultGap
return defaultGap
}
due, err := p.Store.DueForLatestCheck(name, now.Add(-s.Rest).UnixMilli(),
now.Add(-sightingCeilingRests*s.Rest).UnixMilli())
cutoff := now.Add(-p.Cooldown).UnixMilli()
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites(), p.Batch)
if err != nil {
log.Printf("latest poll %s: due query: %v", name, err)
st.Gap = defaultGap
return defaultGap
}
st.Due = len(due)
if s.Browser != nil && f == p.BrowserFetch && !browserWakeDue(due, now, s.Rest) {
// Below both thresholds Chrome stays asleep (ADR-0005 on-demand
// browser): waking it for a single Poll would cost a challenge solve
// per request. The Lane still paces at the default gap, which is what
// the owner's page must show rather than a zero.
st.Gap, st.Asleep = defaultGap, true
return defaultGap
}
if s.Browser != nil {
// Browser Lanes share one tab, so their combined ceiling is about 360
// Polls an hour. When they cannot keep up, the wait past the rest time
// grows — log by how much, every pass, so the decision to give them
// more pages is made from a measurement rather than a guess.
if behind := maxSeriesWait(due, now, s.Rest) - s.Rest; behind > 0 {
log.Printf("latest poll %s: browser lane behind by %s (browser Sites cannot keep up with the hour)", name, behind)
}
log.Printf("latest poll: due query: %v", err)
return
}
eligible, err := p.Store.EligibleSeriesCount(name)
if err != nil {
log.Printf("latest poll %s: eligible count: %v", name, err)
st.Gap = defaultGap
return defaultGap
}
gap, clamped := effectiveGap(s, eligible)
st.Gap, st.Clamped = gap, clamped
if clamped {
log.Printf("latest poll %s: gap clamped to %s floor (eligible series=%d)", name, minGap, eligible)
}
if eligible == 0 {
// Nothing to poll for the foreseeable future; sleep a full rest instead
// of re-querying every gap.
return s.Rest
}
refusals := 0
checked := 0
for i, sr := range due {
if ctx.Err() != nil {
break
}
if refusals >= 2 {
// This Site refused twice in a row: the remaining Series are left
// unstamped and due, and the Lane waits refuseBackoff before
// trying it again.
break
}
if paced && i > 0 {
// Staggered rather than fired together: a burst of simultaneous requests
// from one server IP is the traffic shape most likely to move that IP's
// bot score. This is the server-side analogue of the userscript's "one
// series per navigation ... indistinguishable from browsing" (L455-456).
stopped := false
if i > 0 && p.Stagger > 0 {
select {
case <-ctx.Done():
break
case <-time.After(gap):
}
if ctx.Err() != nil {
break
stopped = true
case <-time.After(p.Stagger):
}
}
if err := p.checkOne(ctx, sr); err != nil {
switch {
case errors.Is(err, errChallengeHeld):
refusals++
case errors.Is(err, errBrowserInterrupted):
p.setBrowserDown(now)
log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, refuseBackoff)
return gap
default:
refusals = 0
}
} else {
refusals = 0
if stopped {
break
}
st.Checked++
p.checkOne(ctx, sr)
checked++
}
if st.Checked > 0 {
log.Printf("latest poll %s: due=%d checked=%d", name, len(due), st.Checked)
}
if refusals >= 2 {
p.setRefusalBackoff(name, now.Add(refuseBackoff))
log.Printf("latest poll %s: refused twice this run, waiting %s", name, refuseBackoff)
return refuseBackoff
}
return gap
}
func (p *Poller) refusalBackoff(name string) time.Time {
p.mu.Lock()
defer p.mu.Unlock()
return p.refuseUntil[name]
}
func (p *Poller) setRefusalBackoff(name string, until time.Time) {
p.mu.Lock()
defer p.mu.Unlock()
if p.refuseUntil == nil {
p.refuseUntil = make(map[string]time.Time)
}
p.refuseUntil[name] = until
}
// setBrowserDown records when a browser Lane lost the sidecar. It is Poller
// state rather than pass state so the other browser Lanes see it too.
func (p *Poller) setBrowserDown(now time.Time) {
p.mu.Lock()
p.browserDownAt = now
p.mu.Unlock()
}
// browserDownFor reports how long the sidecar has been down and that it is
// down at all — the zero time means never down, which must not read as a
// zero-duration loss. The window decays: once refuseBackoff passes without a
// fresh loss, Lanes probe again.
func (p *Poller) browserDownFor(now time.Time) (time.Duration, bool) {
p.mu.Lock()
defer p.mu.Unlock()
if p.browserDownAt.IsZero() {
return 0, false
}
return now.Sub(p.browserDownAt), true
}
// isBrowserSite reports whether the registry routes this Site's page through
// the browser sidecar.
func isBrowserSite(name string) bool {
return sites[name].Browser != nil
}
// browserWakeDue reports whether a browser Lane may start a run: five or more
// of its Series are due, or any one of them has been due for browserWakeAge.
// Below both thresholds the Lane leaves Chrome asleep — Series Polled together
// become due together, so the group naturally stays clustered, and the age
// rule exists to stop a Series that drifted out of the group from starving.
func browserWakeDue(due []store.Series, now time.Time, rest time.Duration) bool {
if len(due) >= browserWakeCount {
return true
}
return maxSeriesWait(due, now, rest) >= browserWakeAge
}
// maxSeriesWait returns how long the most-overdue of the due Series has been
// waiting past its due moment (0 when due is empty).
func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.Duration {
var oldest time.Duration
for _, sr := range due {
if w := now.Sub(time.UnixMilli(sr.LatestCheckedAt).Add(rest)); w > oldest {
oldest = w
}
}
return oldest
// due vs checked is how you tell which constraint is binding: ticks that
// report due=0 mean the cooldown is the limit, ticks that report due==batch
// every time mean throughput is.
log.Printf("latest poll: due=%d checked=%d", len(due), checked)
}
// checkOne re-checks one series. Every failure path here is "log and move on":
// the poller is a best-effort enhancement, and no single bad series may stall a
// Lane or take down the process. The returned error is the page read's
// classified outcome so the Lane can tell a refusal from a loss of the
// browser; non-classified failures still return nil-equivalent behaviour.
func (p *Poller) checkOne(ctx context.Context, sr store.Series) error {
// batch or take down the process.
func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
defer func() {
if r := recover(); r != nil {
log.Printf("latest poll %q: recovered from panic: %v", sr.Key(), r)
@@ -420,59 +216,52 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error {
}()
// Stamped before the fetch, not after, so an error, a timeout, or a shutdown
// mid-request still consumes the rest. Otherwise a renamed or deleted
// series would be retried on every single pass forever. The userscript
// stamps in the same order and for the same reason (L471-473). A Series
// never reaches checkOne without a fetcher — runLanePass skips those — so
// the stamp means "attempted", and an untried Series stays due.
// mid-request still consumes the cooldown. Otherwise a renamed or deleted
// series would be retried on every single tick forever. The userscript
// stamps in the same order and for the same reason (L471-473).
if err := p.Store.MarkLatestChecked(sr.Site, sr.SeriesID, p.Now().UnixMilli()); err != nil {
log.Printf("latest poll %q: mark checked: %v", sr.Key(), err)
return nil
return
}
facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, p.BrowserFetch, p.Fetch)
if err != nil {
switch {
case errors.Is(err, errNotFetchable):
// The rest above is already consumed, so a row that never
// passes the gate is retried at rest pace rather than
// The cooldown above is already consumed, so a row that never
// passes the gate is retried at cooldown pace rather than
// hot-looping.
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
return err
return
case errors.Is(err, errNoFetcher):
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
return err
return
}
// A legacy cover heals independently of the page read: its source may
// answer — a CDN — while the origin does not, so a fetch failure does
// not skip the heal, matching the order the shared read replaced.
p.healCover(ctx, sr)
p.prefetchCover(ctx, sr)
log.Printf("latest poll %q: %v", sr.Key(), err)
return err
return
}
// A legacy cover source is healed independently of the page read.
p.healCover(ctx, sr)
p.prefetchCover(ctx, sr)
// Cover fill is independent of the chapter signal: a page that lost its
// chapter list may keep its og:image, and a blank Series heals either way.
p.fillBlankCover(ctx, sr, facts.Cover)
if !facts.HasLatest {
// Most likely a challenge page or a layout change. Either way the row is
// already stamped, so this waits out a rest instead of hot-looping.
// already stamped, so this waits out a cooldown instead of hot-looping.
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
return nil
return
}
// The Poll is the oracle for whatever Sighting last raised this Series
// (issue #103), and the judgement is free: the comparison below already
// exists, and no extra request is made to reach it.
p.judgeSighting(sr, facts.Latest.Num)
// Equality, not >, mirroring the userscript (L427): a site that retracts a
// chapter should correct the stored number downward. The comparison is
// against the due-query snapshot; a concurrent write in between only costs
// one redundant UPDATE of the same absolute value, never a wrong one.
if sr.LatestChapterNum != nil && *sr.LatestChapterNum == facts.Latest.Num {
return nil
return
}
// Series-level write: the row is shared, so one update refreshes every
@@ -481,66 +270,9 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error {
// the list.
if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil {
log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err)
return nil
return
}
log.Printf("latest poll %q: latest is now %s", sr.Key(), facts.Latest.Label)
return nil
}
// judgeSighting settles the Sighting the Series' stored Latest Chapter is owed
// to, if any, against what the Site actually publishes. The asymmetry is the
// whole of the detection rule and is what keeps it free of false alarms: a Poll
// finding a *lower* number than stored means the Reader who raised it reported
// a chapter that does not exist, while a Poll finding a higher one is only the
// Site publishing since and means nothing about the report. Equality confirms
// the report, which is how an honest Reader earns back a mark.
//
// A Series with no attribution — the stored value is a Poll's own, or a
// previous Poll already judged the report — is nobody's to answer for.
func (p *Poller) judgeSighting(sr store.Series, found float64) {
if sr.LatestRaisedBy == nil || sr.LatestChapterNum == nil {
return
}
stored := *sr.LatestChapterNum
if found > stored {
// The report is neither confirmed nor contradicted, but it is answered:
// the value about to be stored is the Poll's own, so leaving the
// attribution would credit this Reader with the next Poll's agreement
// and blame them if the Site later retracts.
if err := p.Store.ClearSightingAttribution(sr.Site, sr.SeriesID, *sr.LatestRaisedBy); err != nil {
log.Printf("latest poll %q: clear sighting attribution: %v", sr.Key(), err)
}
return
}
if found < stored {
// Logged with both numbers and the Reader, because that is what tells a
// broken Site adapter (which marks every Reader of that Site at once)
// from one Reader deliberately lying.
log.Printf("latest poll %q: sighting contradicted: reader %d raised it to %v, site publishes %v",
sr.Key(), *sr.LatestRaisedBy, stored, found)
}
if err := p.Store.RecordSightingOutcome(sr.Site, sr.SeriesID, *sr.LatestRaisedBy, found == stored); err != nil {
log.Printf("latest poll %q: record sighting outcome: %v", sr.Key(), err)
}
}
// healCover runs prefetchCover in the background. Cover bytes come from a
// different host — often a CDN — and heal once in a Series's life, so they
// must not consume a Lane's gap: a large import with many blanks would
// otherwise make every Latest Chapter go stale behind a slow image host
// (issue #100).
func (p *Poller) healCover(ctx context.Context, sr store.Series) {
p.coverWG.Add(1)
go func() {
defer p.coverWG.Done()
p.prefetchCover(ctx, sr)
}()
}
// waitCovers blocks until every in-flight cover heal finishes. Tests call it
// after a round before asserting on cover fetches.
func (p *Poller) waitCovers() {
p.coverWG.Wait()
}
// fetchableSeriesURL reports whether site is a Site the registry knows and
+114 -514
View File
@@ -5,7 +5,6 @@ import (
"crypto/sha256"
"database/sql"
"errors"
"fmt"
"log"
"os"
"strings"
@@ -157,16 +156,19 @@ func (f *fakeBytesCoverFetcher) callCount() int {
return len(f.calls)
}
// newTestPoller wires a poller with a frozen clock. Rest and gap come from the
// Site registry, so tests seed checked_at relative to the one-hour rest; the
// round entry point (runOnce) runs every Lane back to back with no real
// pacing, so tests stay instant and deterministic.
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
// instantly and deterministically.
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
t.Helper()
return &Poller{
Store: s,
Fetch: f,
Now: func() time.Time { return at },
Store: s,
Fetch: f,
Now: func() time.Time { return at },
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute,
Stagger: 0,
Batch: 14,
}
}
@@ -206,10 +208,9 @@ func TestRunOncePrefetchesPublicCover(t *testing.T) {
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got)
@@ -240,10 +241,9 @@ func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
@@ -350,6 +350,23 @@ func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
}
}
func TestRunOnceRespectsBatchLimit(t *testing.T) {
s, _ := newTestStore(t)
for i := 0; i < 20; i++ {
key := "asura:s" + string(rune('a'+i))
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
}
f := &fakeFetcher{body: "", status: 200}
p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
p.Batch = 5
p.runOnce(context.Background())
if got := f.callCount(); got != 5 {
t.Fatalf("fetched %d series, want 5 (batch limit)", got)
}
}
// The point of the split (ADR-0003): a series referenced by several bookmarks
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
// series when two readers track it (issue #22).
@@ -392,8 +409,8 @@ func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
}
}
// One unreachable series must not abandon the rest of the Lane.
func TestRunOnceOneBadSeriesDoesNotStallLane(t *testing.T) {
// One unreachable series must not abandon the rest of the batch.
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
s, _ := newTestStore(t)
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
for _, k := range keys {
@@ -419,9 +436,7 @@ func TestRunOnceOneBadSeriesDoesNotStallLane(t *testing.T) {
}
}
// Pace now lives in the registry, not config: Run logs the lane defaults so a
// deployment can see what the poller is doing without reading the source.
func TestRunLogsLaneDefaults(t *testing.T) {
func TestRunLogsCooldowns(t *testing.T) {
var logs strings.Builder
previous := log.Writer()
log.SetOutput(&logs)
@@ -429,19 +444,21 @@ func TestRunLogsLaneDefaults(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
(&Poller{Now: func() time.Time { return time.Now() }}).Run(ctx)
(&Poller{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: time.Hour,
}).Run(ctx)
got := logs.String()
for _, want := range []string{"6 lanes", "rest=1h0m0s", "gap=10s"} {
if !strings.Contains(got, want) {
t.Fatalf("startup log = %q, want %q", got, want)
}
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
!strings.Contains(got, "browser-cooldown=6h") {
t.Fatalf("startup log = %q, want both cooldowns", got)
}
}
// The rest is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the Lane's sleep independent of it.
func TestRunOnceHonoursRestAcrossPasses(t *testing.T) {
// The cooldown is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the tick interval independent of it.
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/x"
seedForCheck(t, s, "asura:x", url, 0)
@@ -454,18 +471,56 @@ func TestRunOnceHonoursRestAcrossPasses(t *testing.T) {
if got := f.callCount(); got != 1 {
t.Fatalf("first pass fetched %d, want 1", got)
}
// Same instant, and again 59 minutes later: both inside the 1h rest.
// Same instant, and again 59 minutes later: both inside the 1h cooldown.
p.runOnce(context.Background())
p.Now = func() time.Time { return now.Add(59 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times inside the rest, want 1", got)
t.Fatalf("fetched %d times inside the cooldown, want 1", got)
}
// Past the rest, it is due again.
// Past the cooldown, it is due again.
p.Now = func() time.Time { return now.Add(61 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 2 {
t.Fatalf("fetched %d times after the rest, want 2", got)
t.Fatalf("fetched %d times after the cooldown, want 2", got)
}
}
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
s, _ := newTestStore(t)
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
hour := time.Hour
now := time.Unix(2*int64(hour/time.Second), 0)
tls := &fakeFetcher{status: 200}
browser := &fakeFetcher{status: 200}
p := &Poller{
Store: s,
Fetch: tls,
BrowserFetch: browser,
Now: func() time.Time { return now },
Cooldown: hour,
BrowserCooldown: 6 * hour,
Batch: 10,
}
p.runOnce(context.Background())
if got := tls.callCount(); got != 1 {
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
}
if got := browser.callCount(); got != 0 {
t.Fatalf("browser fetches after 2h = %d, want 0", got)
}
now = time.Unix(7*int64(hour/time.Second), 0)
p.runOnce(context.Background())
if got := tls.callCount(); got != 2 {
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
}
if got := browser.callCount(); got != 1 {
t.Fatalf("browser fetches after 7h = %d, want 1", got)
}
}
@@ -610,18 +665,17 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s,
Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Store: s,
Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(f.calls) != 0 {
t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
}
if got := readLatestCheckedAt(t, s, "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); got != 0 {
t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got)
}
}
// novelfull without a browser is not skipped outright: its challenge is a
@@ -645,10 +699,11 @@ func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if len(tlsF.calls) != 1 {
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
@@ -686,7 +741,9 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
@@ -732,10 +789,11 @@ func TestComixUsesBrowserFetcher(t *testing.T) {
p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF,
CoverFetch: covers, CoverBytesFetch: tlsCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if len(tlsF.calls) != 0 {
t.Errorf("TLS fetcher was called for comix: %v", tlsF.calls)
@@ -773,16 +831,15 @@ func TestComixSkippedWhenNoBrowserFetcher(t *testing.T) {
f := &fakeFetcher{body: comixSeriesFixture, status: 200}
p := &Poller{
Store: s, Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(f.calls) != 0 {
t.Errorf("TLS fetcher was called for comix: %v", f.calls)
}
if got := readLatestCheckedAt(t, s, "comix:n8we-dungeons-and-crayons"); got != 0 {
t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got)
}
}
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
@@ -803,10 +860,10 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
if err != nil || !ok {
@@ -841,13 +898,11 @@ func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return at },
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
p.waitCovers()
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
@@ -872,10 +927,9 @@ func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
Now: func() time.Time { return now },
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
@@ -906,10 +960,9 @@ func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
@@ -932,10 +985,9 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
@@ -958,10 +1010,9 @@ func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
CoverBytesFetch: publicCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if got := publicCovers.callCount(); got != 1 {
t.Fatalf("public cover fetch calls = %d, want 1", got)
@@ -1077,10 +1128,10 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
p := &Poller{
Store: s, Fetch: page, BrowserFetch: page,
CoverBytesFetch: public, CoverFetch: browser,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
got := readBookmark(t, s, tc.key)
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
@@ -1130,7 +1181,7 @@ func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: public,
Now: func() time.Time { return at },
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
at = at.Add(2 * time.Hour)
@@ -1165,10 +1216,9 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: public,
Now: func() time.Time { return at },
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
if got := readBookmark(t, s, key); got.Cover != "" {
t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover)
}
@@ -1181,7 +1231,6 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
public.contentType = "image/jpeg"
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
p.waitCovers()
if got := public.callCount(); got != 2 {
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
@@ -1215,10 +1264,9 @@ func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) {
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")},
Now: func() time.Time { return now },
Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
p.waitCovers()
got := readBookmark(t, s, key)
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
@@ -1241,451 +1289,3 @@ const kaganeAPIFixtureWithCover = `
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
`
func TestEffectiveGap(t *testing.T) {
asura := sites["asura"]
tests := []struct {
eligible int
want time.Duration
clamped bool
}{
{0, 10 * time.Second, false},
{5, 10 * time.Second, false},
{360, 10 * time.Second, false},
{720, 5 * time.Second, false},
{3600, time.Second, false},
{4000, time.Second, true},
}
for _, tt := range tests {
got, clamped := effectiveGap(asura, tt.eligible)
if got != tt.want || clamped != tt.clamped {
t.Errorf("effectiveGap(asura, %d) = (%s, %v), want (%s, %v)",
tt.eligible, got, clamped, tt.want, tt.clamped)
}
}
}
// The due query orders by sharedness first, then age: a series two readers
// track is polled before a single-reader series that has waited far longer
// (ADR-0003, issue #100).
func TestRunOnceOrdersBySharednessThenAge(t *testing.T) {
s, url := newTestStore(t)
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
if err != nil {
t.Fatalf("Open second reader: %v", err)
}
t.Cleanup(func() { other.Close() })
now := time.UnixMilli(5_000_000)
// popular: two readers, due for 2 minutes. loner: one reader, due for 23
// minutes. Popularity must win — the loner waited far longer.
const (
popularKey = "asura:popular"
lonerKey = "asura:loner"
popularURL = "https://asurascans.com/comics/popular"
lonerURL = "https://asurascans.com/comics/loner"
)
seedForCheck(t, s, popularKey, popularURL, now.Add(-62*time.Minute).UnixMilli())
seedForCheck(t, s, lonerKey, lonerURL, 0)
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
Key: popularKey, Site: "asura", SeriesID: "popular", UpdatedAt: 2000,
}); err != nil {
t.Fatalf("seed second reader: %v", err)
}
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).runOnce(context.Background())
if len(f.calls) != 2 {
t.Fatalf("fetches = %d, want 2", len(f.calls))
}
if f.calls[0] != popularURL {
t.Fatalf("first fetch = %q, want the shared series %q", f.calls[0], popularURL)
}
}
// Two refusals in one pass stop the Lane: the remaining Series stay unstamped
// and due, and the Lane backs off for refuseBackoff before trying the Site
// again. One hostile Site burns only its own Lane's budget (issue #100).
func TestRunOnceSiteRefusalSkipsRestOfLaneAndBacksOff(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
const n = 4
for i := 0; i < n; i++ {
key := fmt.Sprintf("kagane:s%d", i)
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0)
}
browser := &fakeFetcher{status: 403}
tls := &fakeFetcher{status: 200}
// An asura Series sits on its own Lane: it must still be polled while
// kagane's Lane burns its budget on refusals (story 4).
seedForCheck(t, s, "asura:still-polls", "https://asurascans.com/series/still-polls", 0)
p := &Poller{
Store: s, Fetch: tls, BrowserFetch: browser,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
if got := browser.callCount(); got != 2 {
t.Fatalf("browser fetches after first pass = %d, want 2 (refused twice)", got)
}
if got := tls.callCount(); got != 1 {
t.Fatalf("asura fetches after first pass = %d, want 1 (its Lane is independent)", got)
}
stamped := 0
for i := 0; i < n; i++ {
if readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)) == now.UnixMilli() {
stamped++
}
}
if stamped != 2 {
t.Fatalf("stamped series = %d, want 2; the remaining two stay due", stamped)
}
// Inside the backoff window nothing is attempted.
p.Now = func() time.Time { return now.Add(14 * time.Minute) }
p.runOnce(context.Background())
if got := browser.callCount(); got != 2 {
t.Fatalf("browser fetches inside backoff = %d, want still 2", got)
}
// Past the backoff the Lane resumes and the two untried Series are polled.
p.Now = func() time.Time { return now.Add(16 * time.Minute) }
p.runOnce(context.Background())
if got := browser.callCount(); got != 4 {
t.Fatalf("browser fetches after backoff = %d, want 4", got)
}
for i := 0; i < n; i++ {
if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 {
t.Fatalf("kagane:s%d still untried after backoff", i)
}
}
}
// Below five due Series with none waiting long, a browser Lane leaves Chrome
// asleep; five due, or one waiting browserWakeAge, wakes it (ADR-0005).
func TestBrowserLaneWakeThresholds(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
// Freshly due: checked two minutes before the rest elapses, so the wait
// is far below browserWakeAge.
seed := func(i int) {
key := fmt.Sprintf("kagane:w%d", i)
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli())
}
for i := 0; i < 3; i++ {
seed(i)
}
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
p := &Poller{
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
if got := browser.callCount(); got != 0 {
t.Fatalf("browser fetches with 3 freshly-due series = %d, want 0 (Chrome stays asleep)", got)
}
// The owner's page reads this state off the snapshot, and Due-without-
// Checked has to be distinguishable there from a Lane that has stopped.
if lane := laneByName(t, p, "kagane"); !lane.Asleep || lane.Due != 3 || lane.Checked != 0 {
t.Fatalf("asleep kagane lane = %+v, want Asleep with 3 due and 0 checked", lane)
}
// 5 due crosses the count threshold.
for i := 3; i < 5; i++ {
seed(i)
}
p.runOnce(context.Background())
if got := browser.callCount(); got != 5 {
t.Fatalf("browser fetches with 5 due series = %d, want 5", got)
}
if lane := laneByName(t, p, "kagane"); lane.Asleep {
t.Fatalf("woken kagane lane still reports Asleep: %+v", lane)
}
// A single long-neglected series wakes the browser by age alone.
seedForCheck(t, s, "kagane:ancient", "https://kagane.to/series/ancient", 0)
p.runOnce(context.Background())
if got := browser.callCount(); got != 6 {
t.Fatalf("browser fetches with one ancient series = %d, want 6", got)
}
}
// laneByName pulls one Lane out of the poller's snapshot, failing rather than
// returning a zero LaneState a caller would assert against by accident.
func laneByName(t *testing.T, p *Poller, site string) LaneState {
t.Helper()
for _, lane := range p.LaneStatus().Lanes {
if lane.Site == site {
return lane
}
}
t.Fatalf("no %q lane in the snapshot", site)
return LaneState{}
}
// When one browser Lane loses the sidecar, the round's remaining browser
// Lanes are skipped: every fetch would fail anyway, and their Series must not
// burn their stamps on a dead Chrome (issue #100).
func TestRunOnceUnreachableBrowserStopsBrowserLanes(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0)
seedForCheck(t, s, "kagane:k", "https://kagane.to/series/k", 0)
seedForCheck(t, s, "novelfull:n", "https://novelfull.com/n.html", 0)
interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart"))
browser := &fakeFetcher{status: 200, err: interrupted}
p := &Poller{
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
// Sorted lane order: comix, kagane, novelfull. Only comix attempted.
if got := browser.callCount(); got != 1 {
t.Fatalf("browser fetches = %d, want 1 (only the first browser lane)", got)
}
if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() {
t.Fatalf("comix stamp = %d, want %d", got, now.UnixMilli())
}
for _, key := range []string{"kagane:k", "novelfull:n"} {
if got := readLatestCheckedAt(t, s, key); got != 0 {
t.Fatalf("%s stamp = %d, want 0 (untried)", key, got)
}
}
// The shared flag decays after refuseBackoff: the next round probes
// again. comix's Series is resting (stamped last round), so the probe
// falls to kagane — the only Lane with something due — and its fresh
// loss re-gates the Lanes behind it.
p.Now = func() time.Time { return now.Add(16 * time.Minute) }
p.runOnce(context.Background())
if got := browser.callCount(); got != 2 {
t.Fatalf("browser fetches after backoff decay = %d, want 2 (kagane probes again)", got)
}
if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() {
t.Fatalf("comix stamp after decay = %d, want %d (resting, untouched)", got, now.UnixMilli())
}
if got := readLatestCheckedAt(t, s, "kagane:k"); got != now.Add(16*time.Minute).UnixMilli() {
t.Fatalf("kagane stamp after decay = %d, want %d (the probe)", got, now.Add(16*time.Minute).UnixMilli())
}
if got := readLatestCheckedAt(t, s, "novelfull:n"); got != 0 {
t.Fatalf("novelfull stamp after decay = %d, want 0 (gated by kagane's fresh loss)", got)
}
}
// A browser Lane that cannot keep up logs the backlog every pass, so the
// decision to give browser Sites more pages is measured, not guessed.
func TestRunOnceLogsBrowserLaneBehind(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
// Checked three rests ago: two rests' worth of wait past the due moment.
seedForCheck(t, s, "kagane:old", "https://kagane.to/series/old", now.Add(-3*time.Hour).UnixMilli())
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
p := &Poller{
Store: s, Fetch: &fakeFetcher{status: 200},
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
Now: func() time.Time { return now },
}
p.runOnce(context.Background())
if got := logs.String(); !strings.Contains(got, "latest poll kagane: browser lane behind by 1h0m0s") {
t.Fatalf("behind log = %q, want it to name kagane and the backlog", got)
}
}
// gatedCoverFetcher blocks every Fetch on a gate, so a test can hold a cover
// heal in flight and prove a Lane does not wait for it.
type gatedCoverFetcher struct {
inner *fakeBytesCoverFetcher
gate chan struct{}
started chan struct{}
once sync.Once
}
func (g *gatedCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
g.once.Do(func() { g.started <- struct{}{} })
<-g.gate
return g.inner.Fetch(ctx, sourceURL)
}
// Cover heals run in the background: a heal stuck on a slow CDN must not
// delay the Lane's next Series-page Poll, or a large import with many blanks
// would make every Latest Chapter go stale (issue #100).
func TestRunOnceCoverFetchDoesNotDelayNextPoll(t *testing.T) {
s, dbURL := newTestStore(t)
// Two Series whose legacy cover sources still need healing.
for i := 0; i < 2; i++ {
key := fmt.Sprintf("asura:cover-%d", i)
seriesID := fmt.Sprintf("cover-%d", i)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: seriesID,
SeriesURL: "https://asurascans.com/comics/" + seriesID, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "asura", seriesID, fmt.Sprintf("https://cdn.example/covers/%d.jpg", i))
}
pages := &fakeFetcher{body: asuraSeriesFixture, status: 200}
gated := &gatedCoverFetcher{
inner: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
gate: make(chan struct{}),
started: make(chan struct{}, 1),
}
p := &Poller{
Store: s, Fetch: pages, CoverBytesFetch: gated,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
}
done := make(chan struct{})
go func() {
p.runOnce(context.Background())
close(done)
}()
<-gated.started // the first cover heal is now stuck on its CDN
select {
case <-done:
// The Lane finished its page fetches without waiting for the cover.
case <-time.After(5 * time.Second):
t.Fatal("runOnce blocked on an in-flight cover fetch")
}
close(gated.gate)
p.waitCovers()
if got := pages.callCount(); got != 2 {
t.Fatalf("series page fetches = %d, want 2", got)
}
if got := gated.inner.callCount(); got != 2 {
t.Fatalf("cover fetches = %d, want 2", got)
}
}
// At 3601 eligible Series the effective gap falls below the one-second floor;
// the clamp warning must name the Site so the operator knows which Lane is
// outrunning its plan.
func TestRunOnceClampWarningNamesTheSite(t *testing.T) {
s, dbURL := newTestStore(t)
db, err := sql.Open("pgx", dbURL)
if err != nil {
t.Fatalf("open %s: %v", dbURL, err)
}
defer db.Close()
if _, err := db.Exec(`INSERT INTO series (site, series_id, series_url, latest_checked_at)
SELECT 'asura', 'bulk-' || g, 'https://asurascans.com/comics/bulk-' || g, 0
FROM generate_series(1, 3601) AS g`); err != nil {
t.Fatalf("bulk seed series: %v", err)
}
if _, err := db.Exec(`INSERT INTO bookmarks (reader_id, site, series_id, updated_at)
SELECT (SELECT id FROM readers ORDER BY id LIMIT 1), 'asura', 'bulk-' || g, 1000
FROM generate_series(1, 3601) AS g`); err != nil {
t.Fatalf("bulk seed bookmarks: %v", err)
}
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
newTestPoller(t, s, &fakeFetcher{body: "<html></html>", status: 200}, time.UnixMilli(5_000_000)).
runOnce(context.Background())
if got := logs.String(); !strings.Contains(got, "latest poll asura: gap clamped to 1s floor (eligible series=3601)") {
t.Fatalf("clamp warning = %q, want it to name asura and 3601", got)
}
}
// The owner's admin page (issue #102) reads Lane state out of the poller.
// Before any pass the snapshot is empty — a restart must render "no data
// yet", not zeroes — and each pass records what it saw: the frozen clock,
// the due count and the pace, with refusal backoff derived at snapshot time.
func TestLaneStatus(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now)
if st := p.LaneStatus(); len(st.Lanes) != 0 {
t.Fatalf("lanes before any pass = %d, want 0 (nothing has run)", len(st.Lanes))
} else if st.BrowserConfigured || st.BrowserReachable {
t.Fatalf("browser before any pass = configured=%v reachable=%v, want false without a browser fetcher", st.BrowserConfigured, st.BrowserReachable)
}
seedForCheck(t, s, "asura:chronicles", "https://asurascans.com/series/chronicles", 0)
// Two refusals put kagane's Lane into backoff; asura sits on its own Lane.
browser := &fakeFetcher{status: 403}
p.BrowserFetch = browser
for i := range 2 {
key := fmt.Sprintf("kagane:s%d", i)
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0)
}
p.runOnce(context.Background())
st := p.LaneStatus()
var asura, kagane LaneState
for _, lane := range st.Lanes {
switch lane.Site {
case "asura":
asura = lane
case "kagane":
kagane = lane
}
}
if st.Lanes[0].Site != "asura" {
t.Fatalf("first lane = %q, want asura (snapshot sorted by Site)", st.Lanes[0].Site)
}
if asura.Site == "" {
t.Fatalf("asura missing from snapshot: %+v", st.Lanes)
}
if !asura.LastRun.Equal(now) {
t.Fatalf("asura LastRun = %s, want the frozen clock %s", asura.LastRun, now)
}
if asura.Due != 1 {
t.Fatalf("asura Due = %d, want 1", asura.Due)
}
if asura.Gap == 0 {
t.Fatal("asura Gap = 0, want the Lane's pace")
}
if asura.Browser || asura.Refusing {
t.Fatalf("asura = %+v, want a TLS Lane that is not refusing", asura)
}
if !kagane.Browser || !kagane.Refusing {
t.Fatalf("kagane = %+v, want a browser Lane in refusal backoff", kagane)
}
if !st.BrowserConfigured || !st.BrowserReachable {
t.Fatalf("browser after round = configured=%v reachable=%v, want true/true (sidecar never lost)", st.BrowserConfigured, st.BrowserReachable)
}
if asura.Checked != 1 {
t.Fatalf("asura Checked = %d, want the one Series it read", asura.Checked)
}
// A pass that declines to look (kagane is now in backoff) must not restate
// the figures it never gathered as zeroes: the last real pass's due count
// and pace stand until a pass replaces them.
before := kagane
if before.Due == 0 || before.Gap == 0 {
t.Fatalf("kagane after its refusing pass = %+v, want the figures that pass gathered", before)
}
p.runOnce(context.Background())
for _, lane := range p.LaneStatus().Lanes {
if lane.Site != "kagane" {
continue
}
if lane.Due != before.Due || lane.Gap != before.Gap {
t.Fatalf("kagane after a skipped pass = due %d gap %s, want the previous pass's %d / %s",
lane.Due, lane.Gap, before.Due, before.Gap)
}
}
// A lost sidecar reads as unreachable for the same window the Lanes skip.
p.setBrowserDown(now)
if st := p.LaneStatus(); !st.BrowserConfigured || st.BrowserReachable {
t.Fatalf("browser after loss = configured=%v reachable=%v, want true/false", st.BrowserConfigured, st.BrowserReachable)
}
}
+1 -16
View File
@@ -9,7 +9,7 @@ import (
// seriesRead carries the two facts the poll and the acquirer both extract
// from a series page. Persistence, stamps and scheduling stay with the
// callers, so the policies that keep the two flows distinct (stamp order,
// rests) are not swallowed by the module.
// cooldowns) are not swallowed by the module.
type seriesRead struct {
Latest latestChapter
HasLatest bool
@@ -22,9 +22,6 @@ type seriesRead struct {
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
// failures so each caller keeps its own distinct log line for all three.
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
// is how a Lane tells a refusal from an ordinary failure (issue #100).
var (
errNotFetchable = errors.New("series url not fetchable")
errNoFetcher = errors.New("no fetcher for site")
@@ -52,21 +49,9 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
if err != nil {
return seriesRead{}, fmt.Errorf("fetch %s: %w", seriesURL, err)
}
if status == 403 {
// Cloudflare's challenge response for these Sites (cf-mitigated). The
// browser fetcher returns exactly this on a held interstitial, and a
// plain-TLS 403 means the same: the Site is refusing.
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
}
if status != 200 {
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
}
if isInterstitial(body) {
// A 200 that is the challenge page, not the payload: the TLS route can
// receive this where the browser would have kept re-reading. Same
// refusal as the 403.
return seriesRead{}, fmt.Errorf("%w: fetch %s: interstitial body", errChallengeHeld, seriesURL)
}
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
cover, hasCover := coverFrom(site, seriesURL, body)
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
-494
View File
@@ -1,494 +0,0 @@
package latest
import (
"context"
"crypto/sha256"
"fmt"
"log"
"strings"
"testing"
"time"
"bookmarkmanager/backend/internal/store"
)
// Sightings (issue #103) are specified at the Poller seam, with the store as
// the way in: a Sighting is seeded the way handlers.Put performs one, a round
// is run against the injected fetcher and a frozen clock, and the assertions
// are the two observable facts — whether the Series was fetched, and what the
// stored Latest Chapter is afterwards. Nothing here asserts counter arithmetic
// through an internal call or reads how a deferral is represented in a row.
const (
sightingSlug = "chronicles-of-the-demon-faction-f886a8af"
sightingKey = "asura:" + sightingSlug
sightingURL = "https://asurascans.com/comics/" + sightingSlug
)
// sightingFixtureLatest is the newest chapter asuraSeriesFixture publishes.
const sightingFixtureLatest = 181.0
// sight performs one Sighting exactly as the JSON API does (handlers.Put):
// RecordSighting against the row as stored, then the Upsert that stores the
// reported value. The order is load-bearing — the raise comparison has nothing
// to compare against once the Upsert has landed — and the bookmark's own fields
// are carried over untouched, which is what a userscript PUT does when it
// echoes back the row it cached.
func sight(t *testing.T, s *store.Store, readerID int64, key string, num float64, at time.Time) {
t.Helper()
site, seriesID, ok := strings.Cut(key, ":")
if !ok {
t.Fatalf("key %q: no ':' separator", key)
}
b, found, err := s.Get(readerID, key)
if err != nil || !found {
t.Fatalf("sight %q: get: %v found=%v", key, err, found)
}
if err := s.RecordSighting(readerID, site, seriesID, &num, at.UnixMilli()); err != nil {
t.Fatalf("sight %q: %v", key, err)
}
b.LatestChapter = fmt.Sprintf("Chapter %v", num)
b.LatestChapterNum = &num
b.UpdatedAt = at.UnixMilli()
if _, err := s.Upsert(readerID, b); err != nil {
t.Fatalf("sight %q: upsert: %v", key, err)
}
}
// secondReader is another Reader on the same database. The owner seed is the
// only reader-creation path in this package, so a second Open as a different
// owner is how a test gets one (as TestRunOnceFetchesSharedSeriesOnce does).
func secondReader(t *testing.T, dbURL string) *store.Store {
t.Helper()
other, err := store.Open(dbURL,
store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))},
t.TempDir(), testCoverBaseURL)
if err != nil {
t.Fatalf("Open second reader: %v", err)
}
t.Cleanup(func() { other.Close() })
return other
}
func readLatestNum(t *testing.T, s *store.Store, readerID int64, key string) float64 {
t.Helper()
b, ok, err := s.Get(readerID, key)
if err != nil || !ok {
t.Fatalf("Get %q: %v ok=%v", key, err, ok)
}
if b.LatestChapterNum == nil {
t.Fatalf("%q has no latest chapter", key)
}
return *b.LatestChapterNum
}
// A Series only one Reader bookmarks is the case where being wrong can hurt
// nobody but the Reader who reported it, so their Sighting stands in for the
// Poll and the round leaves the Series alone.
func TestSightingOnSolitarySeriesDefersPoll(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).runOnce(context.Background())
if got := f.callCount(); got != 0 {
t.Fatalf("fetched %d times after a Sighting on a solitary Series, want 0", got)
}
}
// On a shared Series the Sighting still writes the Latest Chapter for everyone,
// but the Poll happens on schedule anyway — which is what corrects a wrong
// value within the hour instead of letting it persist.
func TestSightingOnSharedSeriesDoesNotDeferPoll(t *testing.T) {
s, dbURL := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
other := secondReader(t, dbURL)
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
}); err != nil {
t.Fatalf("seed second reader: %v", err)
}
sight(t, s, s.OwnerID(), sightingKey, 200, now.Add(-10*time.Minute))
// The Sighting updated the shared row immediately, before any Poll.
if got := readLatestNum(t, s, s.OwnerID(), sightingKey); got != 200 {
t.Fatalf("latest after the Sighting = %v, want 200", got)
}
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times after a Sighting on a shared Series, want 1", got)
}
if got := readLatestNum(t, s, s.OwnerID(), sightingKey); got != sightingFixtureLatest {
t.Fatalf("latest after the Poll = %v, want the Site's own %v", got, sightingFixtureLatest)
}
}
// Reporting a chapter is not reading one: a Sighting may move the Latest
// Chapter and nothing else. Both the solitary and the shared case, because the
// deferral branch must not be where this guarantee lives.
func TestSightingLeavesProgressAndOrderingUntouched(t *testing.T) {
for _, shared := range []bool{false, true} {
name := "solitary"
if shared {
name = "shared"
}
t.Run(name, func(t *testing.T) {
s, dbURL := newTestStore(t)
read := 5.0
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, SeriesURL: sightingURL,
LastChapter: "Chapter 5", LastChapterNum: read,
LastChapterURL: sightingURL + "/chapter/5", UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
if shared {
other := secondReader(t, dbURL)
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
}); err != nil {
t.Fatalf("seed second reader: %v", err)
}
}
sight(t, s, s.OwnerID(), sightingKey, 200, time.UnixMilli(9_000_000))
b, ok, err := s.Get(s.OwnerID(), sightingKey)
if err != nil || !ok {
t.Fatalf("Get: %v ok=%v", err, ok)
}
if b.LatestChapterNum == nil || *b.LatestChapterNum != 200 {
t.Fatalf("LatestChapterNum = %v, want 200", b.LatestChapterNum)
}
if b.LastChapterNum != read {
t.Fatalf("LastChapterNum = %v, want %v: a Sighting is not Progress", b.LastChapterNum, read)
}
if b.UpdatedAt != 1000 {
t.Fatalf("updated_at moved to %d: a Sighting must not reorder the list", b.UpdatedAt)
}
})
}
}
// The ceiling is what makes trusting a client report safe: however recently a
// Series was sighted, one that has not been Polled in six hours is Polled.
func TestSightingCeilingForcesPoll(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-7*time.Hour).UnixMilli())
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-time.Minute))
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times past the %s ceiling, want 1", got, sightingCeilingRests*defaultRest)
}
}
// Deferral is decided from live facts every round, so a Series that gains a
// second Bookmark stops deferring at once — and one that loses it defers again.
func TestDeferralFollowsTheBookmarkCount(t *testing.T) {
s, dbURL := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
p := newTestPoller(t, s, f, now)
p.runOnce(context.Background())
if got := f.callCount(); got != 0 {
t.Fatalf("solitary Series fetched %d times, want 0", got)
}
other := secondReader(t, dbURL)
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
Key: sightingKey, Site: "asura", SeriesID: sightingSlug, UpdatedAt: 2000,
}); err != nil {
t.Fatalf("seed second reader: %v", err)
}
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("shared Series fetched %d times, want 1", got)
}
// The Poll above consumed the rest, so move past it before asking again.
if err := other.Delete(other.OwnerID(), sightingKey); err != nil {
t.Fatalf("delete second bookmark: %v", err)
}
later := now.Add(2 * time.Hour)
p.Now = func() time.Time { return later }
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, later.Add(-time.Minute))
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("Series fetched %d times after returning to one Bookmark, want 1", got)
}
}
// A Series nobody reports any more returns to the normal schedule on its own:
// the Sighting's standing lasts one rest, not forever.
func TestDeferralExpiresWithoutFurtherSightings(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedForCheck(t, s, sightingKey, sightingURL, now.Add(-2*time.Hour).UnixMilli())
sight(t, s, s.OwnerID(), sightingKey, sightingFixtureLatest, now.Add(-10*time.Minute))
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
p := newTestPoller(t, s, f, now)
p.runOnce(context.Background())
if got := f.callCount(); got != 0 {
t.Fatalf("fetched %d times while the Sighting stood, want 0", got)
}
p.Now = func() time.Time { return now.Add(90 * time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times once the Sighting aged out, want 1", got)
}
}
// demonicFixture publishes one chapter in demonicscans' live page shape, so a
// test can make a Site publish an arbitrary number rather than the one the
// captured fixture froze.
func demonicFixture(num float64) string {
return fmt.Sprintf(
`<a href="/chaptered.php?manga=11799&chapter=%v" class="chplinks" title="Catastrophic Necromancer %v">Chapter %v</a>`,
num, num, num)
}
const (
demonicKey = "demonic:Catastrophic-Necromancer"
demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
)
// contradictOnce reports a chapter that does not exist and then runs the round
// that catches it, returning when that round ran so a caller can chain the
// next one. The wait is one rest and a minute: a Sighting stands in for exactly
// one rest, so that is the first moment this solitary Series is Polled again.
func contradictOnce(t *testing.T, s *store.Store, p *Poller, sightAt time.Time, real float64) time.Time {
t.Helper()
sight(t, s, s.OwnerID(), demonicKey, real+500, sightAt)
at := sightAt.Add(defaultRest + time.Minute)
p.Now = func() time.Time { return at }
p.runOnce(context.Background())
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != real {
t.Fatalf("latest after the Poll = %v, want the Site's own %v", got, real)
}
return at
}
func seedDemonic(t *testing.T, s *store.Store, checkedAt int64) {
t.Helper()
seedForCheck(t, s, demonicKey, demonicURL, checkedAt)
}
// A Poll finding a lower number than stored means the Sighting that raised it
// was false. The Reader is named — not the Series flagged — and both numbers are
// logged, because that is what tells a broken adapter from a deliberate lie.
func TestPollContradictingASightingNamesTheReaderAndBothNumbers(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, now)
contradictOnce(t, s, p, now, 296)
got := logs.String()
for _, want := range []string{
fmt.Sprintf("reader %d", s.OwnerID()), "796", "296", demonicKey,
} {
if !strings.Contains(got, want) {
t.Fatalf("contradiction log = %q, want it to name %q", got, want)
}
}
}
// Three contradictions cost the Reader the right to defer. Nothing here writes
// a counter: the marks are earned through Polls, which is the only way
// production produces them.
func TestThreeContradictionsStopDeferral(t *testing.T) {
s, _ := newTestStore(t)
start := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, start)
at := start
for range store.SightingDisagreementLimit {
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
}
fetchesSoFar := f.callCount()
// The marked Reader sights the same solitary Series again. It still writes
// the Latest Chapter — the penalty removes a privilege, it does not silence
// anyone — but the Poll is no longer postponed: the round below runs while a
// trusted Reader's Sighting would still be standing, and fetches anyway.
sight(t, s, s.OwnerID(), demonicKey, 900, at.Add(31*time.Minute))
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != 900 {
t.Fatalf("latest after a marked Reader's Sighting = %v, want 900", got)
}
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != fetchesSoFar+1 {
t.Fatalf("marked Reader's Sighting still deferred the Poll (fetches %d, want %d)",
got, fetchesSoFar+1)
}
}
// The owner's remedy for a mark a broken Site adapter produced restores the
// privilege without a wait and without SQL.
func TestClearingMarksRestoresDeferral(t *testing.T) {
s, _ := newTestStore(t)
start := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, start)
at := start
for range store.SightingDisagreementLimit {
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
}
if err := s.ClearReaderMarks(s.OwnerID()); err != nil {
t.Fatalf("ClearReaderMarks: %v", err)
}
fetchesSoFar := f.callCount()
sight(t, s, s.OwnerID(), demonicKey, 900, at.Add(31*time.Minute))
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != fetchesSoFar {
t.Fatalf("fetched %d times after the marks were cleared, want %d: deferral must resume",
got, fetchesSoFar)
}
}
// Recovery is automatic but expensive: twenty Polls that each confirm a
// Sighting of this Reader's clear the marks. Each round needs a new chapter,
// because only a report that raises the stored number is attributed and so only
// that one can be confirmed.
func TestTwentyAgreementsClearTheMarks(t *testing.T) {
s, _ := newTestStore(t)
start := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, start.Add(-2*time.Hour).UnixMilli())
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, start)
at := start
for range store.SightingDisagreementLimit {
at = contradictOnce(t, s, p, at.Add(time.Minute), 296)
}
chapter := 296.0
for range store.SightingAgreementsToClear {
chapter++
sight(t, s, s.OwnerID(), demonicKey, chapter, at.Add(time.Minute))
f.body = demonicFixture(chapter) // the Site publishes what was reported
at = at.Add(defaultRest + time.Minute)
p.Now = func() time.Time { return at }
p.runOnce(context.Background())
}
fetchesSoFar := f.callCount()
chapter++
sight(t, s, s.OwnerID(), demonicKey, chapter, at.Add(31*time.Minute))
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
p.runOnce(context.Background())
if got := f.callCount(); got != fetchesSoFar {
t.Fatalf("fetched %d times after %d confirmations, want %d: the marks must be forgiven",
got, store.SightingAgreementsToClear, fetchesSoFar)
}
}
// A Poll finding a higher number is the Site publishing since the Sighting and
// means nothing about the Reader — no mark, and no credit either.
func TestPollFindingHigherNumberIsNotAContradiction(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
// Reported truthfully, then the Site published one more.
sight(t, s, s.OwnerID(), demonicKey, 295, now.Add(-10*time.Minute))
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, now)
// One rest on, the Sighting has lapsed and the Poll happens.
p.Now = func() time.Time { return now.Add(7 * time.Hour) }
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times past the ceiling, want 1", got)
}
// Unmarked, so a fresh Sighting still defers.
at := now.Add(9 * time.Hour)
sight(t, s, s.OwnerID(), demonicKey, 296, at.Add(-time.Minute))
p.Now = func() time.Time { return at }
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("a Reader whose report the Site overtook lost the right to defer (fetches %d, want 1)", got)
}
}
// A Poll that overtakes a Sighting takes ownership of the row: the value stored
// afterwards is the Poll's own, so a later retraction is not the Reader's fault
// and must not be charged to them.
func TestAttributionDoesNotSurviveAPollThatOvertookIt(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
sight(t, s, s.OwnerID(), demonicKey, 295, now.Add(-10*time.Minute))
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, now)
at := now.Add(defaultRest + time.Minute)
p.Now = func() time.Time { return at }
p.runOnce(context.Background())
if got := readLatestNum(t, s, s.OwnerID(), demonicKey); got != 296 {
t.Fatalf("latest after the Poll = %v, want the Site's own 296", got)
}
var logs strings.Builder
prev := log.Writer()
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(prev) })
f.body = demonicFixture(290) // the Site retracts what only the Poll wrote
p.Now = func() time.Time { return at.Add(defaultRest + time.Minute) }
p.runOnce(context.Background())
if strings.Contains(logs.String(), "sighting contradicted") {
t.Fatalf("a retraction of the Poll's own value was charged to a Reader: %s", logs.String())
}
}
// A PUT with no Latest Chapter in it — a favourite toggle, progress written
// from a chapter page — is nobody looking at the Series page, so it buys no
// deferral. Otherwise a client could suppress a Series' Polls while reporting
// nothing, and with nothing reported there would be nothing to judge.
func TestPutWithoutALatestChapterDoesNotDefer(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(20 * time.Hour.Milliseconds())
seedDemonic(t, s, now.Add(-2*time.Hour).UnixMilli())
// The handler's own call, with the field the client omitted.
if err := s.RecordSighting(s.OwnerID(), "demonic", "Catastrophic-Necromancer",
nil, now.Add(-time.Minute).UnixMilli()); err != nil {
t.Fatalf("RecordSighting: %v", err)
}
f := &fakeFetcher{body: demonicFixture(296), status: 200}
p := newTestPoller(t, s, f, now)
p.runOnce(context.Background())
if got := f.callCount(); got != 1 {
t.Fatalf("fetched %d times after a PUT carrying no chapter, want 1", got)
}
}
+3 -73
View File
@@ -9,7 +9,6 @@ import (
"sort"
"strconv"
"strings"
"time"
"github.com/chromedp/chromedp"
)
@@ -33,11 +32,6 @@ type site struct {
LatestChapter func(seriesURL, body string) (latestChapter, bool)
// Cover finds the Cover address in a fetched body.
Cover func(seriesURL, body string) (string, bool)
// Rest is how long a Series of this Site rests between Polls.
Rest time.Duration
// Gap is the Lane's strictest pace: at least one second must pass between
// two consecutive Series-page Polls of this Site (issue #100).
Gap time.Duration
// Browser reads this Site's payload from a cleared browser tab; nil
// means the page is fetched over plain TLS.
Browser *browserRead
@@ -386,58 +380,6 @@ func publishedCoverURL(value string) string {
return strings.ReplaceAll(value, " ", "%20")
}
// Poll Lane constants (issue #100). The per-Site structure is deliberately
// uniform at first — every Site rests an hour and gaps ten seconds — but it
// exists so a single Site can be slowed if it turns hostile, and the numbers
// stay in the registry so the structure has a place to differ.
const (
// defaultRest is how long every Series rests between Polls.
defaultRest = time.Hour
// defaultGap is the strictest pace of every Lane unless the eligible
// Series count forces it tighter.
defaultGap = 10 * time.Second
// minGap floors the effective gap. One request per second is already an
// order of magnitude past the strictest rate rule a free-plan Site can
// express (docs/research/cloudflare-bot-scoring-and-poll-cadence.md);
// below it the Lane is outrunning its own plan and says so loudly.
minGap = time.Second
// refuseBackoff is how long a Lane waits after its Site refused twice in
// one run before attempting it again.
refuseBackoff = 15 * time.Minute
// browserWakeCount and browserWakeAge gate a browser Lane's run: five or
// more due Series, or any one of them waiting this long, or Chrome stays
// asleep (ADR-0005 on-demand browser).
browserWakeCount = 5
browserWakeAge = 15 * time.Minute
// sightingCeilingRests caps Sighting deferral (issue #103): however many
// Sightings arrive, a Series unpolled for this many of its Site's rests is
// Polled. It is what makes a client report safe to trust — a wrong Latest
// Chapter dies within the ceiling deterministically, rather than in
// expectation the way a randomised audit would have it. Six, so a Series a
// Reader visits constantly still gets one authoritative check per working
// day-part.
sightingCeilingRests = 6
)
// effectiveGap is a Site's pace: the registry gap, or one rest divided by the
// eligible Series count when that is smaller, never below one second. The
// denominator follows defaultRest rather than a literal hour so a Site whose
// rest is ever changed keeps its per-Series pace in step. The second return is
// true when the one-second floor engaged (and the Lane logs a warning naming
// the Site, every round it does).
func effectiveGap(s site, eligible int) (time.Duration, bool) {
gap := s.Gap
if eligible > 0 {
if perSeries := defaultRest / time.Duration(eligible); perSeries < gap {
gap = perSeries
}
}
if gap < minGap {
return minGap, true
}
return gap, false
}
// sites is the registry: one entry per Site, keyed by the stored site string.
// Adding a Site means adding an entry here and nowhere else — the dispatch
// functions above and the poller's route list are lookups into this map. An
@@ -448,22 +390,16 @@ var sites = map[string]site{
Host: "asurascans.com",
LatestChapter: asuraLatestChapter,
Cover: ogImageCover,
Rest: defaultRest,
Gap: defaultGap,
},
"demonic": {
Host: "demonicscans.org",
LatestChapter: demonicLatestChapter,
Cover: ogImageCover,
Rest: defaultRest,
Gap: defaultGap,
},
"comix": {
Host: "comix.to",
LatestChapter: comixLatestChapter,
Cover: comixCoverEntry,
Rest: defaultRest,
Gap: defaultGap,
Browser: &browserRead{
Read: comixRead,
// The interstitial is served in place of the page, so "arrived"
@@ -478,8 +414,6 @@ var sites = map[string]site{
Host: "kagane.to",
LatestChapter: kaganeLatestChapter,
Cover: kaganeCoverEntry,
Rest: defaultRest,
Gap: defaultGap,
Browser: &browserRead{
Read: kaganeRead,
Done: func(body string) bool { return body != "" },
@@ -492,8 +426,6 @@ var sites = map[string]site{
Host: "novelfull.com",
LatestChapter: novelfullLatestChapter,
Cover: novelfullCoverEntry,
Rest: defaultRest,
Gap: defaultGap,
Browser: &browserRead{
Read: novelfullRead,
// The interstitial has a DOM too, so "the payload arrived" has to
@@ -506,15 +438,13 @@ var sites = map[string]site{
Host: "lightnovelworld.net",
LatestChapter: lnwLatestChapter,
Cover: ogImageCover,
Rest: defaultRest,
Gap: defaultGap,
},
}
// browserBackedSites is derived from the registry: the Sites whose pages are
// read through the browser sidecar. Sorted so callers that range it (the
// browser fetcher's dispatch) see a stable order instead of map-iteration
// noise.
// read through the browser sidecar, which are also the ones granted the longer
// cooldown. Sorted so callers that range it (the due query, the browser
// fetcher's dispatch) see a stable order instead of map-iteration noise.
func browserBackedSites() []string {
out := make([]string, 0, len(sites))
for name, s := range sites {
-79
View File
@@ -1,79 +0,0 @@
package latest
import "time"
// LaneState is the administrative page's view of one Poll Lane (issue #102):
// what the Lane's last pass saw. Due, Gap and Checked are filled in as the
// pass computes them; a pass that returned before reaching a figure (refusal
// backoff, sidecar down) carries the previous pass's figures forward rather
// than overwriting them with zeroes the page would state as fact.
type LaneState struct {
Site string
Due int
LastRun time.Time
Gap time.Duration
// Checked is how many Series this pass actually read. A Lane with Series
// due and nothing checked has stopped working; one with nothing due is
// merely quiet, and the page must not draw the two the same (story 13).
Checked int
Clamped bool
Refusing bool
Browser bool
// Asleep marks a browser Lane whose last pass declined to wake Chrome
// because it was under both wake thresholds (ADR-0005). Due without
// Checked then means "waiting for the group to gather", not "stopped", and
// the page must not draw it as a stall.
Asleep bool
}
// Status is the owner's page snapshot of the whole poller (issue #102).
type Status struct {
Lanes []LaneState
BrowserConfigured bool
BrowserReachable bool
}
// LaneStatus returns a copy of the poller's Lane state for the owner's page.
// Only Sites that have completed a pass appear — a restart therefore renders
// "no data yet" instead of confident zeroes — in the same order Run iterates.
// Refusing is derived at snapshot time from the refusal backoff, not stored,
// so a Lane that cooled down between passes reports false without a new pass.
// BrowserReachable mirrors the Lanes' own gate: the sidecar is down only
// within the refuseBackoff window since its last loss.
func (p *Poller) LaneStatus() Status {
p.mu.Lock()
defer p.mu.Unlock()
lanes := make([]LaneState, 0, len(p.laneStates))
now := p.Now()
for _, name := range laneNames() {
st, ok := p.laneStates[name]
if !ok {
continue
}
st.Refusing = now.Before(p.refuseUntil[name])
lanes = append(lanes, st)
}
configured := p.BrowserFetch != nil
reachable := configured
if reachable && !p.browserDownAt.IsZero() && now.Sub(p.browserDownAt) < refuseBackoff {
reachable = false
}
return Status{Lanes: lanes, BrowserConfigured: configured, BrowserReachable: reachable}
}
// recordLaneState stores one Lane's last pass for LaneStatus. Called deferred
// from runLanePass so every return path records, even a pass that refused.
// A pass that never reached the pace (Gap zero) keeps the last pass's figures:
// the Lane's due count and gap did not become zero because this pass declined
// to look, and the row's own marks say why it declined.
func (p *Poller) recordLaneState(st LaneState) {
p.mu.Lock()
defer p.mu.Unlock()
if p.laneStates == nil {
p.laneStates = make(map[string]LaneState)
}
if prev, ok := p.laneStates[st.Site]; ok && st.Gap == 0 {
st.Due, st.Gap, st.Clamped, st.Checked = prev.Due, prev.Gap, prev.Clamped, prev.Checked
}
p.laneStates[st.Site] = st
}
@@ -1,6 +0,0 @@
-- The owner's administrative page (issue #102) renders these counters and
-- offers a control to clear them, deliberately shipped before the Sighting
-- feature (issue #103) that fills them, so a false mark never needs SQL
-- against production. Zero counters mean a trusted Reader.
ALTER TABLE readers ADD COLUMN sighting_agreements integer NOT NULL DEFAULT 0;
ALTER TABLE readers ADD COLUMN sighting_disagreements integer NOT NULL DEFAULT 0;
@@ -1,10 +0,0 @@
-- Sighting deferral (issue #103). latest_sighted_at is when a Reader's report
-- last stood in for a Poll; it is separate from latest_checked_at because the
-- six-hour ceiling has to know when the Series was last really fetched, and a
-- Sighting writing the Poll's own column would erase that.
-- latest_raised_by is attribution: whoever last raised this Series' Latest
-- Chapter by Sighting, so a Poll that contradicts the value downwards names a
-- Reader rather than flagging a row. Cleared by the Poll that judges it, NULL
-- whenever the stored value is the Poll's own.
ALTER TABLE series ADD COLUMN latest_sighted_at bigint NOT NULL DEFAULT 0;
ALTER TABLE series ADD COLUMN latest_raised_by bigint REFERENCES readers(id) ON DELETE SET NULL;
+26 -211
View File
@@ -16,6 +16,7 @@ import (
"strconv"
"strings"
"github.com/jackc/pgx/v5/pgtype"
_ "github.com/jackc/pgx/v5/stdlib"
)
@@ -38,7 +39,7 @@ type Bookmark struct {
// origin once the bytes exist, and "" until they do — never a third-party
// address and never an address that 404s (ADR-0007). A client may still
// send this field and it is discarded on the way in; see Upsert.
Cover string `json:"cover"`
Cover string `json:"cover"`
LastChapter string `json:"last_chapter"`
LastChapterNum float64 `json:"last_chapter_num"`
LastChapterURL string `json:"last_chapter_url"`
@@ -79,11 +80,6 @@ type Series struct {
LatestChapter string
LatestChapterNum *float64 // nil until first captured
LatestCheckedAt int64 // unix ms; see MarkLatestChecked
// LatestRaisedBy is the Reader whose Sighting last raised LatestChapter,
// and nil when the stored value is a Poll's own finding. It is what lets a
// Poll that contradicts the value downwards name a Reader instead of
// merely flagging the row (issue #103); the Poll that judges it clears it.
LatestRaisedBy *int64
// readerCount is the number of bookmarks referencing this series, filled
// only by the due-queue query that orders on it.
@@ -200,7 +196,7 @@ const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_add
// due query. latest_checked_at lives only on series — see MarkLatestChecked
// for why it stays off every client-visible write.
const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.cover_address,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by`
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at`
// Owner is the person running the service: the first Reader, seeded at startup
// so a fresh deployment has a library before anyone logs in. The seed makes
@@ -320,42 +316,25 @@ func (s *Store) EnsureReader(discordID string, epochZeroHash [32]byte) (int64, e
return id, nil
}
// SightingDisagreementLimit is the number of contradictions that stop that
// Reader's Sightings from deferring a Poll (issue #103). The counters exist
// before the mechanism that moves them, so the admin page (issue #102) can
// clear a false mark without waiting for the Sighting feature.
const SightingDisagreementLimit = 3
// Blocked reports whether this Reader's Sighting marks have reached the
// disagreement limit, which stops their Sightings from deferring a Poll.
func (r ReaderSummary) Blocked() bool {
return r.Disagreements >= SightingDisagreementLimit
}
// ReaderSummary is one Reader as the owner's administration panel sees them:
// who they are, how many live sessions they hold, and their Sighting marks.
// No credential material, hashed or otherwise, is exposed.
// who they are and how many live sessions they hold. No credential material,
// hashed or otherwise, is exposed.
type ReaderSummary struct {
ID int64
DiscordID string
// Sessions counts unexpired session rows — what the owner revokes.
Sessions int
// Agreements and Disagreements are the Sighting counters (issue #102);
// zero means a trusted Reader.
Agreements int
Disagreements int
}
// Readers lists every Reader with their live session count and Sighting
// marks, oldest first, so the owner row (always the oldest) heads the list.
// Readers lists every Reader with their live session count, oldest first, so
// the owner row (always the oldest) heads the list.
func (s *Store) Readers() ([]ReaderSummary, error) {
rows, err := s.db.Query(`
SELECT r.id, r.discord_id,
r.sighting_agreements, r.sighting_disagreements,
count(sess.id) FILTER (WHERE sess.expires_at > now()) AS sessions
FROM readers r
LEFT JOIN sessions sess ON sess.reader_id = r.id
GROUP BY r.id, r.discord_id, r.sighting_agreements, r.sighting_disagreements
GROUP BY r.id, r.discord_id
ORDER BY r.id`)
if err != nil {
return nil, fmt.Errorf("query readers: %w", err)
@@ -365,7 +344,7 @@ func (s *Store) Readers() ([]ReaderSummary, error) {
out := []ReaderSummary{}
for rows.Next() {
var r ReaderSummary
if err := rows.Scan(&r.ID, &r.DiscordID, &r.Agreements, &r.Disagreements, &r.Sessions); err != nil {
if err := rows.Scan(&r.ID, &r.DiscordID, &r.Sessions); err != nil {
return nil, fmt.Errorf("scan reader: %w", err)
}
out = append(out, r)
@@ -373,18 +352,6 @@ func (s *Store) Readers() ([]ReaderSummary, error) {
return out, rows.Err()
}
// ClearReaderMarks zeroes a Reader's Sighting counters. It is the owner's
// remedy for a mark produced by a broken Site adapter rather than a dishonest
// Reader: it restores a privilege, it is not destruction.
func (s *Store) ClearReaderMarks(readerID int64) error {
if _, err := s.db.Exec(`
UPDATE readers SET sighting_agreements = 0, sighting_disagreements = 0
WHERE id = $1`, readerID); err != nil {
return fmt.Errorf("clear reader marks for reader %d: %w", readerID, err)
}
return nil
}
// readersMigration is the version that creates the readers table. The owner
// seed runs between two migrate passes, so that the run-once migration which
// attaches existing bookmarks (0004) finds the owner row.
@@ -589,17 +556,16 @@ func (s *Store) scanBookmark(scan func(...any) error) (Bookmark, error) {
}
// scanSeries reads one row in seriesColumns order, plus the due query's
// reader_count column. latest_chapter_num and latest_raised_by are both
// nullable, same as latest_chapter_num on the bookmark read path.
// reader_count column. latest_chapter_num is NULL until the first capture,
// same as on the bookmark read path.
func scanSeries(scan func(...any) error) (Series, error) {
var (
sr Series
latestChapterNum sql.NullFloat64
latestRaisedBy sql.NullInt64
)
if err := scan(
&sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress,
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy,
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt,
&sr.readerCount,
); err != nil {
return Series{}, err
@@ -607,9 +573,6 @@ func scanSeries(scan func(...any) error) (Series, error) {
if latestChapterNum.Valid {
sr.LatestChapterNum = &latestChapterNum.Float64
}
if latestRaisedBy.Valid {
sr.LatestRaisedBy = &latestRaisedBy.Int64
}
return sr, nil
}
@@ -934,13 +897,10 @@ func (s *Store) Delete(readerID int64, key string) error {
return nil
}
// DueForLatestCheck returns one Site's series whose server-side
// latest-chapter check has aged past cutoffMs, ordered by how many bookmarks
// reference them (descending) then least-recently-checked first. One Site per
// query, because each Poll Lane asks for its own list: the query carries one
// Site and one cut-off instead of parallel lists (issue #100). There is no
// limit — the Lane's own gap paces the fetches, and the batch size that used
// to cap this query is gone with the shared pace.
// DueForLatestCheck returns series whose server-side latest-chapter check has
// aged past the appropriate cutoff, ordered by how many bookmarks reference
// them (descending) then least-recently-checked first, at most limit of them.
// Browser-backed sites use browserCutoffMs; every other site uses cutoffMs.
//
// The reader_count ordering is the point of the split (ADR-0003): a series
// shared by several readers is fetched once per due cycle, and the popular
@@ -956,33 +916,20 @@ func (s *Store) Delete(readerID int64, key string) error {
// burns requests. Archived bookmarks still count — knowing what a shelved
// series is up to is the whole reason for archiving instead of deleting.
// A series with no bookmarks at all never appears: the join excludes it.
//
// ceilingMs is the Sighting deferral ceiling (issue #103): a Series whose last
// real Poll is older than it appears however recently it was sighted. That is
// what bounds the whole mechanism — a wrong Latest Chapter dies within the
// ceiling deterministically rather than in expectation. Deferral itself is
// decided here, from two facts the query already computes, so a Lane gains no
// query per round: a Sighting younger than cutoffMs holds the Series back, but
// only while COUNT(*) is 1. A Series a second Reader bookmarks is Polled on
// schedule, so a wrong value the whole guild can see is corrected by a check
// that was never postponed; on a solitary Series the only person a wrong value
// reaches is the Reader who reported it. Whether the reporting Reader is
// allowed to defer at all was settled when the Sighting was recorded — see
// RecordSighting.
func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Series, error) {
func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites []string, limit int) ([]Series, error) {
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
FROM series s
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
WHERE s.site = $1
AND s.series_url <> ''
AND s.latest_checked_at <= $2::bigint
WHERE s.series_url <> ''
AND s.latest_checked_at <= CASE
WHEN s.site = ANY($3::text[]) THEN $2::bigint
ELSE $1::bigint
END
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
AND (COUNT(*) > 1
OR s.latest_sighted_at <= $2::bigint
OR s.latest_checked_at <= $3::bigint)
ORDER BY reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs)
ORDER BY reader_count DESC, s.latest_checked_at ASC
LIMIT $4`, cutoffMs, browserCutoffMs, pgtype.FlatArray[string](browserSites), limit)
if err != nil {
return nil, fmt.Errorf("query due series: %w", err)
}
@@ -999,30 +946,6 @@ func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Ser
return out, rows.Err()
}
// EligibleSeriesCount returns how many of a Site's Series still have at least
// one bookmark outside the finished bucket. It is the denominator of the
// Lane's pace (issue #100): the effective gap is the smaller of the registry
// gap and one hour divided by this count, so Series that will never be Polled
// do not make the Lane faster than it needs to be, and counting every eligible
// Series rather than only those currently due keeps the pace steady — the
// single worst moment to be fastest is startup, when everything is due at
// once.
func (s *Store) EligibleSeriesCount(site string) (int, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM (
SELECT 1
FROM series s
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
WHERE s.site = $1
GROUP BY s.site, s.series_id
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
) e`, site).Scan(&n)
if err != nil {
return 0, fmt.Errorf("count eligible series %s: %w", site, err)
}
return n, nil
}
// MarkLatestChecked records that the server looked at a series at ts, whatever
// the look turned up. Marking a missing series is not an error: the row may
// have been orphaned while a fetch was in flight.
@@ -1031,7 +954,7 @@ func (s *Store) EligibleSeriesCount(site string) (int, error) {
// out of the client-visible read path on purpose. PUT /bookmarks/{key} decodes
// a whole Bookmark from the client and Upsert writes every series column it
// knows about, so a userscript PUT — which has no idea this field exists —
// would write a zero and reset the rest, making the poller re-fetch that
// would write a zero and reset the cooldown, making the poller re-fetch that
// series every tick for as long as the user kept reading it.
func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
if _, err := s.db.Exec(
@@ -1043,7 +966,7 @@ func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
}
// LatestCheckedAt reads the column MarkLatestChecked writes. It exists for
// tests outside this package (the poller's own tests assert on rest
// tests outside this package (the poller's own tests assert on cooldown
// bookkeeping) — see MarkLatestChecked for why the field stays off the
// client-visible row.
func (s *Store) LatestCheckedAt(site, seriesID string) (int64, error) {
@@ -1069,111 +992,3 @@ func (s *Store) SetLatestChapter(site, seriesID, label string, num float64) erro
}
return nil
}
// RecordSighting notes that a Reader's browser reported this Series' Latest
// Chapter, which is the half of a Sighting the client body cannot express
// (issue #103). It must be called *before* the Upsert that stores the reported
// value: the raise test compares against what is still on the row, and after
// the Upsert there is nothing left to compare with. A Series that does not
// exist yet — the first Bookmark of it — is not a Sighting at all: nothing has
// ever been Polled, so there is nothing to defer and nobody to attribute.
//
// Two independent effects, hence the two CASE arms. The deferral stamp is only
// written for a Reader below the disagreement limit, so a marked Reader's
// reports keep updating the Latest Chapter but stop postponing anything, and
// clearing their marks restores the privilege on their next Sighting. The
// attribution is written whenever the report raises the stored number,
// including for a marked Reader — their Sightings are still judged, which is
// how they earn the privilege back.
//
// num is the reported chapter number. A PUT that carries none — a favourite
// toggle, or progress written from a chapter page — is no Sighting at all:
// nobody read the Series page, so there is nothing to stand in for a Poll and
// nothing that could later be judged.
func (s *Store) RecordSighting(readerID int64, site, seriesID string, num *float64, ts int64) error {
if num == nil {
return nil
}
if _, err := s.db.Exec(`
UPDATE series SET
latest_sighted_at = CASE
WHEN (SELECT sighting_disagreements FROM readers WHERE id = $3) < $6
THEN $4::bigint ELSE latest_sighted_at END,
latest_raised_by = CASE
WHEN latest_chapter_num IS NULL OR $5::double precision > latest_chapter_num
THEN $3::bigint ELSE latest_raised_by END
WHERE site = $1 AND series_id = $2`,
site, seriesID, readerID, ts, *num, SightingDisagreementLimit); err != nil {
return fmt.Errorf("record sighting %s:%s: %w", site, seriesID, err)
}
return nil
}
// SightingAgreementsToClear is how many Polls must confirm a Reader's
// Sightings in a row before their disagreements are forgiven. An agreement is
// only recorded when a Poll later confirms a Sighting, so this is twenty Polls
// of Series that Reader bookmarks — hours to days, not twenty page views. That
// is the intended price: recovery is automatic but cannot be outwaited, and a
// disagreement resets the run to zero, so credit cannot be banked in advance.
const SightingAgreementsToClear = 20
// RecordSightingOutcome settles what a Poll decided about the Reader whose
// Sighting last raised this Series' Latest Chapter, and clears the attribution
// in the same transaction so one Sighting is judged exactly once. agreed is
// the Poll confirming the stored value; its opposite is the Poll finding a
// lower number, which means the raise was false.
//
// A Poll finding a *higher* number is neither — the Site published — and takes
// ClearSightingAttribution instead.
func (s *Store) RecordSightingOutcome(site, seriesID string, readerID int64, agreed bool) error {
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("begin sighting outcome %s:%s: %w", site, seriesID, err)
}
defer tx.Rollback()
// The run length is what "consecutive" means: a disagreement zeroes the
// agreements, and completing a run zeroes both, so the next run starts
// from nothing rather than forgiving every later disagreement instantly.
q := `UPDATE readers SET sighting_disagreements = sighting_disagreements + 1,
sighting_agreements = 0
WHERE id = $1`
args := []any{readerID}
if agreed {
q = `UPDATE readers SET
sighting_agreements = CASE WHEN sighting_agreements + 1 >= $2 THEN 0
ELSE sighting_agreements + 1 END,
sighting_disagreements = CASE WHEN sighting_agreements + 1 >= $2 THEN 0
ELSE sighting_disagreements END
WHERE id = $1`
args = append(args, SightingAgreementsToClear)
}
if _, err := tx.Exec(q, args...); err != nil {
return fmt.Errorf("record sighting outcome for reader %d: %w", readerID, err)
}
if _, err := tx.Exec(clearAttributionSQL, site, seriesID, readerID); err != nil {
return fmt.Errorf("clear sighting attribution %s:%s: %w", site, seriesID, err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit sighting outcome %s:%s: %w", site, seriesID, err)
}
return nil
}
// ClearSightingAttribution answers a Sighting without judging it: the Poll
// found a higher number, so the value about to be stored is its own and this
// Reader is no longer answerable for the row. Without it the next Poll's
// agreement would be credited to a Reader who did not earn it.
func (s *Store) ClearSightingAttribution(site, seriesID string, readerID int64) error {
if _, err := s.db.Exec(clearAttributionSQL, site, seriesID, readerID); err != nil {
return fmt.Errorf("clear sighting attribution %s:%s: %w", site, seriesID, err)
}
return nil
}
// clearAttributionSQL drops the attribution only while it still names the
// Reader being judged: a Sighting landing between the due query's snapshot and
// this write is a fresh, unjudged one and must not be erased by the previous
// one's verdict.
const clearAttributionSQL = `UPDATE series SET latest_raised_by = NULL
WHERE site = $1 AND series_id = $2 AND latest_raised_by = $3`
+12 -131
View File
@@ -277,10 +277,6 @@ func seedForCheck(t *testing.T, s *Store, key, seriesURL string, checkedAt int64
}
}
// noCeiling is a Sighting deferral ceiling no Series can reach, for the tests
// that predate the ceiling and are about rest, ordering or buckets instead.
const noCeiling = int64(-1)
func TestDueForLatestCheck(t *testing.T) {
const hour = int64(3600_000)
now := 10 * hour
@@ -302,7 +298,7 @@ func TestDueForLatestCheck(t *testing.T) {
s := newTestStore(t)
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
due, err := s.DueForLatestCheck("asura", now-hour, noCeiling)
due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -313,25 +309,22 @@ func TestDueForLatestCheck(t *testing.T) {
}
}
func TestDueForLatestCheckOldestFirstAndScopedToSite(t *testing.T) {
func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
s := newTestStore(t)
// Insert newest-checked first so a correct ORDER BY has to reverse it.
seedForCheck(t, s, "asura:c", "https://asurascans.com/comics/c", 300)
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
// A second Site's due series must not appear in asura's list: each Lane
// asks for one Site, and no Lane may see another's queue.
seedForCheck(t, s, "demonic:z", "https://demonicscans.org/manga/z", 0)
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 3 {
t.Fatalf("got %d rows, want 3 (all of asura's, none of demonic's)", len(due))
if len(due) != 2 {
t.Fatalf("got %d rows, want 2 (limit)", len(due))
}
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" || due[2].Key() != "asura:c" {
t.Fatalf("got %q,%q,%q; want asura:a,asura:b,asura:c (oldest first)", due[0].Key(), due[1].Key(), due[2].Key())
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" {
t.Fatalf("got %q,%q; want asura:a,asura:b (oldest first)", due[0].Key(), due[1].Key())
}
}
@@ -503,7 +496,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
}
}
due, err := store.DueForLatestCheck("asura", time.Now().UnixMilli(), noCeiling)
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -519,38 +512,6 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
}
}
// The gap's denominator counts every Series the Lane will ever Poll: a
// finished Series must not make the Lane faster than it needs to be, and
// another Site's Series must not leak into this Site's count.
func TestEligibleSeriesCount(t *testing.T) {
store := newTestStore(t)
seedForCheck(t, store, "asura:reading", "https://asurascans.com/comics/reading", 0)
seedForCheck(t, store, "asura:archived", "https://asurascans.com/comics/archived", 0)
if _, err := store.Upsert(store.OwnerID(), Bookmark{
Key: "asura:finished", Site: "asura", SeriesID: "finished",
SeriesURL: "https://asurascans.com/comics/finished",
Status: StatusFinished, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed finished: %v", err)
}
seedForCheck(t, store, "demonic:z", "https://demonicscans.org/manga/z", 0)
n, err := store.EligibleSeriesCount("asura")
if err != nil {
t.Fatalf("EligibleSeriesCount: %v", err)
}
if n != 2 {
t.Fatalf("eligible = %d, want 2 (finished excluded, demonic excluded)", n)
}
n, err = store.EligibleSeriesCount("demonic")
if err != nil {
t.Fatalf("EligibleSeriesCount(demonic): %v", err)
}
if n != 1 {
t.Fatalf("eligible(demonic) = %d, want 1", n)
}
}
func TestDisplayChapter(t *testing.T) {
cases := []struct {
name string
@@ -1009,7 +970,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -1035,7 +996,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
t.Fatalf("seed orphan series: %v", err)
}
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -1338,86 +1299,6 @@ func TestReadersAndSessionRevocation(t *testing.T) {
}
}
// The Sighting counters ship before the mechanism that fills them (issue
// #102 before #103), so the admin page depends on their default: a fresh
// Reader reads back trusted. Marking one directly proves Readers() reports
// the counters and Blocked() flips at the limit, and that ClearReaderMarks —
// the owner's remedy for a false mark — zeroes them again.
func TestReaderSightingMarks(t *testing.T) {
s := newTestStore(t)
other := secondReader(t, s)
readers, err := s.Readers()
if err != nil {
t.Fatalf("Readers: %v", err)
}
if len(readers) != 2 {
t.Fatalf("readers = %d, want the owner and the second Reader", len(readers))
}
for _, r := range readers {
if r.Agreements != 0 || r.Disagreements != 0 || r.Blocked() {
t.Fatalf("fresh reader %d has marks: %+v", r.ID, r)
}
}
// The counters' default is the trusted state; writing them directly is
// the only way to exercise the read path until issue #103 moves them.
if _, err := s.db.Exec(`
UPDATE readers SET sighting_agreements = 5, sighting_disagreements = 2
WHERE id = $1`, other); err != nil {
t.Fatalf("mark reader: %v", err)
}
readers, err = s.Readers()
if err != nil {
t.Fatalf("Readers: %v", err)
}
var marked *ReaderSummary
for i := range readers {
if readers[i].ID == other {
marked = &readers[i]
}
}
if marked == nil || marked.Agreements != 5 || marked.Disagreements != 2 {
t.Fatalf("marked reader = %+v, want agreements 5, disagreements 2", marked)
}
if marked.Blocked() {
t.Fatalf("reader with 2 disagreements is blocked; limit is %d", SightingDisagreementLimit)
}
if _, err := s.db.Exec(`
UPDATE readers SET sighting_disagreements = 3 WHERE id = $1`, other); err != nil {
t.Fatalf("block reader: %v", err)
}
readers, err = s.Readers()
if err != nil {
t.Fatalf("Readers: %v", err)
}
for _, r := range readers {
if r.ID == other && !r.Blocked() {
t.Fatalf("reader at the disagreement limit is not blocked: %+v", r)
}
if r.ID == s.OwnerID() && r.Blocked() {
t.Fatalf("untouched owner became blocked: %+v", r)
}
}
if err := s.ClearReaderMarks(other); err != nil {
t.Fatalf("ClearReaderMarks: %v", err)
}
if err := s.ClearReaderMarks(other + 9999); err != nil {
t.Fatalf("ClearReaderMarks(unknown id): %v", err)
}
readers, err = s.Readers()
if err != nil {
t.Fatalf("Readers: %v", err)
}
for _, r := range readers {
if r.Agreements != 0 || r.Disagreements != 0 || r.Blocked() {
t.Fatalf("reader %d not cleared: %+v", r.ID, r)
}
}
}
// Two Readers on one Series: one series row, two independent progresses. The
// second Reader starts at zero however far the first has read, and the shared
// row is still due exactly once.
@@ -1453,7 +1334,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
}
due, err := s.DueForLatestCheck("asura", time.Now().UnixMilli(), noCeiling)
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -1469,7 +1350,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
}
due, err = s.DueForLatestCheck("asura", time.Now().UnixMilli(), noCeiling)
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
if err != nil {
t.Fatalf("DueForLatestCheck after delete: %v", err)
}
-256
View File
@@ -1,256 +0,0 @@
package web
import (
"log"
"net/http"
"strconv"
"time"
"bookmarkmanager/backend/internal/latest"
"bookmarkmanager/backend/internal/store"
)
// LaneReporter is the administrative page's whole window onto the running
// poller: one snapshot of Poll Lane state, copied out of memory on request.
// The Poller satisfies it in production and a fake with fixed values satisfies
// it in tests, so the page's tests need neither a poller nor a Site.
type LaneReporter interface {
LaneStatus() latest.Status
}
// adminView is what the administrative page and the roster fragment receive.
type adminView struct {
Readers []store.ReaderSummary
// OwnerID travels with the roster so it can tell the owner's own row from
// the Readers they may act on.
OwnerID int64
Lanes lanesView
}
// lanesView is the Lane status block: one row per Site that has run, plus the
// browser fact, which is shared by the three browser Sites rather than held
// once per Site.
type lanesView struct {
Rows []laneRow
// PollerOff means no poller is running at all (disabled by config, or its
// client could not be built). The browser line must not answer "not
// configured" then: the sidecar is not the reason nothing is polled.
PollerOff bool
BrowserConfigured bool
BrowserReachable bool
}
// laneRow is one Lane formatted for reading rather than for arithmetic: the
// template renders strings and flags, and every judgement about what they mean
// is made here.
type laneRow struct {
Site string
Due int
Ran string
// Checked is how many Series the last pass read. Due without Checked is a
// Lane that has stopped working; the two figures side by side are what
// separate that from a Lane with nothing to do.
Checked int
// Gap is empty when no pass has reached the pace yet, so the row omits the
// figure instead of stating a zero.
Gap string
Clamped bool
Refusing bool
// BrowserLost marks a Lane whose pages can only be read through the
// sidecar while the sidecar is unreachable — including the case where none
// is configured, which stops those Series just as completely.
BrowserLost bool
// Stalled marks a Lane with Series waiting that its last pass did not read
// — the difference between a stopped Lane and a quiet one (story 13). A
// browser Lane holding Chrome asleep under the wake thresholds is neither,
// so it carries Asleep instead and never Stalled.
Stalled bool
Asleep bool
// Attention is the one flag the template colours on, so an unhealthy Lane
// is found at a glance rather than read for.
Attention bool
}
// adminRoute pairs a route pattern with its handler so the route list and the
// gate cannot drift apart.
type adminRoute struct {
pattern string
handler http.HandlerFunc
}
// adminRoutes is every route that reaches past the acting Reader. Register
// wraps each one in requireOwner, so a new administrative route is gated by
// being listed here rather than by remembering to write a check inside it.
func (h *Handler) adminRoutes() []adminRoute {
return []adminRoute{
{"GET /admin", h.admin},
{"GET /ui/admin/lanes", h.uiLanes},
{"POST /readers/{id}/revoke", h.revokeReaderSessions},
{"POST /readers/{id}/clear-marks", h.clearReaderMarks},
}
}
// AdminPatterns names every administrative route, so one test can prove the
// owner gate covers all of them rather than one test per route. The receiver is
// nil because only the patterns are read; the bound handlers are never called.
func AdminPatterns() []string {
routes := (*Handler)(nil).adminRoutes()
out := make([]string, 0, len(routes))
for _, rt := range routes {
out = append(out, rt.pattern)
}
return out
}
// requireOwner is the owner test, in one place, layered on the session gate: no
// session is still 401, and a signed-in Reader who is not the owner gets 404
// rather than 403 — a refusal that confirms the address exists is a refusal
// that helps whoever is probing for it.
func (h *Handler) requireOwner(next http.HandlerFunc) http.HandlerFunc {
return h.requireSession(func(w http.ResponseWriter, r *http.Request) {
if readerOf(r) != h.store.OwnerID() {
http.NotFound(w, r)
return
}
next(w, r)
})
}
// admin renders the owner's page: the Reader roster and Poll Lane status.
func (h *Handler) admin(w http.ResponseWriter, r *http.Request) {
readers, err := h.store.Readers()
if err != nil {
log.Printf("admin: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.render(w, http.StatusOK, "admin", adminView{
Readers: readers,
OwnerID: h.store.OwnerID(),
Lanes: h.lanesView(),
})
}
// uiLanes answers the status block's own refresh. Only the block refreshes on a
// timer; the roster re-renders after an action, as it always has.
func (h *Handler) uiLanes(w http.ResponseWriter, r *http.Request) {
h.render(w, http.StatusOK, "lanes", h.lanesView())
}
// lanesView copies the poller's snapshot into display form. A nil reporter (no
// poller running) and a poller no Lane has reported to yet are the same thing
// to the page: no data, which it must say rather than draw as confident zeroes
// — an empty page a few seconds after a restart must not read as a stopped one.
func (h *Handler) lanesView() lanesView {
if h.lanes == nil {
return lanesView{PollerOff: true}
}
snap := h.lanes.LaneStatus()
v := lanesView{
Rows: make([]laneRow, 0, len(snap.Lanes)),
BrowserConfigured: snap.BrowserConfigured,
BrowserReachable: snap.BrowserReachable,
}
now := time.Now()
for _, l := range snap.Lanes {
lost := l.Browser && !snap.BrowserReachable
// Series waiting and none read is the shape of a Lane that has stopped
// working, as distinct from one that is quiet for want of work — or one
// deliberately leaving Chrome asleep until its group gathers.
stalled := l.Due > 0 && l.Checked == 0 && !l.Asleep
gap := ""
if l.Gap > 0 {
gap = l.Gap.Truncate(time.Second).String()
}
v.Rows = append(v.Rows, laneRow{
Site: l.Site,
Due: l.Due,
Ran: since(now, l.LastRun),
Checked: l.Checked,
Gap: gap,
Clamped: l.Clamped,
Refusing: l.Refusing,
BrowserLost: lost,
Stalled: stalled,
Asleep: l.Asleep,
Attention: l.Clamped || l.Refusing || lost || stalled,
})
}
return v
}
// since formats how long ago a Lane last ran, at second resolution: the block
// refreshes every thirty seconds, so anything finer is noise the owner would
// have to ignore.
func since(now, then time.Time) string {
d := now.Sub(then).Truncate(time.Second)
if d < time.Second {
return "just now"
}
return d.String() + " ago"
}
// revokeReaderSessions logs one Reader out of every browser they are signed in
// on. The owner gate is the route's, not this handler's.
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
target, ok := readerPathID(w, r)
if !ok {
return
}
// The owner is not one of the Readers this endpoint reaches: revoking
// themselves would sign out the browser making the request, which is what
// logout is for. The roster hides the button; this refuses the hand-rolled
// POST behind it.
if target == h.store.OwnerID() {
http.NotFound(w, r)
return
}
if err := h.store.DeleteReaderSessions(target); err != nil {
log.Printf("revoke sessions: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.renderRoster(w, "revoke sessions")
}
// clearReaderMarks zeroes one Reader's Sighting counters. The guard those
// counters feed has one known false positive — a Site changing its page shape
// makes a correct adapter read a wrong high number and marks every honest
// Reader of that Site at once (issue #103) — and this is its remedy. It
// restores a privilege rather than destroying anything, so the control is
// confirmed but never wears the destruction accent.
func (h *Handler) clearReaderMarks(w http.ResponseWriter, r *http.Request) {
target, ok := readerPathID(w, r)
if !ok {
return
}
if err := h.store.ClearReaderMarks(target); err != nil {
log.Printf("clear marks: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.renderRoster(w, "clear marks")
}
// readerPathID reads the Reader a route names, answering the request itself
// when there is nobody to act on.
func readerPathID(w http.ResponseWriter, r *http.Request) (int64, bool) {
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil {
http.Error(w, "bad reader id", http.StatusBadRequest)
return 0, false
}
return id, true
}
// renderRoster answers an action with the whole roster, so the counts and marks
// it shows cannot describe the state before the tap.
func (h *Handler) renderRoster(w http.ResponseWriter, what string) {
readers, err := h.store.Readers()
if err != nil {
log.Printf("%s: %v", what, err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.render(w, http.StatusOK, "readers", adminView{Readers: readers, OwnerID: h.store.OwnerID()})
}
+4 -53
View File
@@ -87,11 +87,6 @@
--moss: #7fae86; /* finished */
--clay: #b5906f; /* set chapter */
--trash: #977671; /* remove, resting — icons need 3:1, not 4.5:1 */
/* A Lane needing attention: the admin page's only accent. Verdigris — cool,
the far side of the wheel from ember's crimson, and clear of the archive
blue. Neither ember (new chapter) nor danger (destruction) may say
"system unhealthy". */
--patina: #5fb3a6;
/* Desktop cell borders for the two coloured action states. */
--play-hot-line: #3a1d18;
@@ -151,7 +146,6 @@
--moss: #3d6c46;
--clay: #7c5533;
--trash: #8c6558;
--patina: #1f6f66;
--play-hot-line: #f0cfc6;
--fav-line: #e3d3a4;
@@ -296,21 +290,9 @@ button { cursor: pointer; }
letter-spacing: .04em;
}
/* ---- admin page: two sections on the same measured sheet, no cards ----
The reading page is a list of series; this is a list of facts. Both are
sheets of hairline-separated rows, so the roster keeps the shape it had as
a fold-out and the Lane block copies it. */
.readers, .lanes { margin: 0 20px; padding: 12px 0 16px; border-bottom: 1px solid var(--rule); }
.readers h2, .lanes h2 {
margin: 0;
padding: 8px 0;
font: 500 10px/1 var(--font-mono);
letter-spacing: .2em;
text-transform: uppercase;
color: var(--mute-2);
}
.readerlist, .lanelist { margin: 0; padding: 0; list-style: none; }
.readerlist li, .lanelist li {
/* ---- reader roster (owner only): same hairline panel, one row per Reader ---- */
.readerlist { margin: 0; padding: 0; list-style: none; }
.readerlist li {
display: flex;
align-items: center;
flex-wrap: wrap;
@@ -318,8 +300,7 @@ button { cursor: pointer; }
min-height: 44px;
border-top: 1px solid var(--rule);
}
.reader-actions { display: flex; gap: 18px; margin-left: auto; }
.readerlist form { margin: 0; }
.readerlist form { margin: 0 0 0 auto; }
.reader-id {
font: 500 13px/1.4 var(--font-mono);
letter-spacing: .04em;
@@ -331,30 +312,6 @@ button { cursor: pointer; }
text-transform: uppercase;
color: var(--mute);
}
.reader-sightings, .lane-fact {
font: 500 10px/1 var(--font-mono);
letter-spacing: .14em;
text-transform: uppercase;
color: var(--mute-2);
}
/* Two states the owner is meant to find rather than read for: a Reader whose
reports no longer defer a Poll, and a Lane that is not keeping its promise.
Both wear --patina — never ember, which means one thing, and never danger,
which is destruction. */
.reader-blocked, .lane-mark {
font: 500 10px/1 var(--font-mono);
letter-spacing: .14em;
text-transform: uppercase;
color: var(--patina);
}
.lane-site {
font: 400 19px/1.2 var(--font-display);
color: var(--paper-dim);
}
/* The whole row leans patina when the Lane needs attention, so the scan is one
pass down the left edge rather than a read of every mark. */
.lanelist li.attention .lane-site { color: var(--patina); }
.lane-browser { padding: 12px 0 0; }
/* Revocation cuts someone off, so it wears --danger. Ember stays reserved for
the new-chapter signal. */
.ghost.danger { color: var(--danger); }
@@ -643,9 +600,6 @@ button { cursor: pointer; }
box-shadow: inset 0 -2px 0 var(--ember);
}
.topbar form { margin-left: 18px; }
/* The admin page's topbar has no switch to fill the middle, so its back link
keeps company with Log out at the right edge instead of floating centre. */
.topbar .back { margin-left: auto; }
/* At phone width brand + switch + Log out do not fit on one line, so the
switch takes its own row under the wordmark rather than pushing Log out
off-screen. */
@@ -655,9 +609,6 @@ button { cursor: pointer; }
.libswitch { order: 3; margin-left: 0; }
.libswitch a { flex: 1; text-align: center; padding: 8px 14px; }
.topbar form { margin-left: 12px; }
/* The admin page has no switch to take the second row, so its brand claims
the first outright and the back link keeps Log out company below. */
.topbar:has(.back) .brand { flex: 1 1 100%; }
}
/* ---- action strip: full-width on a phone, hairline-divided cells ---- */
-39
View File
@@ -1,39 +0,0 @@
{{/* The owner's administrative page: everything that reaches past one Reader,
at its own address so it can be bookmarked rather than hunted for inside
the reading page. Owner-only at route registration (requireOwner), which
is why nothing in here re-tests who is asking. */}}
{{define "admin"}}
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
<meta name="color-scheme" content="dark light">
<title>BookmarkManager — Admin</title>
<link rel="icon" href="/static/logo.svg" type="image/svg+xml">
<link rel="stylesheet" href="/static/style.css">
<link rel="preload" href="/static/fonts/instrument-serif-400-latin.woff2" as="font" type="font/woff2" crossorigin>
<script src="/static/htmx.min.js" defer></script>
</head>
<body>
<div class="sheet">
<header class="topbar">
<h1 class="brand">{{template "mark" .}}<span>Bookmark<em>Manager</em></span></h1>
{{/* Back to the library, no switch: this page belongs to neither library,
and the ember-lit switch says which library you are reading. */}}
<a class="ghost back" href="/">Library</a>
<form method="post" action="/logout">
<button type="submit" class="ghost">Log out</button>
</form>
</header>
{{/* The live region wraps the swapped block rather than being it: the
refresh replaces the section wholesale, and a region recreated on every
update is never announced. */}}
<div aria-live="polite">{{template "lanes" .Lanes}}</div>
{{template "readers" .}}
</div>
</body>
</html>
{{end}}
+2 -4
View File
@@ -28,10 +28,6 @@
<a href="/?lib=novel&amp;tab=all" class="{{if eq .Lib "novel"}}active{{end}}"
{{if eq .Lib "novel"}}aria-current="page"{{end}}>Novels</a>
</nav>
{{/* The owner's only difference on this page: a link out to the
administrative one. It sits beside Log out rather than in the library
switch — that switch says which library, not which page. */}}
{{if .Owner}}<a class="ghost" href="/admin">Admin</a>{{end}}
<form method="post" action="/logout">
<button type="submit" class="ghost">Log out</button>
</form>
@@ -80,6 +76,8 @@
{{template "setup" .}}
{{if .Owner}}{{template "readers" .}}{{end}}
{{template "keyrow" .}}
{{template "recent" .}}
-42
View File
@@ -1,42 +0,0 @@
{{/* Poll Lane status: one row per Site, refreshing itself so a run can be
watched rather than sampled by reloading. The refresh is one attribute on
the fragment root and the endpoint answers with this same fragment, so the
swap replaces the element that asked for it.
Every figure here is read out of the running poller, never out of a table:
a Site absent from Rows has not completed a pass since the last restart,
which the empty state must say — zeroes would read as a stopped Lane. */}}
{{define "lanes"}}
<section class="lanes" id="lanes"
hx-get="/ui/admin/lanes" hx-trigger="every 30s" hx-swap="outerHTML">
<h2>Poll Lanes</h2>
{{if .Rows}}
<ul class="lanelist">
{{range .Rows}}
<li{{if .Attention}} class="attention"{{end}}>
<span class="lane-site">{{.Site}}</span>
<span class="lane-fact">{{.Due}} due</span>
<span class="lane-fact">{{.Checked}} checked</span>
<span class="lane-fact">ran {{.Ran}}</span>
{{if .Gap}}<span class="lane-fact">gap {{.Gap}}</span>{{end}}
{{if .Clamped}}<span class="lane-mark">gap at floor</span>{{end}}
{{if .Refusing}}<span class="lane-mark">refusing</span>{{end}}
{{if .BrowserLost}}<span class="lane-mark">no browser</span>{{end}}
{{if .Stalled}}<span class="lane-mark">not checking</span>{{end}}
{{if .Asleep}}<span class="lane-mark">browser asleep</span>{{end}}
</li>
{{end}}
</ul>
{{else}}
<p class="setup-copy">No data yet — no Lane has completed a pass since the
backend started.</p>
{{end}}
<p class="setup-copy lane-browser">
{{if .PollerOff}}Polling is switched off in this deployment: no Lane runs,
and Latest Chapter comes from the userscripts alone.
{{else}}Browser sidecar:
{{if not .BrowserConfigured}}not configured — comix, kagane and novelfull
pages are not fetched through it{{else if .BrowserReachable}}reachable
{{else}}unreachable{{end}}.{{end}}</p>
</section>
{{end}}
+17 -36
View File
@@ -1,48 +1,29 @@
{{/* The Reader roster, on the owner's administrative page. Re-rendered whole
as the response to an action so the counts and marks it shows cannot
describe the state before the tap. Both actions are confirm-gated: one
signs a Reader out of every device at once, the other wipes a record.
Owner-only at route registration, so nothing here re-tests who is asking. */}}
{{/* The owner's Reader roster. Rendered only for the owner (listView.Owner),
and re-rendered whole as the response to a revocation so the session
counts cannot describe the state before the tap. Revocation is
confirm-gated: it signs someone out of every device at once. */}}
{{define "readers"}}
<section class="readers" id="readers">
<h2>Readers</h2>
<details class="setup" id="readers">
<summary>Readers</summary>
<p class="setup-copy">Everyone who has signed in through Discord. Revoking
signs a Reader out of every device; their library and bookmarks are
untouched, and they can sign in again. The Sighting counters record how
often a later Poll confirmed or contradicted what that Reader's browser
reported; enough contradictions stop their reports deferring a Poll, and
clearing the marks gives that back.</p>
untouched, and they can sign in again.</p>
<ul class="readerlist">
{{range .Readers}}
<li>
<span class="reader-id">{{.DiscordID}}</span>
<span class="reader-sessions">{{.Sessions}} session{{if ne .Sessions 1}}s{{end}}</span>
<span class="reader-sightings">{{.Agreements}} confirmed / {{.Disagreements}} contradicted</span>
{{/* Blocked is spelled out rather than left to be worked out from two
numbers and a threshold. */}}
{{if .Blocked}}<span class="reader-blocked">deferral blocked</span>{{end}}
<span class="reader-actions">
{{/* Clearing restores a privilege, so it is a plain ghost button —
the destruction accent belongs to revocation alone. It is offered
on every row, including one reading zero: the remedy must be
findable before the counters climb, not after. */}}
<form hx-post="/readers/{{.ID}}/clear-marks" hx-target="#readers" hx-swap="outerHTML"
hx-confirm="Clearing wipes this Reader's whole Sighting record, confirmations included. Clear?">
<button type="submit" class="ghost">Clear marks</button>
</form>
{{/* The owner's own row never offers Revoke: it is the one row where the
button would sign the tapping browser out, and the endpoint refuses
it anyway. Logout is the deliberate way to do that. */}}
{{if and .Sessions (ne .ID $.OwnerID)}}
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
<button type="submit" class="ghost danger">Revoke sessions</button>
</form>
{{end}}
</span>
{{/* The owner's own row never offers Revoke: it is the one row where the
button would sign the tapping browser out, and the endpoint refuses
it anyway. Logout is the deliberate way to do that. */}}
{{if and .Sessions (ne .ID $.OwnerID)}}
<form hx-post="/readers/{{.ID}}/revoke" hx-target="#readers" hx-swap="outerHTML"
hx-confirm="Revoking signs this Reader out on every device immediately. Revoke?">
<button type="submit" class="ghost danger">Revoke sessions</button>
</form>
{{end}}
</li>
{{end}}
</ul>
</section>
</details>
{{end}}
+56 -18
View File
@@ -50,9 +50,6 @@ type Handler struct {
// httpClient is the plain stdlib client that talks to Discord. It is not
// an injected interface: tests point APIBase at a stub server instead.
httpClient *http.Client
// lanes is the Poll Lane snapshot source the administrative page reads.
// Nil is a running deployment with no poller, not a bug.
lanes LaneReporter
}
// listView is what every list-rendering template receives.
@@ -80,9 +77,14 @@ type listView struct {
// It is not "newly registered": a Reader who deletes their last bookmark is
// in the same position and needs the same links.
EmptyLibrary bool
// Owner marks the acting Reader as the deployment's owner, which offers
// the link to the administrative page. Nothing else in the UI differs.
// Owner marks the acting Reader as the deployment's owner, which unlocks
// the Readers panel. Nothing else in the UI differs.
Owner bool
// Readers is the owner's roster, populated only for the owner's own page
// render and the revocation fragment. OwnerID travels with it so the roster
// can tell the owner's own row apart from the Readers they may revoke.
Readers []store.ReaderSummary
OwnerID int64
}
// PageURL and ListURL are the two link shapes every tab needs. Building them
@@ -109,10 +111,7 @@ type loginView struct {
// New parses every template up front so a broken one kills the process at
// startup rather than the first request that touches it.
//
// lanes is the administrative page's window onto the running Poller; nil means
// nothing is polling, which the page reports rather than hides.
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string, lanes LaneReporter) (*Handler, error) {
func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string) (*Handler, error) {
tmpl, err := template.ParseFS(templateFS, "templates/*.html")
if err != nil {
return nil, err
@@ -127,7 +126,6 @@ func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, nove
states: newOAuthStates(),
limiter: session.NewLoginLimiter(),
httpClient: &http.Client{Timeout: discordTimeout},
lanes: lanes,
}, nil
}
@@ -151,11 +149,9 @@ func (h *Handler) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /install/manga-bookmark.user.js", h.requireSession(h.installUserscript("manga-bookmark.user.js")))
mux.HandleFunc("GET /install/novel-bookmark.user.js", h.requireSession(h.installUserscript("novel-bookmark.user.js")))
mux.HandleFunc("POST /rotate-token", h.requireSession(h.rotateToken))
// Owner-only: every route that reaches past the acting Reader is gated in
// one place, so a missing gate is visible in the route list.
for _, rt := range h.adminRoutes() {
mux.HandleFunc(rt.pattern, h.requireOwner(rt.handler))
}
// Owner-only: the one place the UI crosses the Reader boundary.
mux.HandleFunc("POST /readers/{id}/revoke", h.requireSession(h.revokeReaderSessions))
}
// staticHandler serves the embedded assets. An hour, not longer: assets are
@@ -244,9 +240,14 @@ func (h *Handler) index(w http.ResponseWriter, r *http.Request) {
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
// The owner's page differs only by the link to the administrative page:
// the roster lives there now, so the page read every day is only reading.
view.Owner = readerID == h.store.OwnerID()
if readerID == h.store.OwnerID() {
view.Owner, view.OwnerID = true, readerID
if view.Readers, err = h.store.Readers(); err != nil {
log.Printf("index readers: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
}
h.render(w, http.StatusOK, "app", view)
}
@@ -617,3 +618,40 @@ func (h *Handler) rotateToken(w http.ResponseWriter, r *http.Request) {
view := listView{Lib: store.KindManga, Rotated: true}
h.render(w, http.StatusOK, "setup", view)
}
// revokeReaderSessions logs one Reader out of every browser they are signed
// in on. Owner-only: it reaches across the Reader boundary every other handler
// respects, so the guard is a comparison against the seeded owner rather than
// a role a Reader could acquire. A non-owner gets 404 — the panel does not
// exist for them, so neither should the endpoint.
func (h *Handler) revokeReaderSessions(w http.ResponseWriter, r *http.Request) {
if readerOf(r) != h.store.OwnerID() {
http.NotFound(w, r)
return
}
target, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil {
http.Error(w, "bad reader id", http.StatusBadRequest)
return
}
// The owner is not one of the Readers this endpoint reaches: revoking
// themselves would sign out the browser making the request, which is what
// logout is for. The roster hides the button; this refuses the hand-rolled
// POST behind it.
if target == h.store.OwnerID() {
http.NotFound(w, r)
return
}
if err := h.store.DeleteReaderSessions(target); err != nil {
log.Printf("revoke sessions: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
readers, err := h.store.Readers()
if err != nil {
log.Printf("revoke sessions: %v", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
h.render(w, http.StatusOK, "readers", listView{Owner: true, Readers: readers, OwnerID: h.store.OwnerID()})
}
+95 -31
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
@@ -60,15 +61,30 @@ type Config struct {
// LatestPoll configures the background latest-chapter poller.
//
// Only the kill switch lives here. Pace is per Site — rest time and gap are
// registry properties (internal/latest/sites.go, issue #100), because each
// Lane has to be able to differ from the others. The five environment
// settings that used to size a shared pace (cooldown, browser cooldown,
// interval, stagger, batch) are gone with it: no deployed .env may carry them.
// Sizing: batch x (cooldown / interval) is how many series hold a true cooldown
// cadence — 14 x (1h / 10m) = 84 with these defaults, which covers this
// deployment. Past that nothing breaks; the effective cadence stretches to
// N x interval / batch and the oldest-checked-first ordering keeps it uniform.
type LatestPoll struct {
Enabled bool
Enabled bool
Cooldown time.Duration
BrowserCooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
const (
defaultPollCooldown = time.Hour
defaultBrowserPollCooldown = 6 * time.Hour
defaultPollInterval = 10 * time.Minute
defaultPollStagger = 20 * time.Second
defaultPollBatch = 14
// minPollCooldown keeps a typo from turning a polite background check into
// a hammer against sites that are already bot-scoring us.
minPollCooldown = 15 * time.Minute
)
func envOr(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
@@ -91,11 +107,66 @@ func envBool(key string, def bool) bool {
}
}
// loadLatestPoll reads the poller's settings. The pace knobs that used to be
// clamped here are registry properties now (issue #100), so there is nothing
// left to clamp.
// envDuration reads a duration env var. An unparseable or non-positive value
// falls back to def and logs rather than failing startup: the poller is an
// enhancement, and a typo in one of its knobs must not stop bookmark sync.
func envDuration(key string, def time.Duration) time.Duration {
raw := strings.TrimSpace(os.Getenv(key))
if raw == "" {
return def
}
d, err := time.ParseDuration(raw)
if err != nil || d <= 0 {
log.Printf("config: %s=%q is not a positive duration, using %s", key, raw, def)
return def
}
return d
}
// envInt reads a positive integer env var, with the same fallback policy.
func envInt(key string, def int) int {
raw := strings.TrimSpace(os.Getenv(key))
if raw == "" {
return def
}
n, err := strconv.Atoi(raw)
if err != nil || n <= 0 {
log.Printf("config: %s=%q is not a positive integer, using %d", key, raw, def)
return def
}
return n
}
func clampPollCooldown(name string, d time.Duration) time.Duration {
if d < minPollCooldown {
log.Printf("config: %s %s is below the %s floor, clamping", name, d, minPollCooldown)
return minPollCooldown
}
return d
}
// loadLatestPoll reads the poller's settings, clamping anything that would make
// it antisocial.
func loadLatestPoll() LatestPoll {
return LatestPoll{Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true)}
p := LatestPoll{
Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true),
Cooldown: envDuration("LATEST_CHAPTER_POLL_COOLDOWN", defaultPollCooldown),
BrowserCooldown: envDuration("LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", defaultBrowserPollCooldown),
Interval: envDuration("LATEST_CHAPTER_POLL_INTERVAL", defaultPollInterval),
Stagger: envDuration("LATEST_CHAPTER_POLL_STAGGER", defaultPollStagger),
Batch: envInt("LATEST_CHAPTER_POLL_BATCH", defaultPollBatch),
}
p.Cooldown = clampPollCooldown("cooldown", p.Cooldown)
p.BrowserCooldown = clampPollCooldown("browser cooldown", p.BrowserCooldown)
// batch x stagger has to fit inside one tick or a batch is still running
// when the next one is due. Run() serialises them, so this degrades to a
// slower cadence rather than to overlapping fetches — worth a warning, not
// a failure.
if span := time.Duration(p.Batch) * p.Stagger; span > p.Interval {
log.Printf("config: batch(%d) x stagger(%s) = %s exceeds interval %s; batches will overrun their tick",
p.Batch, p.Stagger, span, p.Interval)
}
return p
}
func loadConfig() Config {
@@ -129,10 +200,7 @@ func loadConfig() Config {
// newRouter wires routes and middleware. CORS is the outermost layer so
// preflight OPTIONS short-circuits before auth; /bookmarks* is auth-protected,
// /healthz is public.
//
// lanes may be nil — polling disabled, or its client could not be built. The
// admin page reports that rather than pretending Lanes exist.
func newRouter(s *store.Store, cfg Config, lanes web.LaneReporter) http.Handler {
func newRouter(s *store.Store, cfg Config) http.Handler {
mux := http.NewServeMux()
h := &api.Handler{Store: s}
mux.HandleFunc("GET /healthz", api.Healthz)
@@ -163,7 +231,7 @@ func newRouter(s *store.Store, cfg Config, lanes web.LaneReporter) http.Handler
// The browser UI is always registered; signing in is Discord OAuth, so
// there is no password to forget and no gate to leave unset.
wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey),
cfg.UserscriptPath, cfg.NovelUserscriptPath, lanes)
cfg.UserscriptPath, cfg.NovelUserscriptPath)
if err != nil {
log.Fatalf("web handler: %v", err)
}
@@ -278,17 +346,11 @@ func main() {
}
s.OnSeriesCreated = acq.Acquire
}
// A nil *Poller must not become a non-nil interface holding a nil pointer:
// the admin page tests the reporter for nil to decide whether anything is
// polling at all.
var lanes web.LaneReporter
if poller := startLatestPoller(pollCtx, s, cfg.LatestPoll, browser); poller != nil {
lanes = poller
}
startLatestPoller(pollCtx, s, cfg.LatestPoll, browser)
srv := &http.Server{
Addr: ":" + cfg.Port,
Handler: newRouter(s, cfg, lanes),
Handler: newRouter(s, cfg),
ReadHeaderTimeout: 10 * time.Second,
}
@@ -315,8 +377,7 @@ func main() {
}
}
// newLatestPoller wires the fetcher seams into the poller. Pace is registry
// property, not config (issue #100), so there are no knobs to pass through.
// newLatestPoller wires the configured cooldowns and fetchers into the poller.
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
var covers latest.BrowserCoverFetcher
if f, ok := browser.(latest.BrowserCoverFetcher); ok {
@@ -329,23 +390,27 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch
CoverFetch: covers,
CoverBytesFetch: latest.NewCoverFetcher(),
Now: time.Now,
Cooldown: cfg.Cooldown,
BrowserCooldown: cfg.BrowserCooldown,
Interval: cfg.Interval,
Stagger: cfg.Stagger,
Batch: cfg.Batch,
}
}
// startLatestPoller launches the background poller unless it is disabled or its
// HTTP client cannot be built. Any problem here is logged and skipped: this
// feature going missing degrades the service to userscript-only latest-chapter
// tracking, which is exactly how it behaved before. It returns the running
// Poller, or nil when there is none — the admin page's Lane status reads it.
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) *latest.Poller {
// tracking, which is exactly how it behaved before.
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) {
if !cfg.Enabled {
log.Println("latest-chapter poller: disabled by config")
return nil
return
}
f, err := latest.NewTLSFetcher()
if err != nil {
log.Printf("latest-chapter poller: disabled, cannot build client: %v", err)
return nil
return
}
// Nil browser: sites behind a JavaScript challenge are simply not polled,
// and their latest_chapter comes from the userscript alone — which is how
@@ -353,5 +418,4 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, brow
p := newLatestPoller(s, cfg, f, browser)
go p.Run(ctx)
return p
}
+117 -21
View File
@@ -9,15 +9,31 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"bookmarkmanager/backend/internal/latest"
"bookmarkmanager/backend/internal/store"
)
func TestLoadLatestPollDefaults(t *testing.T) {
t.Setenv("LATEST_CHAPTER_POLL_ENABLED", "")
if got := loadLatestPoll(); got != (LatestPoll{Enabled: true}) {
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, LatestPoll{Enabled: true})
for _, k := range []string{
"LATEST_CHAPTER_POLL_ENABLED", "LATEST_CHAPTER_POLL_COOLDOWN",
"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN", "LATEST_CHAPTER_POLL_INTERVAL",
"LATEST_CHAPTER_POLL_STAGGER", "LATEST_CHAPTER_POLL_BATCH",
} {
t.Setenv(k, "")
}
got := loadLatestPoll()
want := LatestPoll{
Enabled: true,
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
Interval: 10 * time.Minute,
Stagger: 20 * time.Second,
Batch: 14,
}
if got != want {
t.Fatalf("loadLatestPoll() = %+v, want %+v", got, want)
}
}
@@ -47,25 +63,105 @@ func TestLoadLatestPollEnabledParsing(t *testing.T) {
}
}
// newLatestPoller wires the fetcher seams; pace lives in the registry, so
// nothing here sizes a cooldown any more.
func TestNewLatestPollerWiresFetchers(t *testing.T) {
tls := &latest.TLSFetcher{}
p := newLatestPoller(nil, LatestPoll{Enabled: true}, tls, nil)
if p.Fetch != tls {
t.Fatalf("Fetch not wired")
func TestLoadLatestPollClampsAndFallsBack(t *testing.T) {
tests := []struct {
name string
env map[string]string
wantFrom func(LatestPoll) any
want any
}{
{
name: "cooldown below the floor is clamped up",
env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "1m"},
wantFrom: func(p LatestPoll) any { return p.Cooldown },
want: 15 * time.Minute,
},
{
name: "cooldown at the floor is kept",
env: map[string]string{"LATEST_CHAPTER_POLL_COOLDOWN": "15m"},
wantFrom: func(p LatestPoll) any { return p.Cooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown below the floor is clamped up",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "1m"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown at the floor is kept",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "15m"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 15 * time.Minute,
},
{
name: "browser cooldown override is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "8h"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 8 * time.Hour,
},
{
name: "browser cooldown unparseable value falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_BROWSER_COOLDOWN": "six hours"},
wantFrom: func(p LatestPoll) any { return p.BrowserCooldown },
want: 6 * time.Hour,
},
{
name: "a valid override is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "5m"},
wantFrom: func(p LatestPoll) any { return p.Interval },
want: 5 * time.Minute,
},
{
name: "an unparseable duration falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_INTERVAL": "ten minutes"},
wantFrom: func(p LatestPoll) any { return p.Interval },
want: 10 * time.Minute,
},
{
name: "a zero duration falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_STAGGER": "0s"},
wantFrom: func(p LatestPoll) any { return p.Stagger },
want: 20 * time.Second,
},
{
name: "a valid batch is honoured",
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "30"},
wantFrom: func(p LatestPoll) any { return p.Batch },
want: 30,
},
{
name: "a negative batch falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "-5"},
wantFrom: func(p LatestPoll) any { return p.Batch },
want: 14,
},
{
name: "a non-numeric batch falls back",
env: map[string]string{"LATEST_CHAPTER_POLL_BATCH": "lots"},
wantFrom: func(p LatestPoll) any { return p.Batch },
want: 14,
},
}
if p.BrowserFetch != nil {
t.Fatalf("BrowserFetch = %v, want nil for a browser-less deployment", p.BrowserFetch)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
for k, v := range tt.env {
t.Setenv(k, v)
}
if got := tt.wantFrom(loadLatestPoll()); got != tt.want {
t.Fatalf("got %v, want %v", got, tt.want)
}
})
}
if p.CoverFetch != nil {
t.Fatalf("CoverFetch = %v, want nil when the browser is absent", p.CoverFetch)
}
if p.CoverBytesFetch == nil {
t.Fatalf("CoverBytesFetch = nil, want the TLS cover fetcher")
}
if p.Now == nil {
t.Fatalf("Now = nil, want the live clock")
}
func TestNewLatestPollerWiresCooldowns(t *testing.T) {
p := newLatestPoller(nil, LatestPoll{
Cooldown: time.Hour,
BrowserCooldown: 6 * time.Hour,
}, nil, nil)
if p.Cooldown != time.Hour || p.BrowserCooldown != 6*time.Hour {
t.Fatalf("poller cooldowns = %s/%s, want 1h/6h", p.Cooldown, p.BrowserCooldown)
}
}
+3 -3
View File
@@ -49,7 +49,7 @@ func withBody(req *http.Request, body string) *http.Request {
// A refused credential is refused however plausible it looks: only a hash the
// readers table holds authenticates anything.
func TestUnknownCredentialRejected(t *testing.T) {
srv := newRouter(newTestStore(t), testConfig(), nil)
srv := newRouter(newTestStore(t), testConfig())
rr := httptest.NewRecorder()
srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("never-registered")))
@@ -69,7 +69,7 @@ func TestUnknownCredentialRejected(t *testing.T) {
func TestPerReaderIsolation(t *testing.T) {
s := newTestStore(t)
registerReader(t, s, "other-reader")
srv := newRouter(s, testConfig(), nil)
srv := newRouter(s, testConfig())
ownerKey := "asura:solo"
putBookmark(t, srv, ownerKey, store.Bookmark{
@@ -267,7 +267,7 @@ func TestRotateCredentialViaWebUI(t *testing.T) {
}
cfg := testConfig()
cfg.UserscriptPath = path
srv := newRouter(s, cfg, nil)
srv := newRouter(s, cfg)
oldCred := ownerCredential()
rr := httptest.NewRecorder()
+16 -277
View File
@@ -1,7 +1,6 @@
package main
import (
"database/sql"
"encoding/json"
"fmt"
"io"
@@ -16,7 +15,6 @@ import (
"testing"
"time"
"bookmarkmanager/backend/internal/latest"
"bookmarkmanager/backend/internal/session"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/web"
@@ -28,17 +26,11 @@ import (
const testOwnerID = "owner-snowflake"
// newWebTestServer returns the full router plus the store behind it, so tests
// can seed rows and assert on what the handlers wrote back. An optional lane
// reporter stands in for the running poller; omitted means none is running,
// which is what every test that is not about the admin page wants.
func newWebTestServer(t *testing.T, cfg Config, lanes ...web.LaneReporter) (http.Handler, *store.Store) {
// can seed rows and assert on what the handlers wrote back.
func newWebTestServer(t *testing.T, cfg Config) (http.Handler, *store.Store) {
t.Helper()
st := newTestStore(t)
var reporter web.LaneReporter
if len(lanes) > 0 {
reporter = lanes[0]
}
return newRouter(st, cfg, reporter), st
return newRouter(st, cfg), st
}
// sessionCookie mints a live session row for the owner and returns the cookie
@@ -149,12 +141,12 @@ func discordConfig(stubURL string) web.DiscordConfig {
// oauthWebTestServer returns the full router, its store, and a Discord stub
// wired as the configured API — the starting point for sign-in tests.
func oauthWebTestServer(t *testing.T, lanes ...web.LaneReporter) (http.Handler, *store.Store, *discordStub) {
func oauthWebTestServer(t *testing.T) (http.Handler, *store.Store, *discordStub) {
t.Helper()
stub, srv := newDiscordStub(t)
cfg := testConfig()
cfg.Discord = discordConfig(srv.URL)
router, st := newWebTestServer(t, cfg, lanes...)
router, st := newWebTestServer(t, cfg)
return router, st, stub
}
@@ -418,7 +410,7 @@ func TestDiscordLoginRefusesNonMember(t *testing.T) {
cfg.Discord = discordConfig(srv.URL)
cfg.Discord.RequiredRole = tc.require
st := newTestStore(t)
router := newRouter(st, cfg, nil)
router := newRouter(st, cfg)
rr := completeSignIn(t, router, startSignIn(t, router))
if rr.Code != http.StatusForbidden {
@@ -634,52 +626,24 @@ func TestOwnerRevokesAnotherReadersSessions(t *testing.T) {
}
}
// fakeLanes is the admin page's poller stand-in: one fixed snapshot, so the
// page's tests need neither a poller nor a Site.
type fakeLanes struct{ status latest.Status }
func (f fakeLanes) LaneStatus() latest.Status { return f.status }
// The roster moved off the reading page onto its own address: the owner gets a
// link, everyone else gets nothing, and the page itself lists every Reader with
// the counters and the two controls.
func TestAdminPageCarriesRosterAndOwnerLink(t *testing.T) {
// The owner's own page carries the roster; nobody else's does.
func TestOwnerSeesReadersPanel(t *testing.T) {
router, st, _ := oauthWebTestServer(t)
theirCookie := signInCookie(t, router)
ownerCookie := sessionCookie(t, st)
signInCookie(t, router)
req := httptest.NewRequest(http.MethodGet, "/", nil)
req.AddCookie(ownerCookie)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
body := rr.Body.String()
if strings.Contains(body, `id="readers"`) {
t.Error("the reading page still carries the roster; it belongs on /admin")
if !strings.Contains(body, `id="readers"`) {
t.Fatal("the owner's page lacks the Readers panel")
}
if !strings.Contains(body, `href="/admin"`) {
t.Error("the owner's reading page offers no link to the admin page")
if !strings.Contains(body, testOwnerID) {
t.Fatalf("the roster does not list the registered Reader:\n%s", body)
}
req = httptest.NewRequest(http.MethodGet, "/", nil)
req.AddCookie(theirCookie)
rr = httptest.NewRecorder()
router.ServeHTTP(rr, req)
if strings.Contains(rr.Body.String(), `href="/admin"`) {
t.Error("a non-owner was offered the admin link")
}
req = httptest.NewRequest(http.MethodGet, "/admin", nil)
req.AddCookie(ownerCookie)
rr = httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("GET /admin status = %d, want 200", rr.Code)
}
body = rr.Body.String()
for _, want := range []string{`id="readers"`, testOwnerID, "Revoke sessions", "Clear marks", "confirmed"} {
if !strings.Contains(body, want) {
t.Errorf("admin page lacks %q:\n%s", want, body)
}
if !strings.Contains(body, "Revoke sessions") {
t.Fatal("the roster offers no revocation control for a signed-in Reader")
}
// Exactly one revocable row: the other Reader's. The owner's own row carries
// the same session count and no button.
@@ -688,231 +652,6 @@ func TestAdminPageCarriesRosterAndOwnerLink(t *testing.T) {
}
}
// Every administrative route is gated the same way, so the test walks the list
// the router registers rather than naming routes by hand: no session is 401,
// a signed-in non-owner is 404, and the address is not confirmed to either.
func TestAdminRoutesAreOwnerOnly(t *testing.T) {
router, st, _ := oauthWebTestServer(t)
theirCookie := signInCookie(t, router)
ownerCookie := sessionCookie(t, st)
target := strconv.FormatInt(st.OwnerID(), 10)
patterns := web.AdminPatterns()
if len(patterns) == 0 {
t.Fatal("no administrative routes to test")
}
for _, pattern := range patterns {
method, path, ok := strings.Cut(pattern, " ")
if !ok {
t.Fatalf("route pattern %q has no method", pattern)
}
path = strings.Replace(path, "{id}", target, 1)
for _, tc := range []struct {
name string
cookie *http.Cookie
want int
}{
{"no session", nil, http.StatusUnauthorized},
{"non-owner", theirCookie, http.StatusNotFound},
} {
req := httptest.NewRequest(method, path, nil)
if tc.cookie != nil {
req.AddCookie(tc.cookie)
}
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != tc.want {
t.Errorf("%s %s as %s: status = %d, want %d", method, path, tc.name, rr.Code, tc.want)
}
}
req := httptest.NewRequest(method, path, nil)
req.AddCookie(ownerCookie)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code == http.StatusUnauthorized {
t.Errorf("%s %s as the owner: status = 401, the gate rejects the owner", method, path)
}
}
}
// The Lane block reports what the poller says, and marks the Lanes that need
// attention — a clamped gap, a refusal, a Site whose pages can only be read
// through a sidecar that is not there, and a Lane with Series waiting that its
// last pass did not read.
func TestAdminPageShowsLaneStatus(t *testing.T) {
lanes := fakeLanes{latest.Status{
Lanes: []latest.LaneState{
{Site: "asura", Due: 12, Checked: 12, LastRun: time.Now().Add(-90 * time.Second), Gap: 40 * time.Second},
{Site: "kagane", Due: 3, Checked: 3, LastRun: time.Now().Add(-time.Minute), Gap: time.Minute, Browser: true},
{Site: "demonic", Due: 400, Checked: 400, LastRun: time.Now(), Gap: 8 * time.Second, Clamped: true},
{Site: "comix", Due: 7, LastRun: time.Now(), Gap: time.Minute, Browser: true},
},
BrowserConfigured: true,
BrowserReachable: true,
}}
router, st, _ := oauthWebTestServer(t, lanes)
req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("GET /ui/admin/lanes status = %d, want 200", rr.Code)
}
body := rr.Body.String()
for _, want := range []string{"asura", "kagane", "12 due", "12 checked", "gap 40s", "ran 1m30s ago", "gap at floor", "not checking", "reachable"} {
if !strings.Contains(body, want) {
t.Errorf("lane status lacks %q:\n%s", want, body)
}
}
// Nothing is refusing and the sidecar is up, so neither mark may appear:
// a mark the owner cannot act on is worse than none.
for _, unwanted := range []string{"refusing", "no browser"} {
if strings.Contains(body, unwanted) {
t.Errorf("lane status marks %q on a healthy run:\n%s", unwanted, body)
}
}
}
// A browser Lane under both wake thresholds holds Chrome asleep (ADR-0005), so
// Series due with none checked is the design working, not a stopped Lane. The
// two must not render the same mark: "not checking" is the owner's cue to go
// looking, and spending it on the commonest healthy browser-Lane state trains
// them to ignore it.
func TestAsleepBrowserLaneIsNotMarkedStalled(t *testing.T) {
lanes := fakeLanes{latest.Status{
Lanes: []latest.LaneState{
{Site: "kagane", Due: 1, LastRun: time.Now(), Gap: 10 * time.Second, Browser: true, Asleep: true},
},
BrowserConfigured: true,
BrowserReachable: true,
}}
router, st, _ := oauthWebTestServer(t, lanes)
req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
body := rr.Body.String()
if strings.Contains(body, "not checking") {
t.Errorf("an asleep browser Lane is marked as stalled:\n%s", body)
}
if !strings.Contains(body, "browser asleep") {
t.Errorf("an asleep browser Lane says nothing about why it read nothing:\n%s", body)
}
if strings.Contains(body, `class="attention"`) {
t.Errorf("an asleep browser Lane is coloured as unhealthy:\n%s", body)
}
}
// A Lane whose pass never reached a figure must not have that figure drawn as
// a zero: a refusing Lane still reports the due count and gap its last real
// pass saw, and a Lane that has never reached one omits it entirely.
func TestLaneStatusOmitsUnknownGap(t *testing.T) {
lanes := fakeLanes{latest.Status{
Lanes: []latest.LaneState{{Site: "comix", LastRun: time.Now(), Refusing: true, Browser: true}},
BrowserConfigured: true,
BrowserReachable: true,
}}
router, st, _ := oauthWebTestServer(t, lanes)
req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
body := rr.Body.String()
if strings.Contains(body, "gap 0s") {
t.Errorf("a Lane with no pace yet states a zero gap:\n%s", body)
}
if !strings.Contains(body, "refusing") {
t.Errorf("a refusing Lane is not marked as such:\n%s", body)
}
}
// No poller and a poller that has not finished a pass both render "no data
// yet" rather than zeroes that read as a stopped backend — but they are not
// the same fact, so the page must not blame the sidecar when nothing polls.
func TestAdminPageWithoutAPollerSaysSo(t *testing.T) {
for _, tc := range []struct {
name string
lanes []web.LaneReporter
want, unwant string
}{
{"no poller", nil, "Polling is switched off", "not configured"},
{"poller, no pass yet", []web.LaneReporter{fakeLanes{}}, "not configured", "Polling is switched off"},
} {
t.Run(tc.name, func(t *testing.T) {
router, st, _ := oauthWebTestServer(t, tc.lanes...)
req := httptest.NewRequest(http.MethodGet, "/admin", nil)
req.AddCookie(sessionCookie(t, st))
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
body := rr.Body.String()
if !strings.Contains(body, "No data yet") {
t.Errorf("admin page with no Lane data does not say so:\n%s", body)
}
if !strings.Contains(body, tc.want) {
t.Errorf("admin page lacks %q:\n%s", tc.want, body)
}
if strings.Contains(body, tc.unwant) {
t.Errorf("admin page states %q, which is not what is wrong:\n%s", tc.unwant, body)
}
})
}
}
// A Reader past the disagreement threshold is rendered as blocked, and
// clearing their marks both zeroes the counters and lifts the block in the
// roster the response carries back.
func TestOwnerClearsReaderMarks(t *testing.T) {
st, dsn := newTestStoreURL(t)
router := newRouter(st, testConfig(), nil)
cookie := sessionCookie(t, st)
// The counters are filled by issue #103; until it lands the only way to
// stand a marked Reader up is to write the columns directly.
db, err := sql.Open("pgx", dsn)
if err != nil {
t.Fatalf("open %s: %v", dsn, err)
}
defer db.Close()
if _, err := db.Exec(`UPDATE readers SET sighting_agreements = 4, sighting_disagreements = 3 WHERE id = $1`, st.OwnerID()); err != nil {
t.Fatalf("mark reader: %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/admin", nil)
req.AddCookie(cookie)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
body := rr.Body.String()
if !strings.Contains(body, "4 confirmed / 3 contradicted") {
t.Errorf("roster does not report the Reader's marks:\n%s", body)
}
if !strings.Contains(body, "deferral blocked") {
t.Errorf("a Reader at the threshold is not rendered as blocked:\n%s", body)
}
req = httptest.NewRequest(http.MethodPost,
"/readers/"+strconv.FormatInt(st.OwnerID(), 10)+"/clear-marks", nil)
req.AddCookie(cookie)
rr = httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("clear marks: status = %d, want 200 (body %s)", rr.Code, rr.Body.String())
}
body = rr.Body.String()
if !strings.Contains(body, `id="readers"`) {
t.Fatalf("clear marks did not re-render the roster:\n%s", body)
}
if !strings.Contains(body, "0 confirmed / 0 contradicted") {
t.Errorf("roster does not report the cleared counters:\n%s", body)
}
if strings.Contains(body, "deferral blocked") {
t.Errorf("a cleared Reader is still marked blocked:\n%s", body)
}
}
func TestDiscordLoginTokenEndpointDown(t *testing.T) {
stub, srv := newDiscordStub(t)
stub.tokenStatus = http.StatusInternalServerError
+7 -4
View File
@@ -58,11 +58,14 @@ services:
# Second script from the same bindmount; the novel library is a separate
# Violentmonkey install.
NOVEL_USERSCRIPT_PATH: ${NOVEL_USERSCRIPT_PATH:-/userscript/novel-bookmark.user.js}
# Latest-chapter poller. LATEST_CHAPTER_POLL_ENABLED=0 in .env is the
# kill switch; it only takes effect because it is listed here. Pace is
# per Site in the registry (one Poll Lane per Site, issue #100) — the
# cooldown/interval/stagger/batch knobs are gone with the shared pace.
# Latest-chapter poller. LATEST_CHAPTER_POLL_ENABLED=0 in .env is the kill
# switch; it only takes effect because these are listed here.
LATEST_CHAPTER_POLL_ENABLED: ${LATEST_CHAPTER_POLL_ENABLED:-1}
LATEST_CHAPTER_POLL_COOLDOWN: ${LATEST_CHAPTER_POLL_COOLDOWN:-1h}
LATEST_CHAPTER_POLL_BROWSER_COOLDOWN: ${LATEST_CHAPTER_POLL_BROWSER_COOLDOWN:-6h}
LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m}
LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14}
LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s}
# CDP endpoint for sites behind a JavaScript challenge (kagane,
# novelfull). The browser is not part of this stack — it runs on the home
# machine as its own unit (chrome/docker-compose.yml) and is reached over
-91
View File
@@ -1,91 +0,0 @@
# ADR-0010: Poll Lanes — one independent Poll stream per Site
Date: 2026-08-16
Status: accepted
## Decision
Replace the single shared polling pace with one **Poll Lane** per Site: an
independent goroutine that polls only that Site's Series, paced by that Site's
registry entry. Pace moves out of config and into the Site registry
(`internal/latest/sites.go`): every entry carries a `Rest` (how long a Series
rests between Polls) and a `Gap` (how long the Lane waits between fetches).
Rest is enforced by the due query's WHERE clause (`latest_checked_at <= now -
Rest`), never by a timer — the same mechanism that enforced the old cooldown.
The Lane enforces its own gap by sleeping between fetches. `effectiveGap` is
the registry gap, or one hour divided by the Site's eligible Series count when
that is smaller, never below one second.
The five environment settings that used to size the shared pace —
`LATEST_CHAPTER_POLL_COOLDOWN`, `_BROWSER_COOLDOWN`, `_INTERVAL`, `_BATCH`,
`_STAGGER` — are deleted. Only the kill switch `LATEST_CHAPTER_POLL_ENABLED`
remains. No deployed `.env` may carry the deleted knobs.
## Why
The shared pace capped the whole backend at roughly 180 Polls an hour (one
20-second stagger across one queue). ~60 Series today, scaling to hundreds or
thousands, would stretch the hour beyond what the New Chapter signal can
tolerate. Worse, the queue mixed Sites with very different costs: kagane and
comix pay seconds of a serialized single-tab Chrome per Poll (a challenged
page, ADR-0005), and one hostile Site burning its challenge timeout made every
other Site's Series wait — "one hostile Site can eat most of an hour".
Lanes fix both at once:
- **Throughput scales per Site.** The six Lanes fetch concurrently; a Lane's
own gap paces it. The browser Lanes' combined ceiling stays about 360 Polls
an hour (one tab), and when they cannot keep up the wait past Rest grows and
is logged every pass — the "behind by X" measurement, so the decision to
give browser Sites more pages is made from data.
- **Hostility is contained.** A refusal (two challenge-held reads in one
pass) stops only that Site's Lane for `refuseBackoff` (15m); the rest of
that Lane's Series stay unstamped and due. A lost browser gates the other
browser Lanes' passes for the same window — the flag is shared Poller
state, so the loss is noticed once instead of once per Lane per pass, and
decays after 15m so the Lanes probe again. One Site can no longer tax the
others.
## Tradeoffs and rejections
- **Per-Site env knobs** (e.g. `KAGANE_POLL_GAP`) rejected: the registry is
the single place pace lives, testable and reviewable; config knobs would
recreate the shared-pace sprawl with six times the surface. All six entries
are deliberately uniform at first — rest an hour, gap ten seconds — so the
structure exists to differ without inventing numbers for Sites that have
not earned them.
- **Dynamic gap** (`rest / eligible`) is the one knob that stays automatic:
a Site with more Series than one per ten seconds would otherwise back up
behind its own gap, and the per-Series share of the hour is the natural
pace. The ten-second default is not arbitrary: one request per ten seconds
is the strictest rate rule a free-plan Site can even express (per-zone
rate limiting, as documented in
`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`), so the
default pace is exactly what the most restrictive Site would demand of us.
The computed gap never goes below one second and logs loudly when the
floor engages.
- **Timer-based pacing** rejected: the old ticker made the poller's rate a
function of wall clock rather than of what was due. The due-query cutoff is
the only rate authority; the Lane sleep just prevents hammering.
- **Batch size** (the old `_BATCH` cap) is gone with the shared pace: a Lane
processes everything due, paced by its gap. There is no global queue left
to bound.
## Constraints preserved
- Stamp-before-fetch ("attempted" semantics): an untried Series stays due, so
a browser that appears after a restart finds its full queue waiting.
- Browser wake gate (ADR-0005): a browser Lane leaves Chrome asleep below
five due Series and 15 minutes of wait, per Lane.
- The browser is not in the API stack (ADR-0006): an unreachable browser
degrades a Lane exactly as an unset `BROWSER_WS_URL` — browser-only Sites
skipped, plain-TLS unaffected, stored covers still served.
- Cover heals moved to background goroutines (joined by the test suite via
`waitCovers`) so a slow cover CDN cannot consume a Lane's gap.
Supersedes the pace mechanics of ADR-0003's "raise throughput instead" note
(the stagger cut it rejected is what the per-Lane gap replaces) and the
6-hour browser cooldown introduced with the browser-backed Sites; the
1-hour browser rest was already cleared as safe by
`docs/research/cloudflare-bot-scoring-and-poll-cadence.md`.
@@ -1,139 +0,0 @@
# ADR-0011: Sightings — a Reader report defers a Poll where being wrong hurts only them
Date: 2026-08-16
Status: accepted
## Decision
A **Sighting** is the Latest Chapter the Reader's own browser read off the
Series page and PUT to the backend. It is now allowed to stand in for a Poll,
under one restriction and one ceiling:
- **Solitary Series only.** A Sighting defers the Poll of a Series exactly one
Bookmark points at. A Series two Readers share is Polled on schedule no matter
how recently it was sighted.
- **One rest of standing.** A Sighting postpones Polls for one Rest
(`defaultRest`, an hour), not forever: a Series nobody visits again returns to
the normal schedule by itself.
- **Six-rest ceiling.** `sightingCeilingRests = 6`, counted in the Site's own
Rest — six hours everywhere today. However many Sightings arrive, a Series
unpolled that long is Polled.
Both live in the due query's HAVING clause (`store.DueForLatestCheck`), beside
the Rest cutoff — the same place the schedule has always been decided, so no
timer and no second code path can disagree with it.
Attribution and judgement:
- `Store.RecordSighting` runs *before* the Upsert that stores the reported
value, because the raise test needs the row as it stands. A report that raises
the stored Latest Chapter names its Reader in `series.latest_raised_by`.
- The Poll is the oracle. `Poller.checkOne` already compares what the Site
publishes against what is stored, so judgement costs no extra request: a lower
number contradicts the Sighting (`sighting_disagreements + 1`, both numbers and
the Reader logged), the same number confirms it (`sighting_agreements + 1`), a
**higher** number is the Site publishing and means nothing either way — but it
does clear the attribution (`Store.ClearSightingAttribution`), because the
value stored afterwards is the Poll's own and nobody must answer for it.
- At `SightingDisagreementLimit` (3) that Reader's Sightings stop deferring
anything. They still write the Latest Chapter — the penalty removes a
privilege, it does not silence anyone.
- `SightingAgreementsToClear` (20) consecutive confirmations forgive the
disagreements. A disagreement resets the run to zero.
- The owner clears marks from the administration page (issue #102, shipped
first precisely so a false mark has a remedy the day the mechanism lands).
One client change was required, and only one. Both userscripts stopped short of
PUTting a read whose number had not moved (`applyLatestChapterIfChanged`), so
the case this whole mechanism exists for — visiting a Series with nothing new —
never reached the backend. `reportLatestChapter` now sends it, skipping only the
local write and the re-render. A numberless PUT (favourite toggle, progress from
a chapter page) is not a Sighting and defers nothing: nobody read the Series
page, so there would be nothing to judge later.
## Why
Most of the backend's work was redundant. The userscript reads the Latest
Chapter on every Series page visit; minutes later the Poll Lane fetches the same
page for the same number. Deferring on a report converts a visit into a Poll
saved, which is Lane capacity handed back to Series nobody is
reading.
The restriction is the whole safety argument, and it is about **blast radius**,
not about trust arithmetic:
- On a solitary Series, a wrong report can only mislead the Reader who made it.
There is nobody else's ember to falsify.
- On a shared Series it could mislead someone else, so a report never postpones
anything there.
The ceiling bounds the damage in time: a false value dies within six hours
whatever happens, because the Poll that finds it is guaranteed. That is also
what makes lying pointless — the six-hour audit is certain, not sampled, so a
determined attacker buys at most three ceilings' worth of a wrong number on
their own Series and then loses deferral entirely.
The cost of recovery is deliberate. An agreement is only recorded when a later
Poll confirms a Sighting, so twenty agreements are twenty Polls of Series that
Reader bookmarks — hours to days of real time, not twenty page views. Waiting is
therefore not a strategy, and credit cannot be banked in advance.
## Tradeoffs and rejections
- **Trusting a Sighting on a shared Series** rejected: it is the only case where
one Reader's mistake reaches another Reader's list, and no amount of
reputation makes that recoverable within the six-hour window.
- **Cross-Reader agreement, voting, weighting, consensus scoring** rejected on
evidence: every truth-discovery method estimates source reliability by
comparing sources on the same object, and the standard survey states outright
that an object provided by very few sources cannot have its confidence
evaluated — Li, Gao, Meng, Li, Su, Zhao, Fan, Han, *A Survey on Truth
Discovery*, SIGMOD Record 45(1), 2016 (arXiv:1505.02463), §"Challenges" on
sparse sources. With the two Readers this backend actually has, a
disagreement is a coin flip. The Poll is an authoritative oracle, so it is
the only judge.
- **A randomised audit** (Poll a fraction of deferred Series) rejected in favour
of the fixed ceiling. Sampling an oracle against untrusted reports is the
gold-question technique from crowdsourcing quality control — Le, Edmonds,
Hester, Biewald, *Ensuring quality in crowdsourced search relevance
evaluation: the effects of training question distribution*, SIGIR 2010
Workshop on Crowdsourcing for Search Evaluation, which inserts known answers
sporadically and adjusts each worker's trust from them. The ceiling is the
same idea made deterministic: sampling prices an attack in expectation, a
guaranteed six-hour audit prices it as a certainty, which is what makes the
solitary-Series rule defensible in one sentence.
- **A trust *ratio*** (agreements over judgements, as that same gold-question
scheme uses) rejected for two thresholds: a ratio lets an attacker bank
credit first and spend it on lies later, and it needs the owner watching a
score to act. Three-and-twenty is a threshold both ways — a disagreement
resets the run to zero, so credit cannot be pre-bought, and recovery happens
without the owner in the loop.
- **Blocking a marked Reader's writes** rejected: the Latest Chapter they report
is still the best available value, and their Sightings must keep being judged
or they could never earn the privilege back.
- **Per-Series flagging** rejected in favour of per-Reader marks: a Series is
not the thing that can be wrong. Naming the Reader and logging both numbers is
also what distinguishes a broken Site adapter (every Reader of that Site
contradicted at once) from one bad actor.
- **Timers or a background reputation job** rejected: deferral is recomputed
from live facts every round — Bookmark count and sighting timestamp — so a
Series that gains a second Bookmark stops deferring at once, with nothing to
invalidate. The Reader's marks are the one input read earlier, when the
Sighting is recorded rather than when the round runs: a Reader who crosses
the threshold, or has their marks cleared, changes behaviour from their next
Sighting on, and the standing they already bought lasts out its rest. That is
bounded by one rest and costs one subselect instead of joining `readers` into
the due query on every round.
## Constraints preserved
- A Sighting is not Progress: it may move the Latest Chapter and nothing else.
`updated_at` never moves, so a report cannot reorder the list (ADR-0004).
- The Latest Chapter is a Series-level fact (ADR-0003): a Sighting writes the
shared row, so every Reader of a shared Series sees it immediately — deferral
is the only thing the solitary rule withholds.
- Ember means new chapter only (`docs/design-system.md`): a marked Reader
renders no differently in their own list, and nothing about the trust model
reaches the Series list's colour.
- The Poll remains authoritative. Where a Sighting and a Poll disagree, the
Poll's value is what gets stored.
+4 -17
View File
@@ -10,7 +10,7 @@ Implemented in:
| Surface | Files |
| --- | --- |
| Web UI (login, list, card, empty, errors, admin) | `backend/internal/web/static/style.css`, `backend/internal/web/templates/{app,admin,lanes,readers,card,list,login,chrome,icons}.html`, `backend/internal/web/static/filter.js` |
| Web UI (login, list, card, empty, errors) | `backend/internal/web/static/style.css`, `backend/internal/web/templates/{app,card,list,login,chrome,icons}.html`, `backend/internal/web/static/filter.js` |
| Userscript panel (Shadow DOM) | `userscript/manga-bookmark.user.js` — `TEMPLATE` and `CSS` at the bottom of the IIFE |
## 1. The one idea
@@ -73,7 +73,6 @@ Defined once in `backend/internal/web/static/style.css` `:root`, mirrored in the
| `--moss` | `#7fae86` | `#3d6c46` | finished accent |
| `--clay` | `#b5906f` | `#7c5533` | set-chapter accent |
| `--trash` | `#977671` | `#8c6558` | remove, at rest — icons need 3:1, not 4.5:1 |
| `--patina` | `#5fb3a6` | `#1f6f66` | admin page only — a Poll Lane needing attention, a Reader whose reports are blocked |
| `--play-hot-line` | `#3a1d18` | `#f0cfc6` | desktop cell border, play when `.is-new` |
| `--fav-line` | `#332b14` | `#e3d3a4` | desktop cell border, favourite when on |
| `--asura` | `#7d93a5` | `#4f6b80` | site tag |
@@ -82,13 +81,9 @@ Defined once in `backend/internal/web/static/style.css` `:root`, mirrored in the
| `--kagane` | `#9a8aa5` | `#6f5f7d` | site tag |
| `--hatch` / `--hatch-dim` | 135° 5px stripe | paper stripe | missing-cover slot |
`--slate`/`--moss`/`--clay`/`--brass`/`--patina` are held at the same weight
deliberately: one accent per meaning, so a press says which lane it belongs to,
with none of them competing with ember. `--patina` is the admin page's only
colour — a cool verdigris, the far side of the wheel from ember's crimson and
clear of the archive blue: system health is neither a new chapter nor
destruction, so it borrows neither `--ember` nor `--danger`.
Dark is the default (`color-scheme: dark light`);
`--slate`/`--moss`/`--clay`/`--brass` are held at the same weight deliberately:
one accent per action, so a press says which lane it belongs to, with none of
them competing with ember. Dark is the default (`color-scheme: dark light`);
light is a `@media (prefers-color-scheme: light)` override of the same names.
**Any new colour must be added in both branches** — light is not a filter over
dark, the hues are re-tuned.
@@ -145,14 +140,6 @@ Recurring specs (copy these rather than inventing sizes):
main#list article.card … | .empty
```
The owner's admin page (`admin.html`) is the same sheet with two sections in
place of the list — `.lanes` (Poll Lane rows) and `.readers` (the roster) —
and no library switch: it belongs to neither library, so its topbar carries a
plain `.ghost.back` link home. Both sections are eyebrow + hairline-separated
rows, the shape the roster already had as a fold-out. `.lanes` refreshes itself
every 30s via `hx-get="/ui/admin/lanes"` with `hx-swap="outerHTML"`; the roster
re-renders only in answer to an action.
**Brand mark**: an inline `<svg class="mark">` (`viewBox="0 0 200 172"`),
defined once in `chrome.html`'s `mark` template and reused by `app.html` and
`login.html` so it takes the page's `--ink`/`currentColor`/`--ember` rather
-344
View File
@@ -1,344 +0,0 @@
# GIF — maximum byte size of a file
Research note for Gitea issue #71 (backend `maxBodyBytes` = 4 MiB rejects the
8,571,192-byte animated cover GIF at
`https://cdn.asurascans.com/asura-images/covers/a-dragonslayers-peerless-regression.gif`).
All facts fetched live on **2026-08-17**: the GIF89a spec at
`https://www.w3.org/Graphics/GIF/spec-gif89a.txt`, Go stdlib `image/gif`
sources at `/usr/local/go/src/image/gif/reader.go` (Go 1.26.5), Chromium
`blink/renderer/platform/image-decoders/` sources via
`chromium.googlesource.com`, Firefox `image/decoders/nsGIFDecoder2.cpp` via
`hg.mozilla.org`, and cover bytes probed with plain `curl` (desktop Chrome UA;
`HEAD`/ranged `GET`). **No Cloudflare challenge was encountered on any CDN
probe** — every request returned real headers, consistent with the AGENTS.md
note of 2026-07-26 that plain `curl` works against both scan sites from the
dev machine and the VPS.
Every claim carries the URL it came from, or a reproducible command.
Interpretation rather than observation is marked `[INFERENCE]`.
---
## 1. Summary answer table
| Question | Answer | Evidence |
|---|---|---|
| Does the GIF89a spec define a maximum file size? | **No.** There is no file-size field anywhere in the format; the only numeric ceilings are per-field (16-bit screen/image dimensions, 255-byte sub-blocks, 12-bit LZW codes). | §2 |
| Maximum logical screen | 65535 × 65535 pixels (unsigned 16-bit width/height). | §2.1 |
| Number of frames / image descriptors | Unbounded — "An unlimited number of images may be present per Data Stream." | §2.2 |
| Formal max byte size of any single GIF | None. Single-frame worst case ≈ **6.44 GB** (12-bit LZW, max canvas); animated GIFs are **unbounded** because frames are unbounded. | §3 |
| Does the backend's decoder (Go `image/gif`) bound size? | **No.** It reads 16-bit dimensions and allocates `width×height` bytes per frame; a 65535² frame forces a ~4 GiB allocation. No total-size or dimension guard. | §4.1 |
| Do browsers bound on-disk GIF size? | Chromium and Firefox: no on-wire size cap in their GIF readers; Chromium caps *decoded* memory at min(4 B × pixels, platform budget). | §4.3, §4.4 |
| Real cover sizes (asurascans, n=25) | min 190,410 B · median 1,275,082 B · p90 4,524,788 B · max 8,571,192 B · **3/25 > 4 MiB** (two JPEGs and the animated GIF) | §5 |
| Real cover sizes (demonicscans/readermc, n=78) | min 13,298 B · median 63,061 B · max 801,200 B · 0/78 > 4 MiB | §5 |
| Comparable service caps | GitHub: 10 MB for images/GIFs. Discord API: default 10 MiB per file. Wikimedia: 100 MiB upload / 5 GiB host. | §6 |
| Recommended cover cap for #71 | **10 MiB** (separate from the 4 MiB series-page cap). Covers 100% of the 103 observed covers; matches GitHub/Discord calibration; ≤ 20 MiB worst-case transient per concurrent fetch+serve on a 1974 MiB swapless VPS. | §7 |
---
## 2. What the GIF89a specification actually bounds
Source: `https://www.w3.org/Graphics/GIF/spec-gif89a.txt` (fetched 2026-08-17).
### 2.1 Fixed-width fields — the only hard ceilings
The format is a stream of fixed-width blocks; the numeric fields that *do*
have a ceiling are all 16-bit unsigned, little-endian ("multi-byte numeric
fields are ordered Least Significant Byte first", §4 of the spec):
- **Logical Screen Width / Height** — "Unsigned" 2-byte fields (§18, Logical
Screen Descriptor) → maximum **65535 × 65535** pixels.
- **Image Left / Top Position, Image Width / Height** — "Unsigned" 2-byte
fields (§20, Image Descriptor). Each image "must fit within the boundaries
of the Logical Screen" (§20a), so an image cannot exceed the 65535² canvas
even though its own fields would allow it.
- **Data sub-blocks** — "A data sub-block may contain from 0 to 255 data
bytes" (§15); each sub-block is preceded by a 1-byte size field and the
stream is terminated by a 0x00 Block Terminator (§16). This bounds a
*chunk*, not the stream.
- **Global/Local Color Tables** — optional, "3 x 2^(Size of Global Color
Table+1)" bytes with a 3-bit size field → at most 3 × 2⁸ = **768 bytes**
each (§19, §21).
- **LZW codes** — "The output codes are of variable length, starting at
<code size>+1 bits per code, **up to 12 bits per code**. This defines a
maximum code value of 4095 (0xFFF)" (Appendix F, COMPRESSION, rule 4).
- **Trailer** — a single byte, fixed value 0x3B, "indicating the end of the
GIF Data Stream" (§27).
### 2.2 What is unbounded
- **Number of images (frames).** §20a, verbatim: "This block is REQUIRED for
an image. Exactly one Image Descriptor must be present per image in the
Data Stream. **An unlimited number of images may be present per Data
Stream.**"
- **The Data Stream itself.** The grammar in Appendix B is
`<GIF Data Stream> ::= Header <Logical Screen> <Data>* Trailer`, and the
spec states "the entity Data … may be repeated any number of times,
including 0 times." There is **no field anywhere that carries a file size,
byte count, frame count, or total-length value**. §13 (Block Sizes) only
defines sizes *within* blocks.
### 2.3 Verdict
**The GIF89a specification defines no maximum file size.** The only hard
bounds are per-field: 65535×65535 pixels per screen/image, 255 bytes per
sub-block, 12 bits per LZW code, and one trailer byte. A compliant decoder
must process whatever stream the blocks describe. Any byte ceiling a
particular GIF actually hits is therefore *implicit* — 16-bit dimensions,
LZW code width, decoder memory, or an external policy — never something the
format itself enforces. `[INFERENCE]` This is why real-world GIFs cap out at
"a few GB at most" and every service that wants a bound has to impose one
itself (see §6; Wikimedia explicitly documents that a 4 GiB host limit was a
storage-representation artifact of 32-bit integers, `phab:T191805`, not a
format limit).
---
## 3. Theoretical worst case
### 3.1 Single frame, maximal canvas, 8-bit pixels
| Quantity | Value | Derivation |
|---|---|---|
| Max pixels | 4,294,836,225 | 65535 × 65535 |
| Raw 8-bit palette-index raster | 4,294,836,225 B ≈ **4.29 GB / 4.00 GiB** | 1 byte per pixel (Table Based Image Data, §22; Go's `image.Paletted` uses exactly 1 byte/pixel) |
| LZW worst case | ≈ **6.44 GB / 6.00 GiB** | codes ≤ 12 bits each (Appendix F), at most ~1 code per pixel for incompressible data → ≤ 12 bits/px = 1.5 B/px → 4,294,836,225 × 1.5 B |
| Sub-block overhead | ≈ +25.3 MB | every ≤255-byte chunk carries a 1-byte size field (§15): ⌈6,442,254,338 / 255⌉ ≈ 25,263,743 size bytes, + 1 block terminator |
| Fixed overhead | ≈ +1.6 KB | header 6 B (§17) + logical screen descriptor 7 B (§18) + global color table ≤ 768 B (§19) + image descriptor 10 B (§20) + local color table ≤ 768 B (§21) + LZW minimum code size 1 B (§22) |
So a **single maximal-frame GIF cannot exceed ≈ 6.47 GB on the wire**
(12-bit LZW bound), and LZW being lossless means the real byte count depends
entirely on image content — the same canvas can be a few KB (flat color) or
~6 GB (noise).
Two caveats, both marked `[INFERENCE]`:
- The "1.5 B/px" figure assumes ~one emitted code per pixel. An encoder is
permitted to emit a Clear code at any point (Appendix F: "The Clear code
can appear at any point in the image data stream"), so a
pathological-but-compliant encoder emitting clear+pixel per pixel reaches
~24 bits/px ≈ 12.9 GB for the max canvas. Real encoders do not do this;
12-bit/px is the practical bound.
- The spec's deferred-clear note (cover sheet) explicitly allows an encoder
to keep using a full table at 12-bit codes without clearing, so the 12-bit
cap holds for the whole stream, it cannot "grow" past 12 bits.
### 3.2 Animated GIFs: unbounded
Every frame is one Image Descriptor, each bounded by the 65535² canvas, but
the *count* of frames is unbounded (§2.2). Total bytes = sum over frames —
therefore **there is no finite maximum byte size for an animated GIF** in
the format. The only thing that stops a real one is decoder memory, a
service cap, or disk space. `[INFERENCE]` This is the category the issue #71
cover falls into: it is an animated GIF (NETSCAPE2.0 loop extension found at
offset 0x310 of the file, verified 2026-08-17 by a ranged GET), and its
8,571,192 bytes are ~2.04× the current 4 MiB backend cap.
---
## 4. Decoder-side real limits
### 4.1 Go `image/gif` (the backend's decoder path, stdlib)
Source: `/usr/local/go/src/image/gif/reader.go`, Go 1.26.5.
- Dimensions are read as little-endian uint16 — `left/top/width/height :=
int(d.tmp[N]) + int(d.tmp[N+1])<<8` (reader.go:490-493) — so the format
ceiling 65535 applies, and nothing smaller is enforced.
- The only geometric check is that each frame fits inside the logical
screen: `if left+width > d.width || top+height > d.height` →
`errors.New("gif: frame bounds larger than image bounds")` (reader.go:512-513).
- **There is no file-size, byte-count, frame-count, or pixel-count guard.**
Each frame allocates `image.NewPaletted(...)` (reader.go:515) — a
`[]byte` of width×height — so decoding one legal 65535² frame attempts a
**~4.29 GB allocation**. `DecodeAll` (reader.go:603-605) additionally
retains every frame's `Pix` slice for the lifetime of the returned `*GIF`.
- `[INFERENCE]` On the 1974 MiB swapless VPS (root AGENTS.md), decoding such
a file would OOM rather than error cleanly; nothing in stdlib protects
the process. This matters for §7: the backend stores cover bytes without
decoding them (see §5.3), so the fetch path never triggers this — but any
future "validate/re-encode server-side" scheme would.
- Grep for `MaxInt|limit|too large|bounds` in reader.go: the only hits are
the frame-bounds check above and the `tmp [1024]byte` scratch buffer
(reader.go:109); no size caps exist.
### 4.2 giflib / libgif
**Not verified from source.** On 2026-08-17 the giflib sources were not
reachable from this network: `github.com/giflib/giflib` returns 404 (repo
gone/moved), `gitlab.com/giflib/giflib/-/raw/...` answers a Cloudflare
"Just a moment…" challenge, and the SourceForge project download path
404s. No limit claim about giflib is made here. `[INFERENCE]` giflib is
widely known to be allocation-driven with no dimension cap, but that is not
checked against source and is not needed for issue #71 (the backend uses Go
stdlib, not giflib).
### 4.3 Chromium (browser behaviour, first-party source)
- `third_party/blink/renderer/platform/image-decoders/gif/gif_image_reader.cc`
(via `chromium.googlesource.com/chromium/src/+/main/...`, fetched
2026-08-17): **no GIF byte-size or dimension cap found** — grep for
`max|limit|too large|dimension|65535|overflow` matches only license text.
- The base `ImageDecoder` caps *decoded memory*, not transfer size:
`CalculateMaxDecodedBytes` computes `min(4 * num_pixels, platform_max_decoded_bytes)`
(8 bytes/pixel for high-bit-depth), and the header comment says "Ignoring
this limit can cause excessive memory use or even crashes on low-memory
devices"
(`image_decoder.cc:94-117`, `image_decoder.h:545-549`). The GIF reader
itself is untouched by this — it is a decoded-buffer budget.
- Practical consequence `[INFERENCE]`: a browser will happily download and
store a multi-GB GIF from its own cache perspective; Chromium only limits
what it *decodes* into pixels.
### 4.4 Firefox
`image/decoders/nsGIFDecoder2.cpp` (via `hg.mozilla.org/mozilla-central/
raw-file/tip/...`, fetched 2026-08-17): **no dimension or size limit**; the
only guards are on LZW code width (`MAX_BITS` = 12, "maximum codeword size
of 12 bits") and the decode stack. Nothing bounds the on-disk byte size.
### 4.5 Summary
No mainstream decoder enforces a byte-size ceiling; they stop at the 16-bit
dimension ceiling (Go, by construction) or at decoded-memory budgets
(Chromium) or nowhere (Firefox). A GIF's byte size is policed only by
*storage* policies — which is what §6 calibrates and §7 sets.
---
## 5. Practical distribution — what real manga covers weigh
Probed **2026-08-17** with `curl -sI` (HEAD) and ranged GETs, desktop Chrome
UA. No Cloudflare block on any request. Sample = covers *as the backend
would fetch them* (the `og:image`/page-listed cover URL), not thumbnails we
chose by hand.
### 5.1 Exact commands
```sh
# asurascans.com — harvest cover URLs from the homepage, then HEAD each
curl -s -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/126.0" https://asurascans.com/ -o home.html
grep -oE 'https://cdn\.asurascans\.com/asura-images/covers/[^"&\\< ]+\.(webp|gif|jpg|jpeg|png)' home.html \
| sort -u | grep -v '\-400\.' | head -25 > sample.txt # one full-res cover per series, no -400 thumbs
while read -r u; do curl -s -A "…Chrome/126.0" -I "$u" | tr -d '\r' \
| grep -iE '^content-length:'; done < sample.txt
# demonicscans.org — covers live on readermc.org (ADR-0007), URLs contain spaces/UTF-8
curl -s -A "…Chrome/126.0" https://demonicscans.org/ -o demonic.html
grep -oE 'src="https://readermc\.org/images/thumbnails/[^"]+"' demonic.html | tr -d 'src="' > demonic.txt
# …plus og:image from 5 manga pages (Catastrophic-Necromancer, Magic-Emperor, …)
# each URL percent-encoded per path segment (urllib.parse.quote, safe=':/') before HEAD
```
### 5.2 asurascans — 25 full-res covers (mixed formats)
Homepage fetched 200 (664,700 B). All 25 returned `200` with a real
`Content-Length`. Distribution:
| Statistic | Bytes |
|---|---|
| n | 25 |
| min | 190,410 |
| median | 1,275,082 |
| p90 | 4,524,788 |
| max | 8,571,192 |
| mean | 1,943,651 |
| **> 4 MiB (4,194,304)** | **3 (12%)** — `a-dragonslayers-peerless-regression.gif` 8,571,192 (the issue #71 cover, animated: NETSCAPE2.0 at 0x310, 550×733, 256 colors); `bad-born-blood.3008f6.webp` 4,524,788 `image/jpeg`; `ending-maker.cfbf53.webp` 4,619,303 `image/jpeg` |
Notes: the CDN serves `Content-Type` by stored bytes, not by URL extension
(the `.webp` URLs return `image/png`, `image/jpeg`, or `image/webp` — the
sample spans all four of `png/jpeg/webp/gif`). Two of the three over-cap
files are **not GIFs**, so the current 4 MiB cap already silently drops 12%
of asura covers of any format. p90 itself (4.52 MB) exceeds the cap.
### 5.3 demonicscans — 78 covers on readermc.org
78 unique cover URLs (73 from the homepage's `/images/thumbnails/` plus 5
`og:image` values from manga pages — demonicscans publishes the thumbnail
file as the full cover, so that is exactly what the backend would fetch).
**78/78 returned 200 with a real Content-Length** (spaces and UTF-8 in the
filenames were percent-encoded per path segment; the homepage's raw HTML
carries `’`-style mojibake for curly quotes, which was repaired by
latin-1→utf-8 re-encoding before probing).
| Statistic | Bytes |
|---|---|
| n | 78 |
| min | 13,298 |
| median | 63,061 |
| p90 | 206,994 |
| max | 801,200 |
| mean | 110,038 |
| > 4 MiB | 0 |
### 5.4 Reading
`[INFERENCE]` asurascans covers are the heavy tail (median 1.3 MB, top
decile > 4 MiB, occasional ~5–9 MB), demonicscans covers are tiny (all
< 0.8 MB). A cover cap must be chosen against the *asura* distribution —
the 8.57 MB animated GIF is not a freak one-off outlier; the 90th
percentile already crosses 4 MiB and two JPEGs sit between 4.5–4.7 MB.
---
## 6. Comparable documented byte caps (first-party docs only)
| Service | Cap | Source (fetched 2026-08-17) |
|---|---|---|
| GitHub (issues/PR comments) | **10 MB for images and gifs**; 25 MB other files; 10/100 MB video | `https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/attaching-files` — "The maximum file size is: 10MB for images and gifs … 25MB for all other files" |
| Discord (API uploads) | default **10 MiB per file**, higher with Nitro / boost tier | `https://discord.com/developers/docs/reference#uploading-files` — "The file upload size limit applies to each file in a request. The default limit is `10 MiB` for all users" (help-center article `support.discord.com/hc/en-us/articles/115002935588` exists but answered 403 from this network on the probe date, so its figures were not verified here) |
| Wikimedia Commons | **100 MiB** upload limit; hosting up to **5 GiB**; GIF thumbnails limited to **100 megapixels**; prior 4 GiB host cap was a 32-bit storage artifact (phab:T191805) | `https://commons.wikimedia.org/wiki/Commons:Maximum_file_size` |
| MDN | nothing — MDN documents no byte-size limit for images; browsers impose none (see §4.3–4.4) | `[INFERENCE]` from absence in the platform docs read in §4 |
Calibration takeaway: two major platforms independently land on **~10 MB**
as the ceiling for an uploadable image/GIF (GitHub exactly 10 MB, Discord
exactly 10 MiB), with Wikimedia the outlier at 100 MiB/5 GiB because it is a
media *archive*. A 10 MiB cover cap is therefore squarely inside industry
normal.
---
## 7. Recommendation for issue #71
**Raise the cover cap to 10 MiB (10,485,760 B) — as a separate constant, not
by moving the shared one.**
Why:
- **Fits the measured reality.** The largest observed cover is 8,571,192 B
(the issue's animated GIF) = 82% of 10 MiB; 10 MiB covers **100% of the
103 sampled covers** and the *entire* asura distribution, including its
heavy tail. 4 MiB rejects 12% of asura covers (two of them plain JPEGs).
- **Matches industry calibration** (§6): GitHub 10 MB images/GIFs, Discord
10 MiB default. A 10 MiB cap is a number every engineer recognizes, and
it leaves ~18% headroom over the current worst observed file.
- **Costs little on the target hardware.** The backend buffers cover bytes
whole during fetch (`backend/internal/latest/cover.go`: `ContentLength >
maxBodyBytes` rejection at :155, then `io.ReadAll(io.LimitReader(…,
maxBodyBytes+1))` at :158) and loads the full body per `GET /covers/…`
(`backend/internal/api/handlers.go`, `Cover` → `w.Write(body)`). Worst
case per concurrent fetch **+** serve is therefore 2 × cap = 20 MiB; even
ten of each concurrently is ~200 MiB of a 1974 MiB swapless VPS (~10%),
and the browser unit (471 MiB, root AGENTS.md) is no longer on that box.
The 4 MiB series-page cap is *not* the issue — measured pages run
100 KB–1.2 MB (`backend/internal/latest/fetch.go` comment) — so keep it.
- **The cap is a separate knob.** Today one `const maxBodyBytes = 4 << 20`
(`backend/internal/latest/fetch.go:17`) gates *both* series pages and
covers (`cover.go` references it). Raising it wholesale would loosen the
page-side memory guard for no benefit; a cover-specific constant (e.g.
`maxCoverBytes = 10 << 20`) keeps the two policies independent. The fetch
already double-checks `ContentLength` and the post-`LimitReader` length,
so a larger constant changes nothing else.
Alternatives and their costs:
| Option | Cost |
|---|---|
| Keep 4 MiB | 12% of asura covers (incl. non-GIF JPEGs) never stored — current bug, silent missing covers. |
| 16 MiB cap | 2× headroom over the observed max for future GIFs; +60% worst-case transient memory vs 10 MiB; diverges from the GitHub/Discord 10 MB calibration. |
| Server-side re-encode / downscale covers | Requires decoding → Go `image/gif` allocates width×height per frame with **no guard** (§4.1); a legal 65535² GIF forces a ~4.29 GB allocation on a 1974 MiB swapless box — OOM, not an error. Also mutates bytes, which the store treats as immutable/content-addressed (ADR-0007). Highest risk, no upside at this scale. |
| No cap | Unbounded transient memory and disk; rejected outright. |
Decision is the user's; on the evidence, **10 MiB for covers, 4 MiB for
pages** is the defensible middle.
+223
View File
@@ -0,0 +1,223 @@
{
"0": "HTMX Library Internals",
"1": "Cover Fetch Test Helpers",
"2": "Manga Userscript Adapters",
"3": "Novel Userscript Adapters",
"4": "Series Acquisition Tests",
"5": "Bookmarks API Tests",
"6": "Storage Choice ADR",
"7": "Cover & Acquire Internals",
"8": "System Architecture Concepts",
"9": "Session Middleware",
"10": "Go Test Helpers",
"11": "Store Tests",
"12": "Bookmarks API Handler",
"13": "Web UI Handlers",
"14": "Go Error Handling",
"15": "CDP Browser Client",
"16": "Cloudflare bot scoring and poll cadence — what is actually documented",
"17": "Go Code Style Guide",
"18": "Agent Skills",
"19": "Store",
"20": "I/O Performance Patterns",
"21": "CPU Optimization",
"22": "Caching Patterns",
"23": "Browser Entrypoint",
"24": "Memory Allocation & GC",
"25": "Cover Fetcher Tests",
"26": "Open",
"27": "Find Skills Guide",
"28": "Allocation Patterns",
"29": "Observability & Alerting",
"30": "AGENTS.md",
"32": "Repo Hard Constraints",
"33": "Go Testing Guide",
"34": "Session Store",
"35": "Web UI Filter Logic",
"36": "Userscript Test Harness",
"37": "Product & Security Context",
"38": "novel-logic.test.js",
"39": "UI Critique 2026-07-26A",
"40": "UI Critique 2026-07-26B",
"41": "sessions_test.go",
"43": "Issue Tracker & Triage",
"44": "Ticket Workflow",
"45": "Go Perf Alert Rules",
"46": "Userscript Display Logic",
"47": "Go Perf Skill Docs",
"48": "Login Page Art",
"49": "BookmarkManager Logo",
"50": "Skills CLI",
"51": "Skills Leaderboard",
"52": "Complex Condition Extraction",
"53": "Sentinel Errors",
"54": "errors.As Patterns",
"55": "errors.Is Patterns",
"56": "errors.Join Patterns",
"57": "Error Wrapping",
"58": "Single Error Handling",
"59": "SIMD Optimizations",
"60": "GOGC Tuning",
"61": "GOMEMLIMIT",
"62": "Bottleneck Decision Tree",
"63": "pprof Profiling",
"64": "Test Timeout Helper",
"65": "httptest Patterns",
"66": "testify Suite Pattern",
"67": "go:embed Fixtures",
"68": "clockwork Time Mocking",
"69": "testify Mocking",
"70": "t.ArtifactDir Helper",
"71": "Subtests Pitfall",
"72": "golang-benchmark Skill",
"73": "golang-concurrency Skill",
"74": "golang-ci Skill",
"75": "golang-database Skill",
"76": "golang-lint Skill",
"77": "testify Skill",
"78": "Build Tag Integration Tests",
"79": "Test Naming Convention",
"80": "UI Critique A Finding",
"81": "UI Critique B Finding",
"82": "P0 Overflow Bug",
"83": "P1 hx-indicator Gap",
"84": "golang-benchmark Skill (ext)",
"85": "golang-concurrency Skill (ext)",
"86": "golang-ci Skill (ext)",
"87": "golang-data-structures Skill (ext)",
"88": "golang-database Skill (ext)",
"89": "golang-design-patterns Skill (ext)",
"90": "golang-documentation Skill (ext)",
"91": "golang-gopls Skill (ext)",
"92": "golang-lint Skill (ext)",
"93": "golang-naming Skill (ext)",
"94": "golang-observability Skill (ext)",
"95": "golang-refactoring Skill (ext)",
"96": "golang-safety Skill (ext)",
"97": "golang-samber-oops Skill (ext)",
"98": "golang-samber-slog Skill (ext)",
"99": "golang-structs-interfaces Skill (ext)",
"100": "golang-troubleshooting Skill (ext)",
"101": "promql-cli Skill",
"102": "Backend Module",
"103": "bookmark-api Service",
"104": "AGENTS.md",
"105": "reviewer.md",
"106": "Redeploy runbook",
"107": "1. Backend",
"108": "Deployment",
"109": "Cinder — BookmarkManager design system",
"110": "Implement tickets",
"111": "SQLite → Postgres cutover runbook",
"112": "Testing the userscript",
"113": "ADR-0007: The backend hosts every Site's Cover bytes",
"114": "Issue tracker: Gitea (`tea` CLI)",
"115": "ADR-0006: The browser runs on the home machine, over the tailnet",
"116": "ADR-0008: A Series identity is discovered from the Site's links, never derived from an address",
"117": "Domain Docs",
"118": "ticket-implementer.md",
"119": "implementer.md",
"120": "Series is a shared entity, and only the Poll may update it",
"121": "Postgres replaces SQLite as the primary datastore",
"122": "Identity comes from Discord OAuth; we store no passwords and send no email",
"123": "The wire format stays flat and deliberately does not mirror the schema",
"124": "ADR-0005: On-demand browser sidecar",
"126": "Bookmark Manager",
"127": "triage-labels.md",
"128": "Cross-Ticket Contract",
"129": "Implement Tickets Skill",
"130": "Orchestrator Role",
"131": "resolving-merge-conflicts Skill",
"132": "tdd Skill",
"133": "Ticket Wave Batching",
"134": "Four-Object Browser Stub",
"135": "Module Export Hook",
"136": "logic.test.js Test Harness",
"137": "manga-bookmark.user.js",
"138": "stripBuildHash",
"139": "Testing the Userscript Skill",
"140": "cr-spec Agent",
"141": "cr-standards Agent",
"142": "Escalate Rather Than Guess",
"143": "Status Contract",
"144": "Ticket Implementer Agent",
"145": "Worktree Isolation",
"146": "Escalate Rather Than Guess (opencode)",
"147": "Implementer Subagent (opencode)",
"148": "Subagent-Driven Development",
"149": "Code Quality Review",
"150": "Reviewer Subagent (opencode)",
"151": "Finding Severity Rubric",
"152": "Spec Compliance Review",
"158": "T",
"161": "Why Use samber/oops",
"162": "singleflight Cache Stampede Prevention",
"163": "Struct Field Alignment",
"164": "testing/synctest Deterministic Goroutine Testing",
"177": "Backend CLAUDE.md Guidance",
"178": "Graphify Knowledge Graph (graphify-out/)",
"179": "CLAUDE.md (Symlink to AGENTS.md)",
"192": "ADR-0001 (Drop modernc.org/sqlite)",
"193": "ADR-0003 (Split Shared Series Facts)",
"194": "SQLite-to-Postgres Cutover Runbook",
"195": "Import SQL Generation Rules",
"196": "Throwaway Import Generator",
"206": "Real scaling limit is the poller outbound fetch budget",
"207": "PostgreSQL (jackc/pgx/v5)",
"208": "SQLite (modernc.org/sqlite)",
"209": "Postgres chosen for future supportability, not concurrency",
"210": "Per-Reader bearer token for userscripts",
"211": "Discord OAuth2 (authorization code grant)",
"212": "ADR-0002: Discord OAuth, no passwords, no email",
"213": "Discord snowflake is the sole identity (lock-in)",
"214": "Bookmark (per-Reader state: Progress, Favourite, Lifecycle)",
"215": "Deduplicate polling per Series (reader_count DESC queue)",
"216": "ADR-0003: Series is shared, only the Poll updates it",
"217": "Only the Poll writes Series fields (security boundary)",
"218": "Series (shared entity keyed site+series_id)",
"219": "ADR-0004: Wire format stays flat, does not mirror schema",
"220": "Flat wire shape is a contract, not an implementation detail",
"221": "Installed userscripts must keep working (14-day grace window)",
"222": "CDP (Chrome DevTools Protocol) endpoint",
"223": "headless-shell service (socat-fronted CDP)",
"224": "Start Chrome on first CDP connection, reap after 300s idle",
"225": "BROWSER_WS_URL configuration seam",
"226": "ADR-0006: Browser runs on the home machine over the tailnet",
"227": "Browser moved home: VPS memory pressure, no requests served",
"228": "Tailnet (Tailscale network)",
"229": "Content-addressed filesystem storage (SHA-256 of source URL)",
"230": "Cover (Series image bytes)",
"231": "Deny-class destination control for outbound fetch",
"232": "ADR-0007: Backend hosts every Site's Cover bytes",
"233": "kagane CORP same-origin cover restriction",
"234": "Backend acquires, stores, serves every Cover (uniformity)",
"235": "a[aria-label='All Chapter'] anchor pointer",
"236": "Series identity is discovered from the Site's links",
"237": "ADR-0008: Series identity discovered, never derived",
"238": "Chapter slug vs series slug divergence (~7% measured)",
"239": "Scan truncated at first wpd-threads marker",
"240": "Surface ADR conflicts explicitly rather than silently overriding",
"241": "Domain docs: single-context layout guidance",
"242": "/domain-modeling skill (lazy CONTEXT.md creation)",
"243": "CONTEXT.md glossary (ubiquitous language)",
"244": "Gitea (tea CLI, gitea.violetcrown.my.id)",
"245": "wayfinder map/ticket mechanism",
"246": "Triage labels: canonical roles to tracker labels",
"247": "Canonical triage role labels (needs-triage ... wontfix)",
"248": "Cinder (BookmarkManager Web UI design system)",
"249": "Heat is typographic: ember reserved for unread chapters",
"250": "Design tokens (dark + light branches, no hardcoded hex)",
"251": "Three type roles: display serif / mono small-caps / sans",
"252": "a[aria-label='All Chapter'] priority pointer",
"253": "Research: lightnovelworld chapter slug vs series slug",
"254": "Gitea issue #77 (chapter vs series slug)",
"255": "Slug divergence measurements (3/41 diverge, 1 split)",
"256": "Unscoped chapter regex is SAFE, truncated at wpd-threads",
"257": "BookmarkManager",
"258": "Bromite (Primary Device)",
"259": "Dark-First Design Constraint",
"260": "Discord Guild Membership",
"261": "Reader Isolation Invariant",
"273": "AGENTS.md",
"279": "Userscript CLAUDE guidance"
}
+1
View File
@@ -0,0 +1 @@
.
+562
View File
@@ -0,0 +1,562 @@
# Graph Report - mangaBookmark (2026-08-16)
## Corpus Check
- 112 files · ~274,851 words
- Verdict: corpus is large enough that graph structure adds value.
## Summary
- 1640 nodes · 3294 edges · 221 communities (67 shown, 154 thin omitted)
- Extraction: 90% EXTRACTED · 10% INFERRED · 0% AMBIGUOUS · INFERRED: 313 edges (avg confidence: 0.77)
- Token cost: 0 input · 0 output
## Graph Freshness
- Built from commit: `f000cc7e`
- Run `git rev-parse HEAD` and compare to check if the graph is stale.
- Run `graphify update .` after code changes (no API cost).
## Community Hubs (Navigation)
- [[_COMMUNITY_HTMX Library Internals|HTMX Library Internals]]
- [[_COMMUNITY_Cover Fetch Test Helpers|Cover Fetch Test Helpers]]
- [[_COMMUNITY_Manga Userscript Adapters|Manga Userscript Adapters]]
- [[_COMMUNITY_Novel Userscript Adapters|Novel Userscript Adapters]]
- [[_COMMUNITY_Series Acquisition Tests|Series Acquisition Tests]]
- [[_COMMUNITY_Bookmarks API Tests|Bookmarks API Tests]]
- [[_COMMUNITY_Storage Choice ADR|Storage Choice ADR]]
- [[_COMMUNITY_Cover & Acquire Internals|Cover & Acquire Internals]]
- [[_COMMUNITY_System Architecture Concepts|System Architecture Concepts]]
- [[_COMMUNITY_Session Middleware|Session Middleware]]
- [[_COMMUNITY_Go Test Helpers|Go Test Helpers]]
- [[_COMMUNITY_Store Tests|Store Tests]]
- [[_COMMUNITY_Bookmarks API Handler|Bookmarks API Handler]]
- [[_COMMUNITY_Web UI Handlers|Web UI Handlers]]
- [[_COMMUNITY_Go Error Handling|Go Error Handling]]
- [[_COMMUNITY_CDP Browser Client|CDP Browser Client]]
- [[_COMMUNITY_Cloudflare bot scoring and poll cadence — what is actually documented|Cloudflare bot scoring and poll cadence — what is actually documented]]
- [[_COMMUNITY_Go Code Style Guide|Go Code Style Guide]]
- [[_COMMUNITY_Agent Skills|Agent Skills]]
- [[_COMMUNITY_Store|Store]]
- [[_COMMUNITY_IO Performance Patterns|I/O Performance Patterns]]
- [[_COMMUNITY_CPU Optimization|CPU Optimization]]
- [[_COMMUNITY_Caching Patterns|Caching Patterns]]
- [[_COMMUNITY_Browser Entrypoint|Browser Entrypoint]]
- [[_COMMUNITY_Memory Allocation & GC|Memory Allocation & GC]]
- [[_COMMUNITY_Cover Fetcher Tests|Cover Fetcher Tests]]
- [[_COMMUNITY_Open|Open]]
- [[_COMMUNITY_Find Skills Guide|Find Skills Guide]]
- [[_COMMUNITY_Allocation Patterns|Allocation Patterns]]
- [[_COMMUNITY_Observability & Alerting|Observability & Alerting]]
- [[_COMMUNITY_Repo Hard Constraints|Repo Hard Constraints]]
- [[_COMMUNITY_Go Testing Guide|Go Testing Guide]]
- [[_COMMUNITY_Session Store|Session Store]]
- [[_COMMUNITY_Web UI Filter Logic|Web UI Filter Logic]]
- [[_COMMUNITY_Userscript Test Harness|Userscript Test Harness]]
- [[_COMMUNITY_Product & Security Context|Product & Security Context]]
- [[_COMMUNITY_novel-logic.test.js|novel-logic.test.js]]
- [[_COMMUNITY_UI Critique 2026-07-26A|UI Critique 2026-07-26A]]
- [[_COMMUNITY_UI Critique 2026-07-26B|UI Critique 2026-07-26B]]
- [[_COMMUNITY_sessions_test.go|sessions_test.go]]
- [[_COMMUNITY_Issue Tracker & Triage|Issue Tracker & Triage]]
- [[_COMMUNITY_Ticket Workflow|Ticket Workflow]]
- [[_COMMUNITY_Go Perf Alert Rules|Go Perf Alert Rules]]
- [[_COMMUNITY_Userscript Display Logic|Userscript Display Logic]]
- [[_COMMUNITY_Go Perf Skill Docs|Go Perf Skill Docs]]
- [[_COMMUNITY_Login Page Art|Login Page Art]]
- [[_COMMUNITY_BookmarkManager Logo|BookmarkManager Logo]]
- [[_COMMUNITY_Skills CLI|Skills CLI]]
- [[_COMMUNITY_Skills Leaderboard|Skills Leaderboard]]
- [[_COMMUNITY_Complex Condition Extraction|Complex Condition Extraction]]
- [[_COMMUNITY_Sentinel Errors|Sentinel Errors]]
- [[_COMMUNITY_errors.As Patterns|errors.As Patterns]]
- [[_COMMUNITY_errors.Is Patterns|errors.Is Patterns]]
- [[_COMMUNITY_errors.Join Patterns|errors.Join Patterns]]
- [[_COMMUNITY_Error Wrapping|Error Wrapping]]
- [[_COMMUNITY_Single Error Handling|Single Error Handling]]
- [[_COMMUNITY_SIMD Optimizations|SIMD Optimizations]]
- [[_COMMUNITY_GOGC Tuning|GOGC Tuning]]
- [[_COMMUNITY_GOMEMLIMIT|GOMEMLIMIT]]
- [[_COMMUNITY_Bottleneck Decision Tree|Bottleneck Decision Tree]]
- [[_COMMUNITY_pprof Profiling|pprof Profiling]]
- [[_COMMUNITY_Test Timeout Helper|Test Timeout Helper]]
- [[_COMMUNITY_httptest Patterns|httptest Patterns]]
- [[_COMMUNITY_testify Suite Pattern|testify Suite Pattern]]
- [[_COMMUNITY_goembed Fixtures|go:embed Fixtures]]
- [[_COMMUNITY_clockwork Time Mocking|clockwork Time Mocking]]
- [[_COMMUNITY_testify Mocking|testify Mocking]]
- [[_COMMUNITY_t.ArtifactDir Helper|t.ArtifactDir Helper]]
- [[_COMMUNITY_Subtests Pitfall|Subtests Pitfall]]
- [[_COMMUNITY_golang-benchmark Skill|golang-benchmark Skill]]
- [[_COMMUNITY_golang-concurrency Skill|golang-concurrency Skill]]
- [[_COMMUNITY_golang-ci Skill|golang-ci Skill]]
- [[_COMMUNITY_golang-database Skill|golang-database Skill]]
- [[_COMMUNITY_golang-lint Skill|golang-lint Skill]]
- [[_COMMUNITY_testify Skill|testify Skill]]
- [[_COMMUNITY_Build Tag Integration Tests|Build Tag Integration Tests]]
- [[_COMMUNITY_Test Naming Convention|Test Naming Convention]]
- [[_COMMUNITY_UI Critique A Finding|UI Critique A Finding]]
- [[_COMMUNITY_UI Critique B Finding|UI Critique B Finding]]
- [[_COMMUNITY_P0 Overflow Bug|P0 Overflow Bug]]
- [[_COMMUNITY_P1 hx-indicator Gap|P1 hx-indicator Gap]]
- [[_COMMUNITY_golang-benchmark Skill (ext)|golang-benchmark Skill (ext)]]
- [[_COMMUNITY_golang-concurrency Skill (ext)|golang-concurrency Skill (ext)]]
- [[_COMMUNITY_golang-ci Skill (ext)|golang-ci Skill (ext)]]
- [[_COMMUNITY_golang-data-structures Skill (ext)|golang-data-structures Skill (ext)]]
- [[_COMMUNITY_golang-database Skill (ext)|golang-database Skill (ext)]]
- [[_COMMUNITY_golang-design-patterns Skill (ext)|golang-design-patterns Skill (ext)]]
- [[_COMMUNITY_golang-documentation Skill (ext)|golang-documentation Skill (ext)]]
- [[_COMMUNITY_golang-gopls Skill (ext)|golang-gopls Skill (ext)]]
- [[_COMMUNITY_golang-lint Skill (ext)|golang-lint Skill (ext)]]
- [[_COMMUNITY_golang-naming Skill (ext)|golang-naming Skill (ext)]]
- [[_COMMUNITY_golang-observability Skill (ext)|golang-observability Skill (ext)]]
- [[_COMMUNITY_golang-refactoring Skill (ext)|golang-refactoring Skill (ext)]]
- [[_COMMUNITY_golang-safety Skill (ext)|golang-safety Skill (ext)]]
- [[_COMMUNITY_golang-samber-oops Skill (ext)|golang-samber-oops Skill (ext)]]
- [[_COMMUNITY_golang-samber-slog Skill (ext)|golang-samber-slog Skill (ext)]]
- [[_COMMUNITY_golang-structs-interfaces Skill (ext)|golang-structs-interfaces Skill (ext)]]
- [[_COMMUNITY_golang-troubleshooting Skill (ext)|golang-troubleshooting Skill (ext)]]
- [[_COMMUNITY_promql-cli Skill|promql-cli Skill]]
- [[_COMMUNITY_Backend Module|Backend Module]]
- [[_COMMUNITY_bookmark-api Service|bookmark-api Service]]
- [[_COMMUNITY_AGENTS|AGENTS.md]]
- [[_COMMUNITY_reviewer|reviewer.md]]
- [[_COMMUNITY_Redeploy runbook|Redeploy runbook]]
- [[_COMMUNITY_1. Backend|1. Backend]]
- [[_COMMUNITY_Deployment|Deployment]]
- [[_COMMUNITY_Cinder — BookmarkManager design system|Cinder — BookmarkManager design system]]
- [[_COMMUNITY_Implement tickets|Implement tickets]]
- [[_COMMUNITY_SQLite → Postgres cutover runbook|SQLite → Postgres cutover runbook]]
- [[_COMMUNITY_Testing the userscript|Testing the userscript]]
- [[_COMMUNITY_ADR-0007 The backend hosts every Site's Cover bytes|ADR-0007: The backend hosts every Site's Cover bytes]]
- [[_COMMUNITY_Issue tracker Gitea (`tea` CLI)|Issue tracker: Gitea (`tea` CLI)]]
- [[_COMMUNITY_ADR-0006 The browser runs on the home machine, over the tailnet|ADR-0006: The browser runs on the home machine, over the tailnet]]
- [[_COMMUNITY_ADR-0008 A Series identity is discovered from the Site's links, never derived from an address|ADR-0008: A Series identity is discovered from the Site's links, never derived from an address]]
- [[_COMMUNITY_Domain Docs|Domain Docs]]
- [[_COMMUNITY_ticket-implementer|ticket-implementer.md]]
- [[_COMMUNITY_implementer|implementer.md]]
- [[_COMMUNITY_Series is a shared entity, and only the Poll may update it|Series is a shared entity, and only the Poll may update it]]
- [[_COMMUNITY_Postgres replaces SQLite as the primary datastore|Postgres replaces SQLite as the primary datastore]]
- [[_COMMUNITY_Identity comes from Discord OAuth; we store no passwords and send no email|Identity comes from Discord OAuth; we store no passwords and send no email]]
- [[_COMMUNITY_The wire format stays flat and deliberately does not mirror the schema|The wire format stays flat and deliberately does not mirror the schema]]
- [[_COMMUNITY_ADR-0005 On-demand browser sidecar|ADR-0005: On-demand browser sidecar]]
- [[_COMMUNITY_Bookmark Manager|Bookmark Manager]]
- [[_COMMUNITY_triage-labels|triage-labels.md]]
- [[_COMMUNITY_Cross-Ticket Contract|Cross-Ticket Contract]]
- [[_COMMUNITY_Implement Tickets Skill|Implement Tickets Skill]]
- [[_COMMUNITY_Orchestrator Role|Orchestrator Role]]
- [[_COMMUNITY_resolving-merge-conflicts Skill|resolving-merge-conflicts Skill]]
- [[_COMMUNITY_tdd Skill|tdd Skill]]
- [[_COMMUNITY_Ticket Wave Batching|Ticket Wave Batching]]
- [[_COMMUNITY_Four-Object Browser Stub|Four-Object Browser Stub]]
- [[_COMMUNITY_Module Export Hook|Module Export Hook]]
- [[_COMMUNITY_logic.test.js Test Harness|logic.test.js Test Harness]]
- [[_COMMUNITY_manga-bookmark.user.js|manga-bookmark.user.js]]
- [[_COMMUNITY_stripBuildHash|stripBuildHash]]
- [[_COMMUNITY_Testing the Userscript Skill|Testing the Userscript Skill]]
- [[_COMMUNITY_cr-spec Agent|cr-spec Agent]]
- [[_COMMUNITY_cr-standards Agent|cr-standards Agent]]
- [[_COMMUNITY_Escalate Rather Than Guess|Escalate Rather Than Guess]]
- [[_COMMUNITY_Status Contract|Status Contract]]
- [[_COMMUNITY_Ticket Implementer Agent|Ticket Implementer Agent]]
- [[_COMMUNITY_Worktree Isolation|Worktree Isolation]]
- [[_COMMUNITY_Escalate Rather Than Guess (opencode)|Escalate Rather Than Guess (opencode)]]
- [[_COMMUNITY_Implementer Subagent (opencode)|Implementer Subagent (opencode)]]
- [[_COMMUNITY_Subagent-Driven Development|Subagent-Driven Development]]
- [[_COMMUNITY_Code Quality Review|Code Quality Review]]
- [[_COMMUNITY_Reviewer Subagent (opencode)|Reviewer Subagent (opencode)]]
- [[_COMMUNITY_Finding Severity Rubric|Finding Severity Rubric]]
- [[_COMMUNITY_Spec Compliance Review|Spec Compliance Review]]
- [[_COMMUNITY_T|T]]
- [[_COMMUNITY_Why Use samberoops|Why Use samber/oops]]
- [[_COMMUNITY_singleflight Cache Stampede Prevention|singleflight Cache Stampede Prevention]]
- [[_COMMUNITY_Struct Field Alignment|Struct Field Alignment]]
- [[_COMMUNITY_testingsynctest Deterministic Goroutine Testing|testing/synctest Deterministic Goroutine Testing]]
- [[_COMMUNITY_Backend CLAUDE.md Guidance|Backend CLAUDE.md Guidance]]
- [[_COMMUNITY_Graphify Knowledge Graph (graphify-out)|Graphify Knowledge Graph (graphify-out/)]]
- [[_COMMUNITY_CLAUDE.md (Symlink to AGENTS.md)|CLAUDE.md (Symlink to AGENTS.md)]]
- [[_COMMUNITY_ADR-0001 (Drop modernc.orgsqlite)|ADR-0001 (Drop modernc.org/sqlite)]]
- [[_COMMUNITY_ADR-0003 (Split Shared Series Facts)|ADR-0003 (Split Shared Series Facts)]]
- [[_COMMUNITY_SQLite-to-Postgres Cutover Runbook|SQLite-to-Postgres Cutover Runbook]]
- [[_COMMUNITY_Import SQL Generation Rules|Import SQL Generation Rules]]
- [[_COMMUNITY_Throwaway Import Generator|Throwaway Import Generator]]
- [[_COMMUNITY_Real scaling limit is the poller outbound fetch budget|Real scaling limit is the poller outbound fetch budget]]
- [[_COMMUNITY_PostgreSQL (jackcpgxv5)|PostgreSQL (jackc/pgx/v5)]]
- [[_COMMUNITY_SQLite (modernc.orgsqlite)|SQLite (modernc.org/sqlite)]]
- [[_COMMUNITY_Postgres chosen for future supportability, not concurrency|Postgres chosen for future supportability, not concurrency]]
- [[_COMMUNITY_Per-Reader bearer token for userscripts|Per-Reader bearer token for userscripts]]
- [[_COMMUNITY_Discord OAuth2 (authorization code grant)|Discord OAuth2 (authorization code grant)]]
- [[_COMMUNITY_ADR-0002 Discord OAuth, no passwords, no email|ADR-0002: Discord OAuth, no passwords, no email]]
- [[_COMMUNITY_Discord snowflake is the sole identity (lock-in)|Discord snowflake is the sole identity (lock-in)]]
- [[_COMMUNITY_Bookmark (per-Reader state Progress, Favourite, Lifecycle)|Bookmark (per-Reader state: Progress, Favourite, Lifecycle)]]
- [[_COMMUNITY_Deduplicate polling per Series (reader_count DESC queue)|Deduplicate polling per Series (reader_count DESC queue)]]
- [[_COMMUNITY_ADR-0003 Series is shared, only the Poll updates it|ADR-0003: Series is shared, only the Poll updates it]]
- [[_COMMUNITY_Only the Poll writes Series fields (security boundary)|Only the Poll writes Series fields (security boundary)]]
- [[_COMMUNITY_Series (shared entity keyed site+series_id)|Series (shared entity keyed site+series_id)]]
- [[_COMMUNITY_ADR-0004 Wire format stays flat, does not mirror schema|ADR-0004: Wire format stays flat, does not mirror schema]]
- [[_COMMUNITY_Flat wire shape is a contract, not an implementation detail|Flat wire shape is a contract, not an implementation detail]]
- [[_COMMUNITY_Installed userscripts must keep working (14-day grace window)|Installed userscripts must keep working (14-day grace window)]]
- [[_COMMUNITY_CDP (Chrome DevTools Protocol) endpoint|CDP (Chrome DevTools Protocol) endpoint]]
- [[_COMMUNITY_headless-shell service (socat-fronted CDP)|headless-shell service (socat-fronted CDP)]]
- [[_COMMUNITY_Start Chrome on first CDP connection, reap after 300s idle|Start Chrome on first CDP connection, reap after 300s idle]]
- [[_COMMUNITY_BROWSER_WS_URL configuration seam|BROWSER_WS_URL configuration seam]]
- [[_COMMUNITY_ADR-0006 Browser runs on the home machine over the tailnet|ADR-0006: Browser runs on the home machine over the tailnet]]
- [[_COMMUNITY_Browser moved home VPS memory pressure, no requests served|Browser moved home: VPS memory pressure, no requests served]]
- [[_COMMUNITY_Tailnet (Tailscale network)|Tailnet (Tailscale network)]]
- [[_COMMUNITY_Content-addressed filesystem storage (SHA-256 of source URL)|Content-addressed filesystem storage (SHA-256 of source URL)]]
- [[_COMMUNITY_Cover (Series image bytes)|Cover (Series image bytes)]]
- [[_COMMUNITY_Deny-class destination control for outbound fetch|Deny-class destination control for outbound fetch]]
- [[_COMMUNITY_ADR-0007 Backend hosts every Site's Cover bytes|ADR-0007: Backend hosts every Site's Cover bytes]]
- [[_COMMUNITY_kagane CORP same-origin cover restriction|kagane CORP same-origin cover restriction]]
- [[_COMMUNITY_Backend acquires, stores, serves every Cover (uniformity)|Backend acquires, stores, serves every Cover (uniformity)]]
- [[_COMMUNITY_aaria-label='All Chapter' anchor pointer|a[aria-label='All Chapter'] anchor pointer]]
- [[_COMMUNITY_Series identity is discovered from the Site's links|Series identity is discovered from the Site's links]]
- [[_COMMUNITY_ADR-0008 Series identity discovered, never derived|ADR-0008: Series identity discovered, never derived]]
- [[_COMMUNITY_Chapter slug vs series slug divergence (~7% measured)|Chapter slug vs series slug divergence (~7% measured)]]
- [[_COMMUNITY_Scan truncated at first wpd-threads marker|Scan truncated at first wpd-threads marker]]
- [[_COMMUNITY_Surface ADR conflicts explicitly rather than silently overriding|Surface ADR conflicts explicitly rather than silently overriding]]
- [[_COMMUNITY_Domain docs single-context layout guidance|Domain docs: single-context layout guidance]]
- [[_COMMUNITY_domain-modeling skill (lazy CONTEXT.md creation)|/domain-modeling skill (lazy CONTEXT.md creation)]]
- [[_COMMUNITY_CONTEXT.md glossary (ubiquitous language)|CONTEXT.md glossary (ubiquitous language)]]
- [[_COMMUNITY_Gitea (tea CLI, gitea.violetcrown.my.id)|Gitea (tea CLI, gitea.violetcrown.my.id)]]
- [[_COMMUNITY_wayfinder mapticket mechanism|wayfinder map/ticket mechanism]]
- [[_COMMUNITY_Triage labels canonical roles to tracker labels|Triage labels: canonical roles to tracker labels]]
- [[_COMMUNITY_Canonical triage role labels (needs-triage ... wontfix)|Canonical triage role labels (needs-triage ... wontfix)]]
- [[_COMMUNITY_Cinder (BookmarkManager Web UI design system)|Cinder (BookmarkManager Web UI design system)]]
- [[_COMMUNITY_Heat is typographic ember reserved for unread chapters|Heat is typographic: ember reserved for unread chapters]]
- [[_COMMUNITY_Design tokens (dark + light branches, no hardcoded hex)|Design tokens (dark + light branches, no hardcoded hex)]]
- [[_COMMUNITY_Three type roles display serif mono small-caps sans|Three type roles: display serif / mono small-caps / sans]]
- [[_COMMUNITY_aaria-label='All Chapter' priority pointer|a[aria-label='All Chapter'] priority pointer]]
- [[_COMMUNITY_Research lightnovelworld chapter slug vs series slug|Research: lightnovelworld chapter slug vs series slug]]
- [[_COMMUNITY_Gitea issue 77 (chapter vs series slug)|Gitea issue #77 (chapter vs series slug)]]
- [[_COMMUNITY_Slug divergence measurements (341 diverge, 1 split)|Slug divergence measurements (3/41 diverge, 1 split)]]
- [[_COMMUNITY_Unscoped chapter regex is SAFE, truncated at wpd-threads|Unscoped chapter regex is SAFE, truncated at wpd-threads]]
- [[_COMMUNITY_BookmarkManager|BookmarkManager]]
- [[_COMMUNITY_Bromite (Primary Device)|Bromite (Primary Device)]]
- [[_COMMUNITY_Dark-First Design Constraint|Dark-First Design Constraint]]
- [[_COMMUNITY_Discord Guild Membership|Discord Guild Membership]]
- [[_COMMUNITY_Reader Isolation Invariant|Reader Isolation Invariant]]
- [[_COMMUNITY_AGENTS|AGENTS.md]]
- [[_COMMUNITY_Userscript CLAUDE guidance|Userscript CLAUDE guidance]]
## God Nodes (most connected - your core abstractions)
1. `testConfig()` - 53 edges
2. `newWebTestServer()` - 49 edges
3. `newTestStore()` - 43 edges
4. `newTestStore()` - 42 edges
5. `e()` - 33 edges
6. `Handler` - 29 edges
7. `ne()` - 28 edges
8. `De()` - 28 edges
9. `Open()` - 27 edges
10. `se()` - 27 edges
## Surprising Connections (you probably didn't know these)
- `el()` --indirect_call--> `c()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `el()` --indirect_call--> `c()` [INFERRED]
userscript/novel-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `latestChapterFromAnchors()` --indirect_call--> `re()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `el()` --indirect_call--> `k()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `el()` --indirect_call--> `k()` [INFERRED]
userscript/novel-bookmark.user.js → backend/internal/web/static/htmx.min.js
## Import Cycles
- None detected.
## Hyperedges (group relationships)
- **Batch Ticket Implementation Pipeline** — _claude_skills_implement_tickets_skill_implement_tickets, _omp_agents_ticket_implementer_ticket_implementer, _omp_agents_ticket_implementer_cr_spec, _omp_agents_ticket_implementer_cr_standards [INFERRED 0.85]
- **Subagent-Driven Development Pipeline** — _opencode_agent_implementer_implementer, _opencode_agent_reviewer_reviewer, _opencode_agent_implementer_subagent_driven_development [INFERRED 0.85]
- **Go HTML Template Family** — backend_internal_web_templates_app_doc, backend_internal_web_templates_card_doc, backend_internal_web_templates_list_doc, backend_internal_web_templates_chrome_doc, backend_internal_web_templates_login_doc, backend_internal_web_templates_setup_doc, backend_internal_web_templates_readers_doc, backend_internal_web_templates_icons_doc [INFERRED 0.95]
- **htmx Fragment Swap Flow** — backend_internal_web_templates_app_doc, backend_internal_web_templates_card_doc, backend_internal_web_templates_chrome_doc, backend_internal_web_templates_setup_doc, backend_internal_web_templates_readers_doc [INFERRED 0.95]
- **Backend owns the truth (single-writer ownership of shared facts)** — docs_adr_0003_series_shared_and_poll_owned_poll_owned_writes, docs_adr_0004_wire_format_does_not_mirror_the_schema_flat_wire_contract, docs_adr_0007_backend_hosts_cover_bytes_server_side_covers [INFERRED 0.85]
- **Headless browser infrastructure (sidecar, on-demand, home deployment)** — docs_adr_0005_on_demand_browser_headless_shell, docs_adr_0005_on_demand_browser_cdp, docs_adr_0005_on_demand_browser_on_demand_start, docs_adr_0006_browser_on_the_home_machine_home_machine_rationale [INFERRED 0.85]
- **lightnovelworld series-identity investigation and fix** — docs_research_lightnovelworld_chapter_vs_series_slug_issue_77, docs_research_lightnovelworld_chapter_vs_series_slug_unscoped_regex, docs_adr_0008_series_identity_is_discovered_not_derived_discovered_identity [INFERRED 0.85]
## Communities (221 total, 154 thin omitted)
### Community 0 - "HTMX Library Internals"
Cohesion: 0.08
Nodes (101): A(), ae(), an(), at(), B(), be(), bn(), bt() (+93 more)
### Community 1 - "Cover Fetch Test Helpers"
Cohesion: 0.10
Nodes (84): floatPtr(), testConfig(), getCover(), Cookie, Handler, ResponseRecorder, T, TestListRendersAcquiredCover() (+76 more)
### Community 2 - "Manga Userscript Adapters"
Cohesion: 0.06
Nodes (77): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+69 more)
### Community 3 - "Novel Userscript Adapters"
Cohesion: 0.06
Nodes (78): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+70 more)
### Community 4 - "Series Acquisition Tests"
Cohesion: 0.09
Nodes (69): bookmarkNewKaganeSeries(), bookmarkNewNovelfullSeries(), bookmarkNewSeries(), Context, Store, T, newAcquirer(), readBookmark() (+61 more)
### Community 5 - "Bookmarks API Tests"
Cohesion: 0.08
Nodes (67): auth(), getBookmarks(), Handler, Request, Store, T, newTestServer(), newTestStore() (+59 more)
### Community 7 - "Cover & Acquire Internals"
Cohesion: 0.10
Nodes (30): Addr, Context, Store, defaultCoverResolver(), fetchCoverBytes(), Client, Context, NewCoverFetcher() (+22 more)
### Community 8 - "System Architecture Concepts"
Cohesion: 0.13
Nodes (20): app.html — App Shell Template, Manga/Novel Library Switch, Bookmark Bucket Tabs, Confirm Row (Archive/Finish/Remove), card.html — Series Card Template, htmx /ui/* Mutation Endpoints, chrome.html — Out-of-Band Regions, Action Key (+12 more)
### Community 9 - "Session Middleware"
Cohesion: 0.07
Nodes (35): ClearCookie(), ClientIP(), Duration, Mutex, Request, ResponseWriter, Time, isHTTPS() (+27 more)
### Community 10 - "Go Test Helpers"
Cohesion: 0.05
Nodes (39): Test Helpers, Test Timeout, Basic Handler Test, HTTP Handler Testing, Query Parameters and Headers, Docker Compose Fixture, Integration Testing, SQL Schema Fixture (+31 more)
### Community 11 - "Store Tests"
Cohesion: 0.15
Nodes (43): Store, T, newTestStore(), readLatestCheckedAt(), readSeries(), secondReader(), seedForCheck(), seedSecondReader() (+35 more)
### Community 12 - "Bookmarks API Handler"
Cohesion: 0.11
Nodes (24): Auth(), compressible(), CORS(), Handler, ResponseWriter, Store, Gzip(), ResolveReader() (+16 more)
### Community 13 - "Web UI Handlers"
Cohesion: 0.14
Nodes (18): currentLib(), currentTab(), filterBookmarks(), Client, HandlerFunc, Request, ResponseWriter, Store (+10 more)
### Community 14 - "Go Error Handling"
Cohesion: 0.06
Nodes (33): Creating Errors, Custom Error Types, Custom types that wrap other errors, Decision table: which error strategy to use, Error Creation, Error String Conventions, Errors as Values, `errors.New` — static error messages (+25 more)
### Community 15 - "CDP Browser Client"
Cohesion: 0.07
Nodes (36): awaitPromise(), browserConnectionLost(), classifyBrowserError(), comixRead(), comixSeriesPageURL(), Action, Context, Mutex (+28 more)
### Community 16 - "Cloudflare bot scoring and poll cadence — what is actually documented"
Cohesion: 0.06
Nodes (33): 1.1 The score itself, 1.2 The detection engines (Enterprise Bot Management), 1.3 Rate limiting is a separate product, 1. What a bot score is and what feeds it, 2.1 What each plan gets, 2.2 Bot Fight Mode specifics (the Free-plan product), 2.3 Does the free tier "score" continuously?, 2. The free-plan reality (+25 more)
### Community 17 - "Go Code Style Guide"
Cohesion: 0.08
Nodes (23): Code Style Details, Extract Complex Conditions, Value vs Pointer Arguments, Code Organization Within Files, Complex Conditions & Init Scope, Composite Literals, Control Flow, Cross-References (+15 more)
### Community 19 - "Store"
Cohesion: 0.09
Nodes (8): coverRelativePath(), coverSourceAddress(), displayChapter(), Store, scanSeries(), TestDisplayChapter(), Bookmark, ReaderSummary
### Community 20 - "I/O Performance Patterns"
Cohesion: 0.11
Nodes (18): Avoid io.ReadAll for large payloads, Batch Operations, Buffered I/O, Cgo Overhead, Channel: batch processing from a stream, Concurrent Multi-Stage Pipelines, Connection pooling, Database: batch inserts over row-by-row (+10 more)
### Community 21 - "CPU Optimization"
Cohesion: 0.13
Nodes (15): Cache Locality, Contiguous 2D allocation, CPU Optimization, False Sharing, Function Inlining, Handling CPU-specific instruction sets, Instruction-Level Parallelism, Monotonic Time (+7 more)
### Community 22 - "Caching Patterns"
Cohesion: 0.13
Nodes (14): Algorithmic Complexity, Avoid iterator chains, Caching Patterns, Compiled Pattern Caching, Early returns and short-circuit loops, LRU caches, Map lookups over slice scanning, Precomputed lookup tables (+6 more)
### Community 23 - "Browser Entrypoint"
Cohesion: 0.35
Nodes (14): browser_alive(), connection(), connection_signal(), finish_connection(), has_connections(), lock(), reaper(), entrypoint.sh script (+6 more)
### Community 24 - "Memory Allocation & GC"
Cohesion: 0.13
Nodes (15): Allocation Rate Reduction, Ballast pattern (pre-Go 1.19), Garbage Collector Tuning, GC pacing, GC Profiling and Diagnostics, GODEBUG=gctrace=1, GOGC (default: 100), GOMAXPROCS in Containers (+7 more)
### Community 25 - "Cover Fetcher Tests"
Cohesion: 0.33
Nodes (12): coverResponse(), Request, T, TestCoverFetcherCanonicalisesJpgAlias(), TestCoverFetcherFetchesPublicHTTPSImage(), TestCoverFetcherRefusesUnsafeDestinationsBeforeRequest(), TestCoverFetcherRejectsNonImage(), TestCoverFetcherRejectsOversizedBody() (+4 more)
### Community 26 - "Open"
Cohesion: 0.20
Nodes (17): applyMigration(), migrate(), Open(), refreshOwnerToken(), seedOwner(), TestCoverIsContentAddressedOnFilesystem(), TestCoverPersistsAcrossReopen(), TestMigration0002BackfillsExistingBookmarks() (+9 more)
### Community 27 - "Find Skills Guide"
Cohesion: 0.14
Nodes (13): Common Skill Categories, Find Skills, How to Help Users Find Skills, Step 1: Understand What They Need, Step 2: Check the Leaderboard First, Step 3: Search for Skills, Step 4: Verify Quality Before Recommending, Step 5: Present Options to the User (+5 more)
### Community 28 - "Allocation Patterns"
Cohesion: 0.11
Nodes (19): Allocation Patterns, Backing Array Leaks, Direct indexing vs append, Eliminate redundant map lookups, Interface boxing, Map never shrinks, Map of pointers for large, frequently updated structs, Map size hints (+11 more)
### Community 29 - "Observability & Alerting"
Cohesion: 0.22
Nodes (9): Alerting rules (examples), CPU saturation, GC pressure, Goroutine leaks, Grafana Dashboards, Memory leaks, Prometheus Metrics for Go, PromQL Queries for Performance Diagnosis (+1 more)
### Community 32 - "Repo Hard Constraints"
Cohesion: 0.18
Nodes (12): M, TestMain(), M, TestMain(), M, Main(), start(), URL() (+4 more)
### Community 33 - "Go Testing Guide"
Cohesion: 0.20
Nodes (10): CI Regression Detection, Common Mistakes, Core Philosophy, Cross-References, Decision Tree: Where Is Time Spent?, Deep Dives, Go Performance Optimization, Iterative Optimization Methodology (+2 more)
### Community 34 - "Session Store"
Cohesion: 0.28
Nodes (4): Duration, Store, Time, Session
### Community 35 - "Web UI Filter Logic"
Cohesion: 0.31
Nodes (5): closeCardPanels(), setActiveTab(), toggleChapterForm(), toggleConfirmRow(), togglePanel()
### Community 37 - "Product & Security Context"
Cohesion: 0.17
Nodes (11): Accessibility & Inclusion, Brand Commitments, Capabilities and Constraints, Evidence on Hand, Operating Context, Platform, Positioning, Product (+3 more)
### Community 38 - "novel-logic.test.js"
Cohesion: 0.33
Nodes (5): ADR-0009: A Site answers fixed questions; an unusual Site owns its own fetch, Consequences, Considered options, Decision, Why
### Community 39 - "UI Critique 2026-07-26A"
Cohesion: 0.29
Nodes (6): Design Health Score, Design Specificity Verdict, Minor Observations, Persona Red Flags, Priority Issues, Questions to Consider
### Community 40 - "UI Critique 2026-07-26B"
Cohesion: 0.29
Nodes (6): Design Health Score, Design Specificity Verdict, Minor Observations, Persona Red Flags, Priority Issues, Questions to Consider
### Community 41 - "sessions_test.go"
Cohesion: 0.48
Nodes (6): T, TestCreateAndGetSession(), TestDeleteSessionIsPerReader(), TestDeleteSessionRevokes(), TestExpiredSessionIsGone(), TestGetSessionUnknownID()
### Community 45 - "Go Perf Alert Rules"
Cohesion: 0.50
Nodes (4): Prometheus Alerting Rules (Go Performance), GoroutineLeak Alert, HighGCPauseTime Alert, MemoryNearLimit Alert
### Community 46 - "Userscript Display Logic"
Cohesion: 0.07
Nodes (27): 1. Summary answer table, 2. The two reference pages, 3. Chapter page → series URL: every in-page pointer, in priority order, 4.1 Sample method, 4.2 Divergence results, 4.3 Is there a derivable rule? **No.**, 4.4 The split case — a slug can change *mid-series*, 4. The reverse direction, and how common divergence is (+19 more)
### Community 47 - "Go Perf Skill Docs"
Cohesion: 0.22
Nodes (5): Continuous Profiling, Production Observability for Performance, Pyroscope pull mode (via Grafana Alloy), Pyroscope push mode, Real-Time Visualization (Development)
### Community 48 - "Login Page Art"
Cohesion: 0.67
Nodes (3): Fantasy Sword, Fiery Volcanic Scene, Login Art: Sword in Volcanic Rock
### Community 49 - "BookmarkManager Logo"
Cohesion: 1.00
Nodes (3): Mirrored Double Bookmark Mark, Ember Flame Accent, BookmarkManager Logo
### Community 103 - "bookmark-api Service"
Cohesion: 0.32
Nodes (8): Browser Sidecar Service, CDP Endpoint (Tailnet-Bound :9222), Persistent Chrome Profile Volume, chrome/docker-compose.yml — Browser Deployable Unit, bookmark-api Prod Override, CDP Never on Shared Proxy Network, docker-compose.prod.yml — Production Override, Traefik Reverse Proxy Labels
### Community 104 - "AGENTS.md"
Cohesion: 0.12
Nodes (14): Agent skills, Architecture, Commands, Comments, Design system, Domain docs, Forge: Gitea, not GitHub, graphify (+6 more)
### Community 105 - "reviewer.md"
Cohesion: 0.12
Nodes (15): Assessment, Calibration, Critical (Must Fix), Do Not Trust the Report, Important (Should Fix), Inputs, Issues, Method (+7 more)
### Community 106 - "Redeploy runbook"
Cohesion: 0.12
Nodes (15): 0. Preflight, 1. Back up the database, 2. Pull the new code, 3. Rebuild and restart, 4. Verify the deploy, 5. Smoke-test the full loop, 6. Rollback, 7. The whole thing, as one block (+7 more)
### Community 107 - "1. Backend"
Cohesion: 0.13
Nodes (14): 1. Backend, 2. Userscript, Adapter reference (verified live 2026-07-24), Config (env), Deploy behind your reverse proxy, Desktop iteration (optional), Develop / test, Endpoints (+6 more)
### Community 108 - "Deployment"
Cohesion: 0.14
Nodes (13): 0. Prerequisites, 1. Configure `.env`, 1b. Web UI, 2. Build + start, 3. Verify over HTTPS, 4. Configure the userscript, 5. Install on Bromite, 6. Smoke-test the full loop (+5 more)
### Community 109 - "Cinder — BookmarkManager design system"
Cohesion: 0.20
Nodes (9): 1. The one idea, 2. Tokens, 3. Type, 4. Components (web UI), 5. Components (userscript panel), 6. Motion, 7. Accessibility floor (not negotiable), 8. Adding something new — checklist (+1 more)
### Community 110 - "Implement tickets"
Cohesion: 0.22
Nodes (8): 1. Collect the tickets, 2. Plan the batch, 3. Get the plan approved, 4. Run a wave, 5. Land the wave, 6. Close the batch, Implement tickets, Ticket #<n> — <title>
### Community 111 - "SQLite → Postgres cutover runbook"
Cohesion: 0.22
Nodes (8): 0. The generator is throwaway, 1. Stop the old API and take a fresh export, 2. Bring up Postgres with the schema and the owner Reader, 3. Generate the import SQL, 4. Review it by eye, 5. Apply it, 6. Afterwards, SQLite → Postgres cutover runbook
### Community 112 - "Testing the userscript"
Cohesion: 0.29
Nodes (6): Adding a test, Commands, Gotchas, How the harness works, Testing the userscript, What is NOT testable here
### Community 113 - "ADR-0007: The backend hosts every Site's Cover bytes"
Cohesion: 0.29
Nodes (6): ADR-0007: The backend hosts every Site's Cover bytes, Consequences, Considered options, Decision, Two deliberate relaxations, Why a future reader will find this surprising
### Community 114 - "Issue tracker: Gitea (`tea` CLI)"
Cohesion: 0.29
Nodes (6): Conventions, Issue tracker: Gitea (`tea` CLI), Pull requests as a triage surface, Wayfinding operations, When a skill says "fetch the relevant ticket", When a skill says "publish to the issue tracker"
### Community 115 - "ADR-0006: The browser runs on the home machine, over the tailnet"
Cohesion: 0.33
Nodes (5): ADR-0006: The browser runs on the home machine, over the tailnet, Consequences, Constraints, Decision, Why
### Community 116 - "ADR-0008: A Series identity is discovered from the Site's links, never derived from an address"
Cohesion: 0.33
Nodes (5): ADR-0008: A Series identity is discovered from the Site's links, never derived from an address, Consequences, Considered options, Decision, Why
### Community 117 - "Domain Docs"
Cohesion: 0.33
Nodes (5): Before exploring, read these, Domain Docs, File structure, Flag ADR conflicts, Use the glossary's vocabulary
### Community 118 - "ticket-implementer.md"
Cohesion: 0.33
Nodes (5): Escalate rather than guess, Order of work, Report, Review, The worktree is your whole world
### Community 119 - "implementer.md"
Cohesion: 0.33
Nodes (5): Before You Begin, Report Format, Self-Review Before Reporting, When You're in Over Your Head, Your Job
### Community 120 - "Series is a shared entity, and only the Poll may update it"
Cohesion: 0.40
Nodes (4): Consequences, Only the Poll writes Series fields, Series is a shared entity, and only the Poll may update it, Why
### Community 121 - "Postgres replaces SQLite as the primary datastore"
Cohesion: 0.50
Nodes (3): Consequences, Considered options, Postgres replaces SQLite as the primary datastore
### Community 122 - "Identity comes from Discord OAuth; we store no passwords and send no email"
Cohesion: 0.50
Nodes (3): Consequences, Considered options, Identity comes from Discord OAuth; we store no passwords and send no email
### Community 123 - "The wire format stays flat and deliberately does not mirror the schema"
Cohesion: 0.50
Nodes (3): Consequence, The wire format stays flat and deliberately does not mirror the schema, Why a future reader will find this surprising
### Community 124 - "ADR-0005: On-demand browser sidecar"
Cohesion: 0.50
Nodes (3): ADR-0005: On-demand browser sidecar, Constraints, Decision
### Community 158 - "T"
Cohesion: 0.08
Nodes (38): Handler, Request, ResponseWriter, Store, Healthz(), writeJSON(), Request, ReaderID() (+30 more)
### Community 273 - "AGENTS.md"
Cohesion: 0.50
Nodes (3): Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trust), Second script: `novel-bookmark.user.js`, Userscript structure (single IIFE, `manga-bookmark.user.js`)
## Knowledge Gaps
- **520 isolated node(s):** `bookmarkmanager/backend`, `ctxKey`, `loginView`, `ctxKey`, `test` (+515 more)
These have ≤1 connection - possible missing edges or undocumented components.
- **154 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes.
## Suggested Questions
_Questions this graph is uniquely positioned to answer:_
- **Why does `New()` connect `Series Acquisition Tests` to `Bookmarks API Tests`, `Cover & Acquire Internals`, `Session Middleware`, `Web UI Handlers`, `Open`?**
_High betweenness centrality (0.051) - this node is a cross-community bridge._
- **Why does `Open()` connect `Open` to `Repo Hard Constraints`, `Cover Fetch Test Helpers`, `Series Acquisition Tests`, `Bookmarks API Tests`, `Store Tests`, `Store`?**
_High betweenness centrality (0.038) - this node is a cross-community bridge._
- **Why does `newRouter()` connect `Bookmarks API Tests` to `Cover Fetch Test Helpers`, `Bookmarks API Handler`, `Series Acquisition Tests`?**
_High betweenness centrality (0.030) - this node is a cross-community bridge._
- **Are the 47 inferred relationships involving `testConfig()` (e.g. with `TestListRendersAcquiredCover()` and `TestPublicCoverNeverEchoesNonImage()`) actually correct?**
_`testConfig()` has 47 INFERRED edges - model-reasoned connections that need verification._
- **Are the 8 inferred relationships involving `newWebTestServer()` (e.g. with `TestListRendersAcquiredCover()` and `TestPublicCoverRejectsUnknownAddress()`) actually correct?**
_`newWebTestServer()` has 8 INFERRED edges - model-reasoned connections that need verification._
- **Are the 12 inferred relationships involving `newTestStore()` (e.g. with `TestAcquireDoesNotBlockTheWrite()` and `TestAcquireFailureLeavesTheBookmarkIntact()`) actually correct?**
_`newTestStore()` has 12 INFERRED edges - model-reasoned connections that need verification._
- **What connects `bookmarkmanager/backend`, `ctxKey`, `loginView` to the rest of the system?**
_553 weakly-connected nodes found - possible documentation gaps or missing edges._
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large Load Diff
+632
View File
@@ -0,0 +1,632 @@
{
".agents/skills/golang-code-style/evals/evals.json": {
"mtime": 1784884678.6627614,
"ast_hash": "bec0e12446e7af3cd05de9b6d42badd8",
"semantic_hash": "bec0e12446e7af3cd05de9b6d42badd8"
},
".agents/skills/golang-error-handling/evals/evals.json": {
"mtime": 1784884678.6655047,
"ast_hash": "275d710b774fba1e1d0bc098d3646c6d",
"semantic_hash": "275d710b774fba1e1d0bc098d3646c6d"
},
".agents/skills/golang-performance/evals/evals.json": {
"mtime": 1784884678.6688662,
"ast_hash": "4f06df87f90aa0e4f6deaa318b47689f",
"semantic_hash": "4f06df87f90aa0e4f6deaa318b47689f"
},
".agents/skills/golang-testing/evals/evals.json": {
"mtime": 1784884678.6721346,
"ast_hash": "60a821bbfd20c6fe8bba996b8b540dd4",
"semantic_hash": "60a821bbfd20c6fe8bba996b8b540dd4"
},
"backend/go.mod": {
"mtime": 1786216141.668644,
"ast_hash": "dac242903b0e98c3e4395159d609e08e",
"semantic_hash": "dac242903b0e98c3e4395159d609e08e"
},
"backend/main.go": {
"mtime": 1786501521.228955,
"ast_hash": "6e98a3ae91aaa132df251e43c4dfca6d",
"semantic_hash": "6e98a3ae91aaa132df251e43c4dfca6d"
},
"skills-lock.json": {
"mtime": 1784884678.6842625,
"ast_hash": "4a94ac85bad6bce330d085bcc0ae3ffd",
"semantic_hash": "4a94ac85bad6bce330d085bcc0ae3ffd"
},
"userscript/manga-bookmark.user.js": {
"mtime": 1786499529.779988,
"ast_hash": "1f8bcddd3632d709f058a8401af8f127",
"semantic_hash": ""
},
".agents/skills/find-skills/SKILL.md": {
"mtime": 1784884338.760326,
"ast_hash": "62b297abdee9aea84c577ab2e04e1974",
"semantic_hash": "62b297abdee9aea84c577ab2e04e1974"
},
".agents/skills/golang-code-style/SKILL.md": {
"mtime": 1784884678.6623824,
"ast_hash": "d6a01e6f64550a5c8d59dac2e948000e",
"semantic_hash": "d6a01e6f64550a5c8d59dac2e948000e"
},
".agents/skills/golang-code-style/references/details.md": {
"mtime": 1784884678.6627865,
"ast_hash": "19891e396a986f1b24bf34b7a2854fcb",
"semantic_hash": "19891e396a986f1b24bf34b7a2854fcb"
},
".agents/skills/golang-error-handling/SKILL.md": {
"mtime": 1784884678.665052,
"ast_hash": "8b7970f472adb240e5bc4bde863d43a6",
"semantic_hash": "8b7970f472adb240e5bc4bde863d43a6"
},
".agents/skills/golang-error-handling/references/error-creation.md": {
"mtime": 1784884678.665524,
"ast_hash": "248dbf75492c68faef2334b8d83bd080",
"semantic_hash": "248dbf75492c68faef2334b8d83bd080"
},
".agents/skills/golang-error-handling/references/error-handling.md": {
"mtime": 1784884678.6655412,
"ast_hash": "c2424ee3999b05963b199e0100727aa3",
"semantic_hash": "c2424ee3999b05963b199e0100727aa3"
},
".agents/skills/golang-error-handling/references/error-wrapping.md": {
"mtime": 1784884678.6655717,
"ast_hash": "4a15d9266a1ea0c8bb1951e6b9c0f586",
"semantic_hash": "4a15d9266a1ea0c8bb1951e6b9c0f586"
},
".agents/skills/golang-performance/SKILL.md": {
"mtime": 1784884678.667833,
"ast_hash": "35a15fd129c5bedaada3fd4df0b6ba8c",
"semantic_hash": "35a15fd129c5bedaada3fd4df0b6ba8c"
},
".agents/skills/golang-performance/assets/prometheus-alerts.yml": {
"mtime": 1784884678.668527,
"ast_hash": "fa9357ffa87c4f894fc21afa9707c4db",
"semantic_hash": "fa9357ffa87c4f894fc21afa9707c4db"
},
".agents/skills/golang-performance/references/caching.md": {
"mtime": 1784884678.6690567,
"ast_hash": "807c42a82994e5548dbf7eb6e30c71e0",
"semantic_hash": "807c42a82994e5548dbf7eb6e30c71e0"
},
".agents/skills/golang-performance/references/cpu.md": {
"mtime": 1784884678.669087,
"ast_hash": "6d52e532ef51a35cc4134694c944517d",
"semantic_hash": "6d52e532ef51a35cc4134694c944517d"
},
".agents/skills/golang-performance/references/io-networking.md": {
"mtime": 1784884678.6691036,
"ast_hash": "95c5dd51f728fd69c945a92ff021d766",
"semantic_hash": "95c5dd51f728fd69c945a92ff021d766"
},
".agents/skills/golang-performance/references/memory.md": {
"mtime": 1784884678.6691158,
"ast_hash": "3b2108df06b4cfb3980fa80bbd9ebcff",
"semantic_hash": "3b2108df06b4cfb3980fa80bbd9ebcff"
},
".agents/skills/golang-performance/references/observability.md": {
"mtime": 1784884678.6691446,
"ast_hash": "0aa8a498e8d55ccdd4990ad187bae828",
"semantic_hash": "0aa8a498e8d55ccdd4990ad187bae828"
},
".agents/skills/golang-performance/references/runtime.md": {
"mtime": 1784884678.669426,
"ast_hash": "26386c33b3a3794aaef0556713bf8f3a",
"semantic_hash": "26386c33b3a3794aaef0556713bf8f3a"
},
".agents/skills/golang-testing/SKILL.md": {
"mtime": 1784884678.6716368,
"ast_hash": "0a9b9793bba2a239db94e980272a393e",
"semantic_hash": "0a9b9793bba2a239db94e980272a393e"
},
".agents/skills/golang-testing/references/helpers.md": {
"mtime": 1784884678.6722167,
"ast_hash": "0aecb7cfbeb9b374bf61c52e324d9d3f",
"semantic_hash": "0aecb7cfbeb9b374bf61c52e324d9d3f"
},
".agents/skills/golang-testing/references/http-testing.md": {
"mtime": 1784884678.67225,
"ast_hash": "9111110c28a7fbbffc3537aad786b390",
"semantic_hash": "9111110c28a7fbbffc3537aad786b390"
},
".agents/skills/golang-testing/references/integration-testing.md": {
"mtime": 1784884678.67225,
"ast_hash": "fcf9861bc36ae56e3fb7a1c18aa0e77f",
"semantic_hash": "fcf9861bc36ae56e3fb7a1c18aa0e77f"
},
".agents/skills/golang-testing/references/mocking.md": {
"mtime": 1784884678.6722653,
"ast_hash": "3a08979e4603aae5c32a58d5b6c39765",
"semantic_hash": "3a08979e4603aae5c32a58d5b6c39765"
},
"CLAUDE.md": {
"mtime": 1786856633.445473,
"ast_hash": "fab288c23be960d9c6afbfe3f21ff41c",
"semantic_hash": ""
},
"DEPLOY.md": {
"mtime": 1786501942.7367291,
"ast_hash": "3c7b785c44badb6dda6234d2b39a9290",
"semantic_hash": ""
},
"README.md": {
"mtime": 1786499529.7651505,
"ast_hash": "9d6be8aa8a2946c23ad48d8f2864b5ca",
"semantic_hash": ""
},
"docker-compose.prod.yml": {
"mtime": 1786292465.8305523,
"ast_hash": "0751998a532297b8ac507a01ec48dc31",
"semantic_hash": "0751998a532297b8ac507a01ec48dc31"
},
"docker-compose.yml": {
"mtime": 1786499529.7725692,
"ast_hash": "124fd581bf0a662ff15012abfdb40a92",
"semantic_hash": ""
},
".claude/settings.json": {
"mtime": 1784951973.1869545,
"ast_hash": "e51077b6a7f1f67afc748f1a32a1557d",
"semantic_hash": "e51077b6a7f1f67afc748f1a32a1557d"
},
"backend/web_test.go": {
"mtime": 1786216141.700644,
"ast_hash": "8f1b093b59eb1ed81bc7fc0c22495c50",
"semantic_hash": "8f1b093b59eb1ed81bc7fc0c22495c50"
},
"backend/main_test.go": {
"mtime": 1786501521.228955,
"ast_hash": "8a165955cf28ad47481fec5ea7afb3d6",
"semantic_hash": "8a165955cf28ad47481fec5ea7afb3d6"
},
".claude/settings.local.json": {
"mtime": 1785697645.350201,
"ast_hash": "9a1ac6369f968e8df4be9dcff0948f70",
"semantic_hash": "9a1ac6369f968e8df4be9dcff0948f70"
},
"PRODUCT.md": {
"mtime": 1786216141.660644,
"ast_hash": "c52072d1978286060087fa0686f9c7f9",
"semantic_hash": "c52072d1978286060087fa0686f9c7f9"
},
"backend/.impeccable/critique/2026-07-26T15-50-42Z__backend-templates-app-html.md": {
"mtime": 1785128576.2412457,
"ast_hash": "d08627c27f22d453125db6c1ab4b71ec",
"semantic_hash": "d08627c27f22d453125db6c1ab4b71ec"
},
"backend/.impeccable/critique/2026-07-26T17-08-41Z__backend-templates-app-html.md": {
"mtime": 1785128576.2452524,
"ast_hash": "e69a8340a371579ca3ea689660f7d7bd",
"semantic_hash": "e69a8340a371579ca3ea689660f7d7bd"
},
"AGENTS.md": {
"mtime": 1786856633.445473,
"ast_hash": "fab288c23be960d9c6afbfe3f21ff41c",
"semantic_hash": ""
},
"userscript/test/logic.test.js": {
"mtime": 1786499529.779988,
"ast_hash": "80d512bfe6fa8b847f3ba6169c321a74",
"semantic_hash": ""
},
".claude/skills/testing-the-userscript/SKILL.md": {
"mtime": 1786363889.5489495,
"ast_hash": "8f3c0132eb4787a2c8736eb99f7689af",
"semantic_hash": "8f3c0132eb4787a2c8736eb99f7689af"
},
"REDEPLOY.md": {
"mtime": 1786856643.0770833,
"ast_hash": "e5e910f0244a040e2ccdc669b17eb4db",
"semantic_hash": ""
},
"docs/design-system.md": {
"mtime": 1786022513.9623306,
"ast_hash": "421cd7e57f02d4b467f120ca6ddd7b6a",
"semantic_hash": "421cd7e57f02d4b467f120ca6ddd7b6a"
},
"backend/api_test.go": {
"mtime": 1786499529.7651505,
"ast_hash": "8e4b9293bc2e45ee3f42027315594fd5",
"semantic_hash": ""
},
"backend/cover_test.go": {
"mtime": 1786363889.552731,
"ast_hash": "c7e313d6c92eb28e6d370e5e89035984",
"semantic_hash": "c7e313d6c92eb28e6d370e5e89035984"
},
"backend/internal/api/handlers.go": {
"mtime": 1786363889.552731,
"ast_hash": "59e6b8767ab19839bb8f82891a7e4616",
"semantic_hash": "59e6b8767ab19839bb8f82891a7e4616"
},
"backend/internal/httpmw/middleware.go": {
"mtime": 1786216141.672644,
"ast_hash": "385b36f58488b7e6d93eb6d6034e9ee3",
"semantic_hash": "385b36f58488b7e6d93eb6d6034e9ee3"
},
"backend/internal/latest/browser.go": {
"mtime": 1786856879.3397639,
"ast_hash": "fbdba4bb56d8c804087bcb6e141e52ad",
"semantic_hash": ""
},
"backend/internal/latest/browser_test.go": {
"mtime": 1786856254.36163,
"ast_hash": "800fa6aa471ada054a3c48943ad17ab7",
"semantic_hash": ""
},
"backend/internal/latest/fetch.go": {
"mtime": 1786499529.7688599,
"ast_hash": "3e20ad86aa46783e9aa95c2b746551ee",
"semantic_hash": ""
},
"backend/internal/latest/poller.go": {
"mtime": 1786856670.129715,
"ast_hash": "6b7060ff52994729832d9fcc3cc97dad",
"semantic_hash": ""
},
"backend/internal/latest/poller_test.go": {
"mtime": 1786856311.8535168,
"ast_hash": "b6837fab377c94fa1d2f20a931975a4a",
"semantic_hash": ""
},
"backend/internal/latest/sites.go": {
"mtime": 1786856865.4529688,
"ast_hash": "2f00456ac9d1d8148ba1bc6cbceb24f6",
"semantic_hash": ""
},
"backend/internal/latest/sites_test.go": {
"mtime": 1786856890.1231265,
"ast_hash": "0db2028a6073f342fee61ad15c2e5f0f",
"semantic_hash": ""
},
"backend/internal/latest/smoke_image_test.go": {
"mtime": 1786363889.5602942,
"ast_hash": "db068cb59575f8c82669acbaf84bcaed",
"semantic_hash": "db068cb59575f8c82669acbaf84bcaed"
},
"backend/internal/pgtest/pgtest.go": {
"mtime": 1786216141.6766438,
"ast_hash": "f60372d41516e66f7aaeb272da227d6e",
"semantic_hash": "f60372d41516e66f7aaeb272da227d6e"
},
"backend/internal/session/session.go": {
"mtime": 1786216141.6766438,
"ast_hash": "9952474ffb22c825d6f075b866ee26f4",
"semantic_hash": "9952474ffb22c825d6f075b866ee26f4"
},
"backend/internal/session/session_test.go": {
"mtime": 1786216141.680644,
"ast_hash": "37ffd00964e7a67350c68ed50c6503c5",
"semantic_hash": "37ffd00964e7a67350c68ed50c6503c5"
},
"backend/internal/store/migrations/0001_bookmarks.sql": {
"mtime": 1786216141.680644,
"ast_hash": "f87ccfb2c25c43f93021177ced0bfae4",
"semantic_hash": "f87ccfb2c25c43f93021177ced0bfae4"
},
"backend/internal/store/migrations/0002_series.sql": {
"mtime": 1786216141.6820722,
"ast_hash": "5dc98771e0c0f6e8416b434c280b0efb",
"semantic_hash": "5dc98771e0c0f6e8416b434c280b0efb"
},
"backend/internal/store/migrations/0003_reader.sql": {
"mtime": 1786216141.6820722,
"ast_hash": "444a97799f38f6222f87d4e9fb2d6258",
"semantic_hash": "444a97799f38f6222f87d4e9fb2d6258"
},
"backend/internal/store/migrations/0004_owner_bookmarks.sql": {
"mtime": 1786216141.684644,
"ast_hash": "e4fa900cc223865d3ecd4c60c5707a65",
"semantic_hash": "e4fa900cc223865d3ecd4c60c5707a65"
},
"backend/internal/store/migrations/0005_sessions.sql": {
"mtime": 1786216141.684644,
"ast_hash": "5158887ebc57cf8c16a7b821b61cd760",
"semantic_hash": "5158887ebc57cf8c16a7b821b61cd760"
},
"backend/internal/store/migrations/0006_reader_token_epoch.sql": {
"mtime": 1786216141.684644,
"ast_hash": "3093cc1c3aae0cd9643d04105d045402",
"semantic_hash": "3093cc1c3aae0cd9643d04105d045402"
},
"backend/internal/store/sessions.go": {
"mtime": 1786216141.684644,
"ast_hash": "eee3510cc6172ef4b1da820474c26b01",
"semantic_hash": "eee3510cc6172ef4b1da820474c26b01"
},
"backend/internal/store/sessions_test.go": {
"mtime": 1786216141.684644,
"ast_hash": "0b6764a0ee20f5cb7748eecd31a1d220",
"semantic_hash": "0b6764a0ee20f5cb7748eecd31a1d220"
},
"backend/internal/store/store.go": {
"mtime": 1786363889.5602942,
"ast_hash": "54367a8ab043983e2491b2eb2650961c",
"semantic_hash": "54367a8ab043983e2491b2eb2650961c"
},
"backend/internal/store/store_test.go": {
"mtime": 1786363889.5602942,
"ast_hash": "dc823fd77bcce2268114e31d759b20a5",
"semantic_hash": "dc823fd77bcce2268114e31d759b20a5"
},
"backend/internal/userscript/userscript.go": {
"mtime": 1786216141.692644,
"ast_hash": "aa13a71b1c9eefe4930fd31f27722328",
"semantic_hash": "aa13a71b1c9eefe4930fd31f27722328"
},
"backend/internal/userscript/userscript_test.go": {
"mtime": 1786216141.692644,
"ast_hash": "6c050968d7b8b67956da1a3136d2c3c7",
"semantic_hash": "6c050968d7b8b67956da1a3136d2c3c7"
},
"backend/internal/web/discord.go": {
"mtime": 1786216141.692644,
"ast_hash": "69e4959c65fa67d7491aedf6a71bb575",
"semantic_hash": "69e4959c65fa67d7491aedf6a71bb575"
},
"backend/internal/web/oauth_test.go": {
"mtime": 1786216141.692644,
"ast_hash": "a3bddeb70dd8d7eb14139da808723ea8",
"semantic_hash": "a3bddeb70dd8d7eb14139da808723ea8"
},
"backend/internal/web/static/filter.js": {
"mtime": 1786022513.9473197,
"ast_hash": "b4ee3306201bfd88b148b96801972617",
"semantic_hash": "b4ee3306201bfd88b148b96801972617"
},
"backend/internal/web/static/htmx.min.js": {
"mtime": 1785873769.5512016,
"ast_hash": "19a573773be4ca22570ca2f8543120c5",
"semantic_hash": "19a573773be4ca22570ca2f8543120c5"
},
"backend/internal/web/web.go": {
"mtime": 1786363889.5678573,
"ast_hash": "8308949c658d3a08ce1c3cdbea6a907c",
"semantic_hash": "8308949c658d3a08ce1c3cdbea6a907c"
},
"backend/reader_credential_test.go": {
"mtime": 1786216141.700644,
"ast_hash": "2a751ea6d9c06635aa3179db0aef1b2b",
"semantic_hash": "2a751ea6d9c06635aa3179db0aef1b2b"
},
"chrome/entrypoint.sh": {
"mtime": 1786363889.5678573,
"ast_hash": "8008a187690764436540fab47ba0cfcc",
"semantic_hash": "8008a187690764436540fab47ba0cfcc"
},
"userscript/novel-bookmark.user.js": {
"mtime": 1786499529.779988,
"ast_hash": "834effb0821f8d6c9f57f6554a5db462",
"semantic_hash": ""
},
"userscript/test/novel-logic.test.js": {
"mtime": 1786499529.779988,
"ast_hash": "b25a7377af210dd0aff8284501fd0252",
"semantic_hash": ""
},
".opencode/agent/implementer.md": {
"mtime": 1785873769.5402634,
"ast_hash": "000de469c18d68352027e10c8ce8acfb",
"semantic_hash": "000de469c18d68352027e10c8ce8acfb"
},
".opencode/agent/reviewer.md": {
"mtime": 1785873769.5416775,
"ast_hash": "e44a2f6f624db044e19508bc5ab05592",
"semantic_hash": "e44a2f6f624db044e19508bc5ab05592"
},
"CONTEXT.md": {
"mtime": 1786855457.9327722,
"ast_hash": "24548f60414b4c5ff58538acaada5345",
"semantic_hash": ""
},
"CUTOVER.md": {
"mtime": 1786216141.660644,
"ast_hash": "6c6f3e4c4c2f57867894280bce728c50",
"semantic_hash": "6c6f3e4c4c2f57867894280bce728c50"
},
"backend/AGENTS.md": {
"mtime": 1786856656.6826186,
"ast_hash": "d5610ba24076b57e17b9d76241acaa65",
"semantic_hash": ""
},
"backend/CLAUDE.md": {
"mtime": 1786856656.6826186,
"ast_hash": "d5610ba24076b57e17b9d76241acaa65",
"semantic_hash": ""
},
"backend/internal/web/templates/app.html": {
"mtime": 1786216141.692644,
"ast_hash": "3965e20e204afb71ba2a3aa86cb7c61c",
"semantic_hash": "3965e20e204afb71ba2a3aa86cb7c61c"
},
"backend/internal/web/templates/card.html": {
"mtime": 1786363889.5678573,
"ast_hash": "ab83ae0dbb34fd40c146a7cc1263173e",
"semantic_hash": "ab83ae0dbb34fd40c146a7cc1263173e"
},
"backend/internal/web/templates/chrome.html": {
"mtime": 1786363889.5678573,
"ast_hash": "d80b27cf3bd9d131075c485dd169dfcc",
"semantic_hash": "d80b27cf3bd9d131075c485dd169dfcc"
},
"backend/internal/web/templates/icons.html": {
"mtime": 1785873769.553139,
"ast_hash": "8e10c507c32934a92463b4bca9e34fe6",
"semantic_hash": "8e10c507c32934a92463b4bca9e34fe6"
},
"backend/internal/web/templates/list.html": {
"mtime": 1786216141.696644,
"ast_hash": "365548aace8c06559a1f66db0ae47256",
"semantic_hash": "365548aace8c06559a1f66db0ae47256"
},
"backend/internal/web/templates/login.html": {
"mtime": 1786216141.696644,
"ast_hash": "bcc3101498a66cf8b79f9d97c6c9cd6b",
"semantic_hash": "bcc3101498a66cf8b79f9d97c6c9cd6b"
},
"backend/internal/web/templates/readers.html": {
"mtime": 1786216141.696644,
"ast_hash": "c5034e76bd20a705d2799cb5ecb328f0",
"semantic_hash": "c5034e76bd20a705d2799cb5ecb328f0"
},
"backend/internal/web/templates/setup.html": {
"mtime": 1786216141.696644,
"ast_hash": "72e93c0b827414063596f7338987d879",
"semantic_hash": "72e93c0b827414063596f7338987d879"
},
"docs/adr/0001-postgresql-over-sqlite.md": {
"mtime": 1786216141.704644,
"ast_hash": "abfb08754cee58be67311377904f8ca4",
"semantic_hash": "abfb08754cee58be67311377904f8ca4"
},
"docs/adr/0002-discord-oauth-no-passwords-no-email.md": {
"mtime": 1786216141.7071996,
"ast_hash": "852a04d86659385085da6ffc8b489933",
"semantic_hash": "852a04d86659385085da6ffc8b489933"
},
"docs/adr/0003-series-shared-and-poll-owned.md": {
"mtime": 1786501942.7367291,
"ast_hash": "6138b113340693e0cc667d1ecbb75f72",
"semantic_hash": ""
},
"docs/adr/0004-wire-format-does-not-mirror-the-schema.md": {
"mtime": 1786216141.7071996,
"ast_hash": "a6ea2770dec2156f78a65b35ba06902a",
"semantic_hash": "a6ea2770dec2156f78a65b35ba06902a"
},
"docs/agents/domain.md": {
"mtime": 1786216141.7071996,
"ast_hash": "6f99318ac6cb9825b613bfde55d76091",
"semantic_hash": "6f99318ac6cb9825b613bfde55d76091"
},
"docs/agents/issue-tracker.md": {
"mtime": 1786216141.7087462,
"ast_hash": "1342e66ccb84a84fd579fb6dc0b8243a",
"semantic_hash": "1342e66ccb84a84fd579fb6dc0b8243a"
},
"docs/agents/triage-labels.md": {
"mtime": 1786216141.7087462,
"ast_hash": "69114d07ed792d6bb1d13758ba5435e1",
"semantic_hash": "69114d07ed792d6bb1d13758ba5435e1"
},
"userscript/AGENTS.md": {
"mtime": 1786499529.7762787,
"ast_hash": "e276ffe9a6e7b55fd3235466a1995c22",
"semantic_hash": ""
},
"userscript/CLAUDE.md": {
"mtime": 1786499529.7762787,
"ast_hash": "e276ffe9a6e7b55fd3235466a1995c22",
"semantic_hash": ""
},
"backend/internal/web/static/login-art.png": {
"mtime": 1786022513.9585779,
"ast_hash": "05d7863cba344a946256719a0c9ef959",
"semantic_hash": "05d7863cba344a946256719a0c9ef959"
},
"backend/internal/web/static/logo.svg": {
"mtime": 1786022513.9585779,
"ast_hash": "d0d34d0f08a25b53176cc55989b7babe",
"semantic_hash": "d0d34d0f08a25b53176cc55989b7babe"
},
"backend/internal/store/migrations/0007_covers.sql": {
"mtime": 1786262323.6964688,
"ast_hash": "6033ce0701be1236ed175362363bd96c",
"semantic_hash": "6033ce0701be1236ed175362363bd96c"
},
"docs/adr/0005-on-demand-browser.md": {
"mtime": 1786292465.8305523,
"ast_hash": "8cf2fb8c0a66c2c7d82ae433b99ca42b",
"semantic_hash": "8cf2fb8c0a66c2c7d82ae433b99ca42b"
},
"chrome/docker-compose.yml": {
"mtime": 1786292465.8305523,
"ast_hash": "5605599395a3f085904e78a2bfec1e58",
"semantic_hash": "5605599395a3f085904e78a2bfec1e58"
},
"docs/adr/0006-browser-on-the-home-machine.md": {
"mtime": 1786501942.7367291,
"ast_hash": "bfcaf39b6c7e96610a7caa00eeb36533",
"semantic_hash": ""
},
"docs/adr/0007-backend-hosts-cover-bytes.md": {
"mtime": 1786292465.8305523,
"ast_hash": "b19e38045b3dcda7dd59634ed9227a68",
"semantic_hash": "b19e38045b3dcda7dd59634ed9227a68"
},
"backend/internal/store/migrations/0008_filesystem_covers.sql": {
"mtime": 1786363889.5602942,
"ast_hash": "46cf7822d4f667e3cab36b547abe5e97",
"semantic_hash": "46cf7822d4f667e3cab36b547abe5e97"
},
"backend/internal/latest/cover.go": {
"mtime": 1786856664.2957466,
"ast_hash": "d5f2248c3d11de74bf5a3977651b17c2",
"semantic_hash": ""
},
"backend/internal/latest/cover_fetch_test.go": {
"mtime": 1786363889.5565126,
"ast_hash": "60d9eb7c59a3751baf4f31c7655217e7",
"semantic_hash": "60d9eb7c59a3751baf4f31c7655217e7"
},
"backend/internal/latest/acquire.go": {
"mtime": 1786499529.7688599,
"ast_hash": "6c1ad34bbe9f5b49d0fd1eae9093f55d",
"semantic_hash": ""
},
"backend/internal/latest/acquire_test.go": {
"mtime": 1786363889.5565126,
"ast_hash": "7bd9f41814f6bf59d8998dfb81cf990a",
"semantic_hash": "7bd9f41814f6bf59d8998dfb81cf990a"
},
"backend/internal/store/migrations/0009_series_cover_address.sql": {
"mtime": 1786363889.5602942,
"ast_hash": "4c1f6328b2e1a95828fad6d88d474c2d",
"semantic_hash": "4c1f6328b2e1a95828fad6d88d474c2d"
},
".claude/skills/implement-tickets/SKILL.md": {
"mtime": 1786417841.7117643,
"ast_hash": "3060e32e19cc571d91871a98f18afe53",
"semantic_hash": "3060e32e19cc571d91871a98f18afe53"
},
".omp/agents/ticket-implementer.md": {
"mtime": 1786417841.7163916,
"ast_hash": "0150a46c0d21c572b71b3d87d21ac925",
"semantic_hash": "0150a46c0d21c572b71b3d87d21ac925"
},
"docs/adr/0008-series-identity-is-discovered-not-derived.md": {
"mtime": 1786417841.7163916,
"ast_hash": "3ce6d64ef6a8a39f27c257b39065389e",
"semantic_hash": "3ce6d64ef6a8a39f27c257b39065389e"
},
"docs/research/lightnovelworld-chapter-vs-series-slug.md": {
"mtime": 1786417841.7163916,
"ast_hash": "74a4e538875f0a7e8ca3d5dc48c0bb53",
"semantic_hash": "74a4e538875f0a7e8ca3d5dc48c0bb53"
},
"backend/internal/latest/smoke_lnw_test.go": {
"mtime": 1786499529.7725692,
"ast_hash": "2d65da8a081759172918fdf159760f45",
"semantic_hash": ""
},
"docs/adr/0009-a-site-answers-questions-its-own-way.md": {
"mtime": 1786499529.7725692,
"ast_hash": "8039012a5b6de2359ff1a47079f51b66",
"semantic_hash": ""
},
"backend/internal/latest/read.go": {
"mtime": 1786499529.7688599,
"ast_hash": "3cf29046ddaef39fafb1df70b9f9ae8c",
"semantic_hash": ""
},
"docs/research/cloudflare-bot-scoring-and-poll-cadence.md": {
"mtime": 1786501942.7367291,
"ast_hash": "1aa17575ab20f2f36583602999a6a60f",
"semantic_hash": ""
},
"backend/internal/latest/smoke_comix_test.go": {
"mtime": 1786856356.3432186,
"ast_hash": "111fdbb75fc68ac2ab1013bc916063cf",
"semantic_hash": ""
}
}
+14 -24
View File
@@ -905,37 +905,27 @@
}
}
// Reports the newest chapter a site has published. Silent: this fires from
// Records the newest chapter a site has published. Silent: this fires from
// page visits and background checks the user did not ask for, and it never
// reorders the list — updated_at is a candidate the server discards unless
// reading progress moved.
//
// An unchanged number is still sent. It is the read that lets the backend
// skip its own poll of this series (a Sighting, issue #103), so the common
// case — visiting a series with nothing new — is exactly the one worth
// reporting. Only the local write and the re-render are skipped.
async function reportLatestChapter(existing, latest) {
async function applyLatestChapterIfChanged(existing, latest) {
if (!existing || !latest) return;
const changed = existing.latest_chapter_num !== latest.num;
let bm = existing;
if (changed) {
bm = Object.assign({}, existing, {
latest_chapter: latest.label,
latest_chapter_num: latest.num,
updated_at: Date.now(),
});
upsertLocal(bm);
render();
}
if (existing.latest_chapter_num === latest.num) return;
const bm = Object.assign({}, existing, {
latest_chapter: latest.label,
latest_chapter_num: latest.num,
updated_at: Date.now(),
});
upsertLocal(bm);
render();
// A queued write owns this row; the drain sends latest_chapter
// with it, carrying the correct bucket.
if (queueGet(bm.key)) return;
try {
const saved = await apiPut(bm.key, bm);
if (changed) {
upsertLocal(saved);
render();
}
upsertLocal(saved);
render();
} catch (e) {
/* offline — the local cache still shows it, retried on a later visit */
}
@@ -947,7 +937,7 @@
if (p.type !== "series") return;
const existing = state.byKey[keyOf(p)];
if (!existing) return;
reportLatestChapter(
applyLatestChapterIfChanged(
existing,
computeLatestChapter(p.site, anchorsFromDocument(document), p.seriesId)
);
@@ -987,7 +977,7 @@
const html = await res.text();
latest = computeLatestChapter(bm.site, anchorsFromHTML(html), bm.series_id);
}
await reportLatestChapter(state.byKey[bm.key] || bm, latest);
await applyLatestChapterIfChanged(state.byKey[bm.key] || bm, latest);
} catch (e) {
/* offline or blocked — try again after the throttle window */
}
+14 -24
View File
@@ -809,37 +809,27 @@
}
}
// Reports the newest chapter a site has published. Silent: this fires from
// Records the newest chapter a site has published. Silent: this fires from
// page visits and background checks the user did not ask for, and it never
// reorders the list — updated_at is a candidate the server discards unless
// reading progress moved.
//
// An unchanged number is still sent. It is the read that lets the backend
// skip its own poll of this series (a Sighting, issue #103), so the common
// case — visiting a series with nothing new — is exactly the one worth
// reporting. Only the local write and the re-render are skipped.
async function reportLatestChapter(existing, latest) {
async function applyLatestChapterIfChanged(existing, latest) {
if (!existing || !latest) return;
const changed = existing.latest_chapter_num !== latest.num;
let bm = existing;
if (changed) {
bm = Object.assign({}, existing, {
latest_chapter: latest.label,
latest_chapter_num: latest.num,
updated_at: Date.now(),
});
upsertLocal(bm);
render();
}
if (existing.latest_chapter_num === latest.num) return;
const bm = Object.assign({}, existing, {
latest_chapter: latest.label,
latest_chapter_num: latest.num,
updated_at: Date.now(),
});
upsertLocal(bm);
render();
// A queued write owns this row; the drain sends latest_chapter
// with it, carrying the correct bucket.
if (queueGet(bm.key)) return;
try {
const saved = await apiPut(bm.key, bm);
if (changed) {
upsertLocal(saved);
render();
}
upsertLocal(saved);
render();
} catch (e) {
/* offline — the local cache still shows it, retried on a later visit */
}
@@ -851,7 +841,7 @@
if (p.type !== "series") return;
const existing = state.byKey[keyOf(p)];
if (!existing) return;
reportLatestChapter(
applyLatestChapterIfChanged(
existing,
computeLatestChapter(p.site, anchorsFromDocument(document), p.seriesId)
);
@@ -893,7 +883,7 @@
if (!res.ok) continue;
const html = await res.text();
const latest = computeLatestChapter(bm.site, anchorsFromHTML(html), bm.series_id);
await reportLatestChapter(state.byKey[bm.key] || bm, latest);
await applyLatestChapterIfChanged(state.byKey[bm.key] || bm, latest);
} catch (e) {
/* offline or blocked — try again after the throttle window */
}