feat: one registry entry per Site, one shared Series-page read #95

Merged
sulthan merged 7 commits from feat/94-site-registry into main 2026-08-12 05:43:39 +07:00
13 changed files with 13296 additions and 12321 deletions
+9
View File
@@ -70,6 +70,15 @@ Reader present. Performed once per Series no matter how many Readers bookmarked
a Poll is work done on behalf of the Series, never on behalf of a Reader.
_Avoid_: scrape, refresh, check, sync
**Acquisition**:
The single read of a Series page made the moment the Series first exists, giving it
both its Latest Chapter and its Cover without waiting out the Poll queue. Distinct
from a Poll in the two ways that matter: a Reader is present — it is triggered by
their first Bookmark of that Series — and it is the only read that establishes a
Cover rather than refreshing facts. It happens once in a Series's life; every later
read of the same page is a Poll.
_Avoid_: initial poll, first fetch, prefetch, warm-up
**New Chapter**:
The state where Latest Chapter is ahead of Progress. The single condition the ember
accent is permitted to signal.
+25 -26
View File
@@ -2,6 +2,7 @@ package latest
import (
"context"
"errors"
"log"
"sync"
"time"
@@ -97,51 +98,49 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
if a.Fetch == nil && a.BrowserFetch == nil {
return
}
// series_url arrives in a client-supplied PUT body, so the same gate the
// poller uses applies here — without it a token-holder chooses what the
// server fetches from its own network position.
if !fetchableSeriesURL(sr.Site, sr.SeriesURL) {
log.Printf("acquire %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
return
}
f := fetcherFor(sr.Site, a.BrowserFetch, a.Fetch)
if f == nil {
log.Printf("acquire %q: no fetcher for site %q", sr.Key(), sr.Site)
return
}
body, status, err := f.Get(ctx, sr.SeriesURL)
facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, a.BrowserFetch, a.Fetch)
if err != nil {
log.Printf("acquire %q: fetch %s: %v", sr.Key(), sr.SeriesURL, err)
return
}
if status != 200 {
log.Printf("acquire %q: fetch %s: status %d", sr.Key(), sr.SeriesURL, status)
switch {
case errors.Is(err, errNotFetchable):
// series_url arrives in a client-supplied PUT body, so without the
// gate a token-holder chooses what the server fetches from its own
// network position.
log.Printf("acquire %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
case errors.Is(err, errNoFetcher):
log.Printf("acquire %q: no fetcher for site %q", sr.Key(), sr.Site)
default:
log.Printf("acquire %q: %v", sr.Key(), err)
}
return
}
// This page just served the same purpose a poll tick would have; without
// the stamp the row stays due and the poller refetches it immediately.
//
// Stamped after success — the reverse of the poller, which stamps before
// the fetch: the Reader is here, watching the Series they just created, so
// a failed acquisition must leave the row due for a fast retry rather than
// consuming the cooldown. The stamp happens even when the page read
// succeeded but produced no facts to persist.
if err := a.Store.MarkLatestChecked(sr.Site, sr.SeriesID, time.Now().UnixMilli()); err != nil {
log.Printf("acquire %q: mark checked: %v", sr.Key(), err)
}
if latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body); ok {
if err := a.Store.SetLatestChapter(sr.Site, sr.SeriesID, latest.Label, latest.Num); err != nil {
if facts.HasLatest {
if err := a.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil {
log.Printf("acquire %q: set latest chapter: %v", sr.Key(), err)
}
}
cover, ok := coverFrom(sr.Site, sr.SeriesURL, body)
if !ok {
if !facts.HasCover {
return
}
bytes, contentType, err := fetchCoverBytes(ctx, cover, a.BrowserCoverFetch, a.Covers)
bytes, contentType, err := fetchCoverBytes(ctx, facts.Cover, a.BrowserCoverFetch, a.Covers)
if err != nil {
log.Printf("acquire %q: fetch cover %s: %v", sr.Key(), cover, err)
log.Printf("acquire %q: fetch cover %s: %v", sr.Key(), facts.Cover, err)
return
}
if err := a.Store.SetSeriesCover(sr.Site, sr.SeriesID, cover, bytes, contentType); err != nil {
if err := a.Store.SetSeriesCover(sr.Site, sr.SeriesID, facts.Cover, bytes, contentType); err != nil {
log.Printf("acquire %q: persist cover: %v", sr.Key(), err)
}
}
+49 -36
View File
@@ -86,46 +86,59 @@ func (f *BrowserFetcher) Close() {
f.cancel()
}
// Get navigates to seriesURL, lets any challenge resolve, then reads either the
// site's JSON API (kagane) from inside the page so the request carries the
// clearance cookie, or the served HTML itself (novelfull) — see
// novelfullSeriesURL for the latter case. The returned body is whatever the
// site's chapter list lives in, which is what latestChapterFrom's per-site
// switch expects.
// Get navigates to seriesURL, lets any challenge resolve, then reads the
// payload the Site's registry entry describes — kagane's chapter-list API from
// inside the page so the request carries the clearance cookie, novelfull's
// served HTML. The returned body is whatever the Site's chapter list lives in,
// which is what the entry's LatestChapter parse expects.
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
apiURL, isKagane := kaganeAPIURL(seriesURL)
if !isKagane && !novelfullSeriesURL(seriesURL) {
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
}
var body string
// kagane's chapter list is only in its JSON API, which must be called from
// inside the page so the request carries the clearance cookie. novelfull
// renders its chapters into the HTML, so the cleared DOM is the answer.
// chromedp.OuterHTML returns a QueryAction and chromedp.Evaluate an
// EvaluateAction, so the variable has to be the interface both implement.
var read chromedp.Action = chromedp.OuterHTML("html", &body, chromedp.ByQuery)
if isKagane {
read = chromedp.Evaluate(
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
&body,
awaitPromise,
)
}
// novelfull's payload is the DOM itself, and the interstitial has a DOM
// too, so "we have an answer" has to exclude it explicitly. kagane's
// in-page fetch just fails while challenged, which is already the signal.
done := func() bool { return body != "" && (isKagane || !isInterstitial(body)) }
if err := f.run(ctx, seriesURL, read, done); err != nil {
// Challenge never cleared, or the API refused. Indistinguishable from
// here and handled identically by the caller.
if errors.Is(err, errChallengeHeld) {
return "", 403, nil
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
// entries' Read funcs are expected to refuse any address owned by another
// Site, and the loop must not depend on that staying true.
for _, name := range browserBackedSites() {
s := sites[name]
read, ok := s.Browser.Read(seriesURL, &body)
if !ok {
continue
}
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
if err := f.run(ctx, seriesURL, read,
func() bool { return s.Browser.Done(body) }); err != nil {
// Challenge never cleared, or the payload was refused.
// Indistinguishable from here and handled identically by the caller.
if errors.Is(err, errChallengeHeld) {
return "", 403, nil
}
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
}
return body, 200, nil
}
return body, 200, nil
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
}
// kaganeRead builds the in-tab fetch of kagane's chapter-list API: the
// request must be made from inside the page so it carries the clearance
// cookie, and the API is the only place the list exists. Refusing any other
// address is the per-Site half of the SSRF gate, kept deliberately behind
// fetchableSeriesURL (see browserRead.Read).
func kaganeRead(seriesURL string, out *string) (chromedp.Action, bool) {
apiURL, ok := kaganeAPIURL(seriesURL)
if !ok {
return nil, false
}
return chromedp.Evaluate(
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
out, awaitPromise), true
}
// novelfullRead reads the cleared DOM. novelfull renders its chapter list
// into the served HTML, so there is no API to call from inside the page — the
// challenge-cleared DOM is the payload.
func novelfullRead(seriesURL string, out *string) (chromedp.Action, bool) {
if !novelfullSeriesURL(seriesURL) {
return nil, false
}
return chromedp.OuterHTML("html", out, chromedp.ByQuery), true
}
// Image retrieves one cover's bytes through the browser sidecar, and its
+63 -93
View File
@@ -2,9 +2,9 @@ package latest
import (
"context"
"errors"
"log"
"net/url"
"slices"
"time"
"bookmarkmanager/backend/internal/store"
@@ -57,25 +57,23 @@ type Poller struct {
Batch int
}
var browserBackedSites = []string{"kagane", "novelfull"}
// fillBlankCover gives a Series its Cover when it has none. The blank state is
// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows
// created before acquisition existed, and rows whose creation-time fetch
// failed, both heal here. A non-blank CoverAddress is left alone — refetching
// would add a request per Series per cycle and change artwork under the Reader
// for no visible reason. A row that already carries a source URL is owned by
// prefetchCover instead; this path only extracts from the series page.
// prefetchCover instead; this path only records a Cover address already
// extracted from the series page.
//
// Failures are logged against the Series and never returned: the chapter poll
// must not notice. A failed fill is retried the next time this Series is due;
// there is no separate retry queue.
func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, body string) {
func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, cover string) {
if sr.CoverAddress != "" || sr.Cover != "" {
return
}
cover, ok := coverFrom(sr.Site, sr.SeriesURL, body)
if !ok {
if cover == "" {
return
}
p.storeCover(ctx, sr, cover)
@@ -120,27 +118,29 @@ func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL stri
}
// fetcherFor returns the fetcher a site's page needs, or nil when the site
// cannot be fetched at all right now. kagane and novelfull pages sit behind a
// Cloudflare JavaScript challenge that no TLS fingerprint clears (kagane
// verified 2026-08-03, novelfull verified 2026-08-05, both against the same
// Chrome_133 profile TLSFetcher uses), so both prefer the browser; novelfull
// alone falls back to the plain-TLS fetcher when no browser is configured,
// because its challenge is a live time-varying fact (AGENTS.md) and its cover
// bytes never need the browser. kagane never falls back: a plain fetch of a
// kagane page or cover would only ever retrieve a challenge page. One routing
// rule for the poll and the acquirer, so the two cannot drift apart.
// cannot be fetched at all right now. A Site whose registry entry carries a
// Browser read — kagane and novelfull, both behind a Cloudflare JavaScript
// challenge no TLS fingerprint clears — prefers the browser; when it is
// absent, the entry's Fallback decides whether plain TLS may take over. One
// routing rule for the poll and the acquirer, so the two cannot drift apart.
func fetcherFor(site string, browser, tls Fetcher) Fetcher {
switch {
case site == "kagane":
return browser
case slices.Contains(browserBackedSites, site): // novelfull
if browser != nil {
return browser
}
return tls
default:
s, known := sites[site]
if !known {
// No registry entry means nothing to fetch or parse; fail closed even
// though the only caller gates first, so a future caller that skips
// the gate cannot hand an arbitrary https URL to the TLS fetcher.
return nil
}
if s.Browser == nil {
return tls
}
if browser != nil {
return browser
}
if s.Browser.Fallback {
return tls
}
return nil
}
// Run polls until ctx is cancelled.
@@ -170,7 +170,7 @@ func (p *Poller) runOnce(ctx context.Context) {
now := p.Now()
cutoff := now.Add(-p.Cooldown).UnixMilli()
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites(), p.Batch)
if err != nil {
log.Printf("latest poll: due query: %v", err)
return
@@ -224,43 +224,35 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
return
}
// series_url is client-supplied (PUT /bookmarks/{key} accepts any string),
// so this is not just an optimisation against burning a request on an
// unknown site: without it, the server would issue a GET from its own
// network position to whatever URL a token-holder writes, including
// link-local/internal addresses or non-https schemes. The cooldown above
// is already consumed, so a row that never passes this check is retried at
// cooldown pace rather than hot-looping.
if !fetchableSeriesURL(sr.Site, sr.SeriesURL) {
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
return
}
f := fetcherFor(sr.Site, p.BrowserFetch, p.Fetch)
if f == nil {
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
return
}
p.prefetchCover(ctx, sr)
body, status, err := f.Get(ctx, sr.SeriesURL)
facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, p.BrowserFetch, p.Fetch)
if err != nil {
log.Printf("latest poll %q: fetch %s: %v", sr.Key(), sr.SeriesURL, err)
switch {
case errors.Is(err, errNotFetchable):
// The cooldown above is already consumed, so a row that never
// passes the gate is retried at cooldown pace rather than
// hot-looping.
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
return
case errors.Is(err, errNoFetcher):
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
return
}
// A legacy cover heals independently of the page read: its source may
// answer — a CDN — while the origin does not, so a fetch failure does
// not skip the heal, matching the order the shared read replaced.
p.prefetchCover(ctx, sr)
log.Printf("latest poll %q: %v", sr.Key(), err)
return
}
if status != 200 {
log.Printf("latest poll %q: fetch %s: status %d", sr.Key(), sr.SeriesURL, status)
return
}
latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body)
// A legacy cover source is healed independently of the page read.
p.prefetchCover(ctx, sr)
// Cover fill is independent of the chapter signal: a page that lost its
// chapter list may keep its og:image, and a blank Series heals either way.
p.fillBlankCover(ctx, sr, body)
if !ok {
p.fillBlankCover(ctx, sr, facts.Cover)
if !facts.HasLatest {
// Most likely a challenge page or a layout change. Either way the row is
// already stamped, so this waits out a cooldown instead of hot-looping.
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), len(body))
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
return
}
@@ -268,7 +260,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
// chapter should correct the stored number downward. The comparison is
// against the due-query snapshot; a concurrent write in between only costs
// one redundant UPDATE of the same absolute value, never a wrong one.
if sr.LatestChapterNum != nil && *sr.LatestChapterNum == latest.Num {
if sr.LatestChapterNum != nil && *sr.LatestChapterNum == facts.Latest.Num {
return
}
@@ -276,52 +268,30 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
// bookmark joining to it, and the bookmark's updated_at is never touched —
// a newly published chapter is not reading progress and must not reorder
// the list.
if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, latest.Label, latest.Num); err != nil {
if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil {
log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err)
return
}
log.Printf("latest poll %q: latest is now %s", sr.Key(), latest.Label)
log.Printf("latest poll %q: latest is now %s", sr.Key(), facts.Latest.Label)
}
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
// to parse and seriesURL is safe to hand to a fetcher: an https URL with a
// non-empty host. series_url comes from client-supplied PUT bodies, so this is
// a defence against the poller being used to probe arbitrary hosts from the
// server's own network position, not just a check against wasted requests.
//
// Three sites are held to a stricter rule, each for a different reason:
//
// - kagane and novelfull are fetched by a headless browser, which executes
// JavaScript and carries cookies, and is therefore a far stronger SSRF
// primitive than an HTTP GET. Their hosts must match exactly, not merely
// be non-empty.
// - lightnovelworld's parser regex hardcodes its host, so a URL anywhere
// else could never yield a match — reject it here rather than burn the
// request.
// fetchableSeriesURL reports whether site is a Site the registry knows and
// seriesURL is safe to hand to a fetcher: an https URL whose host matches the
// Site's pinned hostname exactly. series_url comes from client-supplied PUT
// bodies, so this is a defence against the poller being used to probe
// arbitrary hosts from the server's own network position, not just a check
// against wasted requests. The pin guards different things per Site — a
// browser Site guards a control that executes JavaScript and carries cookies,
// a parser Site guards a wasted request — but the rule is one rule, from the
// registry.
func fetchableSeriesURL(site, seriesURL string) bool {
switch site {
case "asura", "demonic", "comix", "kagane", "novelfull", "lightnovelworld":
default:
s, known := sites[site]
if !known {
return false
}
u, err := url.Parse(seriesURL)
if err != nil {
return false
}
if u.Scheme != "https" || u.Host == "" {
return false
}
switch site {
case "kagane":
return u.Hostname() == "kagane.to"
case "novelfull":
// Fetched by a real browser, same as kagane, so the host is pinned
// rather than merely non-empty.
return u.Hostname() == "novelfull.com"
case "lightnovelworld":
// Its parser regex hardcodes this host, so a URL anywhere else could
// never yield a match — reject it here rather than burn the request.
return u.Hostname() == "lightnovelworld.net"
}
return true
return u.Scheme == "https" && u.Hostname() == s.Host
}
+9
View File
@@ -623,6 +623,13 @@ func TestFetchableSeriesURL(t *testing.T) {
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
// The three plain-TLS Sites are pinned too: a client-supplied
// series_url must not aim a fetcher at a lookalike host, even when
// the fetcher is only an HTTP GET.
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
{"comix on a foreign host", "comix", "https://evil.example/title/x", false},
// The browser fetcher runs JavaScript and carries cookies, so a
// client-supplied series_url must not be able to aim it anywhere else.
{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
@@ -952,6 +959,8 @@ func TestFetcherForRoutesNovelSites(t *testing.T) {
// browser-less deployment: kagane is nothing, novelfull degrades to TLS
{"kagane", nil, tls, nil},
{"novelfull", nil, tls, tls},
// unknown site: fail closed — nothing to fetch or parse
{"mangadex", browser, tls, nil},
}
for _, tc := range cases {
t.Run(tc.site, func(t *testing.T) {
+58
View File
@@ -0,0 +1,58 @@
package latest
import (
"context"
"errors"
"fmt"
)
// seriesRead carries the two facts the poll and the acquirer both extract
// from a series page. Persistence, stamps and scheduling stay with the
// callers, so the policies that keep the two flows distinct (stamp order,
// cooldowns) are not swallowed by the module.
type seriesRead struct {
Latest latestChapter
HasLatest bool
Cover string
HasCover bool
// BodyLen is the fetched body's length, surfaced because the no-chapter
// log uses it to tell a markup change from a body the size cap cut short.
BodyLen int
}
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
// failures so each caller keeps its own distinct log line for all three.
var (
errNotFetchable = errors.New("series url not fetchable")
errNoFetcher = errors.New("no fetcher for site")
)
// readSeriesPage performs the series-page read the poll and the acquirer have
// in common: gate the address, choose the route, fetch the page, extract the
// Latest Chapter and the Cover address. It persists nothing and stamps
// nothing.
//
// series_url arrives in a client-supplied PUT body (PUT /bookmarks/{key}
// accepts any string), so the gate is not an optimisation against burning a
// request on an unknown site: without it, the server would issue a GET from
// its own network position to whatever URL a token-holder writes, including
// link-local/internal addresses or non-https schemes.
func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fetcher) (seriesRead, error) {
if !fetchableSeriesURL(site, seriesURL) {
return seriesRead{}, fmt.Errorf("%w: site=%q url=%q", errNotFetchable, site, seriesURL)
}
f := fetcherFor(site, browser, tls)
if f == nil {
return seriesRead{}, fmt.Errorf("%w: site %q", errNoFetcher, site)
}
body, status, err := f.Get(ctx, seriesURL)
if err != nil {
return seriesRead{}, fmt.Errorf("fetch %s: %w", seriesURL, err)
}
if status != 200 {
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
}
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
cover, hasCover := coverFrom(site, seriesURL, body)
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
}
+231 -93
View File
@@ -6,8 +6,11 @@ import (
"log"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"github.com/chromedp/chromedp"
)
// latestChapter is the newest chapter a series page advertises.
@@ -16,6 +19,40 @@ type latestChapter struct {
Label string
}
// site answers the fixed questions every series-page read asks of its Site
// (ADR-0009): the host its addresses must carry, how to find the Latest
// Chapter and the Cover address in a body, and — for a Site behind a
// JavaScript challenge — how to read its payload from a cleared tab. One
// entry describes everything about one Site, and nowhere else gets to compare
// the site string.
type site struct {
// Host is the exact hostname a series_url for this Site must carry.
Host string
// LatestChapter finds the newest chapter in a fetched body.
LatestChapter func(seriesURL, body string) (latestChapter, bool)
// Cover finds the Cover address in a fetched body.
Cover func(seriesURL, body string) (string, bool)
// Browser reads this Site's payload from a cleared browser tab; nil
// means the page is fetched over plain TLS.
Browser *browserRead
}
type browserRead struct {
// Read builds the tab read for seriesURL, refusing (false) an address
// this Site will not open in a browser — the per-Site half of the SSRF
// gate, kept deliberately behind fetchableSeriesURL: a headless browser
// executes JavaScript and carries cookies, and series_url is
// client-supplied.
Read func(seriesURL string, out *string) (chromedp.Action, bool)
// Done reports whether the payload arrived.
Done func(body string) bool
// Fallback allows the plain-TLS fetcher when no browser is configured.
// False skips the Site instead. kagane is false — a plain fetch would
// only ever retrieve a challenge page — and novelfull is true, because
// its challenge is a live time-varying fact (AGENTS.md).
Fallback bool
}
// asuraSlugRe pulls the series slug out of a stored series_url.
// Shape verified live 2026-07-26: https://asurascans.com/comics/<slug>, where
// the slug carries a trailing build-hash suffix (e.g. "-f886a8af") that
@@ -66,7 +103,7 @@ var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`)
// under several Chapter Slugs (measured 2026-08-11: a sampled novel serves
// 1-99 under one slug and 100-423 under another), so no stored-slug pattern can
// cover a Series' whole list. An unscoped match is safe because
// latestChapterFrom truncates the body at the comment thread before scanning
// lnwLatestChapter truncates the body at the comment thread before scanning
// (lnwCommentMarker); without that, a visitor's comment could set the Latest
// Chapter on the shared Series row.
var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-([0-9.]+)/`)
@@ -80,86 +117,16 @@ var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-(
// body is skipped, never scanned whole.
const lnwCommentMarker = "wpd-threads"
// latestChapterFrom returns the highest chapter number body advertises for this
// series. ok is false when the body yields nothing usable — an unknown site, an
// empty body, a Cloudflare challenge page, and a site redesign all land here,
// and the caller treats all four identically.
//
// Ported from the userscript's latestChapterFromAnchors (asura L123-133,
// demonic L183-193), including its reason for taking a maximum rather than a
// first or last: neither site lists chapters in a dependable order.
// maxChapter returns the highest chapter number the regex finds in body. A
// maximum rather than a first or last, ported from the userscript's
// latestChapterFromAnchors (asura L123-133, demonic L183-193): neither site
// lists chapters in a dependable order.
//
// The userscript's asura rule additionally requires the anchor text to match
// /Chapter\s+[\d.]+/i. That check exists only to skip the "First Chapter"
// shortcut, which points at chapter/1 and therefore can never win a maximum, so
// it is redundant here. For asura, scoping the pattern to this series' own slug
// replaces it with a stronger guarantee: a chapter link belonging to some other
// series cannot contribute even if the page starts carrying them. demonic has no
// such guarantee — demonicChapterRe matches any chaptered.php?manga=<id> anchor
// with no per-series scoping, because the stored series_id for demonic is a
// slug, not the numeric id the URL carries, so it cannot easily be scoped.
// lightnovelworld is unscoped and body-truncated instead — see lnwChapterRe.
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
var re *regexp.Regexp
switch site {
case "asura":
m := asuraSlugRe.FindStringSubmatch(seriesURL)
if m == nil {
return latestChapter{}, false
}
// Stored URLs predating a redeploy may carry a stale build hash;
// chapter hrefs in the fetched body carry the current one. Strip to
// the stable ID and make the hash optional in the pattern, so scoping
// survives rotations.
slug := asuraBuildHash.ReplaceAllString(m[1], "")
// Compiled per call rather than cached: this runs once per fetch, which
// is at most a few times a minute, and the slug varies per series.
re = regexp.MustCompile(`/comics/` + regexp.QuoteMeta(slug) + `(?:-[0-9a-f]{8})?/chapter/([0-9.]+)`)
case "demonic":
re = demonicChapterRe
case "comix":
id, ok := comixSeriesID(seriesURL)
if !ok {
return latestChapter{}, false
}
// comix ships an SPA: the served HTML carries a JSON state blob instead
// of chapter anchors, and latestChapterUrl is the only place the newest
// chapter appears. Scoping to this series' id prefix keeps a
// "recommended" strip's entries from winning the maximum.
re = regexp.MustCompile(`"latestChapterUrl":"/title/` + regexp.QuoteMeta(id) + `-[^"]*-chapter-([0-9.]+)"`)
case "kagane":
re = kaganeChapterRe
case "novelfull":
u, err := url.Parse(seriesURL)
if err != nil {
return latestChapter{}, false
}
m := novelfullSlugRe.FindStringSubmatch(u.Path)
if m == nil {
return latestChapter{}, false
}
// Scoped to this series' slug for the same reason asura is: page 1
// carries a "latest chapters" widget and a "you may also like" strip,
// and neither may contribute to the maximum.
re = regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`)
case "lightnovelworld":
// The comment thread below the chapter list is the one region of the
// page any visitor can write to, so the scan never reads past it (see
// lnwChapterRe). A body without the marker is skipped, never scanned
// whole — a redesign must degrade into staleness, not into a wrong
// shared value; the logged body length tells a markup change from a
// body the size cap cut short.
i := strings.Index(body, lnwCommentMarker)
if i < 0 {
log.Printf("latest poll %q: no %s marker in %d bytes", seriesURL, lnwCommentMarker, len(body))
return latestChapter{}, false
}
body = body[:i]
re = lnwChapterRe
default:
return latestChapter{}, false
}
// shortcut, which points at chapter/1 and therefore can never win a maximum,
// so it is redundant once a maximum is taken.
func maxChapter(re *regexp.Regexp, body string) (latestChapter, bool) {
var best latestChapter
found := false
for _, m := range re.FindAllStringSubmatch(body, -1) {
@@ -178,6 +145,94 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
return best, found
}
// asuraLatestChapter scopes chapter links to this series' own slug, which
// replaces the userscript's anchor-text check with a stronger guarantee: a
// chapter link belonging to some other series cannot contribute even if the
// page starts carrying them.
func asuraLatestChapter(seriesURL, body string) (latestChapter, bool) {
m := asuraSlugRe.FindStringSubmatch(seriesURL)
if m == nil {
return latestChapter{}, false
}
// Stored URLs predating a redeploy may carry a stale build hash; chapter
// hrefs in the fetched body carry the current one. Strip to the stable ID
// and make the hash optional in the pattern, so scoping survives
// rotations.
slug := asuraBuildHash.ReplaceAllString(m[1], "")
// Compiled per call rather than cached: this runs once per fetch, which is
// at most a few times a minute, and the slug varies per series.
re := regexp.MustCompile(`/comics/` + regexp.QuoteMeta(slug) + `(?:-[0-9a-f]{8})?/chapter/([0-9.]+)`)
return maxChapter(re, body)
}
// demonicLatestChapter is not scoped: demonicChapterRe matches any
// chaptered.php?manga=<id> anchor, because the stored series_id is a slug,
// not the numeric id the URL carries, so it cannot be scoped.
func demonicLatestChapter(_, body string) (latestChapter, bool) {
return maxChapter(demonicChapterRe, body)
}
// comixLatestChapter reads comix's SPA: the served HTML carries a JSON state
// blob instead of chapter anchors, and latestChapterUrl is the only place the
// newest chapter appears. Scoping to this series' id prefix keeps a
// "recommended" strip's entries from winning the maximum.
func comixLatestChapter(seriesURL, body string) (latestChapter, bool) {
id, ok := comixSeriesID(seriesURL)
if !ok {
return latestChapter{}, false
}
re := regexp.MustCompile(`"latestChapterUrl":"/title/` + regexp.QuoteMeta(id) + `-[^"]*-chapter-([0-9.]+)"`)
return maxChapter(re, body)
}
// kaganeLatestChapter scans the kagane series API JSON that the browser read
// fetched from inside the page; the match rides on the property name,
// regardless of the surrounding JSON shape.
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
return maxChapter(kaganeChapterRe, body)
}
// novelfullLatestChapter is scoped to this series' slug for the same reason
// asura is: page 1 carries a "latest chapters" widget and a "you may also
// like" strip, and neither may contribute to the maximum.
func novelfullLatestChapter(seriesURL, body string) (latestChapter, bool) {
u, err := url.Parse(seriesURL)
if err != nil {
return latestChapter{}, false
}
m := novelfullSlugRe.FindStringSubmatch(u.Path)
if m == nil {
return latestChapter{}, false
}
re := regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`)
return maxChapter(re, body)
}
// lnwLatestChapter truncates the body at the comment thread before scanning:
// it is the one region of the page any visitor can write to (see lnwChapterRe).
// A body without the marker is skipped, never scanned whole — a redesign must
// degrade into staleness, not into a wrong shared value; the logged body length
// tells a markup change from a body the size cap cut short.
func lnwLatestChapter(seriesURL, body string) (latestChapter, bool) {
i := strings.Index(body, lnwCommentMarker)
if i < 0 {
log.Printf("latest poll %q: no %s marker in %d bytes", seriesURL, lnwCommentMarker, len(body))
return latestChapter{}, false
}
return maxChapter(lnwChapterRe, body[:i])
}
// latestChapterFrom returns the highest chapter number body advertises for this
// series, via the Site's registry entry. ok is false when the body yields
// nothing usable — an unknown site, an empty body, a Cloudflare challenge page,
// and a site redesign all land here, and the caller treats all four identically.
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
if fn := sites[site].LatestChapter; fn != nil {
return fn(seriesURL, body)
}
return latestChapter{}, false
}
var metaTagRe = regexp.MustCompile(`(?is)<meta\b[^>]*>`)
var doubleQuotedMetaAttrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*"([^"]*)"`)
var singleQuotedMetaAttrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*'([^']*)'`)
@@ -257,24 +312,40 @@ func comixCoverURL(seriesURL, body string) string {
return publishedCoverURL(detail.Poster.Medium)
}
// coverFrom reports false for unknown sites, challenge bodies, and pages with
// no usable cover. Metadata extraction keeps scanning after an empty match so
// a later published cover is not hidden by an empty tag.
func coverFrom(site, seriesURL, body string) (string, bool) {
var cover string
switch site {
case "asura", "demonic", "lightnovelworld":
cover = metaContent(body, "property", "og:image")
case "novelfull":
cover = metaContent(body, "name", "image")
case "comix":
cover = comixCoverURL(seriesURL, body)
case "kagane":
cover = kaganeCoverURL(body)
}
// ogImageCover reads the og:image metadata shared by asura, demonic and
// lightnovelworld.
func ogImageCover(_, body string) (string, bool) {
cover := metaContent(body, "property", "og:image")
return cover, cover != ""
}
func novelfullCoverEntry(_, body string) (string, bool) {
cover := metaContent(body, "name", "image")
return cover, cover != ""
}
func comixCoverEntry(seriesURL, body string) (string, bool) {
cover := comixCoverURL(seriesURL, body)
return cover, cover != ""
}
func kaganeCoverEntry(_, body string) (string, bool) {
cover := kaganeCoverURL(body)
return cover, cover != ""
}
// coverFrom reports false for unknown sites, challenge bodies, and pages with
// no usable cover, via the Site's registry entry.
func coverFrom(site, seriesURL, body string) (string, bool) {
if fn := sites[site].Cover; fn != nil {
return fn(seriesURL, body)
}
return "", false
}
// metaContent returns the content of the first <meta> whose attrName is
// attrValue. It keeps scanning after an empty match so a later published cover
// is not hidden by an empty tag.
func metaContent(body, attrName, attrValue string) string {
for _, tag := range metaTagRe.FindAllString(body, -1) {
attrs := make(map[string]string)
@@ -297,3 +368,70 @@ func publishedCoverURL(value string) string {
value = strings.TrimSpace(html.UnescapeString(value))
return strings.ReplaceAll(value, " ", "%20")
}
// sites is the registry: one entry per Site, keyed by the stored site string.
// Adding a Site means adding an entry here and nowhere else — the dispatch
// functions above and the poller's route list are lookups into this map. An
// unknown site string resolves to the zero entry, which fails the existing
// not-fetchable and no-fetcher paths unchanged.
var sites = map[string]site{
"asura": {
Host: "asurascans.com",
LatestChapter: asuraLatestChapter,
Cover: ogImageCover,
},
"demonic": {
Host: "demonicscans.org",
LatestChapter: demonicLatestChapter,
Cover: ogImageCover,
},
"comix": {
Host: "comix.to",
LatestChapter: comixLatestChapter,
Cover: comixCoverEntry,
},
"kagane": {
Host: "kagane.to",
LatestChapter: kaganeLatestChapter,
Cover: kaganeCoverEntry,
Browser: &browserRead{
Read: kaganeRead,
Done: func(body string) bool { return body != "" },
// Never falls back: a plain fetch of a kagane page or cover would
// only ever retrieve a challenge page (verified 2026-08-03).
Fallback: false,
},
},
"novelfull": {
Host: "novelfull.com",
LatestChapter: novelfullLatestChapter,
Cover: novelfullCoverEntry,
Browser: &browserRead{
Read: novelfullRead,
// The interstitial has a DOM too, so "the payload arrived" has to
// exclude it explicitly.
Done: func(body string) bool { return body != "" && !isInterstitial(body) },
Fallback: true,
},
},
"lightnovelworld": {
Host: "lightnovelworld.net",
LatestChapter: lnwLatestChapter,
Cover: ogImageCover,
},
}
// browserBackedSites is derived from the registry: the Sites whose pages are
// read through the browser sidecar, which are also the ones granted the longer
// cooldown. Sorted so callers that range it (the due query, the browser
// fetcher's dispatch) see a stable order instead of map-iteration noise.
func browserBackedSites() []string {
out := make([]string, 0, len(sites))
for name, s := range sites {
if s.Browser != nil {
out = append(out, name)
}
}
sort.Strings(out)
return out
}
@@ -0,0 +1,93 @@
# ADR-0009: A Site answers fixed questions; an unusual Site owns its own fetch
Date: 2026-08-11
Status: accepted
## Decision
Per-Site knowledge lives in one registry in `backend/internal/latest/sites.go`,
keyed by the stored site string. An entry answers a fixed set of questions: the
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
how to find the Cover address in a body, and — for a Site behind a JavaScript
challenge — how to read its payload from a cleared tab, how to tell that the
payload arrived, and whether the plain-TLS fetcher may take over when no
browser is configured (`Fallback`; kagane never falls back, novelfull does,
each on measured evidence).
The question set does not grow to accommodate one Site. When a Site needs
something the set cannot express, that Site gets an optional override and
performs its own fetch, leaving the other entries untouched. The override is a
per-Site escape hatch, not a stage every Site passes through, and it is added to
the registry type when the first Site needs it rather than in advance.
Two things stay outside the registry. Cover *bytes* are routed by address shape
in `fetchCoverBytes`, never by Site name, so the Poll and the Acquisition cannot
drift apart. And the host pin inside a browser entry's read is kept even though
`fetchableSeriesURL` has already pinned the same host: a headless browser is a
strong SSRF primitive and `series_url` arrives in a client-supplied PUT body, so
the second check is deliberate and must not be deduplicated.
## Why
Before the registry, the site string was compared in six places across three
files: the Latest Chapter switch (`sites.go:102`), the Cover switch
(`sites.go:263`), the browser-backed list and the fetcher choice
(`poller.go:60`, `poller.go:132`), the host pins (`poller.go:314-325`), and the
payload read (`browser.go:96-119`). Nothing tied them together, so adding a
seventh Site meant finding all six unaided, and a Site added to five of them
failed at the sixth in production rather than at compile time.
The Sites are not alike and the registry does not ask them to be. asura strips a
rotating build hash from its slug before scoping a regex; comix reads a JSON
blob embedded in server-rendered HTML; kagane's chapter list exists only in its
JSON API, which must be called from inside the page so the request carries the
clearance cookie; lightnovelworld must truncate the body at the comment thread
first. What they have in common is not behaviour, it is the questions they
answer. Arbitrary behaviour behind one entry is the point.
Making a browser Site contribute a read and a completion test, rather than
letting it drive the browser, was chosen because the tab lifecycle in
`BrowserFetcher.run` is load-bearing and shared. It holds one tab open across
re-reads, because a Cloudflare interstitial needs several seconds of live page
to solve itself and write clearance into the shared cookie jar; reading once and
closing the tab, which is what this did before 2026-08-08, never clears
anything. It also serialises the browser, binds the caller's deadline to the
tab, distinguishes a lost browser from a retryable read, and paces re-reads.
Spreading that across per-Site adapters would put one subtle, measured loop
behind six doors.
This costs the adapters little, because `chromedp.Run` takes an Action and
`chromedp.Tasks` is an Action. A Site that must click, wait on a selector, and
then evaluate expresses all of it as its read. Only a Site needing something
outside the per-tab loop — its own cadence, two tabs, a tab held between calls,
cookies set before navigation — falls outside, and that Site takes the override.
## Considered options
**Widen the shared interface whenever a Site needs something new.** Rejected:
one Site's requirement becomes a field on all seven entries, and the entries
that ignore it still have to be read and understood by anyone adding the eighth.
**Give every Site the whole fetch.** Rejected: it makes the browser lifecycle
above a per-Site concern, and pulls `chromedp` into adapters for five Sites that
never open a browser.
**A Go `interface` with a method set instead of a registry of records.**
Rejected: most Sites differ in one or two answers, and three share a single
Cover implementation, so a method set produces near-empty types. A missing
answer is a nil value caught at dispatch, which is where an unknown Site is
already handled.
## Consequences
Adding a Site is one registry entry. The existing dispatch functions —
`latestChapterFrom`, `coverFrom`, `fetchableSeriesURL` — become registry
lookups, so the table tests that drive them by site string are unchanged.
A future architecture review will see an override that only one Site uses and
read it as an inconsistency to collapse. It is not. Collapsing it means either
widening the question set for every Site or moving the shared tab lifecycle into
the adapters, and both were rejected here on the evidence above.
An unknown site string resolves to the zero entry and fails the existing
not-fetchable and no-fetcher paths, which log and skip. That is unchanged.
+4 -16
View File
@@ -25,12 +25,12 @@
"23": "Browser Entrypoint",
"24": "Memory Allocation & GC",
"25": "Cover Fetcher Tests",
"26": "Deployment & Cover ADRs",
"26": "readSeries",
"27": "Find Skills Guide",
"28": "Allocation Patterns",
"29": "Observability & Alerting",
"30": "PG Test Infrastructure",
"31": "Domain Concepts",
"31": "Memory Layout",
"32": "Repo Hard Constraints",
"33": "Go Testing Guide",
"34": "Session Store",
@@ -124,7 +124,7 @@
"122": "Identity comes from Discord OAuth; we store no passwords and send no email",
"123": "The wire format stays flat and deliberately does not mirror the schema",
"124": "ADR-0005: On-demand browser sidecar",
"125": "AGENTS.md",
"125": "Duration",
"126": "Bookmark Manager",
"127": "triage-labels.md",
"128": "Cross-Ticket Contract",
@@ -179,18 +179,6 @@
"177": "Backend CLAUDE.md Guidance",
"178": "Graphify Knowledge Graph (graphify-out/)",
"179": "CLAUDE.md (Symlink to AGENTS.md)",
"180": "Chapter Slug",
"181": "CONTEXT.md Ubiquitous Language",
"182": "Favourite",
"183": "Latest Chapter",
"184": "Library",
"185": "Lifecycle Bucket",
"186": "New Chapter",
"187": "Poll",
"188": "Progress",
"189": "Reader",
"190": "Series",
"191": "Site",
"192": "ADR-0001 (Drop modernc.org/sqlite)",
"193": "ADR-0003 (Split Shared Series Facts)",
"194": "SQLite-to-Postgres Cutover Runbook",
@@ -271,6 +259,6 @@
"269": "pg_dump Hot Backup",
"270": "Redeploy Runbook",
"271": "Rollback Strategy",
"272": "Bearer-token API client (apiGet/apiPut/apiDelete)",
"273": "AGENTS.md",
"279": "Userscript CLAUDE guidance"
}
+54 -69
View File
@@ -1,16 +1,16 @@
# Graph Report - mangaBookmark (2026-08-11)
# Graph Report - mangaBookmark (2026-08-12)
## Corpus Check
- 108 files · ~265,771 words
- 110 files · ~267,753 words
- Verdict: corpus is large enough that graph structure adds value.
## Summary
- 1637 nodes · 3195 edges · 274 communities (66 shown, 208 thin omitted)
- Extraction: 90% EXTRACTED · 10% INFERRED · 0% AMBIGUOUS · INFERRED: 305 edges (avg confidence: 0.77)
- 1651 nodes · 3253 edges · 262 communities (67 shown, 195 thin omitted)
- Extraction: 90% EXTRACTED · 10% INFERRED · 0% AMBIGUOUS · INFERRED: 314 edges (avg confidence: 0.77)
- Token cost: 0 input · 0 output
## Graph Freshness
- Built from commit: `fb65c78e`
- Built from commit: `700de202`
- Run `git rev-parse HEAD` and compare to check if the graph is stale.
- Run `graphify update .` after code changes (no API cost).
@@ -41,12 +41,12 @@
- [[_COMMUNITY_Browser Entrypoint|Browser Entrypoint]]
- [[_COMMUNITY_Memory Allocation & GC|Memory Allocation & GC]]
- [[_COMMUNITY_Cover Fetcher Tests|Cover Fetcher Tests]]
- [[_COMMUNITY_Deployment & Cover ADRs|Deployment & Cover ADRs]]
- [[_COMMUNITY_readSeries|readSeries]]
- [[_COMMUNITY_Find Skills Guide|Find Skills Guide]]
- [[_COMMUNITY_Allocation Patterns|Allocation Patterns]]
- [[_COMMUNITY_Observability & Alerting|Observability & Alerting]]
- [[_COMMUNITY_PG Test Infrastructure|PG Test Infrastructure]]
- [[_COMMUNITY_Domain Concepts|Domain Concepts]]
- [[_COMMUNITY_Memory Layout|Memory Layout]]
- [[_COMMUNITY_Repo Hard Constraints|Repo Hard Constraints]]
- [[_COMMUNITY_Go Testing Guide|Go Testing Guide]]
- [[_COMMUNITY_Session Store|Session Store]]
@@ -140,7 +140,7 @@
- [[_COMMUNITY_Identity comes from Discord OAuth; we store no passwords and send no email|Identity comes from Discord OAuth; we store no passwords and send no email]]
- [[_COMMUNITY_The wire format stays flat and deliberately does not mirror the schema|The wire format stays flat and deliberately does not mirror the schema]]
- [[_COMMUNITY_ADR-0005 On-demand browser sidecar|ADR-0005: On-demand browser sidecar]]
- [[_COMMUNITY_AGENTS|AGENTS.md]]
- [[_COMMUNITY_Duration|Duration]]
- [[_COMMUNITY_Bookmark Manager|Bookmark Manager]]
- [[_COMMUNITY_triage-labels|triage-labels.md]]
- [[_COMMUNITY_Cross-Ticket Contract|Cross-Ticket Contract]]
@@ -194,18 +194,6 @@
- [[_COMMUNITY_Backend CLAUDE.md Guidance|Backend CLAUDE.md Guidance]]
- [[_COMMUNITY_Graphify Knowledge Graph (graphify-out)|Graphify Knowledge Graph (graphify-out/)]]
- [[_COMMUNITY_CLAUDE.md (Symlink to AGENTS.md)|CLAUDE.md (Symlink to AGENTS.md)]]
- [[_COMMUNITY_Chapter Slug|Chapter Slug]]
- [[_COMMUNITY_CONTEXT.md Ubiquitous Language|CONTEXT.md Ubiquitous Language]]
- [[_COMMUNITY_Favourite|Favourite]]
- [[_COMMUNITY_Latest Chapter|Latest Chapter]]
- [[_COMMUNITY_Library|Library]]
- [[_COMMUNITY_Lifecycle Bucket|Lifecycle Bucket]]
- [[_COMMUNITY_New Chapter|New Chapter]]
- [[_COMMUNITY_Poll|Poll]]
- [[_COMMUNITY_Progress|Progress]]
- [[_COMMUNITY_Reader|Reader]]
- [[_COMMUNITY_Series|Series]]
- [[_COMMUNITY_Site|Site]]
- [[_COMMUNITY_ADR-0001 (Drop modernc.orgsqlite)|ADR-0001 (Drop modernc.org/sqlite)]]
- [[_COMMUNITY_ADR-0003 (Split Shared Series Facts)|ADR-0003 (Split Shared Series Facts)]]
- [[_COMMUNITY_SQLite-to-Postgres Cutover Runbook|SQLite-to-Postgres Cutover Runbook]]
@@ -286,7 +274,7 @@
- [[_COMMUNITY_pg_dump Hot Backup|pg_dump Hot Backup]]
- [[_COMMUNITY_Redeploy Runbook|Redeploy Runbook]]
- [[_COMMUNITY_Rollback Strategy|Rollback Strategy]]
- [[_COMMUNITY_Bearer-token API client (apiGetapiPutapiDelete)|Bearer-token API client (apiGet/apiPut/apiDelete)]]
- [[_COMMUNITY_AGENTS|AGENTS.md]]
- [[_COMMUNITY_Userscript CLAUDE guidance|Userscript CLAUDE guidance]]
## God Nodes (most connected - your core abstractions)
@@ -294,7 +282,7 @@
2. `newWebTestServer()` - 49 edges
3. `newTestStore()` - 42 edges
4. `newTestStore()` - 41 edges
5. `e()` - 29 edges
5. `e()` - 33 edges
6. `Handler` - 29 edges
7. `ne()` - 28 edges
8. `De()` - 28 edges
@@ -306,12 +294,12 @@
chrome/docker-compose.yml → backend/AGENTS.md
- `el()` --indirect_call--> `c()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `el()` --indirect_call--> `c()` [INFERRED]
userscript/novel-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `latestChapterFromAnchors()` --indirect_call--> `re()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `el()` --indirect_call--> `k()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
- `makeDraggable()` --indirect_call--> `e()` [INFERRED]
userscript/manga-bookmark.user.js → backend/internal/web/static/htmx.min.js
## Import Cycles
- None detected.
@@ -327,7 +315,7 @@
- **Headless browser infrastructure (sidecar, on-demand, home deployment)** — docs_adr_0005_on_demand_browser_headless_shell, docs_adr_0005_on_demand_browser_cdp, docs_adr_0005_on_demand_browser_on_demand_start, docs_adr_0006_browser_on_the_home_machine_home_machine_rationale [INFERRED 0.85]
- **lightnovelworld series-identity investigation and fix** — docs_research_lightnovelworld_chapter_vs_series_slug_issue_77, docs_research_lightnovelworld_chapter_vs_series_slug_unscoped_regex, docs_adr_0008_series_identity_is_discovered_not_derived_discovered_identity [INFERRED 0.85]
## Communities (274 total, 208 thin omitted)
## Communities (262 total, 195 thin omitted)
### Community 0 - "HTMX Library Internals"
Cohesion: 0.08
@@ -339,47 +327,47 @@ Nodes (84): floatPtr(), testConfig(), getCover(), Cookie, Handler, ResponseRecor
### Community 2 - "Manga Userscript Adapters"
Cohesion: 0.06
Nodes (76): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+68 more)
Nodes (77): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+69 more)
### Community 3 - "Novel Userscript Adapters"
Cohesion: 0.08
Nodes (71): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+63 more)
Cohesion: 0.06
Nodes (78): adapterFor(), anchorsFromDocument(), anchorsFromHTML(), apiDelete(), apiGet(), apiPut(), applyFabPos(), applyLatestChapterIfChanged() (+70 more)
### Community 4 - "Series Acquisition Tests"
Cohesion: 0.10
Nodes (67): bookmarkNewKaganeSeries(), bookmarkNewNovelfullSeries(), bookmarkNewSeries(), Context, Store, T, newAcquirer(), readBookmark() (+59 more)
Cohesion: 0.09
Nodes (68): bookmarkNewKaganeSeries(), bookmarkNewNovelfullSeries(), bookmarkNewSeries(), Context, Store, T, newAcquirer(), readBookmark() (+60 more)
### Community 5 - "Bookmarks API Tests"
Cohesion: 0.08
Nodes (67): auth(), getBookmarks(), Handler, Request, Store, T, newTestServer(), newTestStore() (+59 more)
### Community 7 - "Cover & Acquire Internals"
Cohesion: 0.08
Nodes (40): Addr, Context, Store, defaultCoverResolver(), fetchCoverBytes(), Client, Context, NewCoverFetcher() (+32 more)
Cohesion: 0.10
Nodes (30): Addr, Context, Store, defaultCoverResolver(), fetchCoverBytes(), Client, Context, NewCoverFetcher() (+22 more)
### Community 8 - "System Architecture Concepts"
Cohesion: 0.10
Nodes (26): Confirm-Gated Destructive Actions, Discord OAuth & Guild-Membership Gate, Lifecycle Buckets (reading/archived/finished), Reader-Owned Store, HMAC-Derived Reader Credentials, Web Package (Browser UI + Templates), app.html — App Shell Template, Manga/Novel Library Switch (+18 more)
### Community 9 - "Session Middleware"
Cohesion: 0.08
Nodes (35): ClearCookie(), ClientIP(), Duration, Mutex, Request, ResponseWriter, Time, isHTTPS() (+27 more)
Cohesion: 0.16
Nodes (20): ClearCookie(), ClientIP(), Duration, Mutex, Request, ResponseWriter, Time, isHTTPS() (+12 more)
### Community 10 - "Go Test Helpers"
Cohesion: 0.05
Nodes (39): Test Helpers, Test Timeout, Basic Handler Test, HTTP Handler Testing, Query Parameters and Headers, Docker Compose Fixture, Integration Testing, SQL Schema Fixture (+31 more)
### Community 11 - "Store Tests"
Cohesion: 0.14
Nodes (44): scanSeries(), Store, T, newTestStore(), readLatestCheckedAt(), readSeries(), secondReader(), seedForCheck() (+36 more)
Cohesion: 0.15
Nodes (43): Store, T, newTestStore(), readLatestCheckedAt(), readSeries(), secondReader(), seedForCheck(), seedSecondReader() (+35 more)
### Community 12 - "Bookmarks API Handler"
Cohesion: 0.08
Nodes (32): Handler, Request, ResponseWriter, Store, Healthz(), writeJSON(), Auth(), compressible() (+24 more)
### Community 13 - "Web UI Handlers"
Cohesion: 0.14
Nodes (18): currentLib(), currentTab(), filterBookmarks(), Client, HandlerFunc, Request, ResponseWriter, Store (+10 more)
Cohesion: 0.07
Nodes (33): Context, Mutex, Request, ResponseWriter, Time, Handler, newOAuthStates(), T (+25 more)
### Community 14 - "Go Error Handling"
Cohesion: 0.06
@@ -387,26 +375,26 @@ Nodes (33): Creating Errors, Custom Error Types, Custom types that wrap other er
### Community 15 - "CDP Browser Client"
Cohesion: 0.08
Nodes (29): Action, awaitPromise(), browserConnectionLost(), classifyBrowserError(), Context, Mutex, isInterstitial(), jsString() (+21 more)
Nodes (31): awaitPromise(), browserConnectionLost(), classifyBrowserError(), Action, Context, Mutex, jsString(), kaganeAPIURL() (+23 more)
### Community 16 - "Store"
Cohesion: 0.09
Nodes (7): CoverContentType(), coverRelativePath(), coverSourceAddress(), displayChapter(), Store, TestDisplayChapter(), Bookmark
Cohesion: 0.12
Nodes (4): CoverContentType(), coverRelativePath(), coverSourceAddress(), Store
### Community 17 - "Go Code Style Guide"
Cohesion: 0.08
Nodes (23): Code Style Details, Extract Complex Conditions, Value vs Pointer Arguments, Code Organization Within Files, Complex Conditions & Init Scope, Composite Literals, Control Flow, Cross-References (+15 more)
### Community 19 - "Open"
Cohesion: 0.19
Nodes (18): applyMigration(), migrate(), Open(), refreshOwnerToken(), seedOwner(), TestCoverIsContentAddressedOnFilesystem(), TestCoverPersistsAcrossReopen(), TestMigration0002BackfillsExistingBookmarks() (+10 more)
Cohesion: 0.17
Nodes (19): applyMigration(), migrate(), Open(), refreshOwnerToken(), scanSeries(), seedOwner(), TestCoverIsContentAddressedOnFilesystem(), TestCoverPersistsAcrossReopen() (+11 more)
### Community 20 - "I/O Performance Patterns"
Cohesion: 0.11
Nodes (18): Avoid io.ReadAll for large payloads, Batch Operations, Buffered I/O, Cgo Overhead, Channel: batch processing from a stream, Concurrent Multi-Stage Pipelines, Connection pooling, Database: batch inserts over row-by-row (+10 more)
### Community 21 - "CPU Optimization"
Cohesion: 0.12
Cohesion: 0.13
Nodes (15): Cache Locality, Contiguous 2D allocation, CPU Optimization, False Sharing, Function Inlining, Handling CPU-specific instruction sets, Instruction-Level Parallelism, Monotonic Time (+7 more)
### Community 22 - "Caching Patterns"
@@ -425,13 +413,17 @@ Nodes (15): Allocation Rate Reduction, Ballast pattern (pre-Go 1.19), Garbage Co
Cohesion: 0.33
Nodes (12): coverResponse(), Request, T, TestCoverFetcherCanonicalisesJpgAlias(), TestCoverFetcherFetchesPublicHTTPSImage(), TestCoverFetcherRefusesUnsafeDestinationsBeforeRequest(), TestCoverFetcherRejectsNonImage(), TestCoverFetcherRejectsOversizedBody() (+4 more)
### Community 26 - "readSeries"
Cohesion: 0.13
Nodes (28): asuraLatestChapter(), browserOnlyCoverURL(), comixCoverEntry(), comixCoverURL(), comixLatestChapter(), comixSeriesID(), coverFrom(), demonicLatestChapter() (+20 more)
### Community 27 - "Find Skills Guide"
Cohesion: 0.14
Nodes (13): Common Skill Categories, Find Skills, How to Help Users Find Skills, Step 1: Understand What They Need, Step 2: Check the Leaderboard First, Step 3: Search for Skills, Step 4: Verify Quality Before Recommending, Step 5: Present Options to the User (+5 more)
### Community 28 - "Allocation Patterns"
Cohesion: 0.17
Nodes (12): Backing Array Leaks, Map never shrinks, Map of pointers for large, frequently updated structs, Memory Layout, Memory Optimization, Pointer receivers for large structs, Slice reslicing retains the entire backing array, String and Byte Optimization (+4 more)
Cohesion: 0.14
Nodes (14): Allocation Patterns, Backing Array Leaks, Direct indexing vs append, Eliminate redundant map lookups, Interface boxing, Map never shrinks, Map size hints, Memory Optimization (+6 more)
### Community 29 - "Observability & Alerting"
Cohesion: 0.22
@@ -441,6 +433,10 @@ Nodes (9): Alerting rules (examples), CPU saturation, GC pressure, Goroutine lea
Cohesion: 0.18
Nodes (12): M, TestMain(), M, TestMain(), M, Main(), start(), URL() (+4 more)
### Community 31 - "Memory Layout"
Cohesion: 0.40
Nodes (5): Map of pointers for large, frequently updated structs, Memory Layout, Pointer receivers for large structs, Struct field alignment, Zero-size field at end of struct
### Community 33 - "Go Testing Guide"
Cohesion: 0.20
Nodes (10): CI Regression Detection, Common Mistakes, Core Philosophy, Cross-References, Decision Tree: Where Is Time Spent?, Deep Dives, Go Performance Optimization, Iterative Optimization Methodology (+2 more)
@@ -458,15 +454,8 @@ Cohesion: 0.17
Nodes (11): Accessibility & Inclusion, Brand Commitments, Capabilities and Constraints, Evidence on Hand, Operating Context, Platform, Positioning, Product (+3 more)
### Community 38 - "novel-logic.test.js"
Cohesion: 0.13
Nodes (8): assert, attrEls, elements, metaTags, {
novelfull,
lightnovelworld,
computeLatestChapter,
kindOf,
maxChapter,
repairLnwStaleRow,
}, reset(), store, test
Cohesion: 0.33
Nodes (5): ADR-0009: A Site answers fixed questions; an unusual Site owns its own fetch, Consequences, Considered options, Decision, Why
### Community 39 - "UI Critique 2026-07-26A"
Cohesion: 0.29
@@ -489,7 +478,7 @@ Cohesion: 0.07
Nodes (27): 1. Summary answer table, 2. The two reference pages, 3. Chapter page → series URL: every in-page pointer, in priority order, 4.1 Sample method, 4.2 Divergence results, 4.3 Is there a derivable rule? **No.**, 4.4 The split case — a slug can change *mid-series*, 4. The reverse direction, and how common divergence is (+19 more)
### Community 47 - "Go Perf Skill Docs"
Cohesion: 0.28
Cohesion: 0.22
Nodes (5): Continuous Profiling, Production Observability for Performance, Pyroscope pull mode (via Grafana Alloy), Pyroscope push mode, Real-Time Visualization (Development)
### Community 48 - "Login Page Art"
@@ -588,33 +577,29 @@ Nodes (3): Consequence, The wire format stays flat and deliberately does not mir
Cohesion: 0.50
Nodes (3): ADR-0005: On-demand browser sidecar, Constraints, Decision
### Community 125 - "AGENTS.md"
Cohesion: 0.29
Nodes (7): Allocation Patterns, Direct indexing vs append, Eliminate redundant map lookups, Interface boxing, Map size hints, Reuse slices via append(s[:0], ...), Sentinel errors vs fmt.Errorf
### Community 272 - "Bearer-token API client (apiGet/apiPut/apiDelete)"
### Community 273 - "AGENTS.md"
Cohesion: 0.50
Nodes (3): Live URL shapes (verified 2026-07-26, may drift — re-check against live pages before trust), Second script: `novel-bookmark.user.js`, Userscript structure (single IIFE, `manga-bookmark.user.js`)
## Knowledge Gaps
- **540 isolated node(s):** `test`, `assert`, `store`, `metaTags`, `elements` (+535 more)
- **530 isolated node(s):** `bookmarkmanager/backend`, `ctxKey`, `loginView`, `ctxKey`, `test` (+525 more)
These have ≤1 connection - possible missing edges or undocumented components.
- **208 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes.
- **195 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes.
## Suggested Questions
_Questions this graph is uniquely positioned to answer:_
- **Why does `Open()` connect `Open` to `Cover Fetch Test Helpers`, `Series Acquisition Tests`, `Bookmarks API Tests`, `Store Tests`, `Store`, `PG Test Infrastructure`?**
_High betweenness centrality (0.041) - this node is a cross-community bridge._
- **Why does `New()` connect `Series Acquisition Tests` to `Bookmarks API Tests`, `Cover & Acquire Internals`, `Session Middleware`, `Web UI Handlers`, `Open`?**
_High betweenness centrality (0.031) - this node is a cross-community bridge._
_High betweenness centrality (0.047) - this node is a cross-community bridge._
- **Why does `Open()` connect `Open` to `Cover Fetch Test Helpers`, `Series Acquisition Tests`, `Bookmarks API Tests`, `Store Tests`, `Store`, `PG Test Infrastructure`?**
_High betweenness centrality (0.032) - this node is a cross-community bridge._
- **Why does `newRouter()` connect `Bookmarks API Tests` to `Cover Fetch Test Helpers`, `Bookmarks API Handler`, `Series Acquisition Tests`?**
_High betweenness centrality (0.030) - this node is a cross-community bridge._
_High betweenness centrality (0.025) - this node is a cross-community bridge._
- **Are the 47 inferred relationships involving `testConfig()` (e.g. with `TestListRendersAcquiredCover()` and `TestPublicCoverNeverEchoesNonImage()`) actually correct?**
_`testConfig()` has 47 INFERRED edges - model-reasoned connections that need verification._
- **Are the 8 inferred relationships involving `newWebTestServer()` (e.g. with `TestListRendersAcquiredCover()` and `TestPublicCoverRejectsUnknownAddress()`) actually correct?**
_`newWebTestServer()` has 8 INFERRED edges - model-reasoned connections that need verification._
- **Are the 6 inferred relationships involving `newTestStore()` (e.g. with `TestCreateAndGetSession()` and `TestDeleteSessionIsPerReader()`) actually correct?**
_`newTestStore()` has 6 INFERRED edges - model-reasoned connections that need verification._
- **What connects `test`, `assert`, `store` to the rest of the system?**
_582 weakly-connected nodes found - possible documentation gaps or missing edges._
- **What connects `bookmarkmanager/backend`, `ctxKey`, `loginView` to the rest of the system?**
_572 weakly-connected nodes found - possible documentation gaps or missing edges._
File diff suppressed because one or more lines are too long
+12671 -11968
View File
File diff suppressed because it is too large Load Diff
+26 -16
View File
@@ -245,9 +245,9 @@
"semantic_hash": "385b36f58488b7e6d93eb6d6034e9ee3"
},
"backend/internal/latest/browser.go": {
"mtime": 1786363889.5565126,
"ast_hash": "092dfbd2bc96e7a251672a5a0d922b37",
"semantic_hash": "092dfbd2bc96e7a251672a5a0d922b37"
"mtime": 1786469354.923288,
"ast_hash": "ed129a7f00601ea90c877ff29fa21220",
"semantic_hash": ""
},
"backend/internal/latest/browser_test.go": {
"mtime": 1786262323.6964688,
@@ -260,18 +260,18 @@
"semantic_hash": ""
},
"backend/internal/latest/poller.go": {
"mtime": 1786363889.5565126,
"ast_hash": "ebb0ee4e57d62b6aed97b8d266639a23",
"semantic_hash": "ebb0ee4e57d62b6aed97b8d266639a23"
"mtime": 1786469641.8342345,
"ast_hash": "44fef6074ac2eaffc8233f46aad5236b",
"semantic_hash": ""
},
"backend/internal/latest/poller_test.go": {
"mtime": 1786446006.9219902,
"ast_hash": "91151a7671a3fb1efc957137cfc6839b",
"mtime": 1786469644.9306462,
"ast_hash": "64bc838c822f1bf33bbf9e291215454b",
"semantic_hash": ""
},
"backend/internal/latest/sites.go": {
"mtime": 1786446006.9219902,
"ast_hash": "a424fb3b3e1c5afb4ab545c2e88b167c",
"mtime": 1786469348.5394242,
"ast_hash": "b744cc685363317a526cc3bebceea39e",
"semantic_hash": ""
},
"backend/internal/latest/sites_test.go": {
@@ -415,9 +415,9 @@
"semantic_hash": "e44a2f6f624db044e19508bc5ab05592"
},
"CONTEXT.md": {
"mtime": 1786417841.7163916,
"ast_hash": "1ff47b5340a725b0b7287ee29f63cd34",
"semantic_hash": "1ff47b5340a725b0b7287ee29f63cd34"
"mtime": 1786466512.6719532,
"ast_hash": "544b7d93f1d5cb9d347cb0b92fc3a709",
"semantic_hash": ""
},
"CUTOVER.md": {
"mtime": 1786216141.660644,
@@ -570,9 +570,9 @@
"semantic_hash": "60d9eb7c59a3751baf4f31c7655217e7"
},
"backend/internal/latest/acquire.go": {
"mtime": 1786363889.5565126,
"ast_hash": "760310097141a864f14318ed3be2703e",
"semantic_hash": "760310097141a864f14318ed3be2703e"
"mtime": 1786468950.9432797,
"ast_hash": "6c1ad34bbe9f5b49d0fd1eae9093f55d",
"semantic_hash": ""
},
"backend/internal/latest/acquire_test.go": {
"mtime": 1786363889.5565126,
@@ -608,5 +608,15 @@
"mtime": 1786446006.9219902,
"ast_hash": "2d65da8a081759172918fdf159760f45",
"semantic_hash": ""
},
"docs/adr/0009-a-site-answers-questions-its-own-way.md": {
"mtime": 1786469367.6875648,
"ast_hash": "8039012a5b6de2359ff1a47079f51b66",
"semantic_hash": ""
},
"backend/internal/latest/read.go": {
"mtime": 1786469329.7445614,
"ast_hash": "3cf29046ddaef39fafb1df70b9f9ae8c",
"semantic_hash": ""
}
}