feat: one registry entry per Site, one shared Series-page read #95

Merged
sulthan merged 7 commits from feat/94-site-registry into main 2026-08-12 05:43:39 +07:00
6 changed files with 31 additions and 14 deletions
Showing only changes of commit d998f8725c - Show all commits
+5 -4
View File
@@ -93,10 +93,11 @@ func (f *BrowserFetcher) Close() {
// which is what the entry's LatestChapter parse expects.
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
var body string
for _, s := range sites {
if s.Browser == nil {
continue
}
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
// entries' Read funcs are expected to refuse any address owned by another
// Site, and the loop must not depend on that staying true.
for _, name := range browserBackedSites() {
s := sites[name]
read, ok := s.Browser.Read(seriesURL, &body)
if !ok {
continue
+8 -3
View File
@@ -226,11 +226,16 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
// passes the gate is retried at cooldown pace rather than
// hot-looping.
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
return
case errors.Is(err, errNoFetcher):
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
default:
log.Printf("latest poll %q: %v", sr.Key(), err)
return
}
// A legacy cover heals independently of the page read: its source may
// answer — a CDN — while the origin does not, so a fetch failure does
// not skip the heal, matching the order the shared read replaced.
p.prefetchCover(ctx, sr)
log.Printf("latest poll %q: %v", sr.Key(), err)
return
}
// A legacy cover source is healed independently of the page read.
@@ -241,7 +246,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
if !facts.HasLatest {
// Most likely a challenge page or a layout change. Either way the row is
// already stamped, so this waits out a cooldown instead of hot-looping.
log.Printf("latest poll %q: no chapter links", sr.Key())
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
return
}
+3 -3
View File
@@ -623,9 +623,9 @@ func TestFetchableSeriesURL(t *testing.T) {
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
// The pins added in #94 cover the three plain-TLS Sites too: a
// client-supplied series_url must not aim a fetcher at a lookalike
// host, even when the fetcher is only an HTTP GET.
// The three plain-TLS Sites are pinned too: a client-supplied
// series_url must not aim a fetcher at a lookalike host, even when
// the fetcher is only an HTTP GET.
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
+4 -1
View File
@@ -15,6 +15,9 @@ type seriesRead struct {
HasLatest bool
Cover string
HasCover bool
// BodyLen is the fetched body's length, surfaced because the no-chapter
// log uses it to tell a markup change from a body the size cap cut short.
BodyLen int
}
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
@@ -51,5 +54,5 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
}
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
cover, hasCover := coverFrom(site, seriesURL, body)
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover}, nil
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
}
+7 -1
View File
@@ -6,6 +6,7 @@ import (
"log"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
@@ -184,6 +185,9 @@ func comixLatestChapter(seriesURL, body string) (latestChapter, bool) {
return maxChapter(re, body)
}
// kaganeLatestChapter scans the kagane series API JSON that the browser read
// fetched from inside the page; the match rides on the property name,
// regardless of the surrounding JSON shape.
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
return maxChapter(kaganeChapterRe, body)
}
@@ -419,7 +423,8 @@ var sites = map[string]site{
// browserBackedSites is derived from the registry: the Sites whose pages are
// read through the browser sidecar, which are also the ones granted the longer
// cooldown.
// cooldown. Sorted so callers that range it (the due query, the browser
// fetcher's dispatch) see a stable order instead of map-iteration noise.
func browserBackedSites() []string {
out := make([]string, 0, len(sites))
for name, s := range sites {
@@ -427,5 +432,6 @@ func browserBackedSites() []string {
out = append(out, name)
}
}
sort.Strings(out)
return out
}
@@ -9,8 +9,10 @@ Per-Site knowledge lives in one registry in `backend/internal/latest/sites.go`,
keyed by the stored site string. An entry answers a fixed set of questions: the
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
how to find the Cover address in a body, and — for a Site behind a JavaScript
challenge — how to read its payload from a cleared tab and how to tell that the
payload arrived.
challenge — how to read its payload from a cleared tab, how to tell that the
payload arrived, and whether the plain-TLS fetcher may take over when no
browser is configured (`Fallback`; kagane never falls back, novelfull does,
each on measured evidence).
The question set does not grow to accommodate one Site. When a Site needs
something the set cannot express, that Site gets an optional override and