feat: one registry entry per Site, one shared Series-page read #95
@@ -93,10 +93,11 @@ func (f *BrowserFetcher) Close() {
|
||||
// which is what the entry's LatestChapter parse expects.
|
||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||
var body string
|
||||
for _, s := range sites {
|
||||
if s.Browser == nil {
|
||||
continue
|
||||
}
|
||||
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
|
||||
// entries' Read funcs are expected to refuse any address owned by another
|
||||
// Site, and the loop must not depend on that staying true.
|
||||
for _, name := range browserBackedSites() {
|
||||
s := sites[name]
|
||||
read, ok := s.Browser.Read(seriesURL, &body)
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -226,11 +226,16 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
||||
// passes the gate is retried at cooldown pace rather than
|
||||
// hot-looping.
|
||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
||||
return
|
||||
case errors.Is(err, errNoFetcher):
|
||||
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||
default:
|
||||
log.Printf("latest poll %q: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
// A legacy cover heals independently of the page read: its source may
|
||||
// answer — a CDN — while the origin does not, so a fetch failure does
|
||||
// not skip the heal, matching the order the shared read replaced.
|
||||
p.prefetchCover(ctx, sr)
|
||||
log.Printf("latest poll %q: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
// A legacy cover source is healed independently of the page read.
|
||||
@@ -241,7 +246,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
||||
if !facts.HasLatest {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||
log.Printf("latest poll %q: no chapter links", sr.Key())
|
||||
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -623,9 +623,9 @@ func TestFetchableSeriesURL(t *testing.T) {
|
||||
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
||||
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
||||
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
||||
// The pins added in #94 cover the three plain-TLS Sites too: a
|
||||
// client-supplied series_url must not aim a fetcher at a lookalike
|
||||
// host, even when the fetcher is only an HTTP GET.
|
||||
// The three plain-TLS Sites are pinned too: a client-supplied
|
||||
// series_url must not aim a fetcher at a lookalike host, even when
|
||||
// the fetcher is only an HTTP GET.
|
||||
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
||||
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
||||
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
||||
|
||||
@@ -15,6 +15,9 @@ type seriesRead struct {
|
||||
HasLatest bool
|
||||
Cover string
|
||||
HasCover bool
|
||||
// BodyLen is the fetched body's length, surfaced because the no-chapter
|
||||
// log uses it to tell a markup change from a body the size cap cut short.
|
||||
BodyLen int
|
||||
}
|
||||
|
||||
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
||||
@@ -51,5 +54,5 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
||||
}
|
||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover}, nil
|
||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -184,6 +185,9 @@ func comixLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
||||
return maxChapter(re, body)
|
||||
}
|
||||
|
||||
// kaganeLatestChapter scans the kagane series API JSON that the browser read
|
||||
// fetched from inside the page; the match rides on the property name,
|
||||
// regardless of the surrounding JSON shape.
|
||||
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
|
||||
return maxChapter(kaganeChapterRe, body)
|
||||
}
|
||||
@@ -419,7 +423,8 @@ var sites = map[string]site{
|
||||
|
||||
// browserBackedSites is derived from the registry: the Sites whose pages are
|
||||
// read through the browser sidecar, which are also the ones granted the longer
|
||||
// cooldown.
|
||||
// cooldown. Sorted so callers that range it (the due query, the browser
|
||||
// fetcher's dispatch) see a stable order instead of map-iteration noise.
|
||||
func browserBackedSites() []string {
|
||||
out := make([]string, 0, len(sites))
|
||||
for name, s := range sites {
|
||||
@@ -427,5 +432,6 @@ func browserBackedSites() []string {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -9,8 +9,10 @@ Per-Site knowledge lives in one registry in `backend/internal/latest/sites.go`,
|
||||
keyed by the stored site string. An entry answers a fixed set of questions: the
|
||||
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
|
||||
how to find the Cover address in a body, and — for a Site behind a JavaScript
|
||||
challenge — how to read its payload from a cleared tab and how to tell that the
|
||||
payload arrived.
|
||||
challenge — how to read its payload from a cleared tab, how to tell that the
|
||||
payload arrived, and whether the plain-TLS fetcher may take over when no
|
||||
browser is configured (`Fallback`; kagane never falls back, novelfull does,
|
||||
each on measured evidence).
|
||||
|
||||
The question set does not grow to accommodate one Site. When a Site needs
|
||||
something the set cannot express, that Site gets an optional override and
|
||||
|
||||
Reference in New Issue
Block a user