feat: one registry entry per Site, one shared Series-page read #95
@@ -93,10 +93,11 @@ func (f *BrowserFetcher) Close() {
|
|||||||
// which is what the entry's LatestChapter parse expects.
|
// which is what the entry's LatestChapter parse expects.
|
||||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||||
var body string
|
var body string
|
||||||
for _, s := range sites {
|
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
|
||||||
if s.Browser == nil {
|
// entries' Read funcs are expected to refuse any address owned by another
|
||||||
continue
|
// Site, and the loop must not depend on that staying true.
|
||||||
}
|
for _, name := range browserBackedSites() {
|
||||||
|
s := sites[name]
|
||||||
read, ok := s.Browser.Read(seriesURL, &body)
|
read, ok := s.Browser.Read(seriesURL, &body)
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -226,11 +226,16 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
|||||||
// passes the gate is retried at cooldown pace rather than
|
// passes the gate is retried at cooldown pace rather than
|
||||||
// hot-looping.
|
// hot-looping.
|
||||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
||||||
|
return
|
||||||
case errors.Is(err, errNoFetcher):
|
case errors.Is(err, errNoFetcher):
|
||||||
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||||
default:
|
return
|
||||||
log.Printf("latest poll %q: %v", sr.Key(), err)
|
|
||||||
}
|
}
|
||||||
|
// A legacy cover heals independently of the page read: its source may
|
||||||
|
// answer — a CDN — while the origin does not, so a fetch failure does
|
||||||
|
// not skip the heal, matching the order the shared read replaced.
|
||||||
|
p.prefetchCover(ctx, sr)
|
||||||
|
log.Printf("latest poll %q: %v", sr.Key(), err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// A legacy cover source is healed independently of the page read.
|
// A legacy cover source is healed independently of the page read.
|
||||||
@@ -241,7 +246,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
|||||||
if !facts.HasLatest {
|
if !facts.HasLatest {
|
||||||
// Most likely a challenge page or a layout change. Either way the row is
|
// Most likely a challenge page or a layout change. Either way the row is
|
||||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||||
log.Printf("latest poll %q: no chapter links", sr.Key())
|
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -623,9 +623,9 @@ func TestFetchableSeriesURL(t *testing.T) {
|
|||||||
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
||||||
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
||||||
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
||||||
// The pins added in #94 cover the three plain-TLS Sites too: a
|
// The three plain-TLS Sites are pinned too: a client-supplied
|
||||||
// client-supplied series_url must not aim a fetcher at a lookalike
|
// series_url must not aim a fetcher at a lookalike host, even when
|
||||||
// host, even when the fetcher is only an HTTP GET.
|
// the fetcher is only an HTTP GET.
|
||||||
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
||||||
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
||||||
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
||||||
|
|||||||
@@ -15,6 +15,9 @@ type seriesRead struct {
|
|||||||
HasLatest bool
|
HasLatest bool
|
||||||
Cover string
|
Cover string
|
||||||
HasCover bool
|
HasCover bool
|
||||||
|
// BodyLen is the fetched body's length, surfaced because the no-chapter
|
||||||
|
// log uses it to tell a markup change from a body the size cap cut short.
|
||||||
|
BodyLen int
|
||||||
}
|
}
|
||||||
|
|
||||||
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
||||||
@@ -51,5 +54,5 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
|||||||
}
|
}
|
||||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover}, nil
|
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -184,6 +185,9 @@ func comixLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
|||||||
return maxChapter(re, body)
|
return maxChapter(re, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kaganeLatestChapter scans the kagane series API JSON that the browser read
|
||||||
|
// fetched from inside the page; the match rides on the property name,
|
||||||
|
// regardless of the surrounding JSON shape.
|
||||||
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
|
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
|
||||||
return maxChapter(kaganeChapterRe, body)
|
return maxChapter(kaganeChapterRe, body)
|
||||||
}
|
}
|
||||||
@@ -419,7 +423,8 @@ var sites = map[string]site{
|
|||||||
|
|
||||||
// browserBackedSites is derived from the registry: the Sites whose pages are
|
// browserBackedSites is derived from the registry: the Sites whose pages are
|
||||||
// read through the browser sidecar, which are also the ones granted the longer
|
// read through the browser sidecar, which are also the ones granted the longer
|
||||||
// cooldown.
|
// cooldown. Sorted so callers that range it (the due query, the browser
|
||||||
|
// fetcher's dispatch) see a stable order instead of map-iteration noise.
|
||||||
func browserBackedSites() []string {
|
func browserBackedSites() []string {
|
||||||
out := make([]string, 0, len(sites))
|
out := make([]string, 0, len(sites))
|
||||||
for name, s := range sites {
|
for name, s := range sites {
|
||||||
@@ -427,5 +432,6 @@ func browserBackedSites() []string {
|
|||||||
out = append(out, name)
|
out = append(out, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sort.Strings(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,8 +9,10 @@ Per-Site knowledge lives in one registry in `backend/internal/latest/sites.go`,
|
|||||||
keyed by the stored site string. An entry answers a fixed set of questions: the
|
keyed by the stored site string. An entry answers a fixed set of questions: the
|
||||||
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
|
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
|
||||||
how to find the Cover address in a body, and — for a Site behind a JavaScript
|
how to find the Cover address in a body, and — for a Site behind a JavaScript
|
||||||
challenge — how to read its payload from a cleared tab and how to tell that the
|
challenge — how to read its payload from a cleared tab, how to tell that the
|
||||||
payload arrived.
|
payload arrived, and whether the plain-TLS fetcher may take over when no
|
||||||
|
browser is configured (`Fallback`; kagane never falls back, novelfull does,
|
||||||
|
each on measured evidence).
|
||||||
|
|
||||||
The question set does not grow to accommodate one Site. When a Site needs
|
The question set does not grow to accommodate one Site. When a Site needs
|
||||||
something the set cannot express, that Site gets an optional override and
|
something the set cannot express, that Site gets an optional override and
|
||||||
|
|||||||
Reference in New Issue
Block a user