fix: keep the legacy cover heal and no-chapter byte count (#94)
Restore two review findings against spec claims 'the Poll keeps its own Cover policy' and 'pinning is the only behavioural change': prefetchCover now also runs on ticks where the page fetch itself fails (the heal is independent of the page read, and its source may answer while the origin does not), and the no-chapter log surfaces the fetched body length again via a BodyLen fact on seriesRead. Browser dispatch iterates the sorted browser site list so its outcome cannot depend on map order.
This commit is contained in:
@@ -93,10 +93,11 @@ func (f *BrowserFetcher) Close() {
|
|||||||
// which is what the entry's LatestChapter parse expects.
|
// which is what the entry's LatestChapter parse expects.
|
||||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||||
var body string
|
var body string
|
||||||
for _, s := range sites {
|
// Sorted order (browserBackedSites sorts) makes dispatch deterministic:
|
||||||
if s.Browser == nil {
|
// entries' Read funcs are expected to refuse any address owned by another
|
||||||
continue
|
// Site, and the loop must not depend on that staying true.
|
||||||
}
|
for _, name := range browserBackedSites() {
|
||||||
|
s := sites[name]
|
||||||
read, ok := s.Browser.Read(seriesURL, &body)
|
read, ok := s.Browser.Read(seriesURL, &body)
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -226,11 +226,16 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
|||||||
// passes the gate is retried at cooldown pace rather than
|
// passes the gate is retried at cooldown pace rather than
|
||||||
// hot-looping.
|
// hot-looping.
|
||||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL)
|
||||||
|
return
|
||||||
case errors.Is(err, errNoFetcher):
|
case errors.Is(err, errNoFetcher):
|
||||||
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
|
||||||
default:
|
return
|
||||||
log.Printf("latest poll %q: %v", sr.Key(), err)
|
|
||||||
}
|
}
|
||||||
|
// A legacy cover heals independently of the page read: its source may
|
||||||
|
// answer — a CDN — while the origin does not, so a fetch failure does
|
||||||
|
// not skip the heal, matching the order the shared read replaced.
|
||||||
|
p.prefetchCover(ctx, sr)
|
||||||
|
log.Printf("latest poll %q: %v", sr.Key(), err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// A legacy cover source is healed independently of the page read.
|
// A legacy cover source is healed independently of the page read.
|
||||||
@@ -241,7 +246,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
|||||||
if !facts.HasLatest {
|
if !facts.HasLatest {
|
||||||
// Most likely a challenge page or a layout change. Either way the row is
|
// Most likely a challenge page or a layout change. Either way the row is
|
||||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||||
log.Printf("latest poll %q: no chapter links", sr.Key())
|
log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -623,9 +623,9 @@ func TestFetchableSeriesURL(t *testing.T) {
|
|||||||
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
||||||
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
||||||
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
||||||
// The pins added in #94 cover the three plain-TLS Sites too: a
|
// The three plain-TLS Sites are pinned too: a client-supplied
|
||||||
// client-supplied series_url must not aim a fetcher at a lookalike
|
// series_url must not aim a fetcher at a lookalike host, even when
|
||||||
// host, even when the fetcher is only an HTTP GET.
|
// the fetcher is only an HTTP GET.
|
||||||
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
||||||
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
||||||
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
||||||
|
|||||||
@@ -15,6 +15,9 @@ type seriesRead struct {
|
|||||||
HasLatest bool
|
HasLatest bool
|
||||||
Cover string
|
Cover string
|
||||||
HasCover bool
|
HasCover bool
|
||||||
|
// BodyLen is the fetched body's length, surfaced because the no-chapter
|
||||||
|
// log uses it to tell a markup change from a body the size cap cut short.
|
||||||
|
BodyLen int
|
||||||
}
|
}
|
||||||
|
|
||||||
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
// errNotFetchable and errNoFetcher separate the gate and the route from fetch
|
||||||
@@ -51,5 +54,5 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
|||||||
}
|
}
|
||||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover}, nil
|
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -184,6 +185,9 @@ func comixLatestChapter(seriesURL, body string) (latestChapter, bool) {
|
|||||||
return maxChapter(re, body)
|
return maxChapter(re, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kaganeLatestChapter scans the kagane series API JSON that the browser read
|
||||||
|
// fetched from inside the page; the match rides on the property name,
|
||||||
|
// regardless of the surrounding JSON shape.
|
||||||
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
|
func kaganeLatestChapter(_, body string) (latestChapter, bool) {
|
||||||
return maxChapter(kaganeChapterRe, body)
|
return maxChapter(kaganeChapterRe, body)
|
||||||
}
|
}
|
||||||
@@ -419,7 +423,8 @@ var sites = map[string]site{
|
|||||||
|
|
||||||
// browserBackedSites is derived from the registry: the Sites whose pages are
|
// browserBackedSites is derived from the registry: the Sites whose pages are
|
||||||
// read through the browser sidecar, which are also the ones granted the longer
|
// read through the browser sidecar, which are also the ones granted the longer
|
||||||
// cooldown.
|
// cooldown. Sorted so callers that range it (the due query, the browser
|
||||||
|
// fetcher's dispatch) see a stable order instead of map-iteration noise.
|
||||||
func browserBackedSites() []string {
|
func browserBackedSites() []string {
|
||||||
out := make([]string, 0, len(sites))
|
out := make([]string, 0, len(sites))
|
||||||
for name, s := range sites {
|
for name, s := range sites {
|
||||||
@@ -427,5 +432,6 @@ func browserBackedSites() []string {
|
|||||||
out = append(out, name)
|
out = append(out, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sort.Strings(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,8 +9,10 @@ Per-Site knowledge lives in one registry in `backend/internal/latest/sites.go`,
|
|||||||
keyed by the stored site string. An entry answers a fixed set of questions: the
|
keyed by the stored site string. An entry answers a fixed set of questions: the
|
||||||
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
|
hostname a `series_url` must carry, how to find the Latest Chapter in a body,
|
||||||
how to find the Cover address in a body, and — for a Site behind a JavaScript
|
how to find the Cover address in a body, and — for a Site behind a JavaScript
|
||||||
challenge — how to read its payload from a cleared tab and how to tell that the
|
challenge — how to read its payload from a cleared tab, how to tell that the
|
||||||
payload arrived.
|
payload arrived, and whether the plain-TLS fetcher may take over when no
|
||||||
|
browser is configured (`Fallback`; kagane never falls back, novelfull does,
|
||||||
|
each on measured evidence).
|
||||||
|
|
||||||
The question set does not grow to accommodate one Site. When a Site needs
|
The question set does not grow to accommodate one Site. When a Site needs
|
||||||
something the set cannot express, that Site gets an optional override and
|
something the set cannot express, that Site gets an optional override and
|
||||||
|
|||||||
Reference in New Issue
Block a user