feat: one registry entry per Site, derived browser route and gate (#94)
Collapse the six per-site comparison points into a sites map in sites.go: Latest Chapter parse, Cover parse, browser-backed list, fetcher route, host pins, and the browser payload read all become lookups into it. All six hosts are now pinned in fetchableSeriesURL; asura/demonic/comix were previously accepted on any https host.
This commit is contained in:
@@ -86,46 +86,58 @@ func (f *BrowserFetcher) Close() {
|
||||
f.cancel()
|
||||
}
|
||||
|
||||
// Get navigates to seriesURL, lets any challenge resolve, then reads either the
|
||||
// site's JSON API (kagane) from inside the page so the request carries the
|
||||
// clearance cookie, or the served HTML itself (novelfull) — see
|
||||
// novelfullSeriesURL for the latter case. The returned body is whatever the
|
||||
// site's chapter list lives in, which is what latestChapterFrom's per-site
|
||||
// switch expects.
|
||||
// Get navigates to seriesURL, lets any challenge resolve, then reads the
|
||||
// payload the Site's registry entry describes — kagane's chapter-list API from
|
||||
// inside the page so the request carries the clearance cookie, novelfull's
|
||||
// served HTML. The returned body is whatever the Site's chapter list lives in,
|
||||
// which is what the entry's LatestChapter parse expects.
|
||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||
apiURL, isKagane := kaganeAPIURL(seriesURL)
|
||||
if !isKagane && !novelfullSeriesURL(seriesURL) {
|
||||
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
|
||||
}
|
||||
|
||||
var body string
|
||||
// kagane's chapter list is only in its JSON API, which must be called from
|
||||
// inside the page so the request carries the clearance cookie. novelfull
|
||||
// renders its chapters into the HTML, so the cleared DOM is the answer.
|
||||
// chromedp.OuterHTML returns a QueryAction and chromedp.Evaluate an
|
||||
// EvaluateAction, so the variable has to be the interface both implement.
|
||||
var read chromedp.Action = chromedp.OuterHTML("html", &body, chromedp.ByQuery)
|
||||
if isKagane {
|
||||
read = chromedp.Evaluate(
|
||||
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
|
||||
&body,
|
||||
awaitPromise,
|
||||
)
|
||||
}
|
||||
|
||||
// novelfull's payload is the DOM itself, and the interstitial has a DOM
|
||||
// too, so "we have an answer" has to exclude it explicitly. kagane's
|
||||
// in-page fetch just fails while challenged, which is already the signal.
|
||||
done := func() bool { return body != "" && (isKagane || !isInterstitial(body)) }
|
||||
if err := f.run(ctx, seriesURL, read, done); err != nil {
|
||||
// Challenge never cleared, or the API refused. Indistinguishable from
|
||||
// here and handled identically by the caller.
|
||||
if errors.Is(err, errChallengeHeld) {
|
||||
return "", 403, nil
|
||||
for _, s := range sites {
|
||||
if s.Browser == nil {
|
||||
continue
|
||||
}
|
||||
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
|
||||
read, ok := s.Browser.Read(seriesURL, &body)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := f.run(ctx, seriesURL, read,
|
||||
func() bool { return s.Browser.Done(body) }); err != nil {
|
||||
// Challenge never cleared, or the payload was refused.
|
||||
// Indistinguishable from here and handled identically by the caller.
|
||||
if errors.Is(err, errChallengeHeld) {
|
||||
return "", 403, nil
|
||||
}
|
||||
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
|
||||
}
|
||||
return body, 200, nil
|
||||
}
|
||||
return body, 200, nil
|
||||
return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
|
||||
}
|
||||
|
||||
// kaganeRead builds the in-tab fetch of kagane's chapter-list API: the
|
||||
// request must be made from inside the page so it carries the clearance
|
||||
// cookie, and the API is the only place the list exists. Refusing any other
|
||||
// address is the per-Site half of the SSRF gate, kept deliberately behind
|
||||
// fetchableSeriesURL (see browserRead.Read).
|
||||
func kaganeRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
apiURL, ok := kaganeAPIURL(seriesURL)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return chromedp.Evaluate(
|
||||
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
|
||||
out, awaitPromise), true
|
||||
}
|
||||
|
||||
// novelfullRead reads the cleared DOM. novelfull renders its chapter list
|
||||
// into the served HTML, so there is no API to call from inside the page — the
|
||||
// challenge-cleared DOM is the payload.
|
||||
func novelfullRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
if !novelfullSeriesURL(seriesURL) {
|
||||
return nil, false
|
||||
}
|
||||
return chromedp.OuterHTML("html", out, chromedp.ByQuery), true
|
||||
}
|
||||
|
||||
// Image retrieves one cover's bytes through the browser sidecar, and its
|
||||
|
||||
Reference in New Issue
Block a user