feat(latest): poll novelfull via headless browser, lightnovelworld via TLS
This commit is contained in:
@@ -44,12 +44,13 @@ type Poller struct {
|
||||
}
|
||||
|
||||
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
|
||||
// fetched at all right now. kagane sits behind a Cloudflare JavaScript
|
||||
// challenge that no TLS fingerprint clears — verified 2026-08-03 from the
|
||||
// deployment host with the same Chrome profile TLSFetcher uses — so it is
|
||||
// browser-only or nothing.
|
||||
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
|
||||
// JavaScript challenge that no TLS fingerprint clears — kagane verified
|
||||
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
|
||||
// profile TLSFetcher uses — so they are browser-only or nothing.
|
||||
func (p *Poller) fetcherFor(site string) Fetcher {
|
||||
if site == "kagane" {
|
||||
switch site {
|
||||
case "kagane", "novelfull":
|
||||
return p.BrowserFetch
|
||||
}
|
||||
return p.Fetch
|
||||
@@ -212,13 +213,18 @@ func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
|
||||
// a defence against the poller being used to probe arbitrary hosts from the
|
||||
// server's own network position, not just a check against wasted requests.
|
||||
//
|
||||
// kagane is held to a stricter rule: it is fetched by a headless browser, which
|
||||
// executes JavaScript and carries cookies, and is therefore a far stronger SSRF
|
||||
// primitive than an HTTP GET. Its host must match exactly, not merely be
|
||||
// non-empty.
|
||||
// Three sites are held to a stricter rule, each for a different reason:
|
||||
//
|
||||
// - kagane and novelfull are fetched by a headless browser, which executes
|
||||
// JavaScript and carries cookies, and is therefore a far stronger SSRF
|
||||
// primitive than an HTTP GET. Their hosts must match exactly, not merely
|
||||
// be non-empty.
|
||||
// - lightnovelworld's parser regex hardcodes its host, so a URL anywhere
|
||||
// else could never yield a match — reject it here rather than burn the
|
||||
// request.
|
||||
func fetchableSeriesURL(site, seriesURL string) bool {
|
||||
switch site {
|
||||
case "asura", "demonic", "comix", "kagane":
|
||||
case "asura", "demonic", "comix", "kagane", "novelfull", "lightnovelworld":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -229,8 +235,17 @@ func fetchableSeriesURL(site, seriesURL string) bool {
|
||||
if u.Scheme != "https" || u.Host == "" {
|
||||
return false
|
||||
}
|
||||
if site == "kagane" {
|
||||
switch site {
|
||||
case "kagane":
|
||||
return u.Hostname() == "kagane.to"
|
||||
case "novelfull":
|
||||
// Fetched by a real browser, same as kagane, so the host is pinned
|
||||
// rather than merely non-empty.
|
||||
return u.Hostname() == "novelfull.com"
|
||||
case "lightnovelworld":
|
||||
// Its parser regex hardcodes this host, so a URL anywhere else could
|
||||
// never yield a match — reject it here rather than burn the request.
|
||||
return u.Hostname() == "lightnovelworld.net"
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user