feat(latest): allow comix and kagane, route kagane to a browser fetcher

This commit is contained in:
2026-08-03 17:23:10 +07:00
parent 2fcf882c49
commit 89eaef70d4
2 changed files with 137 additions and 9 deletions
+42 -9
View File
@@ -32,11 +32,27 @@ type Fetcher interface {
type Poller struct {
Store *store.Store
Fetch Fetcher
Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
// BrowserFetch handles sites behind a JavaScript challenge that Fetch
// cannot clear. Nil disables those sites entirely rather than falling back
// to Fetch, which would only ever retrieve a challenge page.
BrowserFetch Fetcher
Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration
Interval time.Duration
Stagger time.Duration
Batch int
}
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane sits behind a Cloudflare JavaScript
// challenge that no TLS fingerprint clears — verified 2026-08-03 from the
// deployment host with the same Chrome profile TLSFetcher uses — so it is
// browser-only or nothing.
func (p *Poller) fetcherFor(site string) Fetcher {
if site == "kagane" {
return p.BrowserFetch
}
return p.Fetch
}
// Run polls until ctx is cancelled.
@@ -130,7 +146,13 @@ func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
return
}
body, status, err := p.Fetch.Get(ctx, b.SeriesURL)
f := p.fetcherFor(b.Site)
if f == nil {
log.Printf("latest poll %q: no fetcher for site %q", b.Key, b.Site)
return
}
body, status, err := f.Get(ctx, b.SeriesURL)
if err != nil {
log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
return
@@ -185,13 +207,18 @@ func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
}
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
// to parse and seriesURL is safe to hand to the fetcher: an https URL with a
// to parse and seriesURL is safe to hand to a fetcher: an https URL with a
// non-empty host. series_url comes from client-supplied PUT bodies, so this is
// a defence against the poller being used to probe arbitrary hosts from the
// server's own network position, not just a check against wasted requests.
//
// kagane is held to a stricter rule: it is fetched by a headless browser, which
// executes JavaScript and carries cookies, and is therefore a far stronger SSRF
// primitive than an HTTP GET. Its host must match exactly, not merely be
// non-empty.
func fetchableSeriesURL(site, seriesURL string) bool {
switch site {
case "asura", "demonic":
case "asura", "demonic", "comix", "kagane":
default:
return false
}
@@ -199,5 +226,11 @@ func fetchableSeriesURL(site, seriesURL string) bool {
if err != nil {
return false
}
return u.Scheme == "https" && u.Host != ""
if u.Scheme != "https" || u.Host == "" {
return false
}
if site == "kagane" {
return u.Hostname() == "kagane.to"
}
return true
}