sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: one registry entry per Site, derived browser route and gate (#94)

Browse Source 
Collapse the six per-site comparison points into a sites map in sites.go:
Latest Chapter parse, Cover parse, browser-backed list, fetcher route,
host pins, and the browser payload read all become lookups into it. All
six hosts are now pinned in fetchableSeriesURL; asura/demonic/comix were
previously accepted on any https host.
This commit is contained in:
Sulthan Zaki
2026-08-12 00:21:28 +07:00
parent 4a92e956cf
commit 2d134fb05c
4 changed files with 307 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/poller.go
+27 -56
View File
@@ -4,7 +4,6 @@ import (
"context"
"log"
"net/url"
"slices"
"time"
"bookmarkmanager/backend/internal/store"
@@ -57,8 +56,6 @@ type Poller struct {
Batch int
}
var browserBackedSites = []string{"kagane", "novelfull"}
// fillBlankCover gives a Series its Cover when it has none. The blank state is
// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows
// created before acquisition existed, and rows whose creation-time fetch
@@ -120,27 +117,23 @@ func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL stri
}
// fetcherFor returns the fetcher a site's page needs, or nil when the site
// cannot be fetched at all right now. kagane and novelfull pages sit behind a
// Cloudflare JavaScript challenge that no TLS fingerprint clears (kagane
// verified 2026-08-03, novelfull verified 2026-08-05, both against the same
// Chrome_133 profile TLSFetcher uses), so both prefer the browser; novelfull
// alone falls back to the plain-TLS fetcher when no browser is configured,
// because its challenge is a live time-varying fact (AGENTS.md) and its cover
// bytes never need the browser. kagane never falls back: a plain fetch of a
// kagane page or cover would only ever retrieve a challenge page. One routing
// rule for the poll and the acquirer, so the two cannot drift apart.
// cannot be fetched at all right now. A Site whose registry entry carries a
// Browser read — kagane and novelfull, both behind a Cloudflare JavaScript
// challenge no TLS fingerprint clears — prefers the browser; when it is
// absent, the entry's Fallback decides whether plain TLS may take over. One
// routing rule for the poll and the acquirer, so the two cannot drift apart.
func fetcherFor(site string, browser, tls Fetcher) Fetcher {
switch {
case site == "kagane":
return browser
case slices.Contains(browserBackedSites, site): // novelfull
if browser != nil {
return browser
}
return tls
default:
s := sites[site]
if s.Browser == nil {
return tls
}
if browser != nil {
return browser
}
if s.Browser.Fallback {
return tls
}
return nil
}
// Run polls until ctx is cancelled.
@@ -170,7 +163,7 @@ func (p *Poller) runOnce(ctx context.Context) {
now := p.Now()
cutoff := now.Add(-p.Cooldown).UnixMilli()
browserCutoff := now.Add(-p.BrowserCooldown).UnixMilli()
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites, p.Batch)
due, err := p.Store.DueForLatestCheck(cutoff, browserCutoff, browserBackedSites(), p.Batch)
if err != nil {
log.Printf("latest poll: due query: %v", err)
return
@@ -283,45 +276,23 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
log.Printf("latest poll %q: latest is now %s", sr.Key(), latest.Label)
}
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
// to parse and seriesURL is safe to hand to a fetcher: an https URL with a
// non-empty host. series_url comes from client-supplied PUT bodies, so this is
// a defence against the poller being used to probe arbitrary hosts from the
// server's own network position, not just a check against wasted requests.
//
// Three sites are held to a stricter rule, each for a different reason:
//
// - kagane and novelfull are fetched by a headless browser, which executes
// JavaScript and carries cookies, and is therefore a far stronger SSRF
// primitive than an HTTP GET. Their hosts must match exactly, not merely
// be non-empty.
// - lightnovelworld's parser regex hardcodes its host, so a URL anywhere
// else could never yield a match — reject it here rather than burn the
// request.
// fetchableSeriesURL reports whether site is a Site the registry knows and
// seriesURL is safe to hand to a fetcher: an https URL whose host matches the
// Site's pinned hostname exactly. series_url comes from client-supplied PUT
// bodies, so this is a defence against the poller being used to probe
// arbitrary hosts from the server's own network position, not just a check
// against wasted requests. The pin guards different things per Site — a
// browser Site guards a control that executes JavaScript and carries cookies,
// a parser Site guards a wasted request — but the rule is one rule, from the
// registry.
func fetchableSeriesURL(site, seriesURL string) bool {
switch site {
case "asura", "demonic", "comix", "kagane", "novelfull", "lightnovelworld":
default:
s, known := sites[site]
if !known {
return false
}
u, err := url.Parse(seriesURL)
if err != nil {
return false
}
if u.Scheme != "https" || u.Host == "" {
return false
}
switch site {
case "kagane":
return u.Hostname() == "kagane.to"
case "novelfull":
// Fetched by a real browser, same as kagane, so the host is pinned
// rather than merely non-empty.
return u.Hostname() == "novelfull.com"
case "lightnovelworld":
// Its parser regex hardcodes this host, so a URL anywhere else could
// never yield a match — reject it here rather than burn the request.
return u.Hostname() == "lightnovelworld.net"
}
return true
return u.Scheme == "https" && u.Hostname() == s.Host
}