feat(latest): allow comix and kagane, route kagane to a browser fetcher

2026-08-03 17:23:10 +07:00
parent 2fcf882c49
commit 89eaef70d4
2 changed files with 137 additions and 9 deletions
@@ -32,6 +32,10 @@ type Fetcher interface {
 type Poller struct {
 	Store    *store.Store
 	Fetch    Fetcher
+	// BrowserFetch handles sites behind a JavaScript challenge that Fetch
+	// cannot clear. Nil disables those sites entirely rather than falling back
+	// to Fetch, which would only ever retrieve a challenge page.
+	BrowserFetch Fetcher
 	Now          func() time.Time // injected so tests can freeze it
 	Cooldown     time.Duration
 	Interval     time.Duration
@@ -39,6 +43,18 @@ type Poller struct {
 	Batch        int
 }

+// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
+// fetched at all right now. kagane sits behind a Cloudflare JavaScript
+// challenge that no TLS fingerprint clears — verified 2026-08-03 from the
+// deployment host with the same Chrome profile TLSFetcher uses — so it is
+// browser-only or nothing.
+func (p *Poller) fetcherFor(site string) Fetcher {
+	if site == "kagane" {
+		return p.BrowserFetch
+	}
+	return p.Fetch
+}
+
 // Run polls until ctx is cancelled.
 //
 // runOnce is called synchronously, so a batch that overruns the tick delays the
@@ -130,7 +146,13 @@ func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
 		return
 	}

-	body, status, err := p.Fetch.Get(ctx, b.SeriesURL)
+	f := p.fetcherFor(b.Site)
+	if f == nil {
+		log.Printf("latest poll %q: no fetcher for site %q", b.Key, b.Site)
+		return
+	}
+
+	body, status, err := f.Get(ctx, b.SeriesURL)
 	if err != nil {
 		log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
 		return
@@ -185,13 +207,18 @@ func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
 }

 // fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
-// to parse and seriesURL is safe to hand to the fetcher: an https URL with a
+// to parse and seriesURL is safe to hand to a fetcher: an https URL with a
 // non-empty host. series_url comes from client-supplied PUT bodies, so this is
 // a defence against the poller being used to probe arbitrary hosts from the
 // server's own network position, not just a check against wasted requests.
+//
+// kagane is held to a stricter rule: it is fetched by a headless browser, which
+// executes JavaScript and carries cookies, and is therefore a far stronger SSRF
+// primitive than an HTTP GET. Its host must match exactly, not merely be
+// non-empty.
 func fetchableSeriesURL(site, seriesURL string) bool {
 	switch site {
-	case "asura", "demonic":
+	case "asura", "demonic", "comix", "kagane":
 	default:
 		return false
 	}
@@ -199,5 +226,11 @@ func fetchableSeriesURL(site, seriesURL string) bool {
 	if err != nil {
 		return false
 	}
-	return u.Scheme == "https" && u.Host != ""
+	if u.Scheme != "https" || u.Host == "" {
+		return false
+	}
+	if site == "kagane" {
+		return u.Hostname() == "kagane.to"
+	}
+	return true
 }
@@ -358,3 +358,98 @@ func TestRunOnceStopsOnCancelledContext(t *testing.T) {
 		t.Fatalf("fetched %d series with a cancelled context, want 0", got)
 	}
 }
+
+func TestFetchableSeriesURL(t *testing.T) {
+	tests := []struct {
+		name      string
+		site      string
+		seriesURL string
+		want      bool
+	}{
+		{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
+		{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
+		{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
+		{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
+		// The browser fetcher runs JavaScript and carries cookies, so a
+		// client-supplied series_url must not be able to aim it anywhere else.
+		{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
+		{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
+		{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
+		{"non-https", "comix", "http://comix.to/title/x", false},
+		{"no host", "comix", "https:///title/x", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
+				t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
+					tt.site, tt.seriesURL, got, tt.want)
+			}
+		})
+	}
+}
+
+// A kagane row must not be handed to the plain TLS fetcher: it would only ever
+// receive a challenge page, and the browser fetcher is the whole reason kagane
+// is pollable at all.
+func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
+	s := newTestStore(t)
+	if _, err := s.Upsert(store.Bookmark{
+		Key:       "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		Site:      "kagane",
+		SeriesID:  "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		UpdatedAt: 1000,
+	}); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
+	p := &Poller{
+		Store: s, Fetch: f,
+		Now:      func() time.Time { return time.UnixMilli(5_000_000) },
+		Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
+	}
+	p.runOnce(context.Background())
+
+	if len(f.calls) != 0 {
+		t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
+	}
+}
+
+// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
+func TestKaganeUsesBrowserFetcher(t *testing.T) {
+	s := newTestStore(t)
+	key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	if _, err := s.Upsert(store.Bookmark{
+		Key:       key,
+		Site:      "kagane",
+		SeriesID:  "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		UpdatedAt: 1000,
+	}); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	tlsF := &fakeFetcher{body: "", status: 200}
+	browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
+	p := &Poller{
+		Store: s, Fetch: tlsF, BrowserFetch: browserF,
+		Now:      func() time.Time { return time.UnixMilli(5_000_000) },
+		Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
+	}
+	p.runOnce(context.Background())
+
+	if len(tlsF.calls) != 0 {
+		t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
+	}
+	if len(browserF.calls) != 1 {
+		t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
+	}
+	got, found, err := s.Get(key)
+	if err != nil || !found {
+		t.Fatalf("Get: %v found=%v", err, found)
+	}
+	if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
+		t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
+	}
+}