feat(latest): add chromedp browser fetcher for challenge-gated sites

backend/go.mod

@@ -1,10 +1,12 @@
 module mangabm/backend
 
-go 1.24.1
+go 1.26
 
 require (
 	github.com/bogdanfinn/fhttp v0.6.8
 	github.com/bogdanfinn/tls-client v1.15.1
+	github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f
+	github.com/chromedp/chromedp v0.16.0
 	modernc.org/sqlite v1.34.4
 )
 
@@ -15,7 +17,12 @@ require (
 	github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect
 	github.com/bogdanfinn/utls v1.7.7-barnius // indirect
 	github.com/bogdanfinn/websocket v1.5.5-barnius // indirect
+	github.com/chromedp/sysutil v1.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.4.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/klauspost/compress v1.18.2 // indirect
@@ -26,7 +33,7 @@ require (
 	github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
 	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/net v0.48.0 // indirect
-	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.32.0 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 	modernc.org/libc v1.55.3 // indirect

backend/go.sum

@@ -14,10 +14,24 @@ github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5Zl
 github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg=
 github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI=
 github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI=
+github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk=
+github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0=
+github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
+github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
+github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
+github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg=
+github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
+github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
+github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
+github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
 github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
 github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -26,10 +40,14 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
 github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
@@ -56,8 +74,8 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=

backend/internal/latest/browser.go

@@ -0,0 +1,146 @@
+package latest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/chromedp/cdproto/runtime"
+	"github.com/chromedp/chromedp"
+)
+
+// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
+// challenge clears in a few seconds when it clears at all; anything longer is a
+// challenge that is not going to pass, and the caller's cooldown was already
+// stamped before this ran.
+const challengeTimeout = 45 * time.Second
+
+var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
+
+// BrowserFetcher retrieves pages through a remote headless Chrome over the
+// DevTools Protocol.
+//
+// It exists for one reason: kagane.to sits behind a Cloudflare JavaScript
+// challenge. Verified 2026-08-03 from the deployment host, plain HTTP and
+// bogdanfinn/tls-client with a Chrome_133 profile both get 403 with
+// cf-mitigated: challenge on every path, including the API, robots.txt and
+// images. Clearing it requires executing the challenge script, which only a
+// real browser does.
+//
+// The request is made *inside* the page rather than by extracting cf_clearance
+// and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent
+// and often the TLS fingerprint, so replaying it means keeping three things in
+// sync that break silently and separately. The browser's own cookie jar
+// persists across polls, so the challenge is solved once every few hours.
+type BrowserFetcher struct {
+	allocCtx context.Context
+	cancel   context.CancelFunc
+	// One page at a time: caps the sidecar's memory and keeps series from
+	// sharing page state.
+	mu sync.Mutex
+}
+
+var _ Fetcher = (*BrowserFetcher)(nil)
+
+// NewBrowserFetcher connects to a headless-shell over CDP. wsURL is the
+// container's websocket endpoint, e.g. ws://headless-shell:9222.
+//
+// NoModifyURL is load-bearing: /json/version advertises a
+// webSocketDebuggerUrl pointing at 127.0.0.1, which is meaningless from another
+// container, and without this option chromedp follows it and hangs.
+func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
+	if wsURL == "" {
+		return nil, fmt.Errorf("empty browser websocket url")
+	}
+	ctx, cancel := chromedp.NewRemoteAllocator(
+		context.Background(), wsURL, chromedp.NoModifyURL)
+	return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil
+}
+
+func (f *BrowserFetcher) Close() {
+	f.cancel()
+}
+
+// Get navigates to seriesURL, lets any challenge resolve, then reads the site's
+// JSON API from inside the page so the request carries the clearance cookie.
+// The returned body is API JSON, which is what latestChapterFrom's kagane case
+// expects — it is not HTML.
+func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
+	apiURL, ok := kaganeAPIURL(seriesURL)
+	if !ok {
+		return "", 0, fmt.Errorf("not a fetchable kagane series url: %q", seriesURL)
+	}
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
+	defer cancel()
+	// A fresh tab per fetch, closed on return, so one wedged page cannot
+	// poison later polls.
+	tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
+	defer cancelTab()
+	// Bind the caller's deadline to the tab.
+	tabCtx, cancelDeadline := context.WithCancel(tabCtx)
+	defer cancelDeadline()
+	go func() {
+		<-ctx.Done()
+		cancelDeadline()
+	}()
+
+	var body string
+	err := chromedp.Run(tabCtx,
+		chromedp.Navigate(seriesURL),
+		// The challenge reloads the page itself when it passes; waiting for the
+		// site's own root element is what tells us we are through it.
+		chromedp.WaitReady("body", chromedp.ByQuery),
+		chromedp.Evaluate(
+			`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
+			&body,
+			awaitPromise,
+		),
+	)
+	if err != nil {
+		return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
+	}
+	if body == "" {
+		// Challenge still up, or the API refused. Indistinguishable from here
+		// and handled identically by the caller.
+		return "", 403, nil
+	}
+	return body, 200, nil
+}
+
+// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
+// chapter list. Returning false for anything else is a second line of defence
+// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
+// series_url is client-supplied, so the host is pinned here too.
+func kaganeAPIURL(seriesURL string) (string, bool) {
+	u, err := url.Parse(seriesURL)
+	if err != nil || u.Scheme != "https" || u.Hostname() != "kagane.to" {
+		return "", false
+	}
+	m := kaganeSeriesRe.FindStringSubmatch(u.Path)
+	if m == nil {
+		return "", false
+	}
+	return "https://kagane.to/api/v2/series/" + m[1], true
+}
+
+// awaitPromise makes Evaluate resolve the promise rather than returning a
+// serialised Promise object.
+func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
+	return p.WithAwaitPromise(true)
+}
+
+// jsString renders s as a JavaScript string literal for embedding in an
+// Evaluate expression. The URL is host-pinned by kaganeAPIURL before it gets
+// here, but quoting it properly is what keeps that guarantee intact.
+func jsString(s string) string {
+	b, _ := json.Marshal(s)
+	return string(b)
+}

backend/internal/latest/browser_test.go

@@ -0,0 +1,38 @@
+package latest
+
+import "testing"
+
+func TestKaganeAPIURL(t *testing.T) {
+	const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	tests := []struct {
+		name      string
+		seriesURL string
+		want      string
+		wantOK    bool
+	}{
+		{
+			name:      "series page maps to its API endpoint",
+			seriesURL: "https://kagane.to/series/" + uuid,
+			want:      "https://kagane.to/api/v2/series/" + uuid,
+			wantOK:    true,
+		},
+		{
+			name:      "trailing slash is tolerated",
+			seriesURL: "https://kagane.to/series/" + uuid + "/",
+			want:      "https://kagane.to/api/v2/series/" + uuid,
+			wantOK:    true,
+		},
+		{"not a series path", "https://kagane.to/search", "", false},
+		{"foreign host", "https://evil.example/series/" + uuid, "", false},
+		{"garbage", "://", "", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := kaganeAPIURL(tt.seriesURL)
+			if ok != tt.wantOK || got != tt.want {
+				t.Errorf("kaganeAPIURL(%q) = %q, %v; want %q, %v",
+					tt.seriesURL, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}