feat(latest): add chromedp browser fetcher for challenge-gated sites

backend/internal/latest/browser.go

@@ -0,0 +1,146 @@
+package latest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/chromedp/cdproto/runtime"
+	"github.com/chromedp/chromedp"
+)
+
+// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
+// challenge clears in a few seconds when it clears at all; anything longer is a
+// challenge that is not going to pass, and the caller's cooldown was already
+// stamped before this ran.
+const challengeTimeout = 45 * time.Second
+
+var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
+
+// BrowserFetcher retrieves pages through a remote headless Chrome over the
+// DevTools Protocol.
+//
+// It exists for one reason: kagane.to sits behind a Cloudflare JavaScript
+// challenge. Verified 2026-08-03 from the deployment host, plain HTTP and
+// bogdanfinn/tls-client with a Chrome_133 profile both get 403 with
+// cf-mitigated: challenge on every path, including the API, robots.txt and
+// images. Clearing it requires executing the challenge script, which only a
+// real browser does.
+//
+// The request is made *inside* the page rather than by extracting cf_clearance
+// and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent
+// and often the TLS fingerprint, so replaying it means keeping three things in
+// sync that break silently and separately. The browser's own cookie jar
+// persists across polls, so the challenge is solved once every few hours.
+type BrowserFetcher struct {
+	allocCtx context.Context
+	cancel   context.CancelFunc
+	// One page at a time: caps the sidecar's memory and keeps series from
+	// sharing page state.
+	mu sync.Mutex
+}
+
+var _ Fetcher = (*BrowserFetcher)(nil)
+
+// NewBrowserFetcher connects to a headless-shell over CDP. wsURL is the
+// container's websocket endpoint, e.g. ws://headless-shell:9222.
+//
+// NoModifyURL is load-bearing: /json/version advertises a
+// webSocketDebuggerUrl pointing at 127.0.0.1, which is meaningless from another
+// container, and without this option chromedp follows it and hangs.
+func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
+	if wsURL == "" {
+		return nil, fmt.Errorf("empty browser websocket url")
+	}
+	ctx, cancel := chromedp.NewRemoteAllocator(
+		context.Background(), wsURL, chromedp.NoModifyURL)
+	return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil
+}
+
+func (f *BrowserFetcher) Close() {
+	f.cancel()
+}
+
+// Get navigates to seriesURL, lets any challenge resolve, then reads the site's
+// JSON API from inside the page so the request carries the clearance cookie.
+// The returned body is API JSON, which is what latestChapterFrom's kagane case
+// expects — it is not HTML.
+func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
+	apiURL, ok := kaganeAPIURL(seriesURL)
+	if !ok {
+		return "", 0, fmt.Errorf("not a fetchable kagane series url: %q", seriesURL)
+	}
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
+	defer cancel()
+	// A fresh tab per fetch, closed on return, so one wedged page cannot
+	// poison later polls.
+	tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
+	defer cancelTab()
+	// Bind the caller's deadline to the tab.
+	tabCtx, cancelDeadline := context.WithCancel(tabCtx)
+	defer cancelDeadline()
+	go func() {
+		<-ctx.Done()
+		cancelDeadline()
+	}()
+
+	var body string
+	err := chromedp.Run(tabCtx,
+		chromedp.Navigate(seriesURL),
+		// The challenge reloads the page itself when it passes; waiting for the
+		// site's own root element is what tells us we are through it.
+		chromedp.WaitReady("body", chromedp.ByQuery),
+		chromedp.Evaluate(
+			`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
+			&body,
+			awaitPromise,
+		),
+	)
+	if err != nil {
+		return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
+	}
+	if body == "" {
+		// Challenge still up, or the API refused. Indistinguishable from here
+		// and handled identically by the caller.
+		return "", 403, nil
+	}
+	return body, 200, nil
+}
+
+// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
+// chapter list. Returning false for anything else is a second line of defence
+// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
+// series_url is client-supplied, so the host is pinned here too.
+func kaganeAPIURL(seriesURL string) (string, bool) {
+	u, err := url.Parse(seriesURL)
+	if err != nil || u.Scheme != "https" || u.Hostname() != "kagane.to" {
+		return "", false
+	}
+	m := kaganeSeriesRe.FindStringSubmatch(u.Path)
+	if m == nil {
+		return "", false
+	}
+	return "https://kagane.to/api/v2/series/" + m[1], true
+}
+
+// awaitPromise makes Evaluate resolve the promise rather than returning a
+// serialised Promise object.
+func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
+	return p.WithAwaitPromise(true)
+}
+
+// jsString renders s as a JavaScript string literal for embedding in an
+// Evaluate expression. The URL is host-pinned by kaganeAPIURL before it gets
+// here, but quoting it properly is what keeps that guarantee intact.
+func jsString(s string) string {
+	b, _ := json.Marshal(s)
+	return string(b)
+}