diff --git a/backend/internal/latest/browser.go b/backend/internal/latest/browser.go index fb856af..b23c244 100644 --- a/backend/internal/latest/browser.go +++ b/backend/internal/latest/browser.go @@ -2,7 +2,9 @@ package latest import ( "context" + "encoding/base64" "encoding/json" + "errors" "fmt" "net/url" "regexp" @@ -22,6 +24,12 @@ const challengeTimeout = 45 * time.Second var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`) +// kaganeImageIDRe pins the only path segment Image interpolates into an +// outbound URL. The id arrives from a stored cover URL, which a client +// supplied, so it is matched rather than trusted: a headless browser is a +// strong SSRF primitive. +var kaganeImageIDRe = regexp.MustCompile(`^[0-9a-f-]{36}$`) + // BrowserFetcher retrieves pages through a remote headless Chrome over the // DevTools Protocol. // @@ -91,23 +99,6 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL) } - f.mu.Lock() - defer f.mu.Unlock() - - ctx, cancel := context.WithTimeout(ctx, challengeTimeout) - defer cancel() - // A fresh tab per fetch, closed on return, so one wedged page cannot - // poison later polls. - tabCtx, cancelTab := chromedp.NewContext(f.allocCtx) - defer cancelTab() - // Bind the caller's deadline to the tab. - tabCtx, cancelDeadline := context.WithCancel(tabCtx) - defer cancelDeadline() - go func() { - <-ctx.Done() - cancelDeadline() - }() - var body string // kagane's chapter list is only in its JSON API, which must be called from // inside the page so the request carries the clearance cookie. novelfull @@ -123,24 +114,134 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int ) } - err := chromedp.Run(tabCtx, - chromedp.Navigate(seriesURL), - // The challenge reloads the page itself when it passes; waiting for the - // site's own root element is what tells us we are through it. - chromedp.WaitReady("body", chromedp.ByQuery), - read, - ) - if err != nil { + // novelfull's payload is the DOM itself, and the interstitial has a DOM + // too, so "we have an answer" has to exclude it explicitly. kagane's + // in-page fetch just fails while challenged, which is already the signal. + done := func() bool { return body != "" && (isKagane || !isInterstitial(body)) } + if err := f.run(ctx, seriesURL, read, done); err != nil { + // Challenge never cleared, or the API refused. Indistinguishable from + // here and handled identically by the caller. + if errors.Is(err, errChallengeHeld) { + return "", 403, nil + } return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err) } - if body == "" { - // Challenge still up, or the API refused. Indistinguishable from here - // and handled identically by the caller. - return "", 403, nil - } return body, 200, nil } +// Image retrieves one kagane cover as raw bytes and its content type. +// +// It exists because kagane serves covers behind the same challenge as its +// pages *and* with `cross-origin-resource-policy: same-origin`, so an on +// the web UI's origin cannot load one even from a browser that already holds +// the clearance cookie (verified 2026-08-08). Proxying is the only route. +// +// The image URL is navigated to rather than fetched from some other kagane +// page: the challenge only runs on a top-level navigation, and once it clears +// the document *is* the image, so a same-origin fetch of location.href reads +// it straight back out of the cache. +// +// The challenge is not solved by the first read: WaitReady("body") is satisfied +// by the interstitial too. run holds the tab open until the in-page fetch +// succeeds, which is what gives the challenge script the seconds it needs. +func (f *BrowserFetcher) Image(ctx context.Context, imageID string) ([]byte, string, error) { + if !kaganeImageIDRe.MatchString(imageID) { + return nil, "", fmt.Errorf("not a kagane image id: %q", imageID) + } + var dataURL string + err := f.run(ctx, "https://kagane.to/api/v2/image/"+imageID+"/compressed", + chromedp.Evaluate(`fetch(location.href).then(r => r.ok + ? r.blob().then(b => new Promise(res => { + const fr = new FileReader(); + fr.onload = () => res(fr.result); + fr.readAsDataURL(b); + })) + : "")`, &dataURL, awaitPromise), + func() bool { return dataURL != "" }) + if err != nil { + return nil, "", fmt.Errorf("browser image %s: %w", imageID, err) + } + // "data:image/webp;base64,". + head, payload, ok := strings.Cut(dataURL, ";base64,") + if !ok { + return nil, "", fmt.Errorf("browser image %s: not a data url", imageID) + } + raw, err := base64.StdEncoding.DecodeString(payload) + if err != nil { + return nil, "", fmt.Errorf("browser image %s: %w", imageID, err) + } + return raw, strings.TrimPrefix(head, "data:"), nil +} + +// errChallengeHeld reports that the budget ran out with the interstitial still +// up. Distinct from a transport failure: it means "this site said no", which +// the poller answers with a 403 and its ordinary cooldown. +var errChallengeHeld = errors.New("challenge held") + +// challengePollInterval paces re-reads while a challenge solves itself. +const challengePollInterval = 2 * time.Second + +// isInterstitial reports whether html is Cloudflare's challenge page rather +// than the site's own. Matched on the challenge runtime's script path, which is +// stable across the interstitial's wording and locale — the visible "Just a +// moment..." title is neither. +func isInterstitial(html string) bool { + return strings.Contains(html, "/cdn-cgi/challenge-platform/") +} + +// run navigates to target and re-reads until done reports an answer, bounded by +// challengeTimeout and by the caller's own deadline, in a tab that is closed on +// return so one wedged page cannot poison later calls. +// +// Holding the tab open across re-reads is the whole point. A Cloudflare +// interstitial needs several seconds of a live page to solve itself and write +// clearance into the browser's shared cookie jar; reading once and closing the +// tab — which is what this did before 2026-08-08 — never gives it that window, +// so every fetch lands on the interstitial and the clearance that would have +// unblocked all the later ones is never obtained. +func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.Action, done func() bool) error { + f.mu.Lock() + defer f.mu.Unlock() + + ctx, cancel := context.WithTimeout(ctx, challengeTimeout) + defer cancel() + tabCtx, cancelTab := chromedp.NewContext(f.allocCtx) + defer cancelTab() + // Bind the caller's deadline to the tab. + tabCtx, cancelDeadline := context.WithCancel(tabCtx) + defer cancelDeadline() + go func() { + <-ctx.Done() + cancelDeadline() + }() + + if err := chromedp.Run(tabCtx, + chromedp.Navigate(target), + chromedp.WaitReady("body", chromedp.ByQuery), + ); err != nil { + return err + } + + var lastErr error + for { + // The challenge reloads the page when it passes, which tears down the + // execution context mid-read. That is a retry, not a failure. + if err := chromedp.Run(tabCtx, read); err != nil { + lastErr = err + } else if done() { + return nil + } + select { + case <-ctx.Done(): + if lastErr != nil { + return fmt.Errorf("%w (last read: %v)", errChallengeHeld, lastErr) + } + return errChallengeHeld + case <-time.After(challengePollInterval): + } + } +} + // kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its // chapter list. Returning false for anything else is a second line of defence // behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and diff --git a/backend/internal/latest/smoke_image_test.go b/backend/internal/latest/smoke_image_test.go new file mode 100644 index 0000000..cb88b8c --- /dev/null +++ b/backend/internal/latest/smoke_image_test.go @@ -0,0 +1,91 @@ +package latest + +import ( + "context" + "net/http" + "os" + "testing" + "time" +) + +// TestSmokeKaganeImage is the live proof that the cover proxy's fetch actually +// clears Cloudflare and returns image bytes. It needs a real headless Chrome +// with outbound network, so it runs only when SMOKE_BROWSER_WS_URL is set: +// +// docker run --rm --shm-size=1gb -p 19222:9222 chromedp/headless-shell:stable +// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:19222 go test -run TestSmokeKaganeImage ./internal/latest +func TestSmokeKaganeImage(t *testing.T) { + ws := os.Getenv("SMOKE_BROWSER_WS_URL") + if ws == "" { + t.Skip("SMOKE_BROWSER_WS_URL unset") + } + const imageID = "019fe11a-84c3-7fc3-a84b-88787374b617" // SP Baby's cover + + // The same URL through a plain client is what the web UI's gets. + // Asserting on it keeps the test honest about why the browser is needed. + req, err := http.NewRequest(http.MethodGet, + "https://kagane.to/api/v2/image/"+imageID+"/compressed", nil) + if err != nil { + t.Fatal(err) + } + if res, err := (&http.Client{Timeout: 15 * time.Second}).Do(req); err == nil { + res.Body.Close() + if res.StatusCode == http.StatusOK { + t.Log("note: kagane answered a plain request 200 — the challenge is not up right now") + } + } + + f, err := NewBrowserFetcher(ws) + if err != nil { + t.Fatalf("NewBrowserFetcher: %v", err) + } + defer f.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + body, contentType, err := f.Image(ctx, imageID) + if err != nil { + t.Fatalf("Image: %v", err) + } + if len(body) < 1000 { + t.Fatalf("body is %d bytes, want a real image", len(body)) + } + if contentType != "image/webp" { + t.Fatalf("content type = %q, want image/webp", contentType) + } + // WebP files start with "RIFF....WEBP". + if string(body[:4]) != "RIFF" || string(body[8:12]) != "WEBP" { + t.Fatalf("body is not a WebP: % x", body[:12]) + } + t.Logf("fetched %d bytes of %s", len(body), contentType) + + if _, _, err := f.Image(ctx, "not-a-uuid"); err == nil { + t.Fatal("Image accepted a non-uuid id") + } +} + +// Control for the test above: the poller's own kagane path, same sidecar. If +// this fails too, the sidecar is not clearing the challenge at all and the +// image result says nothing about Image itself. +func TestSmokeKaganeGet(t *testing.T) { + ws := os.Getenv("SMOKE_BROWSER_WS_URL") + if ws == "" { + t.Skip("SMOKE_BROWSER_WS_URL unset") + } + f, err := NewBrowserFetcher(ws) + if err != nil { + t.Fatalf("NewBrowserFetcher: %v", err) + } + defer f.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + body, status, err := f.Get(ctx, "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787") + if err != nil { + t.Fatalf("Get: %v", err) + } + t.Logf("status=%d bytes=%d head=%.80q", status, len(body), body) + if status != 200 { + t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status) + } +}