sulthan
05e93a4869
BrowserFetcher.run navigated, waited for "body", read once, and closed the tab - about half a second end to end. The Cloudflare interstitial has a body too, so WaitReady was satisfied by the challenge page itself, and the read that followed was of the interstitial rather than the site. That made the challenge unclearable rather than merely slow. An interstitial needs several seconds of a live page to solve itself and write clearance into the browser's shared cookie jar; tearing the tab down first means the clearance that would have unblocked every later fetch is never obtained, so each call is challenged exactly like the one before it. run now holds one tab and re-reads until the caller's predicate reports an answer, bounded by challengeTimeout and by the caller's own deadline. Each caller supplies the predicate that fits its payload: kagane's in-page fetch simply returns nothing while challenged, whereas novelfull's payload is the DOM, and the interstitial has a DOM as well, so that one excludes the challenge markup explicitly. Exhausting the budget is now reported as errChallengeHeld and mapped back to the 403 the poller already expects, keeping a challenged site distinct from a broken transport. Image loses its own retry loop, which run now subsumes. Measured against a real kagane cover from a cold browser profile: no image at all before, 4.9s to a 56710-byte image/webp after. The live proof is TestSmokeKagane* in smoke_image_test.go, which skips unless SMOKE_BROWSER_WS_URL names a sidecar, so `go test ./...` stays hermetic.
92 lines
2.8 KiB
Go
92 lines
2.8 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// TestSmokeKaganeImage is the live proof that the cover proxy's fetch actually
|
|
// clears Cloudflare and returns image bytes. It needs a real headless Chrome
|
|
// with outbound network, so it runs only when SMOKE_BROWSER_WS_URL is set:
|
|
//
|
|
// docker run --rm --shm-size=1gb -p 19222:9222 chromedp/headless-shell:stable
|
|
// SMOKE_BROWSER_WS_URL=ws://127.0.0.1:19222 go test -run TestSmokeKaganeImage ./internal/latest
|
|
func TestSmokeKaganeImage(t *testing.T) {
|
|
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
|
if ws == "" {
|
|
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
|
}
|
|
const imageID = "019fe11a-84c3-7fc3-a84b-88787374b617" // SP Baby's cover
|
|
|
|
// The same URL through a plain client is what the web UI's <img> gets.
|
|
// Asserting on it keeps the test honest about why the browser is needed.
|
|
req, err := http.NewRequest(http.MethodGet,
|
|
"https://kagane.to/api/v2/image/"+imageID+"/compressed", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res, err := (&http.Client{Timeout: 15 * time.Second}).Do(req); err == nil {
|
|
res.Body.Close()
|
|
if res.StatusCode == http.StatusOK {
|
|
t.Log("note: kagane answered a plain request 200 — the challenge is not up right now")
|
|
}
|
|
}
|
|
|
|
f, err := NewBrowserFetcher(ws)
|
|
if err != nil {
|
|
t.Fatalf("NewBrowserFetcher: %v", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
body, contentType, err := f.Image(ctx, imageID)
|
|
if err != nil {
|
|
t.Fatalf("Image: %v", err)
|
|
}
|
|
if len(body) < 1000 {
|
|
t.Fatalf("body is %d bytes, want a real image", len(body))
|
|
}
|
|
if contentType != "image/webp" {
|
|
t.Fatalf("content type = %q, want image/webp", contentType)
|
|
}
|
|
// WebP files start with "RIFF....WEBP".
|
|
if string(body[:4]) != "RIFF" || string(body[8:12]) != "WEBP" {
|
|
t.Fatalf("body is not a WebP: % x", body[:12])
|
|
}
|
|
t.Logf("fetched %d bytes of %s", len(body), contentType)
|
|
|
|
if _, _, err := f.Image(ctx, "not-a-uuid"); err == nil {
|
|
t.Fatal("Image accepted a non-uuid id")
|
|
}
|
|
}
|
|
|
|
// Control for the test above: the poller's own kagane path, same sidecar. If
|
|
// this fails too, the sidecar is not clearing the challenge at all and the
|
|
// image result says nothing about Image itself.
|
|
func TestSmokeKaganeGet(t *testing.T) {
|
|
ws := os.Getenv("SMOKE_BROWSER_WS_URL")
|
|
if ws == "" {
|
|
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
|
}
|
|
f, err := NewBrowserFetcher(ws)
|
|
if err != nil {
|
|
t.Fatalf("NewBrowserFetcher: %v", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
body, status, err := f.Get(ctx, "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787")
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
t.Logf("status=%d bytes=%d head=%.80q", status, len(body), body)
|
|
if status != 200 {
|
|
t.Fatalf("status = %d, want 200 — the sidecar is not clearing the challenge", status)
|
|
}
|
|
}
|