From ba23411a7444120e57a3f7fbfc442542ffe07645 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Mon, 3 Aug 2026 18:21:25 +0700 Subject: [PATCH] fix: address issues found in end-to-end verification Chained defects made the kagane browser-fetch path completely non-functional in Docker Compose: headless-shell's compose command re-declared --remote-debugging-port, colliding with the image's own entrypoint/socat proxy (EOF on every dial); the sidecar was then only reachable by Docker DNS name, which Chrome's DevTools HTTP handler rejects with a 500 (Host-header/DNS-rebinding check); and NewBrowserFetcher's NoModifyURL option skipped /json/version discovery entirely, dialing a bare host:port that Chrome 404s since /devtools/browser/ is minted fresh per Chrome start. Fixed by trimming the redundant command flags, pinning headless-shell to a static IP so BROWSER_WS_URL can name it directly, and removing NoModifyURL so chromedp's discovery (which echoes the request's Host back into webSocketDebuggerUrl) does the right thing on its own. Co-Authored-By: Claude Opus 5 --- backend/internal/latest/browser.go | 22 +++++++++++++++------- docker-compose.prod.yml | 5 ++++- docker-compose.yml | 26 ++++++++++++++++++++++---- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/backend/internal/latest/browser.go b/backend/internal/latest/browser.go index 81c2707..10bcf21 100644 --- a/backend/internal/latest/browser.go +++ b/backend/internal/latest/browser.go @@ -46,18 +46,26 @@ type BrowserFetcher struct { var _ Fetcher = (*BrowserFetcher)(nil) -// NewBrowserFetcher connects to a headless-shell over CDP. wsURL is the -// container's websocket endpoint, e.g. ws://headless-shell:9222. +// NewBrowserFetcher connects to a headless-shell over CDP. wsURL must name the +// sidecar by IP, e.g. ws://172.28.0.10:9222 — not by Docker DNS name. Chrome's +// DevTools HTTP handler 500s any /json/version request whose Host header +// isn't an IP or "localhost" (confirmed 2026-08-03 against +// chromedp/headless-shell:stable), so the compose network pins the sidecar's +// address for this to resolve at all. // -// NoModifyURL is load-bearing: /json/version advertises a -// webSocketDebuggerUrl pointing at 127.0.0.1, which is meaningless from another -// container, and without this option chromedp follows it and hangs. +// Do not add chromedp.NoModifyURL here: that option skips the /json/version +// discovery request entirely and dials wsURL as if it were already the full +// debugger endpoint, but Chrome only accepts connections at +// /devtools/browser/, a path chosen fresh at every Chrome start — dialing +// the bare host:port 404s. The default (discovery) path works precisely +// because Chrome's /json/version response echoes back the Host header of the +// discovery request in webSocketDebuggerUrl, so as long as wsURL is a +// container-reachable IP, the URL chromedp gets back already points at it. func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) { if wsURL == "" { return nil, fmt.Errorf("empty browser websocket url") } - ctx, cancel := chromedp.NewRemoteAllocator( - context.Background(), wsURL, chromedp.NoModifyURL) + ctx, cancel := chromedp.NewRemoteAllocator(context.Background(), wsURL) return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil } diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f27f172..e58e286 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,10 @@ services: # Traffic arrives over the Traefik network, not a published port. ports: !reset [] environment: - BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://headless-shell:9222} + # Must be an IP, not the DNS name — see the base file's comment on this + # same key: Chrome's DevTools HTTP handler 500s any Host header that + # isn't an IP or "localhost". + BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://172.28.0.10:9222} depends_on: - headless-shell # `networks:` here replaces the base file's list entirely, so both must be diff --git a/docker-compose.yml b/docker-compose.yml index 3d53280..75e49df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,13 @@ services: LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s} # CDP endpoint for sites behind a JavaScript challenge (kagane). Unset # disables browser polling for those sites; the userscript still covers them. - BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://headless-shell:9222} + # Must be an IP, not the "headless-shell" DNS name: Chrome's DevTools HTTP + # handler rejects the discovery request (GET /json/version) with a 500 + # unless the Host header is an IP address or "localhost" — confirmed + # 2026-08-03 against chromedp/headless-shell:stable, independent of + # chromedp's own dial logic. The sidecar's static address below exists so + # this URL survives container recreation. + BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://172.28.0.10:9222} depends_on: - headless-shell volumes: @@ -58,13 +64,22 @@ services: init: true # Deliberately no `ports:` — an exposed CDP endpoint is remote code # execution. Only manga-api, via the `browser` network below, may reach it. + # Don't pass --remote-debugging-address/--remote-debugging-port here: the + # image's own entrypoint (/headless-shell/run.sh) already starts Chrome on + # 127.0.0.1:9223 and fronts it with a socat proxy listening on 0.0.0.0:9222. + # Redeclaring the port flag here overrides Chrome's, so it binds 9222 + # directly (IPv6 loopback only) instead of 9223 — collides with socat's own + # bind on 9222 and leaves nothing listening on 9223, so every external + # connection to headless-shell:9222 fails with EOF. Only pass flags the + # entrypoint doesn't already set. command: - - --remote-debugging-address=0.0.0.0 - - --remote-debugging-port=9222 - --disable-gpu - --no-sandbox networks: - - browser + browser: + # Pinned so BROWSER_WS_URL can name an IP (required, see above) that + # survives `docker compose up` recreating this container. + ipv4_address: 172.28.0.10 volumes: bookmarks-data: @@ -74,3 +89,6 @@ networks: # kagane.to. Isolation here comes from membership (only manga-api and # headless-shell join it), not from cutting egress. browser: + ipam: + config: + - subnet: 172.28.0.0/24