Make browser sidecar on-demand (#44) (#50)

Closes #44. Chrome now starts on first CDP connection, tracks concurrent helpers, reaps after 300 seconds idle, preserves the named profile, and classifies reap interruptions. Shutdown stops Chrome's process group so cookie batches flush. ADR-0005 records the measured constraints and decisions. Verification: docker build, live CDP wake, graceful stop cleanup, sh -n, and go test ./... (7 packages, 3 no tests).

Reviewed-on: #50
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>

backend/internal/latest/browser.go

@@ -178,6 +178,36 @@ func (f *BrowserFetcher) Image(ctx context.Context, imageID string) ([]byte, str
 // the poller answers with a 403 and its ordinary cooldown.
 var errChallengeHeld = errors.New("challenge held")
 
+// errBrowserInterrupted distinguishes a remote Chrome restart from the
+// caller's own deadline. chromedp reports both as context.Canceled.
+var errBrowserInterrupted = errors.New("browser interrupted")
+
+func classifyBrowserError(ctx context.Context, browserLost bool, err error) error {
+	if err == nil || ctx.Err() != nil {
+		return err
+	}
+	if !browserLost {
+		return err
+	}
+	if !errors.Is(err, context.Canceled) {
+		return err
+	}
+	return fmt.Errorf("%w: %w", errBrowserInterrupted, err)
+}
+
+func browserConnectionLost(ctx context.Context) bool {
+	c := chromedp.FromContext(ctx)
+	if c == nil || c.Browser == nil {
+		return true
+	}
+	select {
+	case <-c.Browser.LostConnection:
+		return true
+	default:
+		return false
+	}
+}
+
 // challengePollInterval paces re-reads while a challenge solves itself.
 const challengePollInterval = 2 * time.Second
 
@@ -203,6 +233,7 @@ func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.A
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
+	callerCtx := ctx
 	ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
 	defer cancel()
 	tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
@@ -219,20 +250,26 @@ func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.A
 		chromedp.Navigate(target),
 		chromedp.WaitReady("body", chromedp.ByQuery),
 	); err != nil {
-		return err
+		return classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
 	}
-
 	var lastErr error
 	for {
 		// The challenge reloads the page when it passes, which tears down the
 		// execution context mid-read. That is a retry, not a failure.
 		if err := chromedp.Run(tabCtx, read); err != nil {
+			err = classifyBrowserError(callerCtx, browserConnectionLost(tabCtx), err)
+			if errors.Is(err, errBrowserInterrupted) {
+				return err
+			}
 			lastErr = err
 		} else if done() {
 			return nil
 		}
 		select {
 		case <-ctx.Done():
+			if err := callerCtx.Err(); err != nil {
+				return err
+			}
 			if lastErr != nil {
 				return fmt.Errorf("%w (last read: %v)", errChallengeHeld, lastErr)
 			}

backend/internal/latest/browser_test.go

@@ -1,6 +1,10 @@
 package latest
 
-import "testing"
+import (
+	"context"
+	"errors"
+	"testing"
+)
 
 func TestKaganeAPIURL(t *testing.T) {
 	const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
@@ -57,3 +61,17 @@ func TestNovelfullSeriesURL(t *testing.T) {
 		})
 	}
 }
+func TestClassifyBrowserInterruption(t *testing.T) {
+	if err := classifyBrowserError(context.Background(), true, context.Canceled); !errors.Is(err, errBrowserInterrupted) {
+		t.Fatalf("classifyBrowserError(context.Canceled) = %v, want browser interruption", err)
+	}
+	if err := classifyBrowserError(context.Background(), false, context.Canceled); errors.Is(err, errBrowserInterrupted) {
+		t.Fatalf("ordinary cancellation misclassified as browser interruption: %v", err)
+	}
+
+	caller, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := classifyBrowserError(caller, true, context.Canceled); errors.Is(err, errBrowserInterrupted) {
+		t.Fatalf("caller cancellation misclassified as browser interruption: %v", err)
+	}
+}

chrome/Dockerfile

@@ -18,7 +18,7 @@ FROM debian:trixie-slim
 # stale, and a stale browser is exactly what Cloudflare turns away — the 124 in
 # alpine-chrome is the worked example. Rebuild is the upgrade path.
 RUN apt-get update \
- && apt-get install -y --no-install-recommends ca-certificates wget gnupg \
+ && apt-get install -y --no-install-recommends ca-certificates wget gnupg util-linux \
  && wget -qO- https://dl.google.com/linux/linux_signing_key.pub \
       | gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg \
  && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \
@@ -27,9 +27,15 @@ RUN apt-get update \
  && apt-get install -y --no-install-recommends google-chrome-stable socat \
  && rm -rf /var/lib/apt/lists/*
 
+# Keep the profile path present so Docker initializes the named volume with
+# the unprivileged user's ownership.
+
+RUN useradd --create-home --shell /usr/sbin/nologin chrome \
+ && mkdir -p /home/chrome/profile /home/chrome/state \
+ && chown -R chrome:chrome /home/chrome
+
 # Unprivileged: Chrome refuses to run as root, and the CDP endpoint is a shell
 # on whatever user owns it.
-RUN useradd --create-home --shell /usr/sbin/nologin chrome
 USER chrome
 WORKDIR /home/chrome
 

chrome/entrypoint.sh

@@ -16,46 +16,197 @@ set -eu
 [ -n "${TZ:-}" ] || TZ=$(cat /etc/timezone 2>/dev/null || echo UTC)
 export TZ
 
-# Chrome's own UA advertises "HeadlessChrome" under --headless=new, and that one
+state=/home/chrome/state
-# token is the difference between kagane.to's challenge clearing in ~4s and
+profile=/home/chrome/profile
-# never clearing at all (measured 2026-08-08, same host, same Chrome, only the
+lock_file=$state/lock
-# UA changed). Overriding it does not touch the Sec-CH-UA client hints, which
+pid_file=$state/chrome.pid
-# report the real version, so the version is read back out of the binary rather
+connections_dir=$state/connections
-# than hardcoded: a hardcoded one would drift out of step with the hints on the
+last_use_file=$state/last-use
-# next Chrome update and become a fresh tell.
+idle_seconds=300
+
+mkdir -p "$state" "$profile" "$connections_dir"
+exec 9>>"$lock_file"
+
+# Chrome's own UA advertises "HeadlessChrome" under --headless=new, and that
+# one token is the difference between kagane.to's challenge clearing in ~4s and
+# never clearing at all. Read the installed major version so client hints and
+# the UA stay aligned after an image rebuild.
 major=$(google-chrome-stable --version | sed -E 's/[^0-9]*([0-9]+)\..*/\1/')
 ua="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36"
 
-# Chrome binds its DevTools port to loopback and silently ignores
+lock() {
-# --remote-debugging-address (verified 2026-08-08: Chrome 151 with
+	flock 9
-# --remote-debugging-address=0.0.0.0 still listened on 127.0.0.1 only), so the
+}
-# caller — another container — cannot reach it directly. socat fronting the
-# loopback port is how chromedp/headless-shell solved the same problem and is
-# why this image is a drop-in for it.
-#
-# Nothing publishes 9222; reachability is the `browser` network in
-# docker-compose.yml, and an exposed CDP endpoint is remote code execution.
-socat TCP-LISTEN:9222,fork,reuseaddr TCP:127.0.0.1:9223 &
 
-# Chrome stays in the foreground so that its death takes the container down and
+unlock() {
-# compose's restart policy applies; a backgrounded browser behind a live socat
+	flock -u 9
-# would leave the sidecar looking healthy while answering nothing.
+}
-#
+
-# No --enable-automation: it sets navigator.webdriver, the first thing a bot
+browser_alive() {
-# check reads.
+	[ -s "$pid_file" ] || return 1
-#
+	pid=$(cat "$pid_file")
-# --no-sandbox because Chrome's zygote wants user namespaces, which Docker's
+	[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null
-# default profile does not hand out; the alternative is --cap-add=SYS_ADMIN,
+}
-# which gives the container strictly more than it takes away. Containment here
+
-# is the unprivileged user, the isolated network, and the fact that this
+has_connections() {
-# browser only ever navigates to kagane.to and novelfull.com.
+	for marker in "$connections_dir"/*; do
-exec google-chrome-stable \
+		[ -e "$marker" ] || continue
-  --headless=new \
+		pid=${marker##*/}
-  --no-sandbox \
+		if kill -0 "$pid" 2>/dev/null; then
-  --remote-debugging-port=9223 \
+			return 0
-  --user-agent="$ua" \
+		fi
-  --user-data-dir=/home/chrome/profile \
+		# A SIGKILLed helper cannot run its cleanup trap. Reconcile its marker
-  --no-first-run \
+		# here so one dead client cannot pin Chrome forever.
-  --no-default-browser-check \
+		rm -f "$marker"
-  --disable-gpu \
+	done
-  about:blank
+	return 1
+}
+
+start_browser() {
+	# No --enable-automation: it sets navigator.webdriver, the first thing a
+	# bot check reads. setsid gives Chrome a process group so the reaper can
+	# terminate its renderer children with the browser.
+	# --no-sandbox avoids granting SYS_ADMIN solely for Docker's unavailable
+	# user namespaces; containment is the unprivileged user and private network.
+	setsid google-chrome-stable \
+		--headless=new \
+		--no-sandbox \
+		--remote-debugging-port=9223 \
+		--user-agent="$ua" \
+		--user-data-dir="$profile" \
+		--no-first-run \
+		--no-default-browser-check \
+		--disable-gpu \
+		about:blank >/dev/null 2>&1 &
+	printf '%s\n' "$!" >"$pid_file"
+}
+
+stop_browser() {
+	pid=$(cat "$pid_file")
+	kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
+	i=0
+	while kill -0 "$pid" 2>/dev/null && [ "$i" -lt 100 ]; do
+		i=$((i + 1))
+		sleep 0.1
+	done
+	if kill -0 "$pid" 2>/dev/null; then
+		kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true
+	fi
+	rm -f "$pid_file"
+}
+
+wait_for_browser() {
+	i=0
+	while [ "$i" -lt 300 ]; do
+		if wget -qO /dev/null http://127.0.0.1:9223/json/version; then
+			return 0
+		fi
+		browser_alive || return 1
+		i=$((i + 1))
+		sleep 0.1
+	done
+	return 1
+}
+
+finish_connection() {
+	lock
+	rm -f "$connection_marker"
+	date +%s >"$last_use_file"
+	unlock
+}
+
+connection_signal() {
+	trap - INT TERM HUP
+	finish_connection
+	exit 143
+}
+
+connection() {
+	connection_marker=$connections_dir/$$
+	lock
+	: >"$connection_marker"
+	if ! browser_alive; then
+		rm -f "$pid_file"
+		start_browser
+	fi
+	date +%s >"$last_use_file"
+	unlock
+
+	trap connection_signal INT TERM HUP
+	if wait_for_browser; then
+		if socat STDIO TCP:127.0.0.1:9223; then
+			result=0
+		else
+			result=$?
+		fi
+	else
+		result=1
+	fi
+	finish_connection
+	return "$result"
+}
+
+reaper() {
+	while :; do
+		sleep 10
+		lock
+		if ! has_connections && browser_alive; then
+			now=$(date +%s)
+			last=$(cat "$last_use_file" 2>/dev/null || printf '%s' "$now")
+			if [ $((now - last)) -ge "$idle_seconds" ]; then
+				stop_browser
+			fi
+		fi
+		unlock
+	done
+}
+
+if [ "${1:-}" = connection ]; then
+	connection
+	exit $?
+fi
+
+# The files are process state, not the Chrome profile. The profile is a named
+# volume in Compose, so clearance survives both a reap and a container rebuild.
+for marker in "$connections_dir"/*; do
+	[ -e "$marker" ] || continue
+	rm -f "$marker"
+done
+rm -f "$pid_file" "$last_use_file"
+
+# Chrome binds DevTools to loopback and silently ignores
+# --remote-debugging-address. socat remains the network front-end, but each
+# accepted connection now starts a browser on demand and is tracked by a
+# per-helper marker. A connection held by Go's transport delays reap by its
+# idle timeout; the 300-second threshold starts once the last connection closes.
+reaper &
+reaper_pid=$!
+socat TCP-LISTEN:9222,reuseaddr,fork EXEC:'/entrypoint.sh connection',nofork &
+front_pid=$!
+
+stop_browser_gracefully() {
+	lock
+	if browser_alive; then
+		# Chrome is a separate session, so stop its process group explicitly;
+		# this gives its cookie batch time to flush before the container exits.
+		stop_browser
+	fi
+	unlock
+}
+
+shutdown() {
+	trap - INT TERM HUP
+	stop_browser_gracefully
+	kill "$front_pid" "$reaper_pid" 2>/dev/null || true
+	exit 143
+}
+trap shutdown INT TERM HUP
+
+if wait "$front_pid"; then
+	status=0
+else
+	status=$?
+fi
+stop_browser_gracefully
+kill "$reaper_pid" 2>/dev/null || true
+exit "$status"

docker-compose.yml

@@ -122,6 +122,8 @@ services:
       # The zone *name*, which is what Chrome's ICU needs — see chrome/entrypoint.sh.
       # Absent on a non-Debian host, which the entrypoint handles by falling back to UTC.
       - /etc/timezone:/etc/timezone:ro
+      # Cloudflare clearance must survive Chrome reaping and image recreation.
+      - chrome-profile:/home/chrome/profile
     # Chrome allocates shared memory per tab and dies on Docker's 64MB default.
     shm_size: '1gb'
     # Reaps zombie renderer processes, which otherwise accumulate for the
@@ -143,6 +145,7 @@ volumes:
   # declared here: undeclared means `docker compose down -v` cannot take it
   # with the rest, so the old database survives the cutover until someone
   # removes it by hand.
+  chrome-profile:
 
 networks:
   # Not `internal: true`: headless Chrome still needs outbound access to reach

docs/adr/0005-on-demand-browser.md

@@ -0,0 +1,48 @@
+# ADR-0005: On-demand browser sidecar
+
+Date: 2026-08-09
+Status: accepted
+
+## Decision
+
+Keep the `headless-shell` service and its CDP port alive, but start Google Chrome
+only when the first CDP connection arrives. The entrypoint supervises a `socat`
+front-end, serializes browser start/reap state with `flock`, and tracks each
+connection with a marker named for its helper PID. A reaper stops Chrome after
+300 seconds with no live markers. Marker reconciliation covers a helper killed
+before its cleanup trap runs.
+
+Chrome runs in its own process group so reap sends the termination signal to
+Chrome and its renderer children. The explicit `/home/chrome/profile` user-data
+directory remains: Chrome remaps remote debugging to loopback on modern builds,
+and Chrome ignores remote-debugging flags on a default profile. `socat` therefore
+continues to front Chrome's loopback CDP port.
+
+The profile is a named Compose volume. Clearance cookies survive both a reap and
+`docker compose up --build`; the browser still starts with a fresh debugger UUID,
+so chromedp must keep endpoint discovery enabled and must not use
+`chromedp.NoModifyURL`.
+
+The socat front-end and explicit profile are retained because Chromium remaps a
+non-loopback debugging address to loopback since M113, while Chrome ignores the
+remote-debugging flags on a default profile since Chrome 136. Flag tuning is
+deliberately not adopted: its roughly 30% idle-footprint saving is irrelevant
+to a browser that exists for seconds per wake and risks an untested fingerprint.
+
+## Constraints
+
+The 300-second floor is deliberate. Chromium batches cookie persistence on a
+roughly 31-second timer, and Go's default HTTP transport can keep the discovery
+connection parked for about 90 seconds after use. Reaping only with zero live
+connections holds Chrome through both windows and through the poller's staggered
+batch plus cover prefetch.
+
+The anti-bot properties remain unchanged: a plausible non-UTC timezone, a
+Chrome-version-derived User-Agent without `HeadlessChrome`, and no automation
+flag. A remote browser restart can surface as `context.Canceled`, the same error
+as a caller deadline, so the backend wraps cancellation observed with a closed
+CDP connection as `browser interrupted`; the focused test asserts that
+classification without killing a real browser.
+The same process-group stop runs during supervisor shutdown, not only during
+idle reap, so Chrome can flush its cookie batch before a container rebuild or
+graceful stop.