From 2c7b4952f31c5c1cd6fc48ada0c8c5a330a92116 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Mon, 3 Aug 2026 17:46:29 +0700 Subject: [PATCH] feat: wire headless-shell sidecar for challenge-gated polling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also bumps backend/Dockerfile's build stage to golang:1.26-alpine — chromedp v0.16.0 and cdproto both require go 1.26, and the pinned 1.24-alpine base no longer builds the module. Co-Authored-By: Claude Opus 5 --- .env.example | 4 ++++ CLAUDE.md | 9 +++++++-- DEPLOY.md | 6 ++++++ README.md | 7 ++++--- backend/Dockerfile | 2 +- backend/main.go | 15 +++++++++++++++ docker-compose.prod.yml | 10 ++++++++++ docker-compose.yml | 21 +++++++++++++++++++++ 8 files changed, 68 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 3ea053b..68000f1 100644 --- a/.env.example +++ b/.env.example @@ -52,3 +52,7 @@ WEB_PASSWORD= # BATCH x (COOLDOWN / INTERVAL) series hold the cooldown cadence — 84 with these # defaults. Beyond that the cadence stretches uniformly rather than breaking; # raise BATCH or lower INTERVAL. Keep BATCH x STAGGER under INTERVAL. + +# Headless-shell CDP endpoint for sites behind a JavaScript challenge (kagane). +# Unset disables browser polling; those sites then rely on the userscript alone. +BROWSER_WS_URL=ws://headless-shell:9222 diff --git a/CLAUDE.md b/CLAUDE.md index 7f41000..2dc42d4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ Violentmonkey userscript (isolated world, per-site adapters, localStorage cache) packages together into `newRouter`. Root-level `*_test.go` hold integration tests that exercise the full router; unit tests for a package live beside it under `internal/`. -- **Single-user store.** One `bookmarks` table keyed `:` (`asura`|`demonic`). Sync **last-write-wins**. Schema and endpoint list in plan. +- **Single-user store.** One `bookmarks` table keyed `:` (`asura`|`demonic`|`comix`|`kagane`). Sync **last-write-wins**. Schema and endpoint list in plan. - **Endpoints:** `GET /bookmarks`, `PUT /bookmarks/{key}` (upsert; see `updated_at` rule below), `DELETE /bookmarks/{key}`, `GET /healthz` (no auth). - **Web UI:** same binary serve password-gated browser UI on second hostname — `GET /` (list, or login page when no session), @@ -75,7 +75,10 @@ Violentmonkey userscript (isolated world, per-site adapters, localStorage cache) cooldown instead of retrying every tick, and writes go through `Store.Get` + `Store.Upsert` so new chapter never reorders list. Fetches use `bogdanfinn/tls-client` with Chrome profile as defence in depth - against fingerprint-based blocking; any failure log and skip. See + against fingerprint-based blocking; any failure log and skip. kagane sits + behind a Cloudflare JavaScript challenge the TLS client can't clear, so it is + browser-only: fetched over CDP via `BROWSER_WS_URL`, and simply not polled + when that's unset. See `docs/superpowers/specs/2026-07-26-server-latest-chapter-polling-design.md`. Poller's `Store.Get` + `Store.Upsert` not wrapped in transaction, so userscript `PUT` that commits between the two can get overwritten by @@ -100,6 +103,8 @@ Violentmonkey userscript (isolated world, per-site adapters, localStorage cache) (background latest-chapter poller; defaults on, `1h`/`10m`/`14`/`20s`). `USERSCRIPT_PATH` (file served at `/u/{token}/manga-bookmark.user.js`, default `/userscript/manga-bookmark.user.js`, supplied by bindmount). + `BROWSER_WS_URL` (headless-shell CDP endpoint for kagane; unset disables + browser polling and leaves that site to the userscript alone). ### Userscript structure (single IIFE, `manga-bookmark.user.js`) diff --git a/DEPLOY.md b/DEPLOY.md index 98bb97f..8f82aea 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -116,6 +116,12 @@ This merges the base file (build/image/env/volume) with the prod override (no host port, Traefik network + router labels). Always pass **both** `-f` flags — the prod file is not standalone. +Two services come up: `manga-api` (the backend) and `headless-shell`, a CDP +sidecar the poller uses to fetch kagane (behind a Cloudflare JS challenge). +It has no published port — only `manga-api` can reach it, over +`BROWSER_WS_URL`. Missing or unreachable, the poller just skips kagane and +logs it; nothing else is affected. + Check it's up and healthy: ```bash diff --git a/README.md b/README.md index 1e6a83a..6ca2428 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # Manga Bookmark -Track manga read-progress on **asurascans.com** (a.k.a. asuracomic.net) and -**demonicscans.org** from a phone (Bromite / mobile Chromium), synced to a -self-hosted Go backend so bookmarks unify across both sites and all devices. +Track manga read-progress on **asurascans.com** (a.k.a. asuracomic.net), +**demonicscans.org**, **comix.to**, and **kagane.to** from a phone (Bromite / +mobile Chromium), synced to a self-hosted Go backend so bookmarks unify across +all four sites and all devices. Two parts: diff --git a/backend/Dockerfile b/backend/Dockerfile index f91faf4..15afbd4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # --- build stage: compile a static, CGO-free binary --- -FROM golang:1.24-alpine AS build +FROM golang:1.26-alpine AS build WORKDIR /src # Dependencies first for layer caching (changes rarely). diff --git a/backend/main.go b/backend/main.go index 75db207..a4241c3 100644 --- a/backend/main.go +++ b/backend/main.go @@ -274,5 +274,20 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll) { Stagger: cfg.Stagger, Batch: cfg.Batch, } + + // Optional: without it, sites behind a JavaScript challenge are simply not + // polled, and their latest_chapter comes from the userscript alone — which + // is how the service behaved before the sidecar existed. + if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" { + bf, err := latest.NewBrowserFetcher(ws) + if err != nil { + log.Printf("latest-chapter poller: browser fetcher disabled: %v", err) + } else { + p.BrowserFetch = bf + context.AfterFunc(ctx, bf.Close) + log.Printf("latest-chapter poller: browser fetcher at %s", ws) + } + } + go p.Run(ctx) } diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 78bc45b..9444cf8 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -17,6 +17,10 @@ services: manga-api: # Traffic arrives over the Traefik network, not a published port. ports: !reset [] + environment: + BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://headless-shell:9222} + depends_on: + - headless-shell networks: - proxy labels: @@ -36,6 +40,12 @@ services: - "traefik.http.routers.mangaweb.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}" - "traefik.http.routers.mangaweb.service=mangabm" + # manga-api only joins `proxy` in this override (see above), so headless-shell + # has to follow it there too or DNS resolution of headless-shell:9222 breaks. + headless-shell: + networks: + - proxy + networks: proxy: external: true diff --git a/docker-compose.yml b/docker-compose.yml index c8adc6b..304ea69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,11 @@ services: LATEST_CHAPTER_POLL_INTERVAL: ${LATEST_CHAPTER_POLL_INTERVAL:-10m} LATEST_CHAPTER_POLL_BATCH: ${LATEST_CHAPTER_POLL_BATCH:-14} LATEST_CHAPTER_POLL_STAGGER: ${LATEST_CHAPTER_POLL_STAGGER:-20s} + # CDP endpoint for sites behind a JavaScript challenge (kagane). Unset + # disables browser polling for those sites; the userscript still covers them. + BROWSER_WS_URL: ${BROWSER_WS_URL:-ws://headless-shell:9222} + depends_on: + - headless-shell volumes: - bookmarks-data:/data # The userscript is served from here, read fresh on every request. Editing @@ -41,5 +46,21 @@ services: ports: - "127.0.0.1:8080:8080" + headless-shell: + image: chromedp/headless-shell:stable + restart: unless-stopped + # Chrome allocates shared memory per tab and dies on Docker's 64MB default. + shm_size: '1gb' + # Reaps zombie renderer processes, which otherwise accumulate for the + # container's lifetime. + init: true + # Deliberately no `ports:` — an exposed CDP endpoint is remote code + # execution. Only the backend on the internal network may reach it. + command: + - --remote-debugging-address=0.0.0.0 + - --remote-debugging-port=9222 + - --disable-gpu + - --no-sandbox + volumes: bookmarks-data: