Add comix.to and kagane.to support (#13)

Tracks read progress on comix.to and kagane.to alongside asura and demonic, in both the userscript and the backend. Implements `docs/superpowers/plans/2026-08-03-comix-kagane-support.md`. ## Userscript - `comix` adapter — `/title/<id>-<slug>`; only the id prefix is identity (the slug follows the title). No `og:image`, so the cover is matched by `alt`. - `kagane` adapter — reader URLs are uuids with no chapter number, so it comes out of `og:title`; anchor scanning is structurally impossible, replaced by `latestChapterFromApi` against kagane's same-origin JSON API. - `seriesId` threaded through `latestChapterFromAnchors` so comix can scope its scan to its own series and a recommendation strip cannot win the maximum. - `@match` for both hosts, panel chips, v1.6.0. ## Backend - `latestChapterFrom` cases: comix parses the SSR JSON state blob (`latestChapterUrl`, scoped to the series id); kagane parses API JSON (`chapter_no`). - Poller allowlist extended; `Poller.BrowserFetch` with `fetcherFor(site)` routes kagane to a browser fetcher. Nil means kagane is not polled at all — never a fallback to the TLS fetcher, which would only ever retrieve a challenge page. - `BrowserFetcher`: chromedp against a `headless-shell` sidecar. kagane sits behind a Cloudflare JS challenge that no TLS fingerprint clears, and the request is made inside the page rather than by replaying `cf_clearance`. - `BROWSER_WS_URL` wiring, sidecar in both compose files (no `ports:`, dedicated non-external network), Dockerfile on `golang:1.26-alpine` — chromedp requires go 1.26. - Web UI `--comix` / `--kagane` tokens in both colour branches. ## Notes for review - `series_url` is client-supplied and a headless browser is a strong SSRF primitive, so kagane's host is pinned twice: in `fetchableSeriesURL` and again in `kaganeAPIURL`. - Three chained defects found during verification made the browser path dead under Compose (sidecar flag collision, Chrome's Host-header DNS-rebinding check, the wrong chromedp option). Fixed; the compose comments record the wrong configurations too, so they don't get "simplified" back. - `ALLOWED_ORIGINS` now includes both new origins. Without it every write from comix/kagane silently fails CORS preflight, parks in the retry queue, and drops at the cap. ## Verification 221 backend tests, 32 userscript tests, static `CGO_ENABLED=0` build, both compose configs. Two gaps, both real: 1. The userscript on live pages via Violentmonkey needs a human browser profile — not run. Check: comix series page (title/cover, no chapter), comix chapter page (records the number; an *older* chapter must not regress it), comix SPA navigation without reload, kagane series page (og:image cover), kagane reader (number from `og:title`), both chips opening the right sites. 2. The kagane browser path has not completed end-to-end anywhere. Dial/navigate/fetch is confirmed, but Cloudflare 403'd headless-shell's Chrome on every attempt from the dev sandbox, and comix's poll-through-Docker was blocked by that environment's TLS interception. Both environment-dependent rather than branch defects — the first real deploy is the actual verification. Reviewed-on: #13 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
2026-08-03 19:53:45 +07:00
parent 3c935ba7c3
commit 180ee78b1f
49 changed files with 2233 additions and 1027 deletions
@@ -0,0 +1,455 @@
+package latest
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"mangabm/backend/internal/store"
+)
+
+// newTestStore opens a fresh SQLite store in a temp dir.
+func newTestStore(t *testing.T) *store.Store {
+	t.Helper()
+	s, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	t.Cleanup(func() { s.Close() })
+	return s
+}
+
+// seedForCheck inserts a bookmark and forces its latest_checked_at.
+func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
+	t.Helper()
+	if _, err := s.Upsert(store.Bookmark{
+		Key:       key,
+		Site:      "asura",
+		SeriesID:  key,
+		SeriesURL: seriesURL,
+		UpdatedAt: 1000,
+	}); err != nil {
+		t.Fatalf("seed %q: %v", key, err)
+	}
+	if err := s.MarkLatestChecked(key, checkedAt); err != nil {
+		t.Fatalf("seed mark %q: %v", key, err)
+	}
+}
+
+func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
+	t.Helper()
+	ts, err := s.LatestCheckedAt(key)
+	if err != nil {
+		t.Fatalf("LatestCheckedAt %q: %v", key, err)
+	}
+	return ts
+}
+
+// fakeFetcher stands in for the network. Every poller test uses it, so nothing
+// in this file can reach tls-client or a real site.
+type fakeFetcher struct {
+	mu     sync.Mutex
+	calls  []string
+	body   string
+	status int
+	err    error
+	// perURL overrides body/status/err for specific URLs.
+	perURL map[string]fakeResponse
+}
+
+type fakeResponse struct {
+	body   string
+	status int
+	err    error
+}
+
+func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
+	f.mu.Lock()
+	f.calls = append(f.calls, url)
+	f.mu.Unlock()
+	if r, ok := f.perURL[url]; ok {
+		return r.body, r.status, r.err
+	}
+	return f.body, f.status, f.err
+}
+
+func (f *fakeFetcher) callCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.calls)
+}
+
+// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
+// instantly and deterministically.
+func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
+	t.Helper()
+	return &Poller{
+		Store:    s,
+		Fetch:    f,
+		Now:      func() time.Time { return at },
+		Cooldown: time.Hour,
+		Interval: 10 * time.Minute,
+		Stagger:  0,
+		Batch:    14,
+	}
+}
+
+func TestRunOnceRecordsLatestChapter(t *testing.T) {
+	s := newTestStore(t)
+	const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
+	seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
+
+	now := time.UnixMilli(5_000_000)
+	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
+	newTestPoller(t, s, f, now).runOnce(context.Background())
+
+	b, ok, err := s.Get("asura:chronicles-of-the-demon-faction-f886a8af")
+	if err != nil || !ok {
+		t.Fatalf("Get: %v ok=%v", err, ok)
+	}
+	if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
+		t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
+	}
+	if b.LatestChapter != "Chapter 181" {
+		t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
+	}
+	if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
+		t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
+	}
+}
+
+// The whole point of the updated_at CASE in Upsert: a newly published chapter is
+// not reading progress and must not move the series up the list.
+func TestRunOnceDoesNotReorderList(t *testing.T) {
+	s := newTestStore(t)
+	const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
+	const key = "asura:chronicles-of-the-demon-faction-f886a8af"
+
+	// "other" is the most recently read, so it must stay at the top of List().
+	if _, err := s.Upsert(store.Bookmark{
+		Key: "asura:other", Site: "asura", SeriesID: "other",
+		SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
+	}); err != nil {
+		t.Fatalf("seed other: %v", err)
+	}
+	seedForCheck(t, s, key, url, 0)
+	before, _, err := s.Get(key)
+	if err != nil {
+		t.Fatalf("Get before: %v", err)
+	}
+
+	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
+	newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
+
+	after, _, err := s.Get(key)
+	if err != nil {
+		t.Fatalf("Get after: %v", err)
+	}
+	if after.UpdatedAt != before.UpdatedAt {
+		t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
+			before.UpdatedAt, after.UpdatedAt)
+	}
+	list, err := s.List()
+	if err != nil {
+		t.Fatalf("List: %v", err)
+	}
+	if list[0].Key != "asura:other" {
+		t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
+	}
+}
+
+// A failed fetch must still consume the cooldown, or a renamed series gets
+// retried on every tick forever.
+func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
+	tests := []struct {
+		name string
+		resp fakeResponse
+	}{
+		{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
+		{"non-200", fakeResponse{body: "nope", status: 503}},
+		{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
+		{"empty body", fakeResponse{body: "", status: 200}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := newTestStore(t)
+			const url = "https://asurascans.com/comics/x"
+			seedForCheck(t, s, "asura:x", url, 0)
+
+			now := time.UnixMilli(7_000_000)
+			f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
+			newTestPoller(t, s, f, now).runOnce(context.Background())
+
+			if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
+				t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
+			}
+			b, _, err := s.Get("asura:x")
+			if err != nil {
+				t.Fatalf("Get: %v", err)
+			}
+			if b.LatestChapterNum != nil {
+				t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
+			}
+		})
+	}
+}
+
+func TestRunOnceRespectsBatchLimit(t *testing.T) {
+	s := newTestStore(t)
+	for i := 0; i < 20; i++ {
+		key := "asura:s" + string(rune('a'+i))
+		seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
+	}
+
+	f := &fakeFetcher{body: "", status: 200}
+	p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
+	p.Batch = 5
+	p.runOnce(context.Background())
+
+	if got := f.callCount(); got != 5 {
+		t.Fatalf("fetched %d series, want 5 (batch limit)", got)
+	}
+}
+
+// One unreachable series must not abandon the rest of the batch.
+func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
+	s := newTestStore(t)
+	keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
+	for _, k := range keys {
+		seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
+	}
+
+	now := time.UnixMilli(6_000_000)
+	f := &fakeFetcher{
+		body: "", status: 200,
+		perURL: map[string]fakeResponse{
+			"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
+		},
+	}
+	newTestPoller(t, s, f, now).runOnce(context.Background())
+
+	if got := f.callCount(); got != 5 {
+		t.Fatalf("fetched %d series, want all 5 attempted", got)
+	}
+	for _, k := range keys {
+		if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
+			t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
+		}
+	}
+}
+
+// The cooldown is enforced by the due query, so a second immediate pass must do
+// nothing at all — this is what makes the tick interval independent of it.
+func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
+	s := newTestStore(t)
+	const url = "https://asurascans.com/comics/x"
+	seedForCheck(t, s, "asura:x", url, 0)
+
+	now := time.UnixMilli(8_000_000)
+	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
+	p := newTestPoller(t, s, f, now)
+
+	p.runOnce(context.Background())
+	if got := f.callCount(); got != 1 {
+		t.Fatalf("first pass fetched %d, want 1", got)
+	}
+	// Same instant, and again 59 minutes later: both inside the 1h cooldown.
+	p.runOnce(context.Background())
+	p.Now = func() time.Time { return now.Add(59 * time.Minute) }
+	p.runOnce(context.Background())
+	if got := f.callCount(); got != 1 {
+		t.Fatalf("fetched %d times inside the cooldown, want 1", got)
+	}
+	// Past the cooldown, it is due again.
+	p.Now = func() time.Time { return now.Add(61 * time.Minute) }
+	p.runOnce(context.Background())
+	if got := f.callCount(); got != 2 {
+		t.Fatalf("fetched %d times after the cooldown, want 2", got)
+	}
+}
+
+// A site that retracts a chapter should correct the stored number downward,
+// mirroring the userscript's equality check (L427) rather than a >.
+func TestRunOnceCorrectsDownward(t *testing.T) {
+	s := newTestStore(t)
+	const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
+	const key = "demonic:Catastrophic-Necromancer"
+
+	high := 400.0
+	if _, err := s.Upsert(store.Bookmark{
+		Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
+		SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
+		UpdatedAt: 1000,
+	}); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
+	newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
+
+	b, _, err := s.Get(key)
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
+		t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
+	}
+}
+
+// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
+// reject anything that is not a known site with an https URL before spending a
+// request on it — the cooldown still gets consumed either way.
+func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
+	tests := []struct {
+		name      string
+		site      string
+		seriesURL string
+		wantCalls int
+	}{
+		{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
+		{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
+		{"unparseable url", "asura", "http://[::1", 0},
+		{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := newTestStore(t)
+			key := tt.site + ":x"
+			if _, err := s.Upsert(store.Bookmark{
+				Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
+				UpdatedAt: 1000,
+			}); err != nil {
+				t.Fatalf("seed: %v", err)
+			}
+
+			now := time.UnixMilli(4_000_000)
+			f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
+			newTestPoller(t, s, f, now).checkOne(context.Background(), store.Bookmark{
+				Key: key, Site: tt.site, SeriesURL: tt.seriesURL,
+			})
+
+			if got := f.callCount(); got != tt.wantCalls {
+				t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
+			}
+			if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
+				t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
+			}
+		})
+	}
+}
+
+// A cancelled context must abandon the batch rather than run it to completion.
+func TestRunOnceStopsOnCancelledContext(t *testing.T) {
+	s := newTestStore(t)
+	for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
+		seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	f := &fakeFetcher{body: "", status: 200}
+	newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)
+
+	if got := f.callCount(); got != 0 {
+		t.Fatalf("fetched %d series with a cancelled context, want 0", got)
+	}
+}
+
+func TestFetchableSeriesURL(t *testing.T) {
+	tests := []struct {
+		name      string
+		site      string
+		seriesURL string
+		want      bool
+	}{
+		{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
+		{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
+		{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
+		{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
+		// The browser fetcher runs JavaScript and carries cookies, so a
+		// client-supplied series_url must not be able to aim it anywhere else.
+		{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
+		{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
+		{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
+		{"non-https", "comix", "http://comix.to/title/x", false},
+		{"no host", "comix", "https:///title/x", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
+				t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
+					tt.site, tt.seriesURL, got, tt.want)
+			}
+		})
+	}
+}
+
+// A kagane row must not be handed to the plain TLS fetcher: it would only ever
+// receive a challenge page, and the browser fetcher is the whole reason kagane
+// is pollable at all.
+func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
+	s := newTestStore(t)
+	if _, err := s.Upsert(store.Bookmark{
+		Key:       "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		Site:      "kagane",
+		SeriesID:  "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		UpdatedAt: 1000,
+	}); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
+	p := &Poller{
+		Store: s, Fetch: f,
+		Now:      func() time.Time { return time.UnixMilli(5_000_000) },
+		Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
+	}
+	p.runOnce(context.Background())
+
+	if len(f.calls) != 0 {
+		t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
+	}
+}
+
+// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
+func TestKaganeUsesBrowserFetcher(t *testing.T) {
+	s := newTestStore(t)
+	key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	if _, err := s.Upsert(store.Bookmark{
+		Key:       key,
+		Site:      "kagane",
+		SeriesID:  "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
+		UpdatedAt: 1000,
+	}); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	tlsF := &fakeFetcher{body: "", status: 200}
+	browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
+	p := &Poller{
+		Store: s, Fetch: tlsF, BrowserFetch: browserF,
+		Now:      func() time.Time { return time.UnixMilli(5_000_000) },
+		Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
+	}
+	p.runOnce(context.Background())
+
+	if len(tlsF.calls) != 0 {
+		t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
+	}
+	if len(browserF.calls) != 1 {
+		t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
+	}
+	got, found, err := s.Get(key)
+	if err != nil || !found {
+		t.Fatalf("Get: %v found=%v", err, found)
+	}
+	if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
+		t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
+	}
+}