Browser-backed Sites join the Cover pipeline (#62) (#72)

Fixes #62 Browser-backed Sites join the Cover pipeline: kagane and novelfull Series now get their Covers at creation, through the same acquisition path as every other Site, instead of waiting for a poll pass. ## What changed `latest.Acquirer` (creation-time acquisition, fired by the first Bookmark of a Series) previously skipped kagane and novelfull entirely — their pages only yield a Cloudflare challenge to the TLS client, so the request was spent for nothing. It now routes them like the poller does, with the two Sites split exactly as the issue demands: - **kagane** — page fetched through the browser sidecar, cover URL extracted from the API JSON, bytes fetched through the browser sidecar (the only path that clears the challenge) into the content-addressed store. With no `BROWSER_WS_URL` configured, acquisition is skipped entirely and nothing falls back to a plain fetch. - **novelfull** — page fetched through the browser sidecar, cover URL extracted from the HTML, bytes fetched over plain TLS through the ordinary gated fetcher (its image paths answer 200 with `access-control-allow-origin: *`, measured 2026-08-09). With no browser configured, the page fetch falls back to the TLS client — novelfull's challenge is a live time-varying fact (AGENTS.md), so when the page body answers, the Cover still lands; when it is challenged, nothing happens. The byte-routing rule (kagane → browser, every other Site → TLS) is now one shared function (`latest.fetchCoverBytes`) used by both the Poller and the Acquirer, so the two cannot drift apart. ## Acceptance criteria - [x] kagane cover bytes are fetched through the browser sidecar and stored in the content-addressed store — `TestAcquireKaganeCoverThroughBrowser` - [x] novelfull cover URLs are extracted from the browser-fetched HTML, and its bytes are fetched over plain TLS — `TestAcquireNovelfullCoverOverPlainTLS` - [x] With no browser sidecar configured, kagane Covers are absent and nothing falls back to a plain fetch — `TestAcquireKaganeSkippedWithoutBrowser` - [x] With no browser sidecar configured, novelfull Covers still work if its page body is available — `TestAcquireNovelfullCoverWithoutBrowser` - [x] Manually verified on-device: a kagane Series shows its Cover in the panel, not a broken-image glyph — being run by a separate manual-verification agent against a mocked scenario (no prod data); not part of this PR - [x] `go test ./...` is green, with live-network checks gated behind `SMOKE_BROWSER_WS_URL` like the existing kagane image smoke test — new `TestSmokeAcquireKaganeCover` proves the end-to-end acquire path against the real browser when the env var is set ## Verification - `go test ./...` green across all packages - New unit tests exercise every routing decision with fakes — no network in the default suite - Smoke test gated behind `SMOKE_BROWSER_WS_URL`, skipped by default ## Post-review changes (a66491a) - **One routing rule for pages too** — `fetcherFor` is now a shared function used by both the Poller and the Acquirer; novelfull falls back to the plain-TLS fetcher in *both* when no browser is configured, so pre-existing (client-scraped) novelfull rows get healed by the poll as well, not just Series created after this change (`TestNovelfullUsesTLSWhenNoBrowserFetcher`). - **Byte-level no-fallback proof** — `TestAcquireKaganeBytesNeverFallBackToPlainTLS` pins that kagane cover bytes never route to the TLS fetcher even when the page came through a browser. - **Acquirer wired independent of the TLS client** — if `NewTLSFetcher` fails, kagane/novelfull acquisition still works via the sidecar (`main.go`). - AGENTS.md (root + backend) updated for the novelfull plain-TLS fallback. Reviewed-on: #72 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
2026-08-10 11:06:36 +07:00
parent b9220b3dfc
commit 78234f3c19
9 changed files with 406 additions and 53 deletions
@@ -20,6 +20,29 @@ const (
 	acquireCoverURL  = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
 )

+const (
+	kaganeKey       = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	kaganeSeriesID  = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	kaganeSeriesURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	kaganeImageID   = "019fe11a-84c3-7fc3-a84b-88787374b617"
+	kaganeCoverSrc  = "https://kagane.to/api/v2/image/" + kaganeImageID + "/compressed"
+)
+
+// kagane's browser-fetched body is one JSON object carrying both the chapter
+// list (series_books) and the cover image ids (series_covers), so the single
+// acquisition fetch yields both facts.
+const kaganeSeriesAndCoverFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",` +
+	`"series_books":[{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41}],` +
+	`"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en",` +
+	`"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}]}`
+
+const (
+	novelfullKey       = "novelfull:reverend-insanity"
+	novelfullSeriesID  = "reverend-insanity"
+	novelfullSeriesURI = "https://novelfull.com/reverend-insanity.html"
+	novelfullCoverURL  = "https://novelfull.com/uploads/webp/novel/reverend-insanity-82661d911a.webp"
+)
+
 // newAcquirer wires an acquirer onto the store's creation hook, which is how
 // main wires it: the write path is what starts an acquisition.
 func newAcquirer(s *store.Store, page *fakeFetcher, covers *fakeBytesCoverFetcher) *Acquirer {
@@ -50,6 +73,30 @@ func readBookmark(t *testing.T, s *store.Store, key string) store.Bookmark {
 	return b
 }

+func bookmarkNewKaganeSeries(t *testing.T, s *store.Store) store.Bookmark {
+	t.Helper()
+	stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
+		Key: kaganeKey, Site: "kagane", SeriesID: kaganeSeriesID,
+		Title: "Infinite Decryption", SeriesURL: kaganeSeriesURL, UpdatedAt: 1000,
+	})
+	if err != nil {
+		t.Fatalf("Upsert: %v", err)
+	}
+	return stored
+}
+
+func bookmarkNewNovelfullSeries(t *testing.T, s *store.Store) store.Bookmark {
+	t.Helper()
+	stored, err := s.Upsert(s.OwnerID(), store.Bookmark{
+		Key: novelfullKey, Site: "novelfull", SeriesID: novelfullSeriesID,
+		Title: "Reverend Insanity", SeriesURL: novelfullSeriesURI, UpdatedAt: 1000,
+	})
+	if err != nil {
+		t.Fatalf("Upsert: %v", err)
+	}
+	return stored
+}
+
 // The reported bug: a Reader bookmarks a Series nobody holds and expects the
 // Cover, not a broken image. Both facts come from the one series-page fetch.
 func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
@@ -237,3 +284,147 @@ func TestAcquireDoesNotBlockTheWrite(t *testing.T) {
 	close(release)
 	acq.Wait()
 }
+
+// The second symptom of #47: a kagane Series bookmarked from a chapter page
+// gets its Cover at creation, with the bytes fetched through the browser
+// sidecar — the only path that clears the challenge — into the
+// content-addressed store.
+func TestAcquireKaganeCoverThroughBrowser(t *testing.T) {
+	s, _ := newTestStore(t)
+	tlsPage := &fakeFetcher{body: "", status: 403}
+	browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
+	covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
+	acq := &Acquirer{
+		Store: s, Fetch: tlsPage, BrowserFetch: browserPage,
+		BrowserCoverFetch: covers, Covers: &fakeBytesCoverFetcher{},
+	}
+	s.OnSeriesCreated = acq.Acquire
+
+	bookmarkNewKaganeSeries(t, s)
+	acq.Wait()
+
+	if got := tlsPage.callCount(); got != 0 {
+		t.Fatalf("plain-TLS page fetches = %d, want 0 — kagane pages are browser-only", got)
+	}
+	if got := browserPage.callCount(); got != 1 {
+		t.Fatalf("browser page fetches = %d, want 1", got)
+	}
+	if got := covers.callCount(); got != 1 {
+		t.Fatalf("browser cover fetches = %d, want 1", got)
+	}
+	if got := covers.calls[0]; got != kaganeImageID {
+		t.Fatalf("browser cover fetched image id %q, want %q", got, kaganeImageID)
+	}
+	got := readBookmark(t, s, kaganeKey)
+	if want := testCoverBaseURL + "/covers/" + store.CoverAddress(kaganeCoverSrc); got.Cover != want {
+		t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
+	}
+	body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(kaganeCoverSrc))
+	if err != nil || !ok {
+		t.Fatalf("CoverByAddress = %v, %v", ok, err)
+	}
+	if string(body) != "cover-bytes" || contentType != "image/webp" {
+		t.Fatalf("stored cover = (%q, %q), want the browser-fetched bytes", body, contentType)
+	}
+}
+
+// novelfull needs the browser only for its HTML: the cover URL comes out of
+// the browser-fetched page, but the bytes go over plain TLS through the
+// ordinary gated fetcher, never through the browser (issue #62).
+func TestAcquireNovelfullCoverOverPlainTLS(t *testing.T) {
+	s, _ := newTestStore(t)
+	browserPage := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
+	covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
+	acq := &Acquirer{
+		Store: s, Fetch: &fakeFetcher{body: "", status: 403},
+		BrowserFetch: browserPage, Covers: covers,
+	}
+	s.OnSeriesCreated = acq.Acquire
+
+	bookmarkNewNovelfullSeries(t, s)
+	acq.Wait()
+
+	if got := browserPage.callCount(); got != 1 {
+		t.Fatalf("browser page fetches = %d, want 1", got)
+	}
+	if got := covers.callCount(); got != 1 {
+		t.Fatalf("cover fetches = %d, want 1 — novelfull bytes never touch the browser", got)
+	}
+	if got := covers.calls[0]; got != novelfullCoverURL {
+		t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
+	}
+	got := readBookmark(t, s, novelfullKey)
+	if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
+		t.Fatalf("Cover = %q, want %q", got.Cover, want)
+	}
+}
+
+// With no browser sidecar configured, kagane is simply not acquired: no
+// request is spent on a page that could only ever answer with a challenge,
+// and nothing falls back to a plain fetch.
+func TestAcquireKaganeSkippedWithoutBrowser(t *testing.T) {
+	s, _ := newTestStore(t)
+	tlsPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
+	acq := &Acquirer{
+		Store: s, Fetch: tlsPage,
+		Covers: &fakeBytesCoverFetcher{body: []byte("x"), contentType: "image/webp"},
+	}
+	s.OnSeriesCreated = acq.Acquire
+
+	bookmarkNewKaganeSeries(t, s)
+	acq.Wait()
+
+	if got := tlsPage.callCount(); got != 0 {
+		t.Fatalf("plain-TLS fetches for kagane = %d, want 0", got)
+	}
+	if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
+		t.Fatalf("Cover = %q, want empty without a browser", got.Cover)
+	}
+}
+
+// The byte half of "nothing falls back to a plain fetch": with a browser for
+// the page but none for the bytes, a kagane Cover stays absent and the TLS
+// cover fetcher is never consulted.
+func TestAcquireKaganeBytesNeverFallBackToPlainTLS(t *testing.T) {
+	s, _ := newTestStore(t)
+	browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
+	tlsCovers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
+	acq := &Acquirer{
+		Store: s, Fetch: &fakeFetcher{body: "", status: 403},
+		BrowserFetch: browserPage, Covers: tlsCovers,
+	}
+	s.OnSeriesCreated = acq.Acquire
+
+	bookmarkNewKaganeSeries(t, s)
+	acq.Wait()
+
+	if got := tlsCovers.callCount(); got != 0 {
+		t.Fatalf("plain-TLS cover fetches = %d, want 0 — kagane bytes are browser-only", got)
+	}
+	if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
+		t.Fatalf("Cover = %q, want empty without a browser cover fetcher", got.Cover)
+	}
+}
+
+// novelfull's no-browser degradation differs from kagane's: only its HTML
+// needs the sidecar, so when the page body is available — the challenge is a
+// live time-varying fact that sometimes answers a plain request — the Cover
+// still lands, bytes over plain TLS.
+func TestAcquireNovelfullCoverWithoutBrowser(t *testing.T) {
+	s, _ := newTestStore(t)
+	tlsPage := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
+	covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
+	acq := &Acquirer{Store: s, Fetch: tlsPage, Covers: covers}
+	s.OnSeriesCreated = acq.Acquire
+
+	bookmarkNewNovelfullSeries(t, s)
+	acq.Wait()
+
+	if got := covers.callCount(); got != 1 {
+		t.Fatalf("cover fetches = %d, want 1", got)
+	}
+	got := readBookmark(t, s, novelfullKey)
+	if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
+		t.Fatalf("Cover = %q, want %q", got.Cover, want)
+	}
+}