feat(latest): poll comix through the browser sidecar (#98)

comix.to began answering plain-TLS fetches with a Cloudflare JavaScript
challenge on 2026-08-12, so every poll got a 403 interstitial and its cover
host static.comix.to is gated the same way. comix joins kagane and novelfull
as a browser Site: one registry entry, no plain-TLS fallback, and cover bytes
routed through the browser's image path behind a fully pinned URL pattern.

The read is an in-tab fetch of the Series URL, not a DOM render: comix is an
SPA, so rendering costs ~65 requests for the same server-rendered HTML one
fetch returns (24.5 KB, ~480 ms). Parsers and stored Series identity are
untouched.

Verified live against the real browser unit: page 24793 bytes in one fetch,
chapter 53, cover accepted by the pin and 26862 image bytes retrieved by
direct navigation (comix's Series page sets cross-origin-embedder-policy:
require-corp, so an in-page fetch of the cover host cannot work).
This commit is contained in:
2026-08-16 12:05:11 +07:00
parent 3f53c79cf4
commit 86160c164a
11 changed files with 336 additions and 55 deletions
+80
View File
@@ -762,6 +762,86 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
}
}
// comix joined kagane behind the challenge on 2026-08-12 (#98): its page goes
// to the browser, its Cover bytes go through the browser's image route because
// static.comix.to is gated the same way, and the TLS fetcher is never asked
// for either.
func TestComixUsesBrowserFetcher(t *testing.T) {
s, dbURL := newTestStore(t)
const (
key = "comix:n8we-dungeons-and-crayons"
seriesID = "n8we-dungeons-and-crayons"
seriesURL = "https://comix.to/title/n8we-dungeons-and-crayons"
coverURL = "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "comix", SeriesID: seriesID, SeriesURL: seriesURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
seedCoverSource(t, dbURL, "comix", seriesID, coverURL)
tlsF := &fakeFetcher{body: "", status: 200}
browserF := &fakeFetcher{body: comixSeriesFixture, status: 200}
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
tlsCovers := &fakeBytesCoverFetcher{body: []byte("tls-bytes"), contentType: "image/jpeg"}
p := &Poller{
Store: s, Fetch: tlsF, BrowserFetch: browserF,
CoverFetch: covers, CoverBytesFetch: tlsCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(tlsF.calls) != 0 {
t.Errorf("TLS fetcher was called for comix: %v", tlsF.calls)
}
if len(browserF.calls) != 1 {
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
}
if got := tlsCovers.callCount(); got != 0 {
t.Errorf("TLS cover fetches = %d, want 0: static.comix.to answers a challenge", got)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("browser cover fetches = %d, want 1", got)
}
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != 80 {
t.Errorf("LatestChapterNum = %v, want 80", got.LatestChapterNum)
}
}
// Without a browser, comix is skipped outright rather than handed to plain
// TLS: a plain fetch retrieves only a challenge page (measured 2026-08-12).
func TestComixSkippedWhenNoBrowserFetcher(t *testing.T) {
s, _ := newTestStore(t)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: "comix:n8we-dungeons-and-crayons", Site: "comix",
SeriesID: "n8we-dungeons-and-crayons",
SeriesURL: "https://comix.to/title/n8we-dungeons-and-crayons",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
f := &fakeFetcher{body: comixSeriesFixture, status: 200}
p := &Poller{
Store: s, Fetch: f,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(f.calls) != 0 {
t.Errorf("TLS fetcher was called for comix: %v", f.calls)
}
}
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
s, dbURL := newTestStore(t)
const (