sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Unify page routing and harden acquire wiring after review (#62)

Browse Source
This commit is contained in:
Sulthan Zaki
2026-08-10 10:56:34 +07:00
parent 40ce68b7ab
commit a66491acd2
7 changed files with 127 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/acquire.go
+4 -28
View File
@@ -3,7 +3,6 @@ package latest
import (
"context"
"log"
"slices"
"sync"
"time"
@@ -36,11 +35,9 @@ type Acquirer struct {
// BrowserFetch disables acquisition entirely.
Fetch Fetcher
// BrowserFetch retrieves kagane and novelfull pages through the browser
// sidecar, which is the only thing that clears their Cloudflare
// challenge. Nil leaves those Sites unacquired; kagane never falls back
// to Fetch (a plain request only retrieves a challenge page), while
// novelfull does, because its challenge is a live time-varying fact and
// its cover bytes never need the browser.
// sidecar, the only thing that clears their Cloudflare challenge. The
// per-site fallback policy lives in fetcherFor. Nil leaves those Sites
// unacquired when no fallback applies.
BrowserFetch Fetcher
// Covers retrieves the cover bytes. Nil leaves the Cover blank and the
// chapter half working.
@@ -108,7 +105,7 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
return
}
f := a.fetcherFor(sr.Site)
f := fetcherFor(sr.Site, a.BrowserFetch, a.Fetch)
if f == nil {
log.Printf("acquire %q: no fetcher for site %q", sr.Key(), sr.Site)
return
@@ -148,24 +145,3 @@ func (a *Acquirer) acquire(ctx context.Context, sr store.Series) {
log.Printf("acquire %q: persist cover: %v", sr.Key(), err)
}
}
// fetcherFor returns the fetcher a site's page needs, or nil when the site
// cannot be fetched at all right now. kagane and novelfull pages sit behind a
// Cloudflare JavaScript challenge, so they prefer the browser; novelfull alone
// falls back to the plain-TLS fetcher when no browser is configured, because
// its challenge is a live time-varying fact (AGENTS.md) and its cover bytes
// never need the browser. kagane never falls back: a plain fetch of a kagane
// page or cover would only ever retrieve a challenge page.
func (a *Acquirer) fetcherFor(site string) Fetcher {
switch {
case site == "kagane":
return a.BrowserFetch
case slices.Contains(browserBackedSites, site): // novelfull
if a.BrowserFetch != nil {
return a.BrowserFetch
}
return a.Fetch
default:
return a.Fetch
}
}
backend/internal/latest/acquire_test.go
+24
View File
@@ -382,6 +382,30 @@ func TestAcquireKaganeSkippedWithoutBrowser(t *testing.T) {
}
}
// The byte half of "nothing falls back to a plain fetch": with a browser for
// the page but none for the bytes, a kagane Cover stays absent and the TLS
// cover fetcher is never consulted.
func TestAcquireKaganeBytesNeverFallBackToPlainTLS(t *testing.T) {
s, _ := newTestStore(t)
browserPage := &fakeFetcher{body: kaganeSeriesAndCoverFixture, status: 200}
tlsCovers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
acq := &Acquirer{
Store: s, Fetch: &fakeFetcher{body: "", status: 403},
BrowserFetch: browserPage, Covers: tlsCovers,
}
s.OnSeriesCreated = acq.Acquire
bookmarkNewKaganeSeries(t, s)
acq.Wait()
if got := tlsCovers.callCount(); got != 0 {
t.Fatalf("plain-TLS cover fetches = %d, want 0 — kagane bytes are browser-only", got)
}
if got := readBookmark(t, s, kaganeKey); got.Cover != "" {
t.Fatalf("Cover = %q, want empty without a browser cover fetcher", got.Cover)
}
}
// novelfull's no-browser degradation differs from kagane's: only its HTML
// needs the sidecar, so when the page body is available — the challenge is a
// live time-varying fact that sometimes answers a plain request — the Cover
backend/internal/latest/poller.go
+22 -10
View File
@@ -116,16 +116,28 @@ func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL stri
}
}
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
// JavaScript challenge that no TLS fingerprint clears — kagane verified
// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
// profile TLSFetcher uses — so they are browser-only or nothing.
func (p *Poller) fetcherFor(site string) Fetcher {
if slices.Contains(browserBackedSites, site) {
return p.BrowserFetch
// fetcherFor returns the fetcher a site's page needs, or nil when the site
// cannot be fetched at all right now. kagane and novelfull pages sit behind a
// Cloudflare JavaScript challenge that no TLS fingerprint clears (kagane
// verified 2026-08-03, novelfull verified 2026-08-05, both against the same
// Chrome_133 profile TLSFetcher uses), so both prefer the browser; novelfull
// alone falls back to the plain-TLS fetcher when no browser is configured,
// because its challenge is a live time-varying fact (AGENTS.md) and its cover
// bytes never need the browser. kagane never falls back: a plain fetch of a
// kagane page or cover would only ever retrieve a challenge page. One routing
// rule for the poll and the acquirer, so the two cannot drift apart.
func fetcherFor(site string, browser, tls Fetcher) Fetcher {
switch {
case site == "kagane":
return browser
case slices.Contains(browserBackedSites, site): // novelfull
if browser != nil {
return browser
}
return tls
default:
return tls
}
return p.Fetch
}
// Run polls until ctx is cancelled.
@@ -221,7 +233,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
return
}
f := p.fetcherFor(sr.Site)
f := fetcherFor(sr.Site, p.BrowserFetch, p.Fetch)
if f == nil {
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
return
backend/internal/latest/poller_test.go
+57 -8
View File
@@ -671,6 +671,51 @@ func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
}
}
// novelfull without a browser is not skipped outright: its challenge is a
// live time-varying fact, so the plain-TLS page fetch is attempted and — when
// the body answers — fills both the chapter and the Cover, exactly the
// client-scraped rows #62 wants healed.
func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
s, _ := newTestStore(t)
key := "novelfull:reverend-insanity"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key,
Site: "novelfull",
SeriesID: "reverend-insanity",
SeriesURL: "https://novelfull.com/reverend-insanity.html",
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour,
Interval: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if len(tlsF.calls) != 1 {
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetches = %d, want 1", got)
}
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want)
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum)
}
}
// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
func TestKaganeUsesBrowserFetcher(t *testing.T) {
s, _ := newTestStore(t)
@@ -893,20 +938,24 @@ func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
func TestFetcherForRoutesNovelSites(t *testing.T) {
tls := &fakeFetcher{}
browser := &fakeFetcher{}
p := &Poller{Fetch: tls, BrowserFetch: browser}
cases := []struct {
site string
want Fetcher
site string
browser Fetcher
tls Fetcher
want Fetcher
}{
{"asura", tls},
{"lightnovelworld", tls},
{"kagane", browser},
{"novelfull", browser},
{"asura", browser, tls, tls},
{"lightnovelworld", browser, tls, tls},
{"kagane", browser, tls, browser},
{"novelfull", browser, tls, browser},
// browser-less deployment: kagane is nothing, novelfull degrades to TLS
{"kagane", nil, tls, nil},
{"novelfull", nil, tls, tls},
}
for _, tc := range cases {
t.Run(tc.site, func(t *testing.T) {
if got := p.fetcherFor(tc.site); got != tc.want {
if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want {
t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
}
})