Prefetch Kagane covers during latest polling #51

Merged
sulthan merged 1 commits from issue-45 into main 2026-08-09 14:59:01 +07:00
5 changed files with 258 additions and 15 deletions
+36 -1
View File
@@ -16,6 +16,11 @@ type Fetcher interface {
Get(ctx context.Context, url string) (body string, status int, err error) Get(ctx context.Context, url string) (body string, status int, err error)
} }
// CoverFetcher retrieves one kagane cover through the browser-backed path.
type CoverFetcher interface {
Image(ctx context.Context, imageID string) (body []byte, contentType string, err error)
}
// Poller re-checks each bookmarked series' newest published chapter on a // Poller re-checks each bookmarked series' newest published chapter on a
// schedule, independent of the userscript's own in-browser checks. The two run // schedule, independent of the userscript's own in-browser checks. The two run
// in parallel and report the same observable fact, so whichever writes last wins // in parallel and report the same observable fact, so whichever writes last wins
@@ -36,7 +41,10 @@ type Poller struct {
// BrowserFetch handles sites behind a JavaScript challenge that Fetch // BrowserFetch handles sites behind a JavaScript challenge that Fetch
// cannot clear. Nil disables those sites entirely rather than falling back // cannot clear. Nil disables those sites entirely rather than falling back
// to Fetch, which would only ever retrieve a challenge page. // to Fetch, which would only ever retrieve a challenge page.
BrowserFetch Fetcher BrowserFetch Fetcher
// CoverFetch is optional: it prefetches kagane covers without changing the
// chapter poll's result when the browser or persistence fails.
CoverFetch CoverFetcher
Now func() time.Time // injected so tests can freeze it Now func() time.Time // injected so tests can freeze it
Cooldown time.Duration Cooldown time.Duration
BrowserCooldown time.Duration BrowserCooldown time.Duration
@@ -47,6 +55,32 @@ type Poller struct {
var browserBackedSites = []string{"kagane", "novelfull"} var browserBackedSites = []string{"kagane", "novelfull"}
func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) {
if sr.Site != "kagane" || p.CoverFetch == nil {
return
}
imageID, ok := store.KaganeImageID(sr.Cover)
if !ok {
return
}
_, _, found, err := p.Store.GetKaganeCover(imageID)
if err != nil {
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
return
}
if found {
return
}
body, contentType, err := p.CoverFetch.Image(ctx, imageID)
if err != nil {
log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err)
return
}
if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil {
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
}
}
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be // fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare // fetched at all right now. kagane and novelfull both sit behind a Cloudflare
// JavaScript challenge that no TLS fingerprint clears — kagane verified // JavaScript challenge that no TLS fingerprint clears — kagane verified
@@ -157,6 +191,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site) log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site)
return return
} }
p.prefetchCover(ctx, sr)
body, status, err := f.Get(ctx, sr.SeriesURL) body, status, err := f.Get(ctx, sr.SeriesURL)
if err != nil { if err != nil {
+189
View File
@@ -104,6 +104,30 @@ func (f *fakeFetcher) callCount() int {
return len(f.calls) return len(f.calls)
} }
type fakeCoverFetcher struct {
mu sync.Mutex
calls []string
body []byte
contentType string
err error
}
func (f *fakeCoverFetcher) Image(_ context.Context, imageID string) ([]byte, string, error) {
f.mu.Lock()
f.calls = append(f.calls, imageID)
f.mu.Unlock()
if f.err != nil {
return nil, "", f.err
}
return f.body, f.contentType, nil
}
func (f *fakeCoverFetcher) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run // newTestPoller wires a poller with a frozen clock and no stagger, so tests run
// instantly and deterministically. // instantly and deterministically.
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller { func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
@@ -580,6 +604,171 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
} }
} }
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) },
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
body, contentType, ok, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b")
if err != nil || !ok {
t.Fatalf("GetKaganeCover: %v found=%v", err, ok)
}
if string(body) != "cover-bytes" || contentType != "image/webp" {
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType)
}
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got)
}
}
func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
at := time.UnixMilli(5_000_000)
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
at = at.Add(2 * time.Hour)
p.runOnce(context.Background())
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
}
}
func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
now := time.UnixMilli(5_000_000)
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
got, found, err := s.Get(s.OwnerID(), key)
if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found)
}
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
}
if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() {
t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli())
}
}
func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found {
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
}
}
func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "kagane", SeriesID: seriesID,
SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
p := &Poller{
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found {
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
}
}
func TestRunOnceDoesNotPrefetchNonKaganeCover(t *testing.T) {
s, _ := newTestStore(t)
const key = "asura:solo"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
Cover: "https://asurascans.com/covers/solo.jpg", UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
covers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if got := covers.callCount(); got != 0 {
t.Fatalf("cover fetch calls for asura = %d, want 0", got)
}
}
func TestFetcherForRoutesNovelSites(t *testing.T) { func TestFetcherForRoutesNovelSites(t *testing.T) {
tls := &fakeFetcher{} tls := &fakeFetcher{}
browser := &fakeFetcher{} browser := &fakeFetcher{}
+26 -2
View File
@@ -142,14 +142,35 @@ func (b Bookmark) Initial() string {
// the userscript stores for that site. // the userscript stores for that site.
var kaganeCoverRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`) var kaganeCoverRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`)
// KaganeImageID extracts the validated image id from the cover URL recorded by
// the userscript.
func KaganeImageID(cover string) (string, bool) {
m := kaganeCoverRe.FindStringSubmatch(cover)
if m == nil {
return "", false
}
return m[1], true
}
// IsKaganeCoverContentType reports whether a fetched response is safe to store
// and serve as a cover.
func IsKaganeCoverContentType(contentType string) bool {
switch contentType {
case "image/webp", "image/jpeg", "image/png", "image/avif", "image/gif":
return true
default:
return false
}
}
// CoverURL is the src the web UI puts in an <img>. For every site but kagane // CoverURL is the src the web UI puts in an <img>. For every site but kagane
// that is Cover as stored. kagane serves its images behind a Cloudflare // that is Cover as stored. kagane serves its images behind a Cloudflare
// challenge *and* with `cross-origin-resource-policy: same-origin`, so no page // challenge *and* with `cross-origin-resource-policy: same-origin`, so no page
// on another origin can load one however it asks (verified 2026-08-08); those // on another origin can load one however it asks (verified 2026-08-08); those
// go through the backend's own proxy instead. // go through the backend's own proxy instead.
func (b Bookmark) CoverURL() string { func (b Bookmark) CoverURL() string {
if m := kaganeCoverRe.FindStringSubmatch(b.Cover); m != nil { if imageID, ok := KaganeImageID(b.Cover); ok {
return "/img/kagane/" + m[1] return "/img/kagane/" + imageID
} }
return b.Cover return b.Cover
} }
@@ -551,6 +572,9 @@ func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) {
// PutKaganeCover persists one fetched cover. Image ids are immutable, so a // PutKaganeCover persists one fetched cover. Image ids are immutable, so a
// later fetch cannot replace the bytes already made durable. // later fetch cannot replace the bytes already made durable.
func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error { func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error {
if !IsKaganeCoverContentType(contentType) {
return fmt.Errorf("put kagane cover %q: unsupported content type %q", imageID, contentType)
}
if _, err := s.db.Exec(` if _, err := s.db.Exec(`
INSERT INTO covers (image_id, body, content_type) INSERT INTO covers (image_id, body, content_type)
VALUES ($1, $2, $3) VALUES ($1, $2, $3)
+2 -12
View File
@@ -1,6 +1,7 @@
package web package web
import ( import (
"bookmarkmanager/backend/internal/store"
"context" "context"
"log" "log"
"net/http" "net/http"
@@ -21,17 +22,6 @@ type CoverFetcher interface {
// so it is validated at the boundary rather than passed through. // so it is validated at the boundary rather than passed through.
var coverIDRe = regexp.MustCompile(`^[0-9a-f-]{36}$`) var coverIDRe = regexp.MustCompile(`^[0-9a-f-]{36}$`)
// coverTypes is the set of content types the proxy will echo back. A response
// header sourced from a third party is not repeated verbatim: anything outside
// this set is treated as "not a cover".
var coverTypes = map[string]bool{
"image/webp": true,
"image/jpeg": true,
"image/png": true,
"image/avif": true,
"image/gif": true,
}
// coverTimeout bounds one proxied cover. Shorter than the fetcher's own // coverTimeout bounds one proxied cover. Shorter than the fetcher's own
// challenge budget on purpose: a browser page is waiting on this, and a cover // challenge budget on purpose: a browser page is waiting on this, and a cover
// that has not arrived by now is better left as a broken slot than as a request // that has not arrived by now is better left as a broken slot than as a request
@@ -75,7 +65,7 @@ func (h *Handler) kaganeCover(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r) http.NotFound(w, r)
return return
} }
if !coverTypes[contentType] { if !store.IsKaganeCoverContentType(contentType) {
log.Printf("kagane cover %s: unexpected content type %q", id, contentType) log.Printf("kagane cover %s: unexpected content type %q", id, contentType)
http.NotFound(w, r) http.NotFound(w, r)
return return
+5
View File
@@ -335,10 +335,15 @@ func main() {
// newLatestPoller wires the configured cooldowns and fetchers into the poller. // newLatestPoller wires the configured cooldowns and fetchers into the poller.
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller { func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
var covers latest.CoverFetcher
if f, ok := browser.(latest.CoverFetcher); ok {
covers = f
}
return &latest.Poller{ return &latest.Poller{
Store: s, Store: s,
Fetch: fetch, Fetch: fetch,
BrowserFetch: browser, BrowserFetch: browser,
CoverFetch: covers,
Now: time.Now, Now: time.Now,
Cooldown: cfg.Cooldown, Cooldown: cfg.Cooldown,
BrowserCooldown: cfg.BrowserCooldown, BrowserCooldown: cfg.BrowserCooldown,