diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 87e020b..6514472 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -16,6 +16,11 @@ type Fetcher interface { Get(ctx context.Context, url string) (body string, status int, err error) } +// CoverFetcher retrieves one kagane cover through the browser-backed path. +type CoverFetcher interface { + Image(ctx context.Context, imageID string) (body []byte, contentType string, err error) +} + // Poller re-checks each bookmarked series' newest published chapter on a // schedule, independent of the userscript's own in-browser checks. The two run // in parallel and report the same observable fact, so whichever writes last wins @@ -36,7 +41,10 @@ type Poller struct { // BrowserFetch handles sites behind a JavaScript challenge that Fetch // cannot clear. Nil disables those sites entirely rather than falling back // to Fetch, which would only ever retrieve a challenge page. - BrowserFetch Fetcher + BrowserFetch Fetcher + // CoverFetch is optional: it prefetches kagane covers without changing the + // chapter poll's result when the browser or persistence fails. + CoverFetch CoverFetcher Now func() time.Time // injected so tests can freeze it Cooldown time.Duration BrowserCooldown time.Duration @@ -47,6 +55,32 @@ type Poller struct { var browserBackedSites = []string{"kagane", "novelfull"} +func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) { + if sr.Site != "kagane" || p.CoverFetch == nil { + return + } + imageID, ok := store.KaganeImageID(sr.Cover) + if !ok { + return + } + _, _, found, err := p.Store.GetKaganeCover(imageID) + if err != nil { + log.Printf("latest poll %q: read cover: %v", sr.Key(), err) + return + } + if found { + return + } + body, contentType, err := p.CoverFetch.Image(ctx, imageID) + if err != nil { + log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err) + return + } + if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil { + log.Printf("latest poll %q: persist cover: %v", sr.Key(), err) + } +} + // fetcherFor returns the fetcher a site needs, or nil when the site cannot be // fetched at all right now. kagane and novelfull both sit behind a Cloudflare // JavaScript challenge that no TLS fingerprint clears — kagane verified @@ -157,6 +191,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) { log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site) return } + p.prefetchCover(ctx, sr) body, status, err := f.Get(ctx, sr.SeriesURL) if err != nil { diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 7f422ed..bbba1f5 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -104,6 +104,30 @@ func (f *fakeFetcher) callCount() int { return len(f.calls) } +type fakeCoverFetcher struct { + mu sync.Mutex + calls []string + body []byte + contentType string + err error +} + +func (f *fakeCoverFetcher) Image(_ context.Context, imageID string) ([]byte, string, error) { + f.mu.Lock() + f.calls = append(f.calls, imageID) + f.mu.Unlock() + if f.err != nil { + return nil, "", f.err + } + return f.body, f.contentType, nil +} + +func (f *fakeCoverFetcher) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + // newTestPoller wires a poller with a frozen clock and no stagger, so tests run // instantly and deterministically. func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller { @@ -580,6 +604,171 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) { } } +func TestRunOncePrefetchesKaganeCover(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", + SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", + Cover: coverURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, + BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, + Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + body, contentType, ok, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b") + if err != nil || !ok { + t.Fatalf("GetKaganeCover: %v found=%v", err, ok) + } + if string(body) != "cover-bytes" || contentType != "image/webp" { + t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType) + } + if got := covers.callCount(); got != 1 { + t.Fatalf("cover fetch calls = %d, want 1", got) + } +} + +func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "kagane", SeriesID: seriesID, + SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + at := time.UnixMilli(5_000_000) + covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"} + p := &Poller{ + Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers, + Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + at = at.Add(2 * time.Hour) + p.runOnce(context.Background()) + + if got := covers.callCount(); got != 1 { + t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got) + } +} + +func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "kagane", SeriesID: seriesID, + SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + now := time.UnixMilli(5_000_000) + p := &Poller{ + Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, + CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")}, + Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + got, found, err := s.Get(s.OwnerID(), key) + if err != nil || !found { + t.Fatalf("Get: %v found=%v", err, found) + } + if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 { + t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum) + } + if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() { + t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli()) + } +} + +func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "kagane", SeriesID: seriesID, + SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + p := &Poller{ + Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, + CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"}, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found { + t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err) + } +} + +func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "kagane", SeriesID: seriesID, + SeriesURL: "https://kagane.to/series/" + seriesID, Cover: coverURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + p := &Poller{ + Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found { + t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err) + } +} + +func TestRunOnceDoesNotPrefetchNonKaganeCover(t *testing.T) { + s, _ := newTestStore(t) + const key = "asura:solo" + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo", + Cover: "https://asurascans.com/covers/solo.jpg", UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + covers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"} + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: covers, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + if got := covers.callCount(); got != 0 { + t.Fatalf("cover fetch calls for asura = %d, want 0", got) + } +} + func TestFetcherForRoutesNovelSites(t *testing.T) { tls := &fakeFetcher{} browser := &fakeFetcher{} diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 9c9e918..058bb5f 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -142,14 +142,35 @@ func (b Bookmark) Initial() string { // the userscript stores for that site. var kaganeCoverRe = regexp.MustCompile(`^https://kagane\.to/api/v2/image/([0-9a-f-]{36})/compressed$`) +// KaganeImageID extracts the validated image id from the cover URL recorded by +// the userscript. +func KaganeImageID(cover string) (string, bool) { + m := kaganeCoverRe.FindStringSubmatch(cover) + if m == nil { + return "", false + } + return m[1], true +} + +// IsKaganeCoverContentType reports whether a fetched response is safe to store +// and serve as a cover. +func IsKaganeCoverContentType(contentType string) bool { + switch contentType { + case "image/webp", "image/jpeg", "image/png", "image/avif", "image/gif": + return true + default: + return false + } +} + // CoverURL is the src the web UI puts in an . For every site but kagane // that is Cover as stored. kagane serves its images behind a Cloudflare // challenge *and* with `cross-origin-resource-policy: same-origin`, so no page // on another origin can load one however it asks (verified 2026-08-08); those // go through the backend's own proxy instead. func (b Bookmark) CoverURL() string { - if m := kaganeCoverRe.FindStringSubmatch(b.Cover); m != nil { - return "/img/kagane/" + m[1] + if imageID, ok := KaganeImageID(b.Cover); ok { + return "/img/kagane/" + imageID } return b.Cover } @@ -551,6 +572,9 @@ func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) { // PutKaganeCover persists one fetched cover. Image ids are immutable, so a // later fetch cannot replace the bytes already made durable. func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error { + if !IsKaganeCoverContentType(contentType) { + return fmt.Errorf("put kagane cover %q: unsupported content type %q", imageID, contentType) + } if _, err := s.db.Exec(` INSERT INTO covers (image_id, body, content_type) VALUES ($1, $2, $3) diff --git a/backend/internal/web/cover.go b/backend/internal/web/cover.go index 079ec3b..31cee19 100644 --- a/backend/internal/web/cover.go +++ b/backend/internal/web/cover.go @@ -1,6 +1,7 @@ package web import ( + "bookmarkmanager/backend/internal/store" "context" "log" "net/http" @@ -21,17 +22,6 @@ type CoverFetcher interface { // so it is validated at the boundary rather than passed through. var coverIDRe = regexp.MustCompile(`^[0-9a-f-]{36}$`) -// coverTypes is the set of content types the proxy will echo back. A response -// header sourced from a third party is not repeated verbatim: anything outside -// this set is treated as "not a cover". -var coverTypes = map[string]bool{ - "image/webp": true, - "image/jpeg": true, - "image/png": true, - "image/avif": true, - "image/gif": true, -} - // coverTimeout bounds one proxied cover. Shorter than the fetcher's own // challenge budget on purpose: a browser page is waiting on this, and a cover // that has not arrived by now is better left as a broken slot than as a request @@ -75,7 +65,7 @@ func (h *Handler) kaganeCover(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - if !coverTypes[contentType] { + if !store.IsKaganeCoverContentType(contentType) { log.Printf("kagane cover %s: unexpected content type %q", id, contentType) http.NotFound(w, r) return diff --git a/backend/main.go b/backend/main.go index e657c6e..4b1476a 100644 --- a/backend/main.go +++ b/backend/main.go @@ -335,10 +335,15 @@ func main() { // newLatestPoller wires the configured cooldowns and fetchers into the poller. func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller { + var covers latest.CoverFetcher + if f, ok := browser.(latest.CoverFetcher); ok { + covers = f + } return &latest.Poller{ Store: s, Fetch: fetch, BrowserFetch: browser, + CoverFetch: covers, Now: time.Now, Cooldown: cfg.Cooldown, BrowserCooldown: cfg.BrowserCooldown,