From 9d6d3bde727674fd0f87390576878f7b2c958b7a Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Mon, 10 Aug 2026 00:45:55 +0700 Subject: [PATCH] Add gated cover byte fetcher (#66) ## Summary Adds a plain-TLS cover byte fetcher with a destination-class SSRF gate and wires public cover sources through the content-addressed filesystem store. ## Changes - Resolve hostnames before connecting; refuse non-HTTPS, loopback, private, link-local, unique-local, CGNAT, credentials, and mixed public/private DNS answers. - Re-check every redirect and resolve/classify again at dial time to close DNS rebinding. - Reuse `maxBodyBytes`; reject oversized responses and non-image content types before persistence. - Add generic `Store.GetCover`/`PutCover` source-URL storage while preserving the browser-backed kagane path. - Keep cover prefetch failures isolated from chapter polling. - Add observable tests for TLS, no-connection refusals, all refused address classes, redirect blocking, streaming body caps, non-image rejection, content-addressed persistence, DNS rebinding, and poller routing. ## Verification - `go test -count=1 ./...` - `go vet ./...` Both pass. No test touches the live network. Closes #57 Reviewed-on: https://gitea.violetcrown.my.id/sulthan/mangaBookmark/pulls/66 Co-authored-by: Sulthan Zaki Co-committed-by: Sulthan Zaki --- backend/internal/latest/cover.go | 180 +++++++++++++++++ backend/internal/latest/cover_fetch_test.go | 206 ++++++++++++++++++++ backend/internal/latest/poller.go | 49 +++-- backend/internal/latest/poller_test.go | 101 +++++++++- backend/internal/store/store.go | 19 +- backend/internal/store/store_test.go | 22 +++ backend/internal/web/cover.go | 2 +- backend/main.go | 5 +- 8 files changed, 559 insertions(+), 25 deletions(-) create mode 100644 backend/internal/latest/cover.go create mode 100644 backend/internal/latest/cover_fetch_test.go diff --git a/backend/internal/latest/cover.go b/backend/internal/latest/cover.go new file mode 100644 index 0000000..2487dc2 --- /dev/null +++ b/backend/internal/latest/cover.go @@ -0,0 +1,180 @@ +package latest + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "time" + + "bookmarkmanager/backend/internal/store" +) + +// CoverBytesFetcher retrieves one cover from its source URL. The caller owns +// persistence; this seam keeps network policy independent from the store. +type CoverBytesFetcher interface { + Fetch(ctx context.Context, sourceURL string) (body []byte, contentType string, err error) +} + +// CoverResolver resolves a host before any connection is attempted. Tests +// inject it to exercise hostile DNS results without touching the live network. +type CoverResolver func(context.Context, string) ([]netip.Addr, error) + +// TLSCoverFetcher retrieves image bytes with the standard HTTPS client. Unlike +// TLSFetcher, it does not need a browser fingerprint: cover hosts are public +// CDNs and the response is accepted only after the destination gate passes. +type TLSCoverFetcher struct { + client *http.Client + resolve CoverResolver +} + +var _ CoverBytesFetcher = (*TLSCoverFetcher)(nil) + +const coverRequestTimeout = 30 * time.Second + +var carrierGradeNAT = netip.MustParsePrefix("100.64.0.0/10") + +// NewCoverFetcher builds the production cover client with the real resolver. +func NewCoverFetcher() *TLSCoverFetcher { + return NewCoverFetcherWithResolver(nil) +} + +// NewCoverFetcherWithResolver builds a cover client using resolve, or the real +// system resolver when resolve is nil. +func NewCoverFetcherWithResolver(resolve CoverResolver) *TLSCoverFetcher { + if resolve == nil { + resolve = defaultCoverResolver + } + return newCoverFetcher(newCoverHTTPClient(resolve), resolve) +} + +func newCoverFetcher(client *http.Client, resolve CoverResolver) *TLSCoverFetcher { + f := &TLSCoverFetcher{client: client, resolve: resolve} + client.CheckRedirect = func(req *http.Request, _ []*http.Request) error { + if err := f.validateURL(req.Context(), req.URL); err != nil { + return fmt.Errorf("redirect destination: %w", err) + } + return nil + } + return f +} + +func defaultCoverResolver(ctx context.Context, host string) ([]netip.Addr, error) { + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) +} + +func newCoverHTTPClient(resolve CoverResolver) *http.Client { + base, ok := http.DefaultTransport.(*http.Transport) + if !ok { + base = &http.Transport{} + } + transport := base.Clone() + // A proxy would make the dial target the proxy rather than the cover host, + // defeating destination classification. Cover fetching is direct by design. + transport.Proxy = nil + dialer := &net.Dialer{} + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("split cover address %q: %w", address, err) + } + addrs, err := resolveCoverHost(ctx, host, resolve) + if err != nil { + return nil, err + } + for _, addr := range addrs { + if !publicCoverAddress(addr) { + return nil, fmt.Errorf("cover host resolves to refused address %s", addr) + } + conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(addr.String(), port)) + if err == nil { + return conn, nil + } + } + return nil, fmt.Errorf("cover host %q has no reachable address", host) + } + return &http.Client{Transport: transport, Timeout: coverRequestTimeout} +} + +func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) { + u, err := url.Parse(sourceURL) + if err != nil { + return nil, "", fmt.Errorf("parse cover URL: %w", err) + } + if err := f.validateURL(ctx, u); err != nil { + return nil, "", err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, "", fmt.Errorf("build cover request: %w", err) + } + resp, err := f.client.Do(req) + if err != nil { + return nil, "", fmt.Errorf("fetch cover: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("fetch cover: status %d", resp.StatusCode) + } + contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil || !store.IsCoverContentType(contentType) { + return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", resp.Header.Get("Content-Type")) + } + if resp.ContentLength > maxBodyBytes { + return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes+1)) + if err != nil { + return nil, "", fmt.Errorf("read cover: %w", err) + } + if len(body) > maxBodyBytes { + return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes) + } + return body, contentType, nil +} + +// This gate deliberately differs from fetchableSeriesURL: cover hosts are +// site-independent CDNs, so a Site host allowlist would reject valid covers. +func (f *TLSCoverFetcher) validateURL(ctx context.Context, u *url.URL) error { + if u == nil || u.Scheme != "https" || u.Host == "" || u.User != nil { + return errors.New("cover URL must use HTTPS without credentials") + } + host := u.Hostname() + if host == "" { + return errors.New("cover URL has no host") + } + addrs, err := resolveCoverHost(ctx, host, f.resolve) + if err != nil { + return fmt.Errorf("resolve cover host %q: %w", host, err) + } + if len(addrs) == 0 { + return fmt.Errorf("resolve cover host %q: no addresses", host) + } + for _, addr := range addrs { + if !publicCoverAddress(addr) { + return fmt.Errorf("cover host %q resolves to refused address %s", host, addr) + } + } + return nil +} + +func resolveCoverHost(ctx context.Context, host string, resolve CoverResolver) ([]netip.Addr, error) { + if literal, err := netip.ParseAddr(host); err == nil { + return []netip.Addr{literal.Unmap()}, nil + } + return resolve(ctx, strings.TrimSuffix(host, ".")) +} + +func publicCoverAddress(addr netip.Addr) bool { + addr = addr.Unmap() + return addr.IsValid() && addr.IsGlobalUnicast() && + !addr.IsLoopback() && !addr.IsPrivate() && !addr.IsLinkLocalUnicast() && + !carrierGradeNAT.Contains(addr) +} diff --git a/backend/internal/latest/cover_fetch_test.go b/backend/internal/latest/cover_fetch_test.go new file mode 100644 index 0000000..fd3c188 --- /dev/null +++ b/backend/internal/latest/cover_fetch_test.go @@ -0,0 +1,206 @@ +package latest + +import ( + "bytes" + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "testing" +) + +func TestCoverFetcherFetchesPublicHTTPSImage(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil { + t.Fatal("cover request was not made over TLS") + } + w.Header().Set("Content-Type", "image/jpeg") + io.WriteString(w, "cover-bytes") + })) + defer server.Close() + + transport := server.Client().Transport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // test server certificate + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + client := &http.Client{Transport: transport} + fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil + }) + + body, contentType, err := fetcher.Fetch(context.Background(), "https://cdn.example/cover.jpg") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(body) != "cover-bytes" || contentType != "image/jpeg" { + t.Fatalf("Fetch = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType) + } +} + +func TestNewCoverFetcherRechecksResolverBeforeConnection(t *testing.T) { + var requests int + server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + requests++ + })) + defer server.Close() + _, port, err := net.SplitHostPort(server.Listener.Addr().String()) + if err != nil { + t.Fatalf("server address: %v", err) + } + + resolves := 0 + fetcher := NewCoverFetcherWithResolver(func(context.Context, string) ([]netip.Addr, error) { + resolves++ + if resolves == 1 { + return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil + } + return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil + }) + _, _, err = fetcher.Fetch(context.Background(), "https://cdn.example:"+port+"/cover.jpg") + if err == nil { + t.Fatal("Fetch accepted a destination that became private") + } + if resolves != 2 { + t.Fatalf("resolver calls = %d, want preflight and dial checks", resolves) + } + if requests != 0 { + t.Fatalf("requests = %d, want 0", requests) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func coverResponse(status int, contentType, location string, body []byte) *http.Response { + header := make(http.Header) + if contentType != "" { + header.Set("Content-Type", contentType) + } + if location != "" { + header.Set("Location", location) + } + return &http.Response{ + StatusCode: status, + Status: http.StatusText(status), + Header: header, + Body: io.NopCloser(bytes.NewReader(body)), + ContentLength: int64(len(body)), + } +} + +func TestCoverFetcherRefusesUnsafeDestinationsBeforeRequest(t *testing.T) { + var calls int + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + calls++ + return coverResponse(http.StatusOK, "image/jpeg", "", []byte("must not reach network")), nil + })} + resolve := func(_ context.Context, host string) ([]netip.Addr, error) { + switch host { + case "loopback.example": + return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil + case "private.example": + return []netip.Addr{netip.MustParseAddr("10.0.0.1")}, nil + case "linklocal.example": + return []netip.Addr{netip.MustParseAddr("169.254.1.1")}, nil + case "unique-local.example": + return []netip.Addr{netip.MustParseAddr("fc00::1")}, nil + case "cgnat.example": + return []netip.Addr{netip.MustParseAddr("100.64.0.1")}, nil + default: + return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil + } + } + fetcher := newCoverFetcher(client, resolve) + + tests := []string{ + "http://public.example/cover.jpg", + "https://127.0.0.1/cover.jpg", + "https://10.0.0.1/cover.jpg", + "https://169.254.1.1/cover.jpg", + "https://[fc00::1]/cover.jpg", + "https://100.64.0.1/cover.jpg", + "https://loopback.example/cover.jpg", + "https://private.example/cover.jpg", + "https://linklocal.example/cover.jpg", + "https://unique-local.example/cover.jpg", + "https://cgnat.example/cover.jpg", + } + for _, sourceURL := range tests { + t.Run(sourceURL, func(t *testing.T) { + calls = 0 + if _, _, err := fetcher.Fetch(context.Background(), sourceURL); err == nil { + t.Fatal("Fetch accepted refused destination") + } + if calls != 0 { + t.Fatalf("network calls = %d, want 0", calls) + } + }) + } +} + +func TestCoverFetcherStopsRedirectIntoPrivateAddress(t *testing.T) { + var calls int + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + calls++ + if req.URL.Hostname() != "cdn.example" { + t.Fatalf("redirect reached %s", req.URL) + } + return coverResponse(http.StatusFound, "", "https://internal.example/cover.jpg", nil), nil + })} + fetcher := newCoverFetcher(client, func(_ context.Context, host string) ([]netip.Addr, error) { + if host == "internal.example" { + return []netip.Addr{netip.MustParseAddr("192.168.1.1")}, nil + } + return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil + }) + + if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/cover.jpg"); err == nil { + t.Fatal("Fetch followed redirect into private address") + } + if calls != 1 { + t.Fatalf("network calls = %d, want only public first hop", calls) + } +} + +func TestCoverFetcherRejectsOversizedBody(t *testing.T) { + var calls int + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + calls++ + response := coverResponse(http.StatusOK, "image/webp", "", bytes.Repeat([]byte("x"), maxBodyBytes+1)) + response.ContentLength = -1 + return response, nil + })} + fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil + }) + + if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/large.webp"); err == nil { + t.Fatal("Fetch accepted oversized body") + } + if calls != 1 { + t.Fatalf("network calls = %d, want 1", calls) + } +} + +func TestCoverFetcherRejectsNonImage(t *testing.T) { + var calls int + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + calls++ + return coverResponse(http.StatusOK, "text/html", "", []byte("challenge")), nil + })} + fetcher := newCoverFetcher(client, func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("198.51.100.10")}, nil + }) + + if _, _, err := fetcher.Fetch(context.Background(), "https://cdn.example/challenge"); err == nil { + t.Fatal("Fetch accepted non-image response") + } + if calls != 1 { + t.Fatalf("network calls = %d, want 1", calls) + } +} diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 6514472..e867ade 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -16,8 +16,8 @@ type Fetcher interface { Get(ctx context.Context, url string) (body string, status int, err error) } -// CoverFetcher retrieves one kagane cover through the browser-backed path. -type CoverFetcher interface { +// BrowserCoverFetcher retrieves one kagane cover through the browser-backed path. +type BrowserCoverFetcher interface { Image(ctx context.Context, imageID string) (body []byte, contentType string, err error) } @@ -42,9 +42,11 @@ type Poller struct { // cannot clear. Nil disables those sites entirely rather than falling back // to Fetch, which would only ever retrieve a challenge page. BrowserFetch Fetcher - // CoverFetch is optional: it prefetches kagane covers without changing the - // chapter poll's result when the browser or persistence fails. - CoverFetch CoverFetcher + // CoverFetch is optional; failures are logged and never affect the chapter poll. + CoverFetch BrowserCoverFetcher + // CoverBytesFetch is optional; it handles non-kagane sources through the same + // failure-isolated prefetch path. + CoverBytesFetch CoverBytesFetcher Now func() time.Time // injected so tests can freeze it Cooldown time.Duration BrowserCooldown time.Duration @@ -56,14 +58,39 @@ type Poller struct { var browserBackedSites = []string{"kagane", "novelfull"} func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) { - if sr.Site != "kagane" || p.CoverFetch == nil { + if sr.Cover == "" { return } - imageID, ok := store.KaganeImageID(sr.Cover) - if !ok { + if sr.Site == "kagane" { + if p.CoverFetch == nil { + return + } + imageID, ok := store.KaganeImageID(sr.Cover) + if !ok { + return + } + _, _, found, err := p.Store.GetKaganeCover(imageID) + if err != nil { + log.Printf("latest poll %q: read cover: %v", sr.Key(), err) + return + } + if found { + return + } + body, contentType, err := p.CoverFetch.Image(ctx, imageID) + if err != nil { + log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err) + return + } + if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil { + log.Printf("latest poll %q: persist cover: %v", sr.Key(), err) + } return } - _, _, found, err := p.Store.GetKaganeCover(imageID) + if p.CoverBytesFetch == nil { + return + } + _, _, found, err := p.Store.GetCover(sr.Cover) if err != nil { log.Printf("latest poll %q: read cover: %v", sr.Key(), err) return @@ -71,12 +98,12 @@ func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) { if found { return } - body, contentType, err := p.CoverFetch.Image(ctx, imageID) + body, contentType, err := p.CoverBytesFetch.Fetch(ctx, sr.Cover) if err != nil { log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err) return } - if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil { + if err := p.Store.PutCover(sr.Cover, body, contentType); err != nil { log.Printf("latest poll %q: persist cover: %v", sr.Key(), err) } } diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 2673f3d..84eefe4 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -128,6 +128,30 @@ func (f *fakeCoverFetcher) callCount() int { return len(f.calls) } +type fakeBytesCoverFetcher struct { + mu sync.Mutex + calls []string + body []byte + contentType string + err error +} + +func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) { + f.mu.Lock() + f.calls = append(f.calls, sourceURL) + f.mu.Unlock() + if f.err != nil { + return nil, "", f.err + } + return f.body, f.contentType, nil +} + +func (f *fakeBytesCoverFetcher) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + // newTestPoller wires a poller with a frozen clock and no stagger, so tests run // instantly and deterministically. func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller { @@ -144,6 +168,63 @@ func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Polle } } +func TestRunOncePrefetchesPublicCover(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + coverURL = "https://cdn.example/covers/chronicles.jpg" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af", + SeriesURL: seriesURL, Cover: coverURL, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + if got := covers.callCount(); got != 1 { + t.Fatalf("cover fetch calls = %d, want 1", got) + } + body, contentType, found, err := s.GetCover(coverURL) + if err != nil || !found { + t.Fatalf("GetCover: %v found=%v", err, found) + } + if string(body) != "cover-bytes" || contentType != "image/jpeg" { + t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType) + } +} + +func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "asura:non-image-cover" + seriesURL = "https://asurascans.com/comics/non-image-cover" + coverURL = "https://cdn.example/covers/challenge" + ) + if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ + Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL, Cover: coverURL, + UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed: %v", err) + } + p := &Poller{ + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, + CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"}, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10, + } + p.runOnce(context.Background()) + + if _, _, found, err := s.GetCover(coverURL); err != nil || found { + t.Fatalf("non-image cover = found %v, err %v; want missing", found, err) + } +} + func TestRunOnceRecordsLatestChapter(t *testing.T) { s, _ := newTestStore(t) const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" @@ -748,24 +829,30 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) { } } -func TestRunOnceDoesNotPrefetchNonKaganeCover(t *testing.T) { +func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) { s, _ := newTestStore(t) const key = "asura:solo" + const coverURL = "https://asurascans.com/covers/solo.jpg" if _, err := s.Upsert(s.OwnerID(), store.Bookmark{ Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo", - Cover: "https://asurascans.com/covers/solo.jpg", UpdatedAt: 1000, + Cover: coverURL, UpdatedAt: 1000, }); err != nil { t.Fatalf("seed: %v", err) } - covers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"} + browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"} + publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"} p := &Poller{ - Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: covers, - Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, + Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers, + CoverBytesFetch: publicCovers, + Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10, } p.runOnce(context.Background()) - if got := covers.callCount(); got != 0 { - t.Fatalf("cover fetch calls for asura = %d, want 0", got) + if got := publicCovers.callCount(); got != 1 { + t.Fatalf("public cover fetch calls = %d, want 1", got) + } + if got := browserCovers.callCount(); got != 0 { + t.Fatalf("browser cover fetch calls for asura = %d, want 0", got) } } diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index bb9016d..50a1366 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -156,9 +156,8 @@ func KaganeImageID(cover string) (string, bool) { return m[1], true } -// IsKaganeCoverContentType reports whether a fetched response is safe to store -// and serve as a cover. -func IsKaganeCoverContentType(contentType string) bool { +// IsCoverContentType reports whether a fetched response is safe to store and serve. +func IsCoverContentType(contentType string) bool { switch contentType { case "image/webp", "image/jpeg", "image/png", "image/avif", "image/gif": return true @@ -609,7 +608,7 @@ func (s *Store) getCover(sourceURL string) ([]byte, string, bool, error) { } func (s *Store) putCover(sourceURL string, body []byte, contentType string) error { - if !IsKaganeCoverContentType(contentType) { + if !IsCoverContentType(contentType) { return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType) } address := coverSourceAddress(sourceURL) @@ -647,6 +646,18 @@ func (s *Store) putCover(sourceURL string, body []byte, contentType string) erro return nil } +// GetCover returns the immutable object addressed by its source URL. Missing +// files are reported with ok=false so callers can retry acquisition later. +func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) { + return s.getCover(sourceURL) +} + +// PutCover persists bytes under the source URL's content address. A later +// write for the same URL cannot replace the immutable object. +func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error { + return s.putCover(sourceURL, body, contentType) +} + // GetKaganeCover returns one persisted cover. Missing covers are reported with // ok=false rather than as an error so the web handler can fetch them once. func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) { diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index 2db7277..01b7645 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -1352,3 +1352,25 @@ func TestKaganeCoverIsContentAddressedOnFilesystem(t *testing.T) { t.Fatalf("covers still has body column") } } + +func TestCoverStoreAcceptsAnySourceURL(t *testing.T) { + s := newTestStore(t) + const sourceURL = "https://cdn.example/covers/series.jpg" + want := []byte("cover-bytes") + if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil { + t.Fatalf("PutCover: %v", err) + } + got, contentType, ok, err := s.GetCover(sourceURL) + if err != nil { + t.Fatalf("GetCover: %v", err) + } + if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" { + t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want) + } + if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil { + t.Fatal("PutCover accepted a non-image") + } + if _, _, ok, err := s.GetCover("https://cdn.example/not-image"); err != nil || ok { + t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err) + } +} diff --git a/backend/internal/web/cover.go b/backend/internal/web/cover.go index 31cee19..cbd1261 100644 --- a/backend/internal/web/cover.go +++ b/backend/internal/web/cover.go @@ -65,7 +65,7 @@ func (h *Handler) kaganeCover(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - if !store.IsKaganeCoverContentType(contentType) { + if !store.IsCoverContentType(contentType) { log.Printf("kagane cover %s: unexpected content type %q", id, contentType) http.NotFound(w, r) return diff --git a/backend/main.go b/backend/main.go index 3c84131..ff61752 100644 --- a/backend/main.go +++ b/backend/main.go @@ -343,8 +343,8 @@ func main() { // newLatestPoller wires the configured cooldowns and fetchers into the poller. func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller { - var covers latest.CoverFetcher - if f, ok := browser.(latest.CoverFetcher); ok { + var covers latest.BrowserCoverFetcher + if f, ok := browser.(latest.BrowserCoverFetcher); ok { covers = f } return &latest.Poller{ @@ -352,6 +352,7 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch Fetch: fetch, BrowserFetch: browser, CoverFetch: covers, + CoverBytesFetch: latest.NewCoverFetcher(), Now: time.Now, Cooldown: cfg.Cooldown, BrowserCooldown: cfg.BrowserCooldown,