sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Add gated cover byte fetcher (#66)

Browse Source 
## Summary

Adds a plain-TLS cover byte fetcher with a destination-class SSRF gate and wires public cover sources through the content-addressed filesystem store.

## Changes

- Resolve hostnames before connecting; refuse non-HTTPS, loopback, private, link-local, unique-local, CGNAT, credentials, and mixed public/private DNS answers.
- Re-check every redirect and resolve/classify again at dial time to close DNS rebinding.
- Reuse `maxBodyBytes`; reject oversized responses and non-image content types before persistence.
- Add generic `Store.GetCover`/`PutCover` source-URL storage while preserving the browser-backed kagane path.
- Keep cover prefetch failures isolated from chapter polling.
- Add observable tests for TLS, no-connection refusals, all refused address classes, redirect blocking, streaming body caps, non-image rejection, content-addressed persistence, DNS rebinding, and poller routing.

## Verification

- `go test -count=1 ./...`
- `go vet ./...`

Both pass. No test touches the live network.

Closes #57

Reviewed-on: #66
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #66.
This commit is contained in:
Sulthan Zaki
2026-08-10 00:45:55 +07:00
committed by sulthan
parent e8d1cba6c5
commit 9d6d3bde72
8 changed files with 559 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/poller_test.go
+94 -7
View File
@@ -128,6 +128,30 @@ func (f *fakeCoverFetcher) callCount() int {
return len(f.calls)
}
type fakeBytesCoverFetcher struct {
mu sync.Mutex
calls []string
body []byte
contentType string
err error
}
func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) {
f.mu.Lock()
f.calls = append(f.calls, sourceURL)
f.mu.Unlock()
if f.err != nil {
return nil, "", f.err
}
return f.body, f.contentType, nil
}
func (f *fakeBytesCoverFetcher) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
// instantly and deterministically.
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
@@ -144,6 +168,63 @@ func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Polle
}
}
func TestRunOncePrefetchesPublicCover(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "asura:chronicles-of-the-demon-faction-f886a8af"
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
coverURL = "https://cdn.example/covers/chronicles.jpg"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af",
SeriesURL: seriesURL, Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got)
}
body, contentType, found, err := s.GetCover(coverURL)
if err != nil || !found {
t.Fatalf("GetCover: %v found=%v", err, found)
}
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
}
}
func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
s, _ := newTestStore(t)
const (
key = "asura:non-image-cover"
seriesURL = "https://asurascans.com/comics/non-image-cover"
coverURL = "https://cdn.example/covers/challenge"
)
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL, Cover: coverURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
}
}
func TestRunOnceRecordsLatestChapter(t *testing.T) {
s, _ := newTestStore(t)
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
@@ -748,24 +829,30 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
}
}
func TestRunOnceDoesNotPrefetchNonKaganeCover(t *testing.T) {
func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
s, _ := newTestStore(t)
const key = "asura:solo"
const coverURL = "https://asurascans.com/covers/solo.jpg"
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
Cover: "https://asurascans.com/covers/solo.jpg", UpdatedAt: 1000,
Cover: coverURL, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
covers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"}
p := &Poller{
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: covers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
CoverBytesFetch: publicCovers,
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
}
p.runOnce(context.Background())
if got := covers.callCount(); got != 0 {
t.Fatalf("cover fetch calls for asura = %d, want 0", got)
if got := publicCovers.callCount(); got != 1 {
t.Fatalf("public cover fetch calls = %d, want 1", got)
}
if got := browserCovers.callCount(); got != 0 {
t.Fatalf("browser cover fetch calls for asura = %d, want 0", got)
}
}