78234f3c19
Fixes #62
Browser-backed Sites join the Cover pipeline: kagane and novelfull Series now get their Covers at creation, through the same acquisition path as every other Site, instead of waiting for a poll pass.
## What changed
`latest.Acquirer` (creation-time acquisition, fired by the first Bookmark of a Series) previously skipped kagane and novelfull entirely — their pages only yield a Cloudflare challenge to the TLS client, so the request was spent for nothing. It now routes them like the poller does, with the two Sites split exactly as the issue demands:
- **kagane** — page fetched through the browser sidecar, cover URL extracted from the API JSON, bytes fetched through the browser sidecar (the only path that clears the challenge) into the content-addressed store. With no `BROWSER_WS_URL` configured, acquisition is skipped entirely and nothing falls back to a plain fetch.
- **novelfull** — page fetched through the browser sidecar, cover URL extracted from the HTML, bytes fetched over plain TLS through the ordinary gated fetcher (its image paths answer 200 with `access-control-allow-origin: *`, measured 2026-08-09). With no browser configured, the page fetch falls back to the TLS client — novelfull's challenge is a live time-varying fact (AGENTS.md), so when the page body answers, the Cover still lands; when it is challenged, nothing happens.
The byte-routing rule (kagane → browser, every other Site → TLS) is now one shared function (`latest.fetchCoverBytes`) used by both the Poller and the Acquirer, so the two cannot drift apart.
## Acceptance criteria
- [x] kagane cover bytes are fetched through the browser sidecar and stored in the content-addressed store — `TestAcquireKaganeCoverThroughBrowser`
- [x] novelfull cover URLs are extracted from the browser-fetched HTML, and its bytes are fetched over plain TLS — `TestAcquireNovelfullCoverOverPlainTLS`
- [x] With no browser sidecar configured, kagane Covers are absent and nothing falls back to a plain fetch — `TestAcquireKaganeSkippedWithoutBrowser`
- [x] With no browser sidecar configured, novelfull Covers still work if its page body is available — `TestAcquireNovelfullCoverWithoutBrowser`
- [x] Manually verified on-device: a kagane Series shows its Cover in the panel, not a broken-image glyph — being run by a separate manual-verification agent against a mocked scenario (no prod data); not part of this PR
- [x] `go test ./...` is green, with live-network checks gated behind `SMOKE_BROWSER_WS_URL` like the existing kagane image smoke test — new `TestSmokeAcquireKaganeCover` proves the end-to-end acquire path against the real browser when the env var is set
## Verification
- `go test ./...` green across all packages
- New unit tests exercise every routing decision with fakes — no network in the default suite
- Smoke test gated behind `SMOKE_BROWSER_WS_URL`, skipped by default
## Post-review changes (a66491a)
- **One routing rule for pages too** — `fetcherFor` is now a shared function used by both the Poller and the Acquirer; novelfull falls back to the plain-TLS fetcher in *both* when no browser is configured, so pre-existing (client-scraped) novelfull rows get healed by the poll as well, not just Series created after this change (`TestNovelfullUsesTLSWhenNoBrowserFetcher`).
- **Byte-level no-fallback proof** — `TestAcquireKaganeBytesNeverFallBackToPlainTLS` pins that kagane cover bytes never route to the TLS fetcher even when the page came through a browser.
- **Acquirer wired independent of the TLS client** — if `NewTLSFetcher` fails, kagane/novelfull acquisition still works via the sidecar (`main.go`).
- AGENTS.md (root + backend) updated for the novelfull plain-TLS fallback.
Reviewed-on: #72
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
1203 lines
42 KiB
Go
1203 lines
42 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"errors"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"bookmarkmanager/backend/internal/pgtest"
|
|
"bookmarkmanager/backend/internal/store"
|
|
)
|
|
|
|
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
|
|
|
// testOwner is the owner every test store seeds. A second reader, where a
|
|
// test needs one, is created by opening the same database as a second owner.
|
|
var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}
|
|
|
|
// testCoverBaseURL is the public origin every stored cover URL is built from.
|
|
const testCoverBaseURL = "https://bookmarks.test"
|
|
|
|
// newTestStore opens a store on a Postgres database of this test's own and
|
|
// returns the URL, for helpers that need a second connection to the same
|
|
// database (see TestRunOnceFetchesSharedSeriesOnce).
|
|
func newTestStore(t *testing.T) (*store.Store, string) {
|
|
t.Helper()
|
|
url := pgtest.URL(t)
|
|
s, err := store.Open(url, testOwner, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
return s, url
|
|
}
|
|
|
|
// seedForCheck inserts a bookmark (and with it its series) and forces the
|
|
// series' latest_checked_at.
|
|
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
|
|
t.Helper()
|
|
site, seriesID, ok := strings.Cut(key, ":")
|
|
if !ok {
|
|
t.Fatalf("key %q: no ':' separator", key)
|
|
}
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: site,
|
|
SeriesID: seriesID,
|
|
SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed %q: %v", key, err)
|
|
}
|
|
if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
|
|
t.Fatalf("seed mark %q: %v", key, err)
|
|
}
|
|
}
|
|
|
|
func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
|
|
t.Helper()
|
|
site, seriesID, ok := strings.Cut(key, ":")
|
|
if !ok {
|
|
t.Fatalf("key %q: no ':' separator", key)
|
|
}
|
|
ts, err := s.LatestCheckedAt(site, seriesID)
|
|
if err != nil {
|
|
t.Fatalf("LatestCheckedAt %q: %v", key, err)
|
|
}
|
|
return ts
|
|
}
|
|
|
|
// fakeFetcher stands in for the network. Every poller test uses it, so nothing
|
|
// in this file can reach tls-client or a real site.
|
|
type fakeFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body string
|
|
status int
|
|
err error
|
|
// perURL overrides body/status/err for specific URLs.
|
|
perURL map[string]fakeResponse
|
|
}
|
|
|
|
type fakeResponse struct {
|
|
body string
|
|
status int
|
|
err error
|
|
}
|
|
|
|
func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, url)
|
|
f.mu.Unlock()
|
|
if r, ok := f.perURL[url]; ok {
|
|
return r.body, r.status, r.err
|
|
}
|
|
return f.body, f.status, f.err
|
|
}
|
|
|
|
func (f *fakeFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
type fakeCoverFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body []byte
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeCoverFetcher) Image(_ context.Context, imageID string) ([]byte, string, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, imageID)
|
|
f.mu.Unlock()
|
|
if f.err != nil {
|
|
return nil, "", f.err
|
|
}
|
|
return f.body, f.contentType, nil
|
|
}
|
|
|
|
func (f *fakeCoverFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
type fakeBytesCoverFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body []byte
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, sourceURL)
|
|
f.mu.Unlock()
|
|
if f.err != nil {
|
|
return nil, "", f.err
|
|
}
|
|
return f.body, f.contentType, nil
|
|
}
|
|
|
|
func (f *fakeBytesCoverFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
|
|
// instantly and deterministically.
|
|
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
|
|
t.Helper()
|
|
return &Poller{
|
|
Store: s,
|
|
Fetch: f,
|
|
Now: func() time.Time { return at },
|
|
Cooldown: time.Hour,
|
|
BrowserCooldown: 6 * time.Hour,
|
|
Interval: 10 * time.Minute,
|
|
Stagger: 0,
|
|
Batch: 14,
|
|
}
|
|
}
|
|
|
|
// seedCoverSource writes a Series' cover source address without any stored
|
|
// bytes. Nothing in production produces that state any more — a client cover
|
|
// is discarded and an acquired one arrives with its bytes — but rows created
|
|
// before covers moved server-side still carry one, and the prefetch is what
|
|
// heals them.
|
|
func seedCoverSource(t *testing.T, dbURL, site, seriesID, coverURL string) {
|
|
t.Helper()
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(
|
|
`UPDATE series SET cover = $3 WHERE site = $1 AND series_id = $2`,
|
|
site, seriesID, coverURL); err != nil {
|
|
t.Fatalf("seed cover source %s:%s: %v", site, seriesID, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOncePrefetchesPublicCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
coverURL = "https://cdn.example/covers/chronicles.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af", coverURL)
|
|
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
body, contentType, found, err := s.GetCover(coverURL)
|
|
if err != nil || !found {
|
|
t.Fatalf("GetCover: %v found=%v", err, found)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "asura:non-image-cover"
|
|
seriesURL = "https://asurascans.com/comics/non-image-cover"
|
|
coverURL = "https://cdn.example/covers/challenge"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "non-image-cover", coverURL)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
|
|
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
|
|
|
|
now := time.UnixMilli(5_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af")
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
|
}
|
|
if b.LatestChapter != "Chapter 181" {
|
|
t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
// The whole point of the updated_at CASE in Upsert: a newly published chapter is
|
|
// not reading progress and must not move the series up the list.
|
|
func TestRunOnceDoesNotReorderList(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
const key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
|
|
// "other" is the most recently read, so it must stay at the top of List().
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "asura:other", Site: "asura", SeriesID: "other",
|
|
SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
|
|
}); err != nil {
|
|
t.Fatalf("seed other: %v", err)
|
|
}
|
|
seedForCheck(t, s, key, url, 0)
|
|
before, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get before: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
|
|
|
|
after, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get after: %v", err)
|
|
}
|
|
if after.UpdatedAt != before.UpdatedAt {
|
|
t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
|
|
before.UpdatedAt, after.UpdatedAt)
|
|
}
|
|
list, err := s.List(s.OwnerID())
|
|
if err != nil {
|
|
t.Fatalf("List: %v", err)
|
|
}
|
|
if list[0].Key != "asura:other" {
|
|
t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
|
|
}
|
|
}
|
|
|
|
// A failed fetch must still consume the cooldown, or a renamed series gets
|
|
// retried on every tick forever.
|
|
func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
resp fakeResponse
|
|
}{
|
|
{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
|
|
{"non-200", fakeResponse{body: "nope", status: 503}},
|
|
{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
|
|
{"empty body", fakeResponse{body: "", status: 200}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/x"
|
|
seedForCheck(t, s, "asura:x", url, 0)
|
|
|
|
now := time.UnixMilli(7_000_000)
|
|
f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
b, _, err := s.Get(s.OwnerID(), "asura:x")
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
if b.LatestChapterNum != nil {
|
|
t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRespectsBatchLimit(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
for i := 0; i < 20; i++ {
|
|
key := "asura:s" + string(rune('a'+i))
|
|
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
|
|
}
|
|
|
|
f := &fakeFetcher{body: "", status: 200}
|
|
p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
|
|
p.Batch = 5
|
|
p.runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 5 {
|
|
t.Fatalf("fetched %d series, want 5 (batch limit)", got)
|
|
}
|
|
}
|
|
|
|
// The point of the split (ADR-0003): a series referenced by several bookmarks
|
|
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
|
|
// series when two readers track it (issue #22).
|
|
func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
|
|
s, url := newTestStore(t)
|
|
// The slug must match the fixture's own anchors: asura's parser scopes
|
|
// chapter links to the stored slug.
|
|
const slug = "chronicles-of-the-demon-faction-f886a8af"
|
|
const seriesURL = "https://asurascans.com/comics/" + slug
|
|
seedForCheck(t, s, "asura:"+slug, seriesURL, 0)
|
|
// A second reader tracks the same series. The seed is the only
|
|
// reader-creation path, so a second Open as a different owner is how a
|
|
// test gets a second reader on the same database.
|
|
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open second reader: %v", err)
|
|
}
|
|
t.Cleanup(func() { other.Close() })
|
|
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
|
Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000,
|
|
}); err != nil {
|
|
t.Fatalf("seed second reader: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("fetched shared series %d times, want 1", got)
|
|
}
|
|
// Both bookmarks join to the same updated series row.
|
|
for _, st := range []*store.Store{s, other} {
|
|
b, ok, err := st.Get(st.OwnerID(), "asura:"+slug)
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One unreachable series must not abandon the rest of the batch.
|
|
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
|
|
for _, k := range keys {
|
|
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
|
}
|
|
|
|
now := time.UnixMilli(6_000_000)
|
|
f := &fakeFetcher{
|
|
body: "", status: 200,
|
|
perURL: map[string]fakeResponse{
|
|
"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
|
|
},
|
|
}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 5 {
|
|
t.Fatalf("fetched %d series, want all 5 attempted", got)
|
|
}
|
|
for _, k := range keys {
|
|
if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
|
|
t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunLogsCooldowns(t *testing.T) {
|
|
var logs strings.Builder
|
|
previous := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(previous) })
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
(&Poller{
|
|
Cooldown: time.Hour,
|
|
BrowserCooldown: 6 * time.Hour,
|
|
Interval: time.Hour,
|
|
}).Run(ctx)
|
|
|
|
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
|
|
!strings.Contains(got, "browser-cooldown=6h") {
|
|
t.Fatalf("startup log = %q, want both cooldowns", got)
|
|
}
|
|
}
|
|
|
|
// The cooldown is enforced by the due query, so a second immediate pass must do
|
|
// nothing at all — this is what makes the tick interval independent of it.
|
|
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/x"
|
|
seedForCheck(t, s, "asura:x", url, 0)
|
|
|
|
now := time.UnixMilli(8_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
p := newTestPoller(t, s, f, now)
|
|
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("first pass fetched %d, want 1", got)
|
|
}
|
|
// Same instant, and again 59 minutes later: both inside the 1h cooldown.
|
|
p.runOnce(context.Background())
|
|
p.Now = func() time.Time { return now.Add(59 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("fetched %d times inside the cooldown, want 1", got)
|
|
}
|
|
// Past the cooldown, it is due again.
|
|
p.Now = func() time.Time { return now.Add(61 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 2 {
|
|
t.Fatalf("fetched %d times after the cooldown, want 2", got)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
|
|
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
|
|
|
|
hour := time.Hour
|
|
now := time.Unix(2*int64(hour/time.Second), 0)
|
|
tls := &fakeFetcher{status: 200}
|
|
browser := &fakeFetcher{status: 200}
|
|
p := &Poller{
|
|
Store: s,
|
|
Fetch: tls,
|
|
BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
Cooldown: hour,
|
|
BrowserCooldown: 6 * hour,
|
|
Batch: 10,
|
|
}
|
|
|
|
p.runOnce(context.Background())
|
|
if got := tls.callCount(); got != 1 {
|
|
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
|
|
}
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser fetches after 2h = %d, want 0", got)
|
|
}
|
|
|
|
now = time.Unix(7*int64(hour/time.Second), 0)
|
|
p.runOnce(context.Background())
|
|
if got := tls.callCount(); got != 2 {
|
|
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
|
|
}
|
|
if got := browser.callCount(); got != 1 {
|
|
t.Fatalf("browser fetches after 7h = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
// A site that retracts a chapter should correct the stored number downward,
|
|
// mirroring the userscript's equality check (L427) rather than a >.
|
|
func TestRunOnceCorrectsDownward(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
|
const key = "demonic:Catastrophic-Necromancer"
|
|
|
|
high := 400.0
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
|
|
SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
|
|
|
b, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
|
|
t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
|
|
// reject anything that is not a known site with an https URL before spending a
|
|
// request on it — the cooldown still gets consumed either way.
|
|
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
site string
|
|
seriesURL string
|
|
wantCalls int
|
|
}{
|
|
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
|
|
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
|
|
{"unparseable url", "asura", "http://[::1", 0},
|
|
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := tt.site + ":x"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
now := time.UnixMilli(4_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{
|
|
Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
|
})
|
|
|
|
if got := f.callCount(); got != tt.wantCalls {
|
|
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A cancelled context must abandon the batch rather than run it to completion.
|
|
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
|
|
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
f := &fakeFetcher{body: "", status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)
|
|
|
|
if got := f.callCount(); got != 0 {
|
|
t.Fatalf("fetched %d series with a cancelled context, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestFetchableSeriesURL(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
site string
|
|
seriesURL string
|
|
want bool
|
|
}{
|
|
{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
|
|
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
|
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
|
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
|
// The browser fetcher runs JavaScript and carries cookies, so a
|
|
// client-supplied series_url must not be able to aim it anywhere else.
|
|
{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
|
|
{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
|
|
{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
|
|
{"non-https", "comix", "http://comix.to/title/x", false},
|
|
{"no host", "comix", "https:///title/x", false},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
|
t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
|
|
tt.site, tt.seriesURL, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A kagane row must not be handed to the plain TLS fetcher: it would only ever
|
|
// receive a challenge page, and the browser fetcher is the whole reason kagane
|
|
// is pollable at all.
|
|
func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
Site: "kagane",
|
|
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
|
|
p := &Poller{
|
|
Store: s,
|
|
Fetch: f,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
|
Interval: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(f.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
|
|
}
|
|
}
|
|
|
|
// novelfull without a browser is not skipped outright: its challenge is a
|
|
// live time-varying fact, so the plain-TLS page fetch is attempted and — when
|
|
// the body answers — fills both the chapter and the Cover, exactly the
|
|
// client-scraped rows #62 wants healed.
|
|
func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := "novelfull:reverend-insanity"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: "novelfull",
|
|
SeriesID: "reverend-insanity",
|
|
SeriesURL: "https://novelfull.com/reverend-insanity.html",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
|
|
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
|
Interval: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(tlsF.calls) != 1 {
|
|
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetches = %d, want 1", got)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
|
t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
|
|
func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: "kagane",
|
|
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
tlsF := &fakeFetcher{body: "", status: 200}
|
|
browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, BrowserFetch: browserF,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
|
Interval: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(tlsF.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
|
|
}
|
|
if len(browserF.calls) != 1 {
|
|
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", coverURL)
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
body, contentType, ok, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b")
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetKaganeCover: %v found=%v", err, ok)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/webp" {
|
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType)
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) {
|
|
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
at := time.UnixMilli(5_000_000)
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
|
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
now := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
|
|
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found {
|
|
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if _, _, found, err := s.GetKaganeCover("019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); err != nil || found {
|
|
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const key = "asura:solo"
|
|
const coverURL = "https://asurascans.com/covers/solo.jpg"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "solo", coverURL)
|
|
browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
|
|
publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
|
|
CoverBytesFetch: publicCovers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if got := publicCovers.callCount(); got != 1 {
|
|
t.Fatalf("public cover fetch calls = %d, want 1", got)
|
|
}
|
|
if got := browserCovers.callCount(); got != 0 {
|
|
t.Fatalf("browser cover fetch calls for asura = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestFetcherForRoutesNovelSites(t *testing.T) {
|
|
tls := &fakeFetcher{}
|
|
browser := &fakeFetcher{}
|
|
|
|
cases := []struct {
|
|
site string
|
|
browser Fetcher
|
|
tls Fetcher
|
|
want Fetcher
|
|
}{
|
|
{"asura", browser, tls, tls},
|
|
{"lightnovelworld", browser, tls, tls},
|
|
{"kagane", browser, tls, browser},
|
|
{"novelfull", browser, tls, browser},
|
|
// browser-less deployment: kagane is nothing, novelfull degrades to TLS
|
|
{"kagane", nil, tls, nil},
|
|
{"novelfull", nil, tls, tls},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.site, func(t *testing.T) {
|
|
if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want {
|
|
t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
site string
|
|
url string
|
|
want bool
|
|
}{
|
|
{"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true},
|
|
{"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false},
|
|
{"novelfull over http", "novelfull", "http://novelfull.com/x.html", false},
|
|
{"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true},
|
|
{"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false},
|
|
{"unknown site", "webnovel", "https://webnovel.com/x", false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
|
t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A Series that has been blank since creation has no source URL to refetch.
|
|
// The poll extracts the Cover from the same series page it already fetched
|
|
// for the chapter signal and stores the bytes — every Site, both Libraries.
|
|
func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
key string
|
|
site string
|
|
seriesID string
|
|
seriesURL string
|
|
kind string
|
|
body string
|
|
wantCover string
|
|
browser bool
|
|
}{
|
|
{
|
|
name: "asura manga",
|
|
key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura",
|
|
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
|
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
|
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
|
},
|
|
{
|
|
name: "lightnovelworld novel",
|
|
key: "lightnovelworld:a-will-eternal", site: "lightnovelworld",
|
|
seriesID: "a-will-eternal", seriesURL: "https://lightnovelworld.net/novel/a-will-eternal/",
|
|
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
|
wantCover: "https://lightnovelworld.net/wp-content/uploads/2026/03/a-will-eternal-1.webp",
|
|
},
|
|
{
|
|
name: "kagane manga",
|
|
key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane",
|
|
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
|
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
|
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
|
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: tc.key, Site: tc.site, SeriesID: tc.seriesID,
|
|
SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
page := &fakeFetcher{body: tc.body, status: 200}
|
|
public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: page, BrowserFetch: page,
|
|
CoverBytesFetch: public, CoverFetch: browser,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
got := readBookmark(t, s, tc.key)
|
|
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
|
|
if got.Cover != wantWire {
|
|
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
|
}
|
|
if tc.browser {
|
|
if got := browser.callCount(); got != 1 {
|
|
t.Fatalf("browser cover fetches = %d, want 1", got)
|
|
}
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("public cover fetches = %d, want 0", got)
|
|
}
|
|
} else {
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("public cover fetches = %d, want 1", got)
|
|
}
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser cover fetches = %d, want 0", got)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Once a Cover exists the poll must leave it alone: refetching every cycle is
|
|
// noise for the Reader and a request per Series against Sites that already
|
|
// bot-score the deployment's single IP.
|
|
func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}
|
|
at := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("cover fetch calls = %d, want 0", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
|
}
|
|
}
|
|
|
|
// A blank Cover whose byte fetch fails is retried the next time the Series is
|
|
// polled. There is no separate retry queue — the due cycle is the queue.
|
|
func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
public := &fakeBytesCoverFetcher{err: errors.New("cdn down")}
|
|
at := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
if got := readBookmark(t, s, key); got.Cover != "" {
|
|
t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover)
|
|
}
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls after fail = %d, want 1", got)
|
|
}
|
|
|
|
public.err = nil
|
|
public.body = []byte("cover-bytes")
|
|
public.contentType = "image/jpeg"
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := public.callCount(); got != 2 {
|
|
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
|
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
|
}
|
|
}
|
|
|
|
// Cover work is cosmetic: a failed blank fill must leave the chapter poll's
|
|
// result intact for every Site, not only kagane.
|
|
func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
now := time.UnixMilli(5_000_000)
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")},
|
|
Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
got := readBookmark(t, s, key)
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
|
}
|
|
if got.Cover != "" {
|
|
t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover)
|
|
}
|
|
if !strings.Contains(logs.String(), key) {
|
|
t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String())
|
|
}
|
|
}
|
|
|
|
// kaganeAPIFixture carries chapter data only. The blank-fill path needs a
|
|
// cover image id in the same body the chapter poll already retrieved.
|
|
const kaganeAPIFixtureWithCover = `
|
|
{"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption",
|
|
"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}],
|
|
"series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1},
|
|
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
|
|
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
|
|
`
|