d998f8725c
Restore two review findings against spec claims 'the Poll keeps its own Cover policy' and 'pinning is the only behavioural change': prefetchCover now also runs on ticks where the page fetch itself fails (the heal is independent of the page read, and its source may answer while the origin does not), and the no-chapter log surfaces the fetched body length again via a BodyLen fact on seriesRead. Browser dispatch iterates the sorted browser site list so its outcome cannot depend on map order.
1210 lines
43 KiB
Go
1210 lines
43 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"errors"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"bookmarkmanager/backend/internal/pgtest"
|
|
"bookmarkmanager/backend/internal/store"
|
|
)
|
|
|
|
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
|
|
|
// testOwner is the owner every test store seeds. A second reader, where a
|
|
// test needs one, is created by opening the same database as a second owner.
|
|
var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}
|
|
|
|
// testCoverBaseURL is the public origin every stored cover URL is built from.
|
|
const testCoverBaseURL = "https://bookmarks.test"
|
|
|
|
// newTestStore opens a store on a Postgres database of this test's own and
|
|
// returns the URL, for helpers that need a second connection to the same
|
|
// database (see TestRunOnceFetchesSharedSeriesOnce).
|
|
func newTestStore(t *testing.T) (*store.Store, string) {
|
|
t.Helper()
|
|
url := pgtest.URL(t)
|
|
s, err := store.Open(url, testOwner, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
return s, url
|
|
}
|
|
|
|
// seedForCheck inserts a bookmark (and with it its series) and forces the
|
|
// series' latest_checked_at.
|
|
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
|
|
t.Helper()
|
|
site, seriesID, ok := strings.Cut(key, ":")
|
|
if !ok {
|
|
t.Fatalf("key %q: no ':' separator", key)
|
|
}
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: site,
|
|
SeriesID: seriesID,
|
|
SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed %q: %v", key, err)
|
|
}
|
|
if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
|
|
t.Fatalf("seed mark %q: %v", key, err)
|
|
}
|
|
}
|
|
|
|
func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
|
|
t.Helper()
|
|
site, seriesID, ok := strings.Cut(key, ":")
|
|
if !ok {
|
|
t.Fatalf("key %q: no ':' separator", key)
|
|
}
|
|
ts, err := s.LatestCheckedAt(site, seriesID)
|
|
if err != nil {
|
|
t.Fatalf("LatestCheckedAt %q: %v", key, err)
|
|
}
|
|
return ts
|
|
}
|
|
|
|
// fakeFetcher stands in for the network. Every poller test uses it, so nothing
|
|
// in this file can reach tls-client or a real site.
|
|
type fakeFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body string
|
|
status int
|
|
err error
|
|
// perURL overrides body/status/err for specific URLs.
|
|
perURL map[string]fakeResponse
|
|
}
|
|
|
|
type fakeResponse struct {
|
|
body string
|
|
status int
|
|
err error
|
|
}
|
|
|
|
func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, url)
|
|
f.mu.Unlock()
|
|
if r, ok := f.perURL[url]; ok {
|
|
return r.body, r.status, r.err
|
|
}
|
|
return f.body, f.status, f.err
|
|
}
|
|
|
|
func (f *fakeFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
type fakeCoverFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body []byte
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeCoverFetcher) Image(_ context.Context, imageURL string) ([]byte, string, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, imageURL)
|
|
f.mu.Unlock()
|
|
if f.err != nil {
|
|
return nil, "", f.err
|
|
}
|
|
return f.body, f.contentType, nil
|
|
}
|
|
|
|
func (f *fakeCoverFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
type fakeBytesCoverFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body []byte
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, sourceURL)
|
|
f.mu.Unlock()
|
|
if f.err != nil {
|
|
return nil, "", f.err
|
|
}
|
|
return f.body, f.contentType, nil
|
|
}
|
|
|
|
func (f *fakeBytesCoverFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
|
|
// instantly and deterministically.
|
|
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
|
|
t.Helper()
|
|
return &Poller{
|
|
Store: s,
|
|
Fetch: f,
|
|
Now: func() time.Time { return at },
|
|
Cooldown: time.Hour,
|
|
BrowserCooldown: 6 * time.Hour,
|
|
Interval: 10 * time.Minute,
|
|
Stagger: 0,
|
|
Batch: 14,
|
|
}
|
|
}
|
|
|
|
// seedCoverSource writes a Series' cover source address without any stored
|
|
// bytes. Nothing in production produces that state any more — a client cover
|
|
// is discarded and an acquired one arrives with its bytes — but rows created
|
|
// before covers moved server-side still carry one, and the prefetch is what
|
|
// heals them.
|
|
func seedCoverSource(t *testing.T, dbURL, site, seriesID, coverURL string) {
|
|
t.Helper()
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(
|
|
`UPDATE series SET cover = $3 WHERE site = $1 AND series_id = $2`,
|
|
site, seriesID, coverURL); err != nil {
|
|
t.Fatalf("seed cover source %s:%s: %v", site, seriesID, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOncePrefetchesPublicCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
coverURL = "https://cdn.example/covers/chronicles.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af", coverURL)
|
|
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
body, contentType, found, err := s.GetCover(coverURL)
|
|
if err != nil || !found {
|
|
t.Fatalf("GetCover: %v found=%v", err, found)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "asura:non-image-cover"
|
|
seriesURL = "https://asurascans.com/comics/non-image-cover"
|
|
coverURL = "https://cdn.example/covers/challenge"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "non-image-cover", coverURL)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
|
|
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
|
|
|
|
now := time.UnixMilli(5_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af")
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
|
}
|
|
if b.LatestChapter != "Chapter 181" {
|
|
t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
// The whole point of the updated_at CASE in Upsert: a newly published chapter is
|
|
// not reading progress and must not move the series up the list.
|
|
func TestRunOnceDoesNotReorderList(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
const key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
|
|
// "other" is the most recently read, so it must stay at the top of List().
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "asura:other", Site: "asura", SeriesID: "other",
|
|
SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
|
|
}); err != nil {
|
|
t.Fatalf("seed other: %v", err)
|
|
}
|
|
seedForCheck(t, s, key, url, 0)
|
|
before, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get before: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
|
|
|
|
after, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get after: %v", err)
|
|
}
|
|
if after.UpdatedAt != before.UpdatedAt {
|
|
t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
|
|
before.UpdatedAt, after.UpdatedAt)
|
|
}
|
|
list, err := s.List(s.OwnerID())
|
|
if err != nil {
|
|
t.Fatalf("List: %v", err)
|
|
}
|
|
if list[0].Key != "asura:other" {
|
|
t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
|
|
}
|
|
}
|
|
|
|
// A failed fetch must still consume the cooldown, or a renamed series gets
|
|
// retried on every tick forever.
|
|
func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
resp fakeResponse
|
|
}{
|
|
{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
|
|
{"non-200", fakeResponse{body: "nope", status: 503}},
|
|
{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
|
|
{"empty body", fakeResponse{body: "", status: 200}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/x"
|
|
seedForCheck(t, s, "asura:x", url, 0)
|
|
|
|
now := time.UnixMilli(7_000_000)
|
|
f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
b, _, err := s.Get(s.OwnerID(), "asura:x")
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
if b.LatestChapterNum != nil {
|
|
t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRespectsBatchLimit(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
for i := 0; i < 20; i++ {
|
|
key := "asura:s" + string(rune('a'+i))
|
|
seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
|
|
}
|
|
|
|
f := &fakeFetcher{body: "", status: 200}
|
|
p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
|
|
p.Batch = 5
|
|
p.runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 5 {
|
|
t.Fatalf("fetched %d series, want 5 (batch limit)", got)
|
|
}
|
|
}
|
|
|
|
// The point of the split (ADR-0003): a series referenced by several bookmarks
|
|
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
|
|
// series when two readers track it (issue #22).
|
|
func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
|
|
s, url := newTestStore(t)
|
|
// The slug must match the fixture's own anchors: asura's parser scopes
|
|
// chapter links to the stored slug.
|
|
const slug = "chronicles-of-the-demon-faction-f886a8af"
|
|
const seriesURL = "https://asurascans.com/comics/" + slug
|
|
seedForCheck(t, s, "asura:"+slug, seriesURL, 0)
|
|
// A second reader tracks the same series. The seed is the only
|
|
// reader-creation path, so a second Open as a different owner is how a
|
|
// test gets a second reader on the same database.
|
|
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open second reader: %v", err)
|
|
}
|
|
t.Cleanup(func() { other.Close() })
|
|
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
|
Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000,
|
|
}); err != nil {
|
|
t.Fatalf("seed second reader: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("fetched shared series %d times, want 1", got)
|
|
}
|
|
// Both bookmarks join to the same updated series row.
|
|
for _, st := range []*store.Store{s, other} {
|
|
b, ok, err := st.Get(st.OwnerID(), "asura:"+slug)
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One unreachable series must not abandon the rest of the batch.
|
|
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
|
|
for _, k := range keys {
|
|
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
|
}
|
|
|
|
now := time.UnixMilli(6_000_000)
|
|
f := &fakeFetcher{
|
|
body: "", status: 200,
|
|
perURL: map[string]fakeResponse{
|
|
"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
|
|
},
|
|
}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 5 {
|
|
t.Fatalf("fetched %d series, want all 5 attempted", got)
|
|
}
|
|
for _, k := range keys {
|
|
if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
|
|
t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunLogsCooldowns(t *testing.T) {
|
|
var logs strings.Builder
|
|
previous := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(previous) })
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
(&Poller{
|
|
Cooldown: time.Hour,
|
|
BrowserCooldown: 6 * time.Hour,
|
|
Interval: time.Hour,
|
|
}).Run(ctx)
|
|
|
|
if got := logs.String(); !strings.Contains(got, "cooldown=1h") ||
|
|
!strings.Contains(got, "browser-cooldown=6h") {
|
|
t.Fatalf("startup log = %q, want both cooldowns", got)
|
|
}
|
|
}
|
|
|
|
// The cooldown is enforced by the due query, so a second immediate pass must do
|
|
// nothing at all — this is what makes the tick interval independent of it.
|
|
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/x"
|
|
seedForCheck(t, s, "asura:x", url, 0)
|
|
|
|
now := time.UnixMilli(8_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
p := newTestPoller(t, s, f, now)
|
|
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("first pass fetched %d, want 1", got)
|
|
}
|
|
// Same instant, and again 59 minutes later: both inside the 1h cooldown.
|
|
p.runOnce(context.Background())
|
|
p.Now = func() time.Time { return now.Add(59 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("fetched %d times inside the cooldown, want 1", got)
|
|
}
|
|
// Past the cooldown, it is due again.
|
|
p.Now = func() time.Time { return now.Add(61 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 2 {
|
|
t.Fatalf("fetched %d times after the cooldown, want 2", got)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceUsesBrowserCooldown(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const browserKey = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seedForCheck(t, s, "asura:plain", "https://asurascans.com/comics/plain", 0)
|
|
seedForCheck(t, s, browserKey, "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", 0)
|
|
|
|
hour := time.Hour
|
|
now := time.Unix(2*int64(hour/time.Second), 0)
|
|
tls := &fakeFetcher{status: 200}
|
|
browser := &fakeFetcher{status: 200}
|
|
p := &Poller{
|
|
Store: s,
|
|
Fetch: tls,
|
|
BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
Cooldown: hour,
|
|
BrowserCooldown: 6 * hour,
|
|
Batch: 10,
|
|
}
|
|
|
|
p.runOnce(context.Background())
|
|
if got := tls.callCount(); got != 1 {
|
|
t.Fatalf("plain-TLS fetches after 2h = %d, want 1", got)
|
|
}
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser fetches after 2h = %d, want 0", got)
|
|
}
|
|
|
|
now = time.Unix(7*int64(hour/time.Second), 0)
|
|
p.runOnce(context.Background())
|
|
if got := tls.callCount(); got != 2 {
|
|
t.Fatalf("plain-TLS fetches after 7h = %d, want 2", got)
|
|
}
|
|
if got := browser.callCount(); got != 1 {
|
|
t.Fatalf("browser fetches after 7h = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
// A site that retracts a chapter should correct the stored number downward,
|
|
// mirroring the userscript's equality check (L427) rather than a >.
|
|
func TestRunOnceCorrectsDownward(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
|
const key = "demonic:Catastrophic-Necromancer"
|
|
|
|
high := 400.0
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
|
|
SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
|
|
|
b, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
|
|
t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
|
|
// reject anything that is not a known site with an https URL before spending a
|
|
// request on it — the cooldown still gets consumed either way.
|
|
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
site string
|
|
seriesURL string
|
|
wantCalls int
|
|
}{
|
|
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
|
|
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
|
|
{"unparseable url", "asura", "http://[::1", 0},
|
|
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := tt.site + ":x"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
now := time.UnixMilli(4_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{
|
|
Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
|
})
|
|
|
|
if got := f.callCount(); got != tt.wantCalls {
|
|
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A cancelled context must abandon the batch rather than run it to completion.
|
|
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
|
|
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
f := &fakeFetcher{body: "", status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)
|
|
|
|
if got := f.callCount(); got != 0 {
|
|
t.Fatalf("fetched %d series with a cancelled context, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestFetchableSeriesURL(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
site string
|
|
seriesURL string
|
|
want bool
|
|
}{
|
|
{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
|
|
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
|
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
|
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
|
// The three plain-TLS Sites are pinned too: a client-supplied
|
|
// series_url must not aim a fetcher at a lookalike host, even when
|
|
// the fetcher is only an HTTP GET.
|
|
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
|
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
|
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
|
{"comix on a foreign host", "comix", "https://evil.example/title/x", false},
|
|
// The browser fetcher runs JavaScript and carries cookies, so a
|
|
// client-supplied series_url must not be able to aim it anywhere else.
|
|
{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
|
|
{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
|
|
{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
|
|
{"non-https", "comix", "http://comix.to/title/x", false},
|
|
{"no host", "comix", "https:///title/x", false},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
|
t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
|
|
tt.site, tt.seriesURL, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A kagane row must not be handed to the plain TLS fetcher: it would only ever
|
|
// receive a challenge page, and the browser fetcher is the whole reason kagane
|
|
// is pollable at all.
|
|
func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
Site: "kagane",
|
|
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
|
|
p := &Poller{
|
|
Store: s,
|
|
Fetch: f,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
|
Interval: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(f.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
|
|
}
|
|
}
|
|
|
|
// novelfull without a browser is not skipped outright: its challenge is a
|
|
// live time-varying fact, so the plain-TLS page fetch is attempted and — when
|
|
// the body answers — fills both the chapter and the Cover, exactly the
|
|
// client-scraped rows #62 wants healed.
|
|
func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := "novelfull:reverend-insanity"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: "novelfull",
|
|
SeriesID: "reverend-insanity",
|
|
SeriesURL: "https://novelfull.com/reverend-insanity.html",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
|
|
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
|
Interval: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(tlsF.calls) != 1 {
|
|
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetches = %d, want 1", got)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
|
t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
|
|
func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: "kagane",
|
|
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
tlsF := &fakeFetcher{body: "", status: 200}
|
|
browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, BrowserFetch: browserF,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour,
|
|
Interval: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(tlsF.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
|
|
}
|
|
if len(browserF.calls) != 1 {
|
|
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", coverURL)
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
|
if err != nil || !ok {
|
|
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/webp" {
|
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType)
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) {
|
|
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
at := time.UnixMilli(5_000_000)
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
|
Now: func() time.Time { return at }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
now := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
|
|
Now: func() time.Time { return now }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
|
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
|
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const key = "asura:solo"
|
|
const coverURL = "https://asurascans.com/covers/solo.jpg"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "solo", coverURL)
|
|
browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
|
|
publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
|
|
CoverBytesFetch: publicCovers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) }, Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if got := publicCovers.callCount(); got != 1 {
|
|
t.Fatalf("public cover fetch calls = %d, want 1", got)
|
|
}
|
|
if got := browserCovers.callCount(); got != 0 {
|
|
t.Fatalf("browser cover fetch calls for asura = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestFetcherForRoutesNovelSites(t *testing.T) {
|
|
tls := &fakeFetcher{}
|
|
browser := &fakeFetcher{}
|
|
|
|
cases := []struct {
|
|
site string
|
|
browser Fetcher
|
|
tls Fetcher
|
|
want Fetcher
|
|
}{
|
|
{"asura", browser, tls, tls},
|
|
{"lightnovelworld", browser, tls, tls},
|
|
{"kagane", browser, tls, browser},
|
|
{"novelfull", browser, tls, browser},
|
|
// browser-less deployment: kagane is nothing, novelfull degrades to TLS
|
|
{"kagane", nil, tls, nil},
|
|
{"novelfull", nil, tls, tls},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.site, func(t *testing.T) {
|
|
if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want {
|
|
t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
site string
|
|
url string
|
|
want bool
|
|
}{
|
|
{"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true},
|
|
{"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false},
|
|
{"novelfull over http", "novelfull", "http://novelfull.com/x.html", false},
|
|
{"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true},
|
|
{"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false},
|
|
{"unknown site", "webnovel", "https://webnovel.com/x", false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
|
t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A Series that has been blank since creation has no source URL to refetch.
|
|
// The poll extracts the Cover from the same series page it already fetched
|
|
// for the chapter signal and stores the bytes — every Site, both Libraries.
|
|
func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
key string
|
|
site string
|
|
seriesID string
|
|
seriesURL string
|
|
kind string
|
|
body string
|
|
wantCover string
|
|
browser bool
|
|
}{
|
|
{
|
|
name: "asura manga",
|
|
key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura",
|
|
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
|
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
|
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
|
},
|
|
{
|
|
name: "lightnovelworld novel",
|
|
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
|
|
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
|
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
|
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
|
|
},
|
|
{
|
|
name: "kagane manga",
|
|
key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane",
|
|
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
|
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
|
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
|
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: tc.key, Site: tc.site, SeriesID: tc.seriesID,
|
|
SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
page := &fakeFetcher{body: tc.body, status: 200}
|
|
public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: page, BrowserFetch: page,
|
|
CoverBytesFetch: public, CoverFetch: browser,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
Cooldown: time.Hour, BrowserCooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
got := readBookmark(t, s, tc.key)
|
|
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
|
|
if got.Cover != wantWire {
|
|
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
|
}
|
|
if tc.browser {
|
|
if got := browser.callCount(); got != 1 {
|
|
t.Fatalf("browser cover fetches = %d, want 1", got)
|
|
}
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("public cover fetches = %d, want 0", got)
|
|
}
|
|
} else {
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("public cover fetches = %d, want 1", got)
|
|
}
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser cover fetches = %d, want 0", got)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Once a Cover exists the poll must leave it alone: refetching every cycle is
|
|
// noise for the Reader and a request per Series against Sites that already
|
|
// bot-score the deployment's single IP.
|
|
func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}
|
|
at := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("cover fetch calls = %d, want 0", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
|
}
|
|
}
|
|
|
|
// A blank Cover whose byte fetch fails is retried the next time the Series is
|
|
// polled. There is no separate retry queue — the due cycle is the queue.
|
|
func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
public := &fakeBytesCoverFetcher{err: errors.New("cdn down")}
|
|
at := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
if got := readBookmark(t, s, key); got.Cover != "" {
|
|
t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover)
|
|
}
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls after fail = %d, want 1", got)
|
|
}
|
|
|
|
public.err = nil
|
|
public.body = []byte("cover-bytes")
|
|
public.contentType = "image/jpeg"
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := public.callCount(); got != 2 {
|
|
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
|
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
|
}
|
|
}
|
|
|
|
// Cover work is cosmetic: a failed blank fill must leave the chapter poll's
|
|
// result intact for every Site, not only kagane.
|
|
func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
now := time.UnixMilli(5_000_000)
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")},
|
|
Now: func() time.Time { return now }, Cooldown: time.Hour, Batch: 10,
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
got := readBookmark(t, s, key)
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
|
}
|
|
if got.Cover != "" {
|
|
t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover)
|
|
}
|
|
if !strings.Contains(logs.String(), key) {
|
|
t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String())
|
|
}
|
|
}
|
|
|
|
// kaganeAPIFixture carries chapter data only. The blank-fill path needs a
|
|
// cover image id in the same body the chapter poll already retrieved.
|
|
const kaganeAPIFixtureWithCover = `
|
|
{"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption",
|
|
"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}],
|
|
"series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1},
|
|
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
|
|
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
|
|
`
|