4aaf1d4f91
Implements spec #135 (spec 2 of 4, derived from wayfinder map #114; decisions settled in #120/#121/#125/#131). Blocked-by #134 is merged, so this lands on `main`. Four owner actions the dashboard can now perform, one ticket each: - **#149** — Latest Chapter correction: one numeric input, overwritten by the next machine write. - **#151** — Series URL repair: owner-typed, gated by the poller's own fetch gate. - **#150 / #153 / #154** — Cover replacement: addresses derived from bytes (`#150`), a Forced Poll replaces the Cover while an ordinary pass still only fills a blank one (`#153`), and byte reclamation is one guarded helper, file first / covers row last (`#154`). - **#155** — Orphan removal: one Series at a time, with the foreign key as the guard. Plus **#152** — Latest Chapter provenance: one derived line naming the actor class, so an owner can tell a hand-edited number from a machine read. - Migration `0015_latest_correction.sql` adds the correction/provenance columns; `0009` now derives cover addresses from bytes. - ADR `0014-cover-addresses-from-bytes.md` records the address scheme. Backend tests cover the store, poller, admin handlers, and web routes (`go test ./...`, needs Docker). Reviewed-on: #156 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
2698 lines
102 KiB
Go
2698 lines
102 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"bookmarkmanager/backend/internal/pgtest"
|
|
"bookmarkmanager/backend/internal/store"
|
|
)
|
|
|
|
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
|
|
|
// testOwner is the owner every test store seeds. A second reader, where a
|
|
// test needs one, is created by opening the same database as a second owner.
|
|
var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}
|
|
|
|
// testCoverBaseURL is the public origin every stored cover URL is built from.
|
|
const testCoverBaseURL = "https://bookmarks.test"
|
|
|
|
// newTestStore opens a store on a Postgres database of this test's own and
|
|
// returns the URL, for helpers that need a second connection to the same
|
|
// database (see TestRunOnceFetchesSharedSeriesOnce).
|
|
func newTestStore(t *testing.T) (*store.Store, string) {
|
|
t.Helper()
|
|
url := pgtest.URL(t)
|
|
s, err := store.Open(url, testOwner, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
return s, url
|
|
}
|
|
|
|
// seedForCheck inserts a bookmark (and with it its series) and forces the
|
|
// series' latest_checked_at.
|
|
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
|
|
t.Helper()
|
|
site, seriesID, ok := strings.Cut(key, ":")
|
|
if !ok {
|
|
t.Fatalf("key %q: no ':' separator", key)
|
|
}
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: site,
|
|
SeriesID: seriesID,
|
|
SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed %q: %v", key, err)
|
|
}
|
|
if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
|
|
t.Fatalf("seed mark %q: %v", key, err)
|
|
}
|
|
}
|
|
|
|
func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
|
|
t.Helper()
|
|
site, seriesID, ok := strings.Cut(key, ":")
|
|
if !ok {
|
|
t.Fatalf("key %q: no ':' separator", key)
|
|
}
|
|
ts, err := s.LatestCheckedAt(site, seriesID)
|
|
if err != nil {
|
|
t.Fatalf("LatestCheckedAt %q: %v", key, err)
|
|
}
|
|
return ts
|
|
}
|
|
|
|
// fakeFetcher stands in for the network. Every poller test uses it, so nothing
|
|
// in this file can reach tls-client or a real site.
|
|
type fakeFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body string
|
|
status int
|
|
err error
|
|
// perURL overrides body/status/err for specific URLs.
|
|
perURL map[string]fakeResponse
|
|
}
|
|
|
|
type fakeResponse struct {
|
|
body string
|
|
status int
|
|
err error
|
|
}
|
|
|
|
func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, url)
|
|
f.mu.Unlock()
|
|
if r, ok := f.perURL[url]; ok {
|
|
return r.body, r.status, r.err
|
|
}
|
|
return f.body, f.status, f.err
|
|
}
|
|
|
|
func (f *fakeFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
type fakeCoverFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body []byte
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeCoverFetcher) Image(_ context.Context, imageURL string) ([]byte, string, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, imageURL)
|
|
f.mu.Unlock()
|
|
if f.err != nil {
|
|
return nil, "", f.err
|
|
}
|
|
return f.body, f.contentType, nil
|
|
}
|
|
|
|
func (f *fakeCoverFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
type fakeBytesCoverFetcher struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
body []byte
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeBytesCoverFetcher) Fetch(_ context.Context, sourceURL string) ([]byte, string, error) {
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, sourceURL)
|
|
f.mu.Unlock()
|
|
if f.err != nil {
|
|
return nil, "", f.err
|
|
}
|
|
return f.body, f.contentType, nil
|
|
}
|
|
|
|
func (f *fakeBytesCoverFetcher) callCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.calls)
|
|
}
|
|
|
|
// newTestPoller wires a poller with a frozen clock. Rest and gap come from the
|
|
// Site registry, so tests seed checked_at relative to the one-hour rest; the
|
|
// round entry point (runOnce) runs every Lane back to back with no real
|
|
// pacing, so tests stay instant and deterministic.
|
|
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
|
|
t.Helper()
|
|
return &Poller{
|
|
Store: s,
|
|
Fetch: f,
|
|
Now: func() time.Time { return at },
|
|
}
|
|
}
|
|
|
|
// seedCoverSource writes a Series' cover source address without any stored
|
|
// bytes. Nothing in production produces that state any more — a client cover
|
|
// is discarded and an acquired one arrives with its bytes — but rows created
|
|
// before covers moved server-side still carry one, and the prefetch is what
|
|
// heals them.
|
|
func seedCoverSource(t *testing.T, dbURL, site, seriesID, coverURL string) {
|
|
t.Helper()
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(
|
|
`UPDATE series SET cover = $3 WHERE site = $1 AND series_id = $2`,
|
|
site, seriesID, coverURL); err != nil {
|
|
t.Fatalf("seed cover source %s:%s: %v", site, seriesID, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOncePrefetchesPublicCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
coverURL = "https://cdn.example/covers/chronicles.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af", coverURL)
|
|
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverBytesFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
body, contentType, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
|
if err != nil || !found {
|
|
t.Fatalf("CoverByAddress: %v found=%v", err, found)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceDoesNotStoreNonImagePublicCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "asura:non-image-cover"
|
|
seriesURL = "https://asurascans.com/comics/non-image-cover"
|
|
coverURL = "https://cdn.example/covers/challenge"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "non-image-cover", SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "non-image-cover", coverURL)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("challenge"), contentType: "text/html"},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if _, _, found, err := s.GetCover(coverURL); err != nil || found {
|
|
t.Fatalf("non-image cover = found %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRecordsLatestChapter(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
|
|
|
|
now := time.UnixMilli(5_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af")
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
|
}
|
|
if b.LatestChapter != "Chapter 181" {
|
|
t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
// The whole point of the updated_at CASE in Upsert: a newly published chapter is
|
|
// not reading progress and must not move the series up the list.
|
|
func TestRunOnceDoesNotReorderList(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
const key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
|
|
// "other" is the most recently read, so it must stay at the top of List().
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "asura:other", Site: "asura", SeriesID: "other",
|
|
SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
|
|
}); err != nil {
|
|
t.Fatalf("seed other: %v", err)
|
|
}
|
|
seedForCheck(t, s, key, url, 0)
|
|
before, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get before: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())
|
|
|
|
after, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get after: %v", err)
|
|
}
|
|
if after.UpdatedAt != before.UpdatedAt {
|
|
t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
|
|
before.UpdatedAt, after.UpdatedAt)
|
|
}
|
|
list, err := s.List(s.OwnerID())
|
|
if err != nil {
|
|
t.Fatalf("List: %v", err)
|
|
}
|
|
if list[0].Key != "asura:other" {
|
|
t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
|
|
}
|
|
}
|
|
|
|
// A failed fetch must still consume the cooldown, or a renamed series gets
|
|
// retried on every tick forever.
|
|
func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
resp fakeResponse
|
|
}{
|
|
{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
|
|
{"non-200", fakeResponse{body: "nope", status: 503}},
|
|
{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
|
|
{"empty body", fakeResponse{body: "", status: 200}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/x"
|
|
seedForCheck(t, s, "asura:x", url, 0)
|
|
|
|
now := time.UnixMilli(7_000_000)
|
|
f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
b, _, err := s.Get(s.OwnerID(), "asura:x")
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
if b.LatestChapterNum != nil {
|
|
t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The point of the split (ADR-0003): a series referenced by several bookmarks
|
|
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
|
|
// series when two readers track it (issue #22).
|
|
func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
|
|
s, url := newTestStore(t)
|
|
// The slug must match the fixture's own anchors: asura's parser scopes
|
|
// chapter links to the stored slug.
|
|
const slug = "chronicles-of-the-demon-faction-f886a8af"
|
|
const seriesURL = "https://asurascans.com/comics/" + slug
|
|
seedForCheck(t, s, "asura:"+slug, seriesURL, 0)
|
|
// A second reader tracks the same series. The seed is the only
|
|
// reader-creation path, so a second Open as a different owner is how a
|
|
// test gets a second reader on the same database.
|
|
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open second reader: %v", err)
|
|
}
|
|
t.Cleanup(func() { other.Close() })
|
|
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
|
Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000,
|
|
}); err != nil {
|
|
t.Fatalf("seed second reader: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("fetched shared series %d times, want 1", got)
|
|
}
|
|
// Both bookmarks join to the same updated series row.
|
|
for _, st := range []*store.Store{s, other} {
|
|
b, ok, err := st.Get(st.OwnerID(), "asura:"+slug)
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One unreachable series must not abandon the rest of the Lane.
|
|
func TestRunOnceOneBadSeriesDoesNotStallLane(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
|
|
for _, k := range keys {
|
|
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
|
}
|
|
|
|
now := time.UnixMilli(6_000_000)
|
|
f := &fakeFetcher{
|
|
body: "", status: 200,
|
|
perURL: map[string]fakeResponse{
|
|
"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
|
|
},
|
|
}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if got := f.callCount(); got != 5 {
|
|
t.Fatalf("fetched %d series, want all 5 attempted", got)
|
|
}
|
|
for _, k := range keys {
|
|
if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
|
|
t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
|
|
}
|
|
}
|
|
}
|
|
|
|
// Pace now lives in the registry, not config: Run logs the lane defaults so a
|
|
// deployment can see what the poller is doing without reading the source.
|
|
func TestRunLogsLaneDefaults(t *testing.T) {
|
|
var logs strings.Builder
|
|
previous := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(previous) })
|
|
|
|
// The pass log writes through the store, so Run needs a real one — a
|
|
// poller without a Store is not a poller.
|
|
s, _ := newTestStore(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
(&Poller{Store: s, Now: func() time.Time { return time.Now() }}).Run(ctx)
|
|
|
|
got := logs.String()
|
|
for _, want := range []string{"6 lanes", "rest=1h0m0s", "gap=10s"} {
|
|
if !strings.Contains(got, want) {
|
|
t.Fatalf("startup log = %q, want %q", got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The rest is enforced by the due query, so a second immediate pass must do
|
|
// nothing at all — this is what makes the Lane's sleep independent of it.
|
|
func TestRunOnceHonoursRestAcrossPasses(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/x"
|
|
seedForCheck(t, s, "asura:x", url, 0)
|
|
|
|
now := time.UnixMilli(8_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
p := newTestPoller(t, s, f, now)
|
|
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("first pass fetched %d, want 1", got)
|
|
}
|
|
// Same instant, and again 59 minutes later: both inside the 1h rest.
|
|
p.runOnce(context.Background())
|
|
p.Now = func() time.Time { return now.Add(59 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 1 {
|
|
t.Fatalf("fetched %d times inside the rest, want 1", got)
|
|
}
|
|
// Past the rest, it is due again.
|
|
p.Now = func() time.Time { return now.Add(61 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := f.callCount(); got != 2 {
|
|
t.Fatalf("fetched %d times after the rest, want 2", got)
|
|
}
|
|
}
|
|
|
|
// A site that retracts a chapter should correct the stored number downward,
|
|
// mirroring the userscript's equality check (L427) rather than a >.
|
|
func TestRunOnceCorrectsDownward(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
|
|
const key = "demonic:Catastrophic-Necromancer"
|
|
|
|
high := 400.0
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
|
|
SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())
|
|
|
|
b, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get: %v", err)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
|
|
t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
|
|
// reject anything that is not a known site with an https URL before spending a
|
|
// request on it — the cooldown still gets consumed either way.
|
|
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
site string
|
|
seriesURL string
|
|
wantCalls int
|
|
}{
|
|
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
|
|
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
|
|
{"unparseable url", "asura", "http://[::1", 0},
|
|
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := tt.site + ":x"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
now := time.UnixMilli(4_000_000)
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{
|
|
Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
|
})
|
|
|
|
if got := f.callCount(); got != tt.wantCalls {
|
|
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A cancelled context must abandon the batch rather than run it to completion.
|
|
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
|
|
seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
f := &fakeFetcher{body: "", status: 200}
|
|
newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)
|
|
|
|
if got := f.callCount(); got != 0 {
|
|
t.Fatalf("fetched %d series with a cancelled context, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestFetchableSeriesURL(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
site string
|
|
seriesURL string
|
|
want bool
|
|
}{
|
|
{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
|
|
{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
|
|
{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
|
|
{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
|
|
// The three plain-TLS Sites are pinned too: a client-supplied
|
|
// series_url must not aim a fetcher at a lookalike host, even when
|
|
// the fetcher is only an HTTP GET.
|
|
{"asura on a foreign host", "asura", "https://asurascans.com.evil.example/comics/x", false},
|
|
{"asura on the dead old domain", "asura", "https://asuracomic.net/comics/x", false},
|
|
{"demonic on a lookalike host", "demonic", "https://demonicscans.org.evil.example/manga/X", false},
|
|
{"comix on a foreign host", "comix", "https://evil.example/title/x", false},
|
|
// The browser fetcher runs JavaScript and carries cookies, so a
|
|
// client-supplied series_url must not be able to aim it anywhere else.
|
|
{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
|
|
{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
|
|
{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
|
|
{"non-https", "comix", "http://comix.to/title/x", false},
|
|
{"no host", "comix", "https:///title/x", false},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := FetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
|
t.Errorf("FetchableSeriesURL(%q, %q) = %v, want %v",
|
|
tt.site, tt.seriesURL, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A kagane row must not be handed to the plain TLS fetcher: it would only ever
|
|
// receive a challenge page, and the browser fetcher is the whole reason kagane
|
|
// is pollable at all.
|
|
func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
Site: "kagane",
|
|
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
|
|
p := &Poller{
|
|
Store: s,
|
|
Fetch: f,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(f.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"); got != 0 {
|
|
t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got)
|
|
}
|
|
}
|
|
|
|
// novelfull without a browser is not skipped outright: its challenge is a
|
|
// live time-varying fact, so the plain-TLS page fetch is attempted and — when
|
|
// the body answers — fills both the chapter and the Cover, exactly the
|
|
// client-scraped rows #62 wants healed.
|
|
func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := "novelfull:reverend-insanity"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: "novelfull",
|
|
SeriesID: "reverend-insanity",
|
|
SeriesURL: "https://novelfull.com/reverend-insanity.html",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
tlsF := &fakeFetcher{body: novelfullSeriesFixture + novelfullCoverFixture, status: 200}
|
|
covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, CoverBytesFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if len(tlsF.calls) != 1 {
|
|
t.Fatalf("TLS fetcher calls = %d, want 1", len(tlsF.calls))
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetches = %d, want 1", got)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
|
t.Fatalf("LatestChapterNum = %v, want 2334", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
|
|
func TestKaganeUsesBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key,
|
|
Site: "kagane",
|
|
SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
tlsF := &fakeFetcher{body: "", status: 200}
|
|
browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, BrowserFetch: browserF,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(tlsF.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
|
|
}
|
|
if len(browserF.calls) != 1 {
|
|
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// comix joined kagane behind the challenge on 2026-08-12 (#98): its page goes
|
|
// to the browser, its Cover bytes go through the browser's image route because
|
|
// static.comix.to is gated the same way, and the TLS fetcher is never asked
|
|
// for either.
|
|
func TestComixUsesBrowserFetcher(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "comix:n8we-dungeons-and-crayons"
|
|
seriesID = "n8we-dungeons-and-crayons"
|
|
seriesURL = "https://comix.to/title/n8we-dungeons-and-crayons"
|
|
coverURL = "https://static.comix.to/039d/i/1/34/6a6742bf15736@280.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "comix", SeriesID: seriesID, SeriesURL: seriesURL,
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "comix", seriesID, coverURL)
|
|
|
|
tlsF := &fakeFetcher{body: "", status: 200}
|
|
browserF := &fakeFetcher{body: comixSeriesFixture, status: 200}
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
tlsCovers := &fakeBytesCoverFetcher{body: []byte("tls-bytes"), contentType: "image/jpeg"}
|
|
p := &Poller{
|
|
Store: s, Fetch: tlsF, BrowserFetch: browserF,
|
|
CoverFetch: covers, CoverBytesFetch: tlsCovers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if len(tlsF.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for comix: %v", tlsF.calls)
|
|
}
|
|
if len(browserF.calls) != 1 {
|
|
t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
|
|
}
|
|
if got := tlsCovers.callCount(); got != 0 {
|
|
t.Errorf("TLS cover fetches = %d, want 0: static.comix.to answers a challenge", got)
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("browser cover fetches = %d, want 1", got)
|
|
}
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 80 {
|
|
t.Errorf("LatestChapterNum = %v, want 80", got.LatestChapterNum)
|
|
}
|
|
}
|
|
|
|
// Without a browser, comix is skipped outright rather than handed to plain
|
|
// TLS: a plain fetch retrieves only a challenge page (measured 2026-08-12).
|
|
func TestComixSkippedWhenNoBrowserFetcher(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: "comix:n8we-dungeons-and-crayons", Site: "comix",
|
|
SeriesID: "n8we-dungeons-and-crayons",
|
|
SeriesURL: "https://comix.to/title/n8we-dungeons-and-crayons",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
f := &fakeFetcher{body: comixSeriesFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: f,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if len(f.calls) != 0 {
|
|
t.Errorf("TLS fetcher was called for comix: %v", f.calls)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "comix:n8we-dungeons-and-crayons"); got != 0 {
|
|
t.Errorf("latest_checked_at = %d, want 0 (untried stays due until a browser appears)", got)
|
|
}
|
|
}
|
|
|
|
func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", coverURL)
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
|
if err != nil || !ok {
|
|
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/webp" {
|
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/webp)", body, contentType)
|
|
}
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddressForBytes([]byte("cover-bytes")) {
|
|
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceDoesNotRefetchKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
at := time.UnixMilli(5_000_000)
|
|
covers := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200}, CoverFetch: covers,
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if got := covers.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1 after two due cycles", got)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceCoverFailureDoesNotBlockChapter(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
now := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
CoverFetch: &fakeCoverFetcher{err: errors.New("browser unavailable")},
|
|
Now: func() time.Time { return now },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
got, found, err := s.Get(s.OwnerID(), key)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: %v found=%v", err, found)
|
|
}
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
if checked := readLatestCheckedAt(t, s, key); checked != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", checked, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
CoverFetch: &fakeCoverFetcher{body: []byte("not an image"), contentType: "text/html"},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("not an image"))); err != nil || found {
|
|
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
coverURL = "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "kagane", seriesID, coverURL)
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || found {
|
|
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
|
}
|
|
}
|
|
|
|
func TestRunOnceRoutesNonKaganeCoverToPublicFetcher(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const key = "asura:solo"
|
|
const coverURL = "https://asurascans.com/covers/solo.jpg"
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: "solo", SeriesURL: "https://asurascans.com/comics/solo",
|
|
UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", "solo", coverURL)
|
|
browserCovers := &fakeCoverFetcher{body: []byte("must not be fetched"), contentType: "image/webp"}
|
|
publicCovers := &fakeBytesCoverFetcher{body: []byte("public cover"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture, status: 200}, CoverFetch: browserCovers,
|
|
CoverBytesFetch: publicCovers,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if got := publicCovers.callCount(); got != 1 {
|
|
t.Fatalf("public cover fetch calls = %d, want 1", got)
|
|
}
|
|
if got := browserCovers.callCount(); got != 0 {
|
|
t.Fatalf("browser cover fetch calls for asura = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestFetcherForRoutesNovelSites(t *testing.T) {
|
|
tls := &fakeFetcher{}
|
|
browser := &fakeFetcher{}
|
|
|
|
cases := []struct {
|
|
site string
|
|
browser Fetcher
|
|
tls Fetcher
|
|
want Fetcher
|
|
}{
|
|
{"asura", browser, tls, tls},
|
|
{"lightnovelworld", browser, tls, tls},
|
|
{"kagane", browser, tls, browser},
|
|
{"novelfull", browser, tls, browser},
|
|
// browser-less deployment: kagane is nothing, novelfull degrades to TLS
|
|
{"kagane", nil, tls, nil},
|
|
{"novelfull", nil, tls, tls},
|
|
// unknown site: fail closed — nothing to fetch or parse
|
|
{"mangadex", browser, tls, nil},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.site, func(t *testing.T) {
|
|
if got := fetcherFor(tc.site, tc.browser, tc.tls); got != tc.want {
|
|
t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
site string
|
|
url string
|
|
want bool
|
|
}{
|
|
{"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true},
|
|
{"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false},
|
|
{"novelfull over http", "novelfull", "http://novelfull.com/x.html", false},
|
|
{"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true},
|
|
{"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false},
|
|
{"unknown site", "webnovel", "https://webnovel.com/x", false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := FetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
|
t.Fatalf("FetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A Series that has been blank since creation has no source URL to refetch.
|
|
// The poll extracts the Cover from the same series page it already fetched
|
|
// for the chapter signal and stores the bytes — every Site, both Libraries.
|
|
func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
key string
|
|
site string
|
|
seriesID string
|
|
seriesURL string
|
|
kind string
|
|
body string
|
|
browser bool
|
|
}{
|
|
{
|
|
name: "asura manga",
|
|
key: "asura:chronicles-of-the-demon-faction-f886a8af", site: "asura",
|
|
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
|
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
|
},
|
|
{
|
|
name: "lightnovelworld novel",
|
|
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
|
|
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
|
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
|
},
|
|
{
|
|
name: "kagane manga",
|
|
key: "kagane:019fe11a-8670-7cf3-8343-0b02057d3787", site: "kagane",
|
|
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
|
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
|
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: tc.key, Site: tc.site, SeriesID: tc.seriesID,
|
|
SeriesURL: tc.seriesURL, Kind: tc.kind, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
page := &fakeFetcher{body: tc.body, status: 200}
|
|
public := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"}
|
|
browser := &fakeCoverFetcher{body: []byte("cover-bytes"), contentType: "image/webp"}
|
|
p := &Poller{
|
|
Store: s, Fetch: page, BrowserFetch: page,
|
|
CoverBytesFetch: public, CoverFetch: browser,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
got := readBookmark(t, s, tc.key)
|
|
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes"))
|
|
if got.Cover != wantWire {
|
|
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
|
}
|
|
if tc.browser {
|
|
if got := browser.callCount(); got != 1 {
|
|
t.Fatalf("browser cover fetches = %d, want 1", got)
|
|
}
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("public cover fetches = %d, want 0", got)
|
|
}
|
|
} else {
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("public cover fetches = %d, want 1", got)
|
|
}
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser cover fetches = %d, want 0", got)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Once a Cover exists the poll must leave it alone: refetching every cycle is
|
|
// noise for the Reader and a request per Series against Sites that already
|
|
// bot-score the deployment's single IP.
|
|
func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}
|
|
at := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("cover fetch calls = %d, want 0", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
|
}
|
|
}
|
|
|
|
// A blank Cover whose byte fetch fails is retried the next time the Series is
|
|
// polled. There is no separate retry queue — the due cycle is the queue.
|
|
func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
coverURL = "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
public := &fakeBytesCoverFetcher{err: errors.New("cdn down")}
|
|
at := time.UnixMilli(5_000_000)
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
if got := readBookmark(t, s, key); got.Cover != "" {
|
|
t.Fatalf("Cover after failed fetch = %q, want blank", got.Cover)
|
|
}
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls after fail = %d, want 1", got)
|
|
}
|
|
|
|
public.err = nil
|
|
public.body = []byte("cover-bytes")
|
|
public.contentType = "image/jpeg"
|
|
at = at.Add(2 * time.Hour)
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
if got := public.callCount(); got != 2 {
|
|
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
|
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
|
}
|
|
}
|
|
|
|
// Cover work is cosmetic: a failed blank fill must leave the chapter poll's
|
|
// result intact for every Site, not only kagane.
|
|
func TestRunOnceBlankCoverFailureDoesNotBlockChapter(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
now := time.UnixMilli(5_000_000)
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{err: errors.New("cdn down")},
|
|
Now: func() time.Time { return now },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
got := readBookmark(t, s, key)
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
|
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
|
}
|
|
if got.Cover != "" {
|
|
t.Fatalf("Cover = %q, want blank after failed fetch", got.Cover)
|
|
}
|
|
if !strings.Contains(logs.String(), key) {
|
|
t.Fatalf("cover failure log missing series key %q; got %q", key, logs.String())
|
|
}
|
|
}
|
|
|
|
// kaganeAPIFixture carries chapter data only. The blank-fill path needs a
|
|
// cover image id in the same body the chapter poll already retrieved.
|
|
const kaganeAPIFixtureWithCover = `
|
|
{"series_id":"019fe11a-8670-7cf3-8343-0b02057d3787","title":"Infinite Decryption",
|
|
"series_covers":[{"cover_id":"019fe11a-84d1-714b-9cf4-2827f277f3c0","language":"en","volume_number":"1","chapter_number":null,"note":null,"image_id":"019fe11a-84c3-7fc3-a84b-88787374b617"}],
|
|
"series_books":[{"book_id":"a","title":"Episode 1","chapter_no":"1","sort_no":1},
|
|
{"book_id":"b","title":"Episode 41","chapter_no":"41","sort_no":41},
|
|
{"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
|
|
`
|
|
|
|
func TestEffectiveGap(t *testing.T) {
|
|
asura := sites["asura"]
|
|
tests := []struct {
|
|
eligible int
|
|
want time.Duration
|
|
clamped bool
|
|
}{
|
|
{0, 10 * time.Second, false},
|
|
{5, 10 * time.Second, false},
|
|
{360, 10 * time.Second, false},
|
|
{720, 5 * time.Second, false},
|
|
{3600, time.Second, false},
|
|
{4000, time.Second, true},
|
|
}
|
|
for _, tt := range tests {
|
|
got, clamped := effectiveGap(asura, tt.eligible)
|
|
if got != tt.want || clamped != tt.clamped {
|
|
t.Errorf("effectiveGap(asura, %d) = (%s, %v), want (%s, %v)",
|
|
tt.eligible, got, clamped, tt.want, tt.clamped)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The due query orders by sharedness first, then age: a series two readers
|
|
// track is polled before a single-reader series that has waited far longer
|
|
// (ADR-0003, issue #100).
|
|
func TestRunOnceOrdersBySharednessThenAge(t *testing.T) {
|
|
s, url := newTestStore(t)
|
|
other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))}, t.TempDir(), testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open second reader: %v", err)
|
|
}
|
|
t.Cleanup(func() { other.Close() })
|
|
|
|
now := time.UnixMilli(5_000_000)
|
|
// popular: two readers, due for 2 minutes. loner: one reader, due for 23
|
|
// minutes. Popularity must win — the loner waited far longer.
|
|
const (
|
|
popularKey = "asura:popular"
|
|
lonerKey = "asura:loner"
|
|
popularURL = "https://asurascans.com/comics/popular"
|
|
lonerURL = "https://asurascans.com/comics/loner"
|
|
)
|
|
seedForCheck(t, s, popularKey, popularURL, now.Add(-62*time.Minute).UnixMilli())
|
|
seedForCheck(t, s, lonerKey, lonerURL, 0)
|
|
if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
|
|
Key: popularKey, Site: "asura", SeriesID: "popular", UpdatedAt: 2000,
|
|
}); err != nil {
|
|
t.Fatalf("seed second reader: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
|
|
if len(f.calls) != 2 {
|
|
t.Fatalf("fetches = %d, want 2", len(f.calls))
|
|
}
|
|
if f.calls[0] != popularURL {
|
|
t.Fatalf("first fetch = %q, want the shared series %q", f.calls[0], popularURL)
|
|
}
|
|
}
|
|
|
|
// Two refusals in one pass stop the Lane: the remaining Series stay unstamped
|
|
// and due, and the Lane backs off for RefuseBackoff before trying the Site
|
|
// again. One hostile Site burns only its own Lane's budget (issue #100).
|
|
func TestRunOnceSiteRefusalSkipsRestOfLaneAndBacksOff(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
const n = 4
|
|
for i := 0; i < n; i++ {
|
|
key := fmt.Sprintf("kagane:s%d", i)
|
|
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0)
|
|
}
|
|
|
|
browser := &fakeFetcher{status: 403}
|
|
tls := &fakeFetcher{status: 200}
|
|
// An asura Series sits on its own Lane: it must still be polled while
|
|
// kagane's Lane burns its budget on refusals (story 4).
|
|
seedForCheck(t, s, "asura:still-polls", "https://asurascans.com/series/still-polls", 0)
|
|
p := &Poller{
|
|
Store: s, Fetch: tls, BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
}
|
|
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 2 {
|
|
t.Fatalf("browser fetches after first pass = %d, want 2 (refused twice)", got)
|
|
}
|
|
if got := tls.callCount(); got != 1 {
|
|
t.Fatalf("asura fetches after first pass = %d, want 1 (its Lane is independent)", got)
|
|
}
|
|
stamped := 0
|
|
for i := 0; i < n; i++ {
|
|
if readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)) == now.UnixMilli() {
|
|
stamped++
|
|
}
|
|
}
|
|
if stamped != 2 {
|
|
t.Fatalf("stamped series = %d, want 2; the remaining two stay due", stamped)
|
|
}
|
|
|
|
// Inside the backoff window nothing is attempted.
|
|
p.Now = func() time.Time { return now.Add(14 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 2 {
|
|
t.Fatalf("browser fetches inside backoff = %d, want still 2", got)
|
|
}
|
|
|
|
// Past the backoff the Lane resumes and the two untried Series are polled.
|
|
p.Now = func() time.Time { return now.Add(16 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 4 {
|
|
t.Fatalf("browser fetches after backoff = %d, want 4", got)
|
|
}
|
|
for i := 0; i < n; i++ {
|
|
if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 {
|
|
t.Fatalf("kagane:s%d still untried after backoff", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Below five due Series with none waiting long, a browser Lane leaves Chrome
|
|
// asleep; five due, or one waiting browserWakeAge, wakes it (ADR-0005).
|
|
func TestBrowserLaneWakeThresholds(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
// Freshly due: checked two minutes before the rest elapses, so the wait
|
|
// is far below browserWakeAge.
|
|
seed := func(i int) {
|
|
key := fmt.Sprintf("kagane:w%d", i)
|
|
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli())
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
seed(i)
|
|
}
|
|
|
|
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
}
|
|
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser fetches with 3 freshly-due series = %d, want 0 (Chrome stays asleep)", got)
|
|
}
|
|
// The skipped pass still records its row, carrying the due count it never
|
|
// read; the Lanes page (issue #145) reads that row — see
|
|
// TestRunLanePassRecordsEveryExit/"asleep".
|
|
// 5 due crosses the count threshold.
|
|
for i := 3; i < 5; i++ {
|
|
seed(i)
|
|
}
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 5 {
|
|
t.Fatalf("browser fetches with 5 due series = %d, want 5", got)
|
|
}
|
|
// A single long-neglected series wakes the browser by age alone.
|
|
seedForCheck(t, s, "kagane:ancient", "https://kagane.to/series/ancient", 0)
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 6 {
|
|
t.Fatalf("browser fetches with one ancient series = %d, want 6", got)
|
|
}
|
|
}
|
|
|
|
// When one browser Lane loses the sidecar, the round's remaining browser
|
|
// Lanes are skipped: every fetch would fail anyway, and their Series must not
|
|
// burn their stamps on a dead Chrome (issue #100).
|
|
func TestRunOnceUnreachableBrowserStopsBrowserLanes(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0)
|
|
seedForCheck(t, s, "kagane:k", "https://kagane.to/series/k", 0)
|
|
seedForCheck(t, s, "novelfull:n", "https://novelfull.com/n.html", 0)
|
|
|
|
interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart"))
|
|
browser := &fakeFetcher{status: 200, err: interrupted}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
// Sorted lane order: comix, kagane, novelfull. Only comix attempted.
|
|
if got := browser.callCount(); got != 1 {
|
|
t.Fatalf("browser fetches = %d, want 1 (only the first browser lane)", got)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() {
|
|
t.Fatalf("comix stamp = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
for _, key := range []string{"kagane:k", "novelfull:n"} {
|
|
if got := readLatestCheckedAt(t, s, key); got != 0 {
|
|
t.Fatalf("%s stamp = %d, want 0 (untried)", key, got)
|
|
}
|
|
}
|
|
|
|
// The shared flag decays after RefuseBackoff: the next round probes
|
|
// again. comix's Series is resting (stamped last round), so the probe
|
|
// falls to kagane — the only Lane with something due — and its fresh
|
|
// loss re-gates the Lanes behind it.
|
|
p.Now = func() time.Time { return now.Add(16 * time.Minute) }
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 2 {
|
|
t.Fatalf("browser fetches after backoff decay = %d, want 2 (kagane probes again)", got)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "comix:c"); got != now.UnixMilli() {
|
|
t.Fatalf("comix stamp after decay = %d, want %d (resting, untouched)", got, now.UnixMilli())
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "kagane:k"); got != now.Add(16*time.Minute).UnixMilli() {
|
|
t.Fatalf("kagane stamp after decay = %d, want %d (the probe)", got, now.Add(16*time.Minute).UnixMilli())
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "novelfull:n"); got != 0 {
|
|
t.Fatalf("novelfull stamp after decay = %d, want 0 (gated by kagane's fresh loss)", got)
|
|
}
|
|
}
|
|
|
|
// A browser Lane that cannot keep up logs the backlog every pass, so the
|
|
// decision to give browser Sites more pages is measured, not guessed.
|
|
func TestRunOnceLogsBrowserLaneBehind(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
// Checked three rests ago: two rests' worth of wait past the due moment.
|
|
seedForCheck(t, s, "kagane:old", "https://kagane.to/series/old", now.Add(-3*time.Hour).UnixMilli())
|
|
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{status: 200},
|
|
BrowserFetch: &fakeFetcher{body: kaganeAPIFixture, status: 200},
|
|
Now: func() time.Time { return now },
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
if got := logs.String(); !strings.Contains(got, "latest poll kagane: browser lane behind by 1h0m0s") {
|
|
t.Fatalf("behind log = %q, want it to name kagane and the backlog", got)
|
|
}
|
|
}
|
|
|
|
// gatedCoverFetcher blocks every Fetch on a gate, so a test can hold a cover
|
|
// heal in flight and prove a Lane does not wait for it.
|
|
type gatedCoverFetcher struct {
|
|
inner *fakeBytesCoverFetcher
|
|
gate chan struct{}
|
|
started chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func (g *gatedCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
|
|
g.once.Do(func() { g.started <- struct{}{} })
|
|
<-g.gate
|
|
return g.inner.Fetch(ctx, sourceURL)
|
|
}
|
|
|
|
// Cover heals run in the background: a heal stuck on a slow CDN must not
|
|
// delay the Lane's next Series-page Poll, or a large import with many blanks
|
|
// would make every Latest Chapter go stale (issue #100).
|
|
func TestRunOnceCoverFetchDoesNotDelayNextPoll(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
// Two Series whose legacy cover sources still need healing.
|
|
for i := 0; i < 2; i++ {
|
|
key := fmt.Sprintf("asura:cover-%d", i)
|
|
seriesID := fmt.Sprintf("cover-%d", i)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID,
|
|
SeriesURL: "https://asurascans.com/comics/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
seedCoverSource(t, dbURL, "asura", seriesID, fmt.Sprintf("https://cdn.example/covers/%d.jpg", i))
|
|
}
|
|
|
|
pages := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
gated := &gatedCoverFetcher{
|
|
inner: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
|
gate: make(chan struct{}),
|
|
started: make(chan struct{}, 1),
|
|
}
|
|
p := &Poller{
|
|
Store: s, Fetch: pages, CoverBytesFetch: gated,
|
|
Now: func() time.Time { return time.UnixMilli(5_000_000) },
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
p.runOnce(context.Background())
|
|
close(done)
|
|
}()
|
|
<-gated.started // the first cover heal is now stuck on its CDN
|
|
select {
|
|
case <-done:
|
|
// The Lane finished its page fetches without waiting for the cover.
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("runOnce blocked on an in-flight cover fetch")
|
|
}
|
|
close(gated.gate)
|
|
p.waitCovers()
|
|
|
|
if got := pages.callCount(); got != 2 {
|
|
t.Fatalf("series page fetches = %d, want 2", got)
|
|
}
|
|
if got := gated.inner.callCount(); got != 2 {
|
|
t.Fatalf("cover fetches = %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
// At 3601 eligible Series the effective gap falls below the one-second floor;
|
|
// the clamp warning must name the Site so the operator knows which Lane is
|
|
// outrunning its plan.
|
|
func TestRunOnceClampWarningNamesTheSite(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(`INSERT INTO series (site, series_id, series_url, latest_checked_at)
|
|
SELECT 'asura', 'bulk-' || g, 'https://asurascans.com/comics/bulk-' || g, 0
|
|
FROM generate_series(1, 3601) AS g`); err != nil {
|
|
t.Fatalf("bulk seed series: %v", err)
|
|
}
|
|
if _, err := db.Exec(`INSERT INTO bookmarks (reader_id, site, series_id, updated_at)
|
|
SELECT (SELECT id FROM readers ORDER BY id LIMIT 1), 'asura', 'bulk-' || g, 1000
|
|
FROM generate_series(1, 3601) AS g`); err != nil {
|
|
t.Fatalf("bulk seed bookmarks: %v", err)
|
|
}
|
|
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
newTestPoller(t, s, &fakeFetcher{body: "<html></html>", status: 200}, time.UnixMilli(5_000_000)).
|
|
runOnce(context.Background())
|
|
|
|
if got := logs.String(); !strings.Contains(got, "latest poll asura: gap clamped to 1s floor (eligible series=3601)") {
|
|
t.Fatalf("clamp warning = %q, want it to name asura and 3601", got)
|
|
}
|
|
}
|
|
|
|
// The Lanes page (issue #145) reads the durable pass log, so the poller's
|
|
// only duty to it is that every return path writes its row; the snapshot it
|
|
// used to mirror into memory is gone. A refusing pass still records why it
|
|
// declined, and the latest row per Site is what the page renders — nothing
|
|
// else is left to assert against here, because the page's seam moved into the
|
|
// web layer (web_test.go, seeded-row tests).
|
|
func TestPassLogIsTheLanesPagesOnlyWindow(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now)
|
|
|
|
// No pass has run: the log is empty, which the page renders as "none
|
|
// observed" rather than confident zeroes.
|
|
if _, ok, err := s.LatestLanePass("asura"); err != nil || ok {
|
|
t.Fatalf("LatestLanePass before any pass = ok %v err %v, want no row", ok, err)
|
|
}
|
|
|
|
seedForCheck(t, s, "asura:chronicles", "https://asurascans.com/series/chronicles", 0)
|
|
// Two refusals put kagane's Lane into backoff; asura sits on its own Lane.
|
|
browser := &fakeFetcher{status: 403}
|
|
p.BrowserFetch = browser
|
|
for i := range 2 {
|
|
key := fmt.Sprintf("kagane:s%d", i)
|
|
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0)
|
|
}
|
|
p.runOnce(context.Background())
|
|
|
|
// Both lanes wrote their rows: asura the pass it read; kagane the
|
|
// mid-loop double-refusal exit, which records an empty skip by design
|
|
// with its two refused counts — the durable row is the whole record, and
|
|
// the page reads it rather than a snapshot.
|
|
asura := latestPassFor(t, s, "asura")
|
|
if asura.Due != 1 || asura.Checked != 1 || asura.GapMS == 0 {
|
|
t.Fatalf("asura row = %+v, want due 1 checked 1 with the Lane's pace", asura)
|
|
}
|
|
kagane := latestPassFor(t, s, "kagane")
|
|
if kagane.Skip != "" || kagane.Refused != 2 || kagane.Due != 2 || kagane.GapMS == 0 {
|
|
t.Fatalf("kagane row = %+v, want an empty-skip double-refusal pass with 2 refused", kagane)
|
|
}
|
|
|
|
// With the refusal now durable, the next pass declines ahead of the loop:
|
|
// it records the refusing skip and carries the previous pass's figures
|
|
// forward rather than restating zeroes it never gathered (the carry logic
|
|
// itself is driven in TestRunLanePassCarryForwardOnlyWhenGapZero).
|
|
before := kagane
|
|
p.Now = func() time.Time { return now.Add(time.Minute) }
|
|
p.runOnce(context.Background())
|
|
again := latestPassFor(t, s, "kagane")
|
|
if again.Skip != SkipRefusing {
|
|
t.Fatalf("kagane refusing pass skip = %q, want %q", again.Skip, SkipRefusing)
|
|
}
|
|
if again.Due != before.Due || again.GapMS != before.GapMS || again.Checked != before.Checked {
|
|
t.Fatalf("kagane after a skipped pass = due %d gap %d, want the previous pass's %d / %d",
|
|
again.Due, again.GapMS, before.Due, before.GapMS)
|
|
}
|
|
}
|
|
|
|
// latestPassFor reads a Site's newest durable pass row, failing rather than
|
|
// returning a zero LanePass a caller would assert against by accident.
|
|
func latestPassFor(t *testing.T, s *store.Store, site string) store.LanePass {
|
|
t.Helper()
|
|
pass, ok, err := s.LatestLanePass(site)
|
|
if err != nil || !ok {
|
|
t.Fatalf("LatestLanePass(%s): ok=%v err=%v", site, ok, err)
|
|
}
|
|
return pass
|
|
}
|
|
|
|
// countPassRows counts a Site's durable pass rows, for asserting that a pass
|
|
// records exactly one.
|
|
func countPassRows(t *testing.T, dbURL, site string) int {
|
|
t.Helper()
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
defer db.Close()
|
|
var n int
|
|
if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, site).Scan(&n); err != nil {
|
|
t.Fatalf("count passes for %s: %v", site, err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// One durable pass row per exit, with the skip value naming the exit. The
|
|
// mid-loop browser-unreachable return also writes one row — but with an empty
|
|
// skip, so the row is stall-shaped (due > 0, checked 0, skip ”), matching the
|
|
// deliberate absence of a tenth skip value.
|
|
func TestRunLanePassRecordsEveryExit(t *testing.T) {
|
|
now := time.UnixMilli(5_000_000)
|
|
|
|
t.Run("paused", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0)
|
|
if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("PauseLane: %v", err)
|
|
}
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
p := newTestPoller(t, s, f, now)
|
|
if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute {
|
|
t.Fatalf("paused pace = %s, want 30m (sleep until the expiry)", pace)
|
|
}
|
|
if f.callCount() != 0 {
|
|
t.Fatalf("fetches while paused = %d, want 0", f.callCount())
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "asura:x"); got != 0 {
|
|
t.Fatalf("stamp while paused = %d, want 0 (Series stay due and unstamped)", got)
|
|
}
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused {
|
|
t.Fatalf("skip = %q, want %q", pass.Skip, SkipPaused)
|
|
}
|
|
if got := countPassRows(t, dbURL, "asura"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("refusing", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0)
|
|
if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("SetLaneRefusal: %v", err)
|
|
}
|
|
browser := &fakeFetcher{status: 200}
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = browser
|
|
if pace := p.runLanePass(context.Background(), "kagane", false); pace != 10*time.Minute {
|
|
t.Fatalf("refusing pace = %s, want 10m", pace)
|
|
}
|
|
if browser.callCount() != 0 {
|
|
t.Fatalf("fetches while refusing = %d, want 0", browser.callCount())
|
|
}
|
|
if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipRefusing {
|
|
t.Fatalf("skip = %q, want %q", pass.Skip, SkipRefusing)
|
|
}
|
|
if got := countPassRows(t, dbURL, "kagane"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("sidecar-down", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0)
|
|
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = browser
|
|
p.setBrowserDown(now) // a sibling Lane lost Chrome within the backoff window
|
|
if pace := p.runLanePass(context.Background(), "kagane", false); pace != RefuseBackoff {
|
|
t.Fatalf("sidecar-down pace = %s, want %s", pace, RefuseBackoff)
|
|
}
|
|
if browser.callCount() != 0 {
|
|
t.Fatalf("fetches with the sidecar down = %d, want 0", browser.callCount())
|
|
}
|
|
if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipSidecarDown {
|
|
t.Fatalf("skip = %q, want %q", pass.Skip, SkipSidecarDown)
|
|
}
|
|
if got := countPassRows(t, dbURL, "kagane"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("no-fetcher", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0)
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap {
|
|
t.Fatalf("no-fetcher pace = %s, want %s", pace, defaultGap)
|
|
}
|
|
pass := latestPassFor(t, s, "comix")
|
|
if pass.Skip != SkipNoFetcher || pass.GapMS != defaultGap.Milliseconds() {
|
|
t.Fatalf("no-fetcher pass = %+v, want skip %q with its own gap %s", pass, SkipNoFetcher, defaultGap)
|
|
}
|
|
if got := countPassRows(t, dbURL, "comix"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("due-query", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0)
|
|
// The due query is the pass's first store read after the gates; making
|
|
// it fail without touching poll_passes takes its tables away.
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
if _, err := db.Exec(`DROP TABLE bookmarks`); err != nil {
|
|
t.Fatalf("drop bookmarks: %v", err)
|
|
}
|
|
db.Close()
|
|
p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now)
|
|
if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap {
|
|
t.Fatalf("due-query pace = %s, want %s", pace, defaultGap)
|
|
}
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipDueQuery {
|
|
t.Fatalf("skip = %q, want %q", pass.Skip, SkipDueQuery)
|
|
}
|
|
if got := countPassRows(t, dbURL, "asura"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("asleep", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
for i := 0; i < 3; i++ {
|
|
key := fmt.Sprintf("kagane:w%d", i)
|
|
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli())
|
|
}
|
|
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = browser
|
|
if pace := p.runLanePass(context.Background(), "kagane", false); pace != defaultGap {
|
|
t.Fatalf("asleep pace = %s, want %s", pace, defaultGap)
|
|
}
|
|
if browser.callCount() != 0 {
|
|
t.Fatalf("fetches while Chrome is asleep = %d, want 0", browser.callCount())
|
|
}
|
|
pass := latestPassFor(t, s, "kagane")
|
|
if pass.Skip != SkipAsleep || pass.Due != 3 || pass.GapMS != defaultGap.Milliseconds() {
|
|
t.Fatalf("asleep pass = %+v, want skip %q with 3 due and the default gap", pass, SkipAsleep)
|
|
}
|
|
if got := countPassRows(t, dbURL, "kagane"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("eligible-count", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
// The eligible query shares the due query's tables, so no real store
|
|
// failure reaches it after a successful due read; the seam is how the
|
|
// path is driven at all (see Poller.eligibleCount).
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.eligibleCount = func(string) (int, error) { return 0, errors.New("count failed") }
|
|
if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap {
|
|
t.Fatalf("eligible-count pace = %s, want %s", pace, defaultGap)
|
|
}
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipEligibleCount {
|
|
t.Fatalf("skip = %q, want %q", pass.Skip, SkipEligibleCount)
|
|
}
|
|
if got := countPassRows(t, dbURL, "asura"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("nothing-eligible", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
if pace := p.runLanePass(context.Background(), "asura", false); pace != sites["asura"].Rest {
|
|
t.Fatalf("nothing-eligible pace = %s, want a full rest %s", pace, sites["asura"].Rest)
|
|
}
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipNothingEligible {
|
|
t.Fatalf("skip = %q, want %q", pass.Skip, SkipNothingEligible)
|
|
}
|
|
if got := countPassRows(t, dbURL, "asura"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("reached the loop", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)
|
|
p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now)
|
|
if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap {
|
|
t.Fatalf("loop pace = %s, want %s", pace, defaultGap)
|
|
}
|
|
pass := latestPassFor(t, s, "asura")
|
|
if pass.Skip != "" || pass.Due != 1 || pass.Checked != 1 {
|
|
t.Fatalf("loop pass = %+v, want an empty skip with 1 due and 1 checked", pass)
|
|
}
|
|
if got := countPassRows(t, dbURL, "asura"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
|
|
t.Run("browser-unreachable mid-loop", func(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0)
|
|
interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart"))
|
|
browser := &fakeFetcher{status: 200, err: interrupted}
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = browser
|
|
if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap {
|
|
t.Fatalf("mid-loop pace = %s, want %s", pace, defaultGap)
|
|
}
|
|
// One row was still written, and it is stall-shaped by construction:
|
|
// empty skip with due > 0 and checked 0 because the return precedes
|
|
// the checked counter. The assertion below pins that shape precisely.
|
|
pass := latestPassFor(t, s, "comix")
|
|
if pass.Skip != "" || pass.Unreachable != 1 || pass.Checked != 0 || pass.Due != 1 {
|
|
t.Fatalf("mid-loop pass = %+v, want empty skip, unreachable 1, checked 0, due 1", pass)
|
|
}
|
|
if got := countPassRows(t, dbURL, "comix"); got != 1 {
|
|
t.Fatalf("pass rows = %d, want exactly 1", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Carry-forward moves verbatim from the in-memory snapshot (issue #141): a
|
|
// pass that never computed its own gap carries the previous pass's due, gap,
|
|
// clamped and checked forward; a pass with a gap of its own records its own
|
|
// figures, zeroes included, beside the skip reason that explains them.
|
|
func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) {
|
|
now := time.UnixMilli(5_000_000)
|
|
|
|
t.Run("no gap of its own carries the previous pass's figures", func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0)
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p.runLanePass(context.Background(), "kagane", false)
|
|
first := latestPassFor(t, s, "kagane")
|
|
if first.Due != 1 || first.Checked != 1 || first.GapMS == 0 {
|
|
t.Fatalf("first pass = %+v, want a measured pass", first)
|
|
}
|
|
|
|
if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("SetLaneRefusal: %v", err)
|
|
}
|
|
// The pass row is keyed (site, ran_at), so the second pass needs its
|
|
// own timestamp: a minute later is still inside the refusal.
|
|
p.Now = func() time.Time { return now.Add(time.Minute) }
|
|
p.runLanePass(context.Background(), "kagane", false)
|
|
second := latestPassFor(t, s, "kagane")
|
|
if second.Skip != SkipRefusing {
|
|
t.Fatalf("second pass skip = %q, want %q", second.Skip, SkipRefusing)
|
|
}
|
|
if second.Due != first.Due || second.Checked != first.Checked ||
|
|
second.GapMS != first.GapMS || second.Clamped != first.Clamped {
|
|
t.Fatalf("carried pass = %+v, want the first pass's figures %+v", second, first)
|
|
}
|
|
})
|
|
|
|
t.Run("a gap of its own records its own figures", func(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0)
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200}
|
|
p.runLanePass(context.Background(), "comix", false)
|
|
first := latestPassFor(t, s, "comix")
|
|
if first.Due != 1 || first.GapMS == 0 {
|
|
t.Fatalf("first pass = %+v, want a measured pass", first)
|
|
}
|
|
|
|
// The no-fetcher exit sets its own gap, so no carry-forward: the due
|
|
// count it never gathered records as zero beside its skip reason. A
|
|
// minute later gives the second pass its own (site, ran_at) key.
|
|
p.BrowserFetch = nil
|
|
p.Now = func() time.Time { return now.Add(time.Minute) }
|
|
p.runLanePass(context.Background(), "comix", false)
|
|
second := latestPassFor(t, s, "comix")
|
|
if second.Skip != SkipNoFetcher {
|
|
t.Fatalf("second pass skip = %q, want %q", second.Skip, SkipNoFetcher)
|
|
}
|
|
if second.GapMS != defaultGap.Milliseconds() {
|
|
t.Fatalf("second pass gap = %d, want its own %d (not carried)", second.GapMS, defaultGap.Milliseconds())
|
|
}
|
|
if second.Due != 0 || second.Checked != 0 {
|
|
t.Fatalf("second pass = %+v, want due 0 checked 0 (its own, not the previous pass's)", second)
|
|
}
|
|
})
|
|
}
|
|
|
|
// The pass row's five outcome counts are the classification the Series read
|
|
// already makes — never a second taxonomy (issue #141). Success is derived,
|
|
// never stored: checked minus the four named counts, unreachable excluded
|
|
// because its exit returns before the checked counter increments.
|
|
func TestRunLanePassCountsOutcomes(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
const (
|
|
successKey = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
successURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
)
|
|
seeds := map[string]string{
|
|
"asura:refused": "https://asurascans.com/series/refused",
|
|
"asura:no-chapter": "https://asurascans.com/comics/no-chapter",
|
|
"asura:unfetchable": "https://evil.example/x",
|
|
"asura:transport": "https://asurascans.com/series/transport",
|
|
successKey: successURL,
|
|
}
|
|
for key, url := range seeds {
|
|
seedForCheck(t, s, key, url, 0)
|
|
}
|
|
|
|
f := &fakeFetcher{perURL: map[string]fakeResponse{
|
|
seeds["asura:refused"]: {status: 403},
|
|
seeds["asura:no-chapter"]: {body: "<html></html>", status: 200},
|
|
seeds["asura:transport"]: {err: errors.New("dial tcp: refused")},
|
|
successURL: {body: asuraSeriesFixture, status: 200},
|
|
}}
|
|
newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false)
|
|
|
|
pass := latestPassFor(t, s, "asura")
|
|
if pass.Skip != "" || pass.Due != 5 || pass.Checked != 5 {
|
|
t.Fatalf("pass = %+v, want a full pass over 5 due Series", pass)
|
|
}
|
|
if pass.Refused != 1 || pass.Unreachable != 0 || pass.NoChapter != 1 ||
|
|
pass.Unfetchable != 1 || pass.Errors != 1 {
|
|
t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1",
|
|
pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors)
|
|
}
|
|
if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 {
|
|
t.Fatalf("derived success = %d, want 1", success)
|
|
}
|
|
// The one genuine read went through: the success Series carries the
|
|
// fixture's newest chapter, and none of the four failures do.
|
|
b, ok, err := s.Get(s.OwnerID(), successKey)
|
|
if err != nil || !ok {
|
|
t.Fatalf("Get: %v ok=%v", err, ok)
|
|
}
|
|
if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
|
|
t.Fatalf("success Series latest = %v, want 181 (the fixture's newest)", b.LatestChapterNum)
|
|
}
|
|
for _, key := range []string{"asura:refused", "asura:no-chapter", "asura:unfetchable", "asura:transport"} {
|
|
b, _, err := s.Get(s.OwnerID(), key)
|
|
if err != nil {
|
|
t.Fatalf("Get %s: %v", key, err)
|
|
}
|
|
if b.LatestChapterNum != nil {
|
|
t.Fatalf("%s latest = %v, want nil (no chapter survived a failed read)", key, *b.LatestChapterNum)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A refusal is the Site's mood and outlives our process: the durable stamp a
|
|
// pass writes is honoured by a freshly constructed poller, which must not
|
|
// re-probe the Site inside its backoff (issue #141).
|
|
func TestDurableRefusalSurvivesFreshPoller(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
// Four due Series: the Lane refuses twice, stamps those two, and leaves
|
|
// the remaining two untried and due — the queue the fresh poller must
|
|
// find intact once the durable backoff lifts.
|
|
for i := 0; i < 4; i++ {
|
|
key := fmt.Sprintf("kagane:s%d", i)
|
|
seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0)
|
|
}
|
|
|
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
|
p.BrowserFetch = &fakeFetcher{status: 403}
|
|
p.runLanePass(context.Background(), "kagane", false)
|
|
if got := p.BrowserFetch.(*fakeFetcher).callCount(); got != 2 {
|
|
t.Fatalf("fetches on the refusing pass = %d, want 2 (refused twice)", got)
|
|
}
|
|
|
|
// A restart: a brand-new poller, no in-memory refusal, same store. The
|
|
// durable stamp gates the pass.
|
|
fresh := newTestPoller(t, s, &fakeFetcher{status: 200}, now.Add(14*time.Minute))
|
|
fresh.BrowserFetch = &fakeFetcher{status: 403}
|
|
fresh.runLanePass(context.Background(), "kagane", false)
|
|
if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 0 {
|
|
t.Fatalf("fetches by a fresh poller inside the backoff = %d, want 0", got)
|
|
}
|
|
if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipRefusing {
|
|
t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, SkipRefusing)
|
|
}
|
|
|
|
// Past the backoff the fresh poller probes again — the two Series the
|
|
// original pass never reached, still due with their stamps untouched.
|
|
fresh.Now = func() time.Time { return now.Add(16 * time.Minute) }
|
|
fresh.runLanePass(context.Background(), "kagane", false)
|
|
if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 2 {
|
|
t.Fatalf("fetches after the backoff = %d, want 2", got)
|
|
}
|
|
for i := 2; i < 4; i++ {
|
|
if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 {
|
|
t.Fatalf("kagane:s%d still untried after the backoff", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The pause is read ahead of the refusal check: a Lane that is both paused
|
|
// and inside a refusal backoff records the paused skip value, not the
|
|
// refusing one. The pause is the owner's order and outranks the Site's mood
|
|
// (issue #147).
|
|
func TestPauseGatePrecedesRefusalGate(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0)
|
|
if err := s.SetLaneRefusal("asura", now.Add(10*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("SetLaneRefusal: %v", err)
|
|
}
|
|
if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("PauseLane: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
p := newTestPoller(t, s, f, now)
|
|
if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute {
|
|
t.Fatalf("paused-while-refusing pace = %s, want 30m (the pause's expiry)", pace)
|
|
}
|
|
if f.callCount() != 0 {
|
|
t.Fatalf("fetches while paused and refusing = %d, want 0", f.callCount())
|
|
}
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused {
|
|
t.Fatalf("skip = %q, want %q (the pause outranks the refusal)", pass.Skip, SkipPaused)
|
|
}
|
|
}
|
|
|
|
// A pause is a fact about the Site, not about the process: a freshly
|
|
// constructed poller against a store holding a pause row stays paused until
|
|
// the expiry, then runs the Lane normally. The restart criterion is the
|
|
// whole point of writing a row instead of commanding a poller (issue #147).
|
|
func TestDurablePauseSurvivesFreshPoller(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0)
|
|
if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("PauseLane: %v", err)
|
|
}
|
|
|
|
// A restart: a brand-new poller, no in-memory state, same store.
|
|
fresh := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now)
|
|
if pace := fresh.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute {
|
|
t.Fatalf("fresh poller's paused pace = %s, want 30m", pace)
|
|
}
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused {
|
|
t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, SkipPaused)
|
|
}
|
|
|
|
// Past the expiry the same fresh poller runs the Lane normally.
|
|
fresh.Now = func() time.Time { return now.Add(31 * time.Minute) }
|
|
fresh.runLanePass(context.Background(), "asura", false)
|
|
if pass := latestPassFor(t, s, "asura"); pass.Skip != "" {
|
|
t.Fatalf("pass after the expiry skip = %q, want the loop reached", pass.Skip)
|
|
}
|
|
if got := readLatestCheckedAt(t, s, "asura:x"); got == 0 {
|
|
t.Fatal("the Series was not checked after the pause lifted")
|
|
}
|
|
}
|
|
|
|
// ResumeLane zeroes the pause and the Lane's next pass finds its full queue
|
|
// waiting: a pause delays work rather than discarding it, so the due Series
|
|
// sit unstamped while paused and are all fetched once the pause lifts
|
|
// (issue #147).
|
|
func TestResumeLaneRestoresTheQueue(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
for i := 0; i < 3; i++ {
|
|
seedForCheck(t, s, fmt.Sprintf("asura:s%d", i), "https://asurascans.com/series/x", 0)
|
|
}
|
|
if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil {
|
|
t.Fatalf("PauseLane: %v", err)
|
|
}
|
|
|
|
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
|
p := newTestPoller(t, s, f, now)
|
|
p.runLanePass(context.Background(), "asura", false)
|
|
if f.callCount() != 0 {
|
|
t.Fatalf("fetches while paused = %d, want 0", f.callCount())
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
if got := readLatestCheckedAt(t, s, fmt.Sprintf("asura:s%d", i)); got != 0 {
|
|
t.Fatalf("asura:s%d stamp while paused = %d, want 0 (due and unstamped)", i, got)
|
|
}
|
|
}
|
|
|
|
if err := s.ResumeLane("asura"); err != nil {
|
|
t.Fatalf("ResumeLane: %v", err)
|
|
}
|
|
before := f.callCount()
|
|
p.runLanePass(context.Background(), "asura", false)
|
|
if got := f.callCount() - before; got != 3 {
|
|
t.Fatalf("fetches after resume = %d, want 3 (the full due queue)", got)
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
if got := readLatestCheckedAt(t, s, fmt.Sprintf("asura:s%d", i)); got == 0 {
|
|
t.Fatalf("asura:s%d still untried after resume", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// RecordLanePass prunes in the same call that inserts, so the retention
|
|
// cutoff the recorder passes is observable in what survives: a row just inside
|
|
// 14 days behind the poller's clock is kept, one just outside is pruned
|
|
// (issue #139, #141).
|
|
func TestPassRetentionCutoffIsFourteenDays(t *testing.T) {
|
|
s, dbURL := newTestStore(t)
|
|
// A real-world clock: the seeded rows sit 14 days back, so they must be
|
|
// positive timestamps or the seed's own prune (ran_at < 0) removes them.
|
|
now := time.UnixMilli(1_800_000_000_000)
|
|
kept := now.Add(-14*24*time.Hour + time.Minute).UnixMilli()
|
|
pruned := now.Add(-14*24*time.Hour - time.Minute).UnixMilli()
|
|
for _, ranAt := range []int64{kept, pruned} {
|
|
if err := s.RecordLanePass(store.LanePass{Site: "asura", RanAt: ranAt}, 0); err != nil {
|
|
t.Fatalf("seed pass at %d: %v", ranAt, err)
|
|
}
|
|
}
|
|
|
|
seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0)
|
|
newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now).
|
|
runLanePass(context.Background(), "asura", false)
|
|
|
|
db, err := sql.Open("pgx", dbURL)
|
|
if err != nil {
|
|
t.Fatalf("open %s: %v", dbURL, err)
|
|
}
|
|
defer db.Close()
|
|
var n int
|
|
if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1 AND ran_at = $2`, "asura", pruned).Scan(&n); err != nil {
|
|
t.Fatalf("count pruned row: %v", err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("row at %d survived, want it pruned (older than 14 days)", pruned)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&n); err != nil {
|
|
t.Fatalf("count passes: %v", err)
|
|
}
|
|
if n != 2 {
|
|
t.Fatalf("passes = %d, want 2 (this pass plus the kept row)", n)
|
|
}
|
|
}
|
|
|
|
// stampRecordingFetcher is a fakeFetcher that records the Series' check stamp
|
|
// at call time — the seam that proves the check stamp is written before the
|
|
// fetch (issue #146). If the order were swapped, the recorded stamp would be
|
|
// the pre-pass value and the ordering assertion would fail.
|
|
type stampRecordingFetcher struct {
|
|
fakeFetcher
|
|
store *store.Store
|
|
site string
|
|
seriesID string
|
|
stampAtCall int64
|
|
}
|
|
|
|
func (f *stampRecordingFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
|
ts, err := f.store.LatestCheckedAt(f.site, f.seriesID)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
f.stampAtCall = ts
|
|
return f.fakeFetcher.Get(ctx, url)
|
|
}
|
|
|
|
// The check stamp is written before the fetch is attempted: the fake fetcher
|
|
// records the stamp it sees at call time, and it must already be the pass's
|
|
// own stamp. The order is load-bearing — a forced request is pending while
|
|
// force_poll_at > latest_checked_at, so stamping after the fetch would make a
|
|
// failed forced request sticky — and the assertion fails if it is swapped.
|
|
func TestCheckStampIsWrittenBeforeFetch(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
)
|
|
seedForCheck(t, s, key, seriesURL, 0)
|
|
now := time.UnixMilli(7_000_000)
|
|
rec := &stampRecordingFetcher{
|
|
fakeFetcher: fakeFetcher{body: asuraSeriesFixture, status: 200},
|
|
store: s, site: "asura", seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
|
}
|
|
newTestPoller(t, s, rec, now).runOnce(context.Background())
|
|
if rec.stampAtCall != now.UnixMilli() {
|
|
t.Fatalf("check stamp at fetch time = %d, want %d (the stamp must be written before the fetch)", rec.stampAtCall, now.UnixMilli())
|
|
}
|
|
}
|
|
|
|
// A forced Series whose attempt fails is no longer pending: the check stamp
|
|
// is written before the fetch, so the first attempt ends the pending state
|
|
// whatever it returns. A naive implementation (stamp only on success, or
|
|
// after the fetch) leaves the request sticky and the row due next pass.
|
|
func TestForcedSeriesSelfClearsOnFailedAttempt(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(7_000_000)
|
|
const url = "https://asurascans.com/comics/x"
|
|
// Freshly checked, so only the force flag makes it due.
|
|
seedForCheck(t, s, "asura:x", url, now.Add(-30*time.Minute).UnixMilli())
|
|
if err := s.ForceSeriesPoll("asura", "x", now.UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
|
|
due, err := s.DueForLatestCheck("asura", now.Add(-time.Hour).UnixMilli(), -1)
|
|
if err != nil {
|
|
t.Fatalf("DueForLatestCheck: %v", err)
|
|
}
|
|
if len(due) != 1 || due[0].Key() != "asura:x" || !due[0].Forced {
|
|
t.Fatalf("forced row not due and flagged before the attempt: %+v", due)
|
|
}
|
|
|
|
// The attempt fails, but the attempt still happened: the row is stamped
|
|
// and no longer pending.
|
|
f := &fakeFetcher{err: errors.New("dial tcp: refused")}
|
|
newTestPoller(t, s, f, now).runOnce(context.Background())
|
|
if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
|
|
}
|
|
due, err = s.DueForLatestCheck("asura", now.Add(-time.Hour).UnixMilli(), -1)
|
|
if err != nil {
|
|
t.Fatalf("DueForLatestCheck: %v", err)
|
|
}
|
|
if len(due) != 0 {
|
|
t.Fatalf("failed attempt left the forced row due: %v", due)
|
|
}
|
|
}
|
|
|
|
// The refusal backoff is a Lane gate, not a Series gate: a forced Series does
|
|
// not override a Site that is actively refusing, because hand-forcing a
|
|
// request into a refusal only makes it worse (issue #146).
|
|
func TestForcedSeriesDoesNotOverrideRefusalBackoff(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
seedForCheck(t, s, "kagane:w", "https://kagane.to/series/w", 0)
|
|
if err := s.ForceSeriesPoll("kagane", "w", now.UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
if err := s.SetLaneRefusal("kagane", now.Add(RefuseBackoff).UnixMilli()); err != nil {
|
|
t.Fatalf("SetLaneRefusal: %v", err)
|
|
}
|
|
|
|
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
}
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser fetches through a refusal backoff = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// A forced Series wakes a sleeping browser Lane: the wake thresholds exist to
|
|
// stop the machine waking itself for one unattended check, and a human asking
|
|
// is not that (issue #146). Below both thresholds the Lane still sleeps when
|
|
// nothing is forced.
|
|
func TestForcedSeriesWakesSleepingBrowser(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
now := time.UnixMilli(5_000_000)
|
|
// Freshly due: checked two minutes before the rest elapses, so the wait
|
|
// is far below browserWakeAge and the count is under browserWakeCount.
|
|
seedForCheck(t, s, "kagane:w1", "https://kagane.to/series/w1", now.Add(-62*time.Minute).UnixMilli())
|
|
seedForCheck(t, s, "kagane:w2", "https://kagane.to/series/w2", now.Add(-62*time.Minute).UnixMilli())
|
|
|
|
browser := &fakeFetcher{body: kaganeAPIFixture, status: 200}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser,
|
|
Now: func() time.Time { return now },
|
|
}
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 0 {
|
|
t.Fatalf("browser fetches without a forced series = %d, want 0 (Chrome stays asleep)", got)
|
|
}
|
|
|
|
if err := s.ForceSeriesPoll("kagane", "w1", now.UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
p.runOnce(context.Background())
|
|
if got := browser.callCount(); got != 2 {
|
|
t.Fatalf("browser fetches with a forced series = %d, want 2 (the lane wakes)", got)
|
|
}
|
|
}
|
|
|
|
// A forced pass accepts the page as it now stands, so it writes the Cover
|
|
// through the replace path; an ordinary pass still only fills a blank one
|
|
// (issue #135).
|
|
func TestRunOnceForcedPassReplacesExistingCover(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
at := time.UnixMilli(5_000_000)
|
|
public := &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"}
|
|
|
|
t.Run("unforced pass leaves the cover alone", func(t *testing.T) {
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
if got := public.callCount(); got != 0 {
|
|
t.Fatalf("cover fetch calls = %d, want 0", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
|
}
|
|
})
|
|
|
|
t.Run("forced pass replaces the cover", func(t *testing.T) {
|
|
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: public,
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
if got := public.callCount(); got != 1 {
|
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("second")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the second one %q", got.Cover, want)
|
|
}
|
|
body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("second")))
|
|
if err != nil || !ok {
|
|
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
|
}
|
|
if string(body) != "second" {
|
|
t.Fatalf("stored cover = %q, want second", body)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Identical artwork re-served is an honest no-op the caller can tell apart
|
|
// from a replacement: the address comes from the bytes, so the row cannot
|
|
// change in substance, and the replace call site reports the three outcomes
|
|
// distinctly (issue #135).
|
|
func TestRunOnceForcedPassIdenticalBytesLogsNoOp(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("cover-bytes"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
at := time.UnixMilli(5_000_000)
|
|
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want unchanged %q", got.Cover, want)
|
|
}
|
|
if body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || !ok || string(body) != "cover-bytes" {
|
|
t.Fatalf("stored cover after no-op: found=%v err=%v", ok, err)
|
|
}
|
|
logged := logs.String()
|
|
if !strings.Contains(logged, "cover unchanged") {
|
|
t.Fatalf("no-op not reported as unchanged; log:\n%s", logged)
|
|
}
|
|
if strings.Contains(logged, "cover replaced") {
|
|
t.Fatalf("no-op reported as a replacement; log:\n%s", logged)
|
|
}
|
|
}
|
|
|
|
// A Series whose sharded Cover file was unlinked out from under it is
|
|
// repaired by one forced pass: same bytes mean the same address and the file
|
|
// re-linked (issue #135, story 32).
|
|
func TestRunOnceForcedPassRelinksUnlinkedCoverFile(t *testing.T) {
|
|
coverDir := t.TempDir()
|
|
url := pgtest.URL(t)
|
|
s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("cover-bytes"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
address := store.CoverAddressForBytes([]byte("cover-bytes"))
|
|
coverPath := filepath.Join(coverDir, filepath.FromSlash(address[:2]+"/"+address[2:4]+"/"+address))
|
|
if err := os.Remove(coverPath); err != nil {
|
|
t.Fatalf("unlink cover file: %v", err)
|
|
}
|
|
if _, _, ok, err := s.CoverByAddress(address); err != nil || ok {
|
|
t.Fatalf("CoverByAddress after unlink = found %v err %v; want missing (file gone)", ok, err)
|
|
}
|
|
|
|
at := time.UnixMilli(5_000_000)
|
|
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"},
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
body, contentType, ok, err := s.CoverByAddress(address)
|
|
if err != nil || !ok {
|
|
t.Fatalf("CoverByAddress after forced pass: found=%v err=%v; want the file re-linked", ok, err)
|
|
}
|
|
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
|
t.Fatalf("re-linked cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
|
}
|
|
}
|
|
|
|
// A forced pass degrades exactly like an ordinary one when the cover sidecar
|
|
// is unreachable: the fetch is skipped and logged, the chapter poll is
|
|
// untouched, and the stored Cover is not moved (issue #135, story 6).
|
|
func TestRunOnceForcedPassWithoutCoverFetcherStillPolls(t *testing.T) {
|
|
s, _ := newTestStore(t)
|
|
const (
|
|
key = "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
seriesID = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "kagane", SeriesID: seriesID,
|
|
SeriesURL: "https://kagane.to/series/" + seriesID, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("kagane", seriesID, "https://kagane.to/api/v2/image/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b/compressed", []byte("existing"), "image/webp"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
at := time.UnixMilli(5_000_000)
|
|
if err := s.ForceSeriesPoll("kagane", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
|
|
p := &Poller{
|
|
Store: s, BrowserFetch: &fakeFetcher{body: kaganeAPIFixtureWithCover, status: 200},
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
got := readBookmark(t, s, key)
|
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
|
|
t.Fatalf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
|
|
}
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("existing")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the existing one %q untouched", got.Cover, want)
|
|
}
|
|
if checked := readLatestCheckedAt(t, s, key); checked != at.UnixMilli() {
|
|
t.Fatalf("latest_checked_at = %d, want %d", checked, at.UnixMilli())
|
|
}
|
|
if !strings.Contains(logs.String(), "fetch cover") {
|
|
t.Fatalf("missing skipped-cover log; log:\n%s", logs.String())
|
|
}
|
|
}
|
|
|
|
// A forced replacement strands the previous address, and the poller reclaims
|
|
// it from the stranded branch: the old sharded file and covers row are both
|
|
// gone once the pass lands while the new bytes read back (issue #154). The
|
|
// identical-bytes no-op that follows reclaims nothing: previous == current
|
|
// there, and a reclamation would delete the Cover the pass just wrote.
|
|
func TestRunOnceForcedPassReclaimsSupersededCover(t *testing.T) {
|
|
coverDir := t.TempDir()
|
|
url := pgtest.URL(t)
|
|
s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
const (
|
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
|
seriesID = "chronicles-of-the-demon-faction-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
stale := store.CoverAddressForBytes([]byte("first"))
|
|
stalePath := filepath.Join(coverDir, filepath.FromSlash(stale[:2]+"/"+stale[2:4]+"/"+stale))
|
|
|
|
at := time.UnixMilli(5_000_000)
|
|
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"},
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
// The stranded address is gone on disk and in SQL; the new one reads back.
|
|
if _, err := os.Stat(stalePath); !errors.Is(err, fs.ErrNotExist) {
|
|
t.Fatalf("stale sharded file after replacement = %v, want fs.ErrNotExist", err)
|
|
}
|
|
if _, _, ok, err := s.CoverByAddress(stale); err != nil || ok {
|
|
t.Fatalf("stale bytes after replacement = found %v err %v, want reclaimed", ok, err)
|
|
}
|
|
if body, _, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("second"))); err != nil || !ok || string(body) != "second" {
|
|
t.Fatalf("new bytes after replacement = found %v err %v, want served", ok, err)
|
|
}
|
|
|
|
// The identical-bytes pass is an honest no-op and reclaims nothing.
|
|
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(2*time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
current := store.CoverAddressForBytes([]byte("second"))
|
|
currentPath := filepath.Join(coverDir, filepath.FromSlash(current[:2]+"/"+current[2:4]+"/"+current))
|
|
if _, err := os.Stat(currentPath); err != nil {
|
|
t.Fatalf("live sharded file after no-op pass = %v, want present", err)
|
|
}
|
|
if body, _, ok, err := s.CoverByAddress(current); err != nil || !ok || string(body) != "second" {
|
|
t.Fatalf("bytes after no-op pass = found %v err %v, want still served", ok, err)
|
|
}
|
|
}
|
|
|
|
// A reclamation that fails must not fail the Poll: the failure is logged
|
|
// against the Series and the replacement still lands, so the stranded bytes
|
|
// stay reachable for a retry and the owner's act succeeded (issue #154).
|
|
func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) {
|
|
coverDir := t.TempDir()
|
|
url := pgtest.URL(t)
|
|
s, err := store.Open(url, testOwner, coverDir, testCoverBaseURL)
|
|
if err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
const (
|
|
key = "asura:chronicles-of-the-fallen-f886a8af"
|
|
seriesID = "chronicles-of-the-fallen-f886a8af"
|
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-fallen-f886a8af"
|
|
first = "https://cdn.example/covers/first.jpg"
|
|
)
|
|
if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
|
|
Key: key, Site: "asura", SeriesID: seriesID, SeriesURL: seriesURL, UpdatedAt: 1000,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
if err := s.SetSeriesCover("asura", seriesID, first, []byte("first"), "image/jpeg"); err != nil {
|
|
t.Fatalf("seed cover: %v", err)
|
|
}
|
|
// Make the stranded file unremovable: a non-empty directory in its place.
|
|
stale := store.CoverAddressForBytes([]byte("first"))
|
|
stalePath := filepath.Join(coverDir, filepath.FromSlash(stale[:2]+"/"+stale[2:4]+"/"+stale))
|
|
if err := os.Remove(stalePath); err != nil {
|
|
t.Fatalf("clear file: %v", err)
|
|
}
|
|
if err := os.Mkdir(stalePath, 0o755); err != nil {
|
|
t.Fatalf("replace file with dir: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(stalePath, "blob"), []byte("x"), 0o644); err != nil {
|
|
t.Fatalf("fill dir: %v", err)
|
|
}
|
|
|
|
at := time.UnixMilli(5_000_000)
|
|
if err := s.ForceSeriesPoll("asura", seriesID, at.Add(time.Hour).UnixMilli()); err != nil {
|
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
|
}
|
|
var logs strings.Builder
|
|
prev := log.Writer()
|
|
log.SetOutput(&logs)
|
|
t.Cleanup(func() { log.SetOutput(prev) })
|
|
p := &Poller{
|
|
Store: s, Fetch: &fakeFetcher{body: asuraSeriesFixture + asuraCoverFixture, status: 200},
|
|
CoverBytesFetch: &fakeBytesCoverFetcher{body: []byte("second"), contentType: "image/jpeg"},
|
|
Now: func() time.Time { return at },
|
|
}
|
|
p.runOnce(context.Background())
|
|
p.waitCovers()
|
|
|
|
logged := logs.String()
|
|
if !strings.Contains(logged, "reclaim cover") {
|
|
t.Fatalf("failed reclamation not logged; log:\n%s", logged)
|
|
}
|
|
if !strings.Contains(logged, "cover replaced") {
|
|
t.Fatalf("replacement not reported after a failed reclaim; log:\n%s", logged)
|
|
}
|
|
got := readBookmark(t, s, key)
|
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("second")); got.Cover != want {
|
|
t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want)
|
|
}
|
|
}
|