mangaBookmark/backend/internal/latest/poller_test.go

package latest

import (
	"context"
	"crypto/sha256"
	"errors"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"bookmarkmanager/backend/internal/pgtest"
	"bookmarkmanager/backend/internal/store"
)

func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }

// testOwner is the owner every test store seeds. A second reader, where a
// test needs one, is created by opening the same database as a second owner.
var testOwner = store.Owner{DiscordID: "test-owner", TokenHash: sha256.Sum256([]byte("owner-token-hash"))}

// newTestStore opens a store on a Postgres database of this test's own and
// returns the URL, for helpers that need a second connection to the same
// database (see TestRunOnceFetchesSharedSeriesOnce).
func newTestStore(t *testing.T) (*store.Store, string) {
	t.Helper()
	url := pgtest.URL(t)
	s, err := store.Open(url, testOwner)
	if err != nil {
		t.Fatalf("Open: %v", err)
	}
	t.Cleanup(func() { s.Close() })
	return s, url
}

// seedForCheck inserts a bookmark (and with it its series) and forces the
// series' latest_checked_at.
func seedForCheck(t *testing.T, s *store.Store, key, seriesURL string, checkedAt int64) {
	t.Helper()
	site, seriesID, ok := strings.Cut(key, ":")
	if !ok {
		t.Fatalf("key %q: no ':' separator", key)
	}
	if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
		Key:       key,
		Site:      site,
		SeriesID:  seriesID,
		SeriesURL: seriesURL,
		UpdatedAt: 1000,
	}); err != nil {
		t.Fatalf("seed %q: %v", key, err)
	}
	if err := s.MarkLatestChecked(site, seriesID, checkedAt); err != nil {
		t.Fatalf("seed mark %q: %v", key, err)
	}
}

func readLatestCheckedAt(t *testing.T, s *store.Store, key string) int64 {
	t.Helper()
	site, seriesID, ok := strings.Cut(key, ":")
	if !ok {
		t.Fatalf("key %q: no ':' separator", key)
	}
	ts, err := s.LatestCheckedAt(site, seriesID)
	if err != nil {
		t.Fatalf("LatestCheckedAt %q: %v", key, err)
	}
	return ts
}

// fakeFetcher stands in for the network. Every poller test uses it, so nothing
// in this file can reach tls-client or a real site.
type fakeFetcher struct {
	mu     sync.Mutex
	calls  []string
	body   string
	status int
	err    error
	// perURL overrides body/status/err for specific URLs.
	perURL map[string]fakeResponse
}

type fakeResponse struct {
	body   string
	status int
	err    error
}

func (f *fakeFetcher) Get(ctx context.Context, url string) (string, int, error) {
	f.mu.Lock()
	f.calls = append(f.calls, url)
	f.mu.Unlock()
	if r, ok := f.perURL[url]; ok {
		return r.body, r.status, r.err
	}
	return f.body, f.status, f.err
}

func (f *fakeFetcher) callCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.calls)
}

// newTestPoller wires a poller with a frozen clock and no stagger, so tests run
// instantly and deterministically.
func newTestPoller(t *testing.T, s *store.Store, f Fetcher, at time.Time) *Poller {
	t.Helper()
	return &Poller{
		Store:    s,
		Fetch:    f,
		Now:      func() time.Time { return at },
		Cooldown: time.Hour,
		Interval: 10 * time.Minute,
		Stagger:  0,
		Batch:    14,
	}
}

func TestRunOnceRecordsLatestChapter(t *testing.T) {
	s, _ := newTestStore(t)
	const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
	seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0)

	now := time.UnixMilli(5_000_000)
	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
	newTestPoller(t, s, f, now).runOnce(context.Background())

	b, ok, err := s.Get(s.OwnerID(), "asura:chronicles-of-the-demon-faction-f886a8af")
	if err != nil || !ok {
		t.Fatalf("Get: %v ok=%v", err, ok)
	}
	if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
		t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
	}
	if b.LatestChapter != "Chapter 181" {
		t.Fatalf("LatestChapter = %q, want %q", b.LatestChapter, "Chapter 181")
	}
	if got := readLatestCheckedAt(t, s, "asura:chronicles-of-the-demon-faction-f886a8af"); got != now.UnixMilli() {
		t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
	}
}

// The whole point of the updated_at CASE in Upsert: a newly published chapter is
// not reading progress and must not move the series up the list.
func TestRunOnceDoesNotReorderList(t *testing.T) {
	s, _ := newTestStore(t)
	const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
	const key = "asura:chronicles-of-the-demon-faction-f886a8af"

	// "other" is the most recently read, so it must stay at the top of List().
	if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
		Key: "asura:other", Site: "asura", SeriesID: "other",
		SeriesURL: "https://asurascans.com/comics/other", UpdatedAt: 9_000_000,
	}); err != nil {
		t.Fatalf("seed other: %v", err)
	}
	seedForCheck(t, s, key, url, 0)
	before, _, err := s.Get(s.OwnerID(), key)
	if err != nil {
		t.Fatalf("Get before: %v", err)
	}

	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
	newTestPoller(t, s, f, time.UnixMilli(9_999_999)).runOnce(context.Background())

	after, _, err := s.Get(s.OwnerID(), key)
	if err != nil {
		t.Fatalf("Get after: %v", err)
	}
	if after.UpdatedAt != before.UpdatedAt {
		t.Fatalf("updated_at moved from %d to %d on a latest-chapter bump",
			before.UpdatedAt, after.UpdatedAt)
	}
	list, err := s.List(s.OwnerID())
	if err != nil {
		t.Fatalf("List: %v", err)
	}
	if list[0].Key != "asura:other" {
		t.Fatalf("list reordered: head is %q, want asura:other", list[0].Key)
	}
}

// A failed fetch must still consume the cooldown, or a renamed series gets
// retried on every tick forever.
func TestRunOnceMarksCheckedOnFailure(t *testing.T) {
	tests := []struct {
		name string
		resp fakeResponse
	}{
		{"network error", fakeResponse{err: errors.New("dial tcp: refused")}},
		{"non-200", fakeResponse{body: "nope", status: 503}},
		{"challenge page", fakeResponse{body: challengeFixture, status: 200}},
		{"empty body", fakeResponse{body: "", status: 200}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			s, _ := newTestStore(t)
			const url = "https://asurascans.com/comics/x"
			seedForCheck(t, s, "asura:x", url, 0)

			now := time.UnixMilli(7_000_000)
			f := &fakeFetcher{perURL: map[string]fakeResponse{url: tt.resp}}
			newTestPoller(t, s, f, now).runOnce(context.Background())

			if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() {
				t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli())
			}
			b, _, err := s.Get(s.OwnerID(), "asura:x")
			if err != nil {
				t.Fatalf("Get: %v", err)
			}
			if b.LatestChapterNum != nil {
				t.Fatalf("LatestChapterNum = %v, want nil on a failed check", *b.LatestChapterNum)
			}
		})
	}
}

func TestRunOnceRespectsBatchLimit(t *testing.T) {
	s, _ := newTestStore(t)
	for i := 0; i < 20; i++ {
		key := "asura:s" + string(rune('a'+i))
		seedForCheck(t, s, key, "https://asurascans.com/comics/"+key, 0)
	}

	f := &fakeFetcher{body: "", status: 200}
	p := newTestPoller(t, s, f, time.UnixMilli(5_000_000))
	p.Batch = 5
	p.runOnce(context.Background())

	if got := f.callCount(); got != 5 {
		t.Fatalf("fetched %d series, want 5 (batch limit)", got)
	}
}

// The point of the split (ADR-0003): a series referenced by several bookmarks
// is fetched once per due cycle, not once per bookmark. Two bookmarks share a
// series when two readers track it (issue #22).
func TestRunOnceFetchesSharedSeriesOnce(t *testing.T) {
	s, url := newTestStore(t)
	// The slug must match the fixture's own anchors: asura's parser scopes
	// chapter links to the stored slug.
	const slug = "chronicles-of-the-demon-faction-f886a8af"
	const seriesURL = "https://asurascans.com/comics/" + slug
	seedForCheck(t, s, "asura:"+slug, seriesURL, 0)
	// A second reader tracks the same series. The seed is the only
	// reader-creation path, so a second Open as a different owner is how a
	// test gets a second reader on the same database.
	other, err := store.Open(url, store.Owner{DiscordID: "second-reader", TokenHash: sha256.Sum256([]byte("second-token-hash"))})
	if err != nil {
		t.Fatalf("Open second reader: %v", err)
	}
	t.Cleanup(func() { other.Close() })
	if _, err := s.Upsert(other.OwnerID(), store.Bookmark{
		Key: "asura:" + slug, Site: "asura", SeriesID: slug, UpdatedAt: 2000,
	}); err != nil {
		t.Fatalf("seed second reader: %v", err)
	}

	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
	newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())

	if got := f.callCount(); got != 1 {
		t.Fatalf("fetched shared series %d times, want 1", got)
	}
	// Both bookmarks join to the same updated series row.
	for _, st := range []*store.Store{s, other} {
		b, ok, err := st.Get(st.OwnerID(), "asura:"+slug)
		if err != nil || !ok {
			t.Fatalf("Get: %v ok=%v", err, ok)
		}
		if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 {
			t.Fatalf("LatestChapterNum = %v, want 181", b.LatestChapterNum)
		}
	}
}

// One unreachable series must not abandon the rest of the batch.
func TestRunOnceOneBadSeriesDoesNotStallBatch(t *testing.T) {
	s, _ := newTestStore(t)
	keys := []string{"asura:a", "asura:b", "asura:c", "asura:d", "asura:e"}
	for _, k := range keys {
		seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
	}

	now := time.UnixMilli(6_000_000)
	f := &fakeFetcher{
		body: "", status: 200,
		perURL: map[string]fakeResponse{
			"https://asurascans.com/comics/asura:b": {err: errors.New("boom")},
		},
	}
	newTestPoller(t, s, f, now).runOnce(context.Background())

	if got := f.callCount(); got != 5 {
		t.Fatalf("fetched %d series, want all 5 attempted", got)
	}
	for _, k := range keys {
		if got := readLatestCheckedAt(t, s, k); got != now.UnixMilli() {
			t.Fatalf("%s latest_checked_at = %d, want %d", k, got, now.UnixMilli())
		}
	}
}

// The cooldown is enforced by the due query, so a second immediate pass must do
// nothing at all — this is what makes the tick interval independent of it.
func TestRunOnceHonoursCooldownAcrossPasses(t *testing.T) {
	s, _ := newTestStore(t)
	const url = "https://asurascans.com/comics/x"
	seedForCheck(t, s, "asura:x", url, 0)

	now := time.UnixMilli(8_000_000)
	f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
	p := newTestPoller(t, s, f, now)

	p.runOnce(context.Background())
	if got := f.callCount(); got != 1 {
		t.Fatalf("first pass fetched %d, want 1", got)
	}
	// Same instant, and again 59 minutes later: both inside the 1h cooldown.
	p.runOnce(context.Background())
	p.Now = func() time.Time { return now.Add(59 * time.Minute) }
	p.runOnce(context.Background())
	if got := f.callCount(); got != 1 {
		t.Fatalf("fetched %d times inside the cooldown, want 1", got)
	}
	// Past the cooldown, it is due again.
	p.Now = func() time.Time { return now.Add(61 * time.Minute) }
	p.runOnce(context.Background())
	if got := f.callCount(); got != 2 {
		t.Fatalf("fetched %d times after the cooldown, want 2", got)
	}
}

// A site that retracts a chapter should correct the stored number downward,
// mirroring the userscript's equality check (L427) rather than a >.
func TestRunOnceCorrectsDownward(t *testing.T) {
	s, _ := newTestStore(t)
	const url = "https://demonicscans.org/manga/Catastrophic-Necromancer"
	const key = "demonic:Catastrophic-Necromancer"

	high := 400.0
	if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
		Key: key, Site: "demonic", SeriesID: "Catastrophic-Necromancer",
		SeriesURL: url, LatestChapter: "Chapter 400", LatestChapterNum: &high,
		UpdatedAt: 1000,
	}); err != nil {
		t.Fatalf("seed: %v", err)
	}

	f := &fakeFetcher{body: demonicSeriesFixture, status: 200}
	newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(context.Background())

	b, _, err := s.Get(s.OwnerID(), key)
	if err != nil {
		t.Fatalf("Get: %v", err)
	}
	if b.LatestChapterNum == nil || *b.LatestChapterNum != 296 {
		t.Fatalf("LatestChapterNum = %v, want 296", b.LatestChapterNum)
	}
}

// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
// reject anything that is not a known site with an https URL before spending a
// request on it — the cooldown still gets consumed either way.
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
	tests := []struct {
		name      string
		site      string
		seriesURL string
		wantCalls int
	}{
		{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
		{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
		{"unparseable url", "asura", "http://[::1", 0},
		{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			s, _ := newTestStore(t)
			key := tt.site + ":x"
			if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
				Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
				UpdatedAt: 1000,
			}); err != nil {
				t.Fatalf("seed: %v", err)
			}

			now := time.UnixMilli(4_000_000)
			f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
			newTestPoller(t, s, f, now).checkOne(context.Background(), store.Series{
				Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
			})

			if got := f.callCount(); got != tt.wantCalls {
				t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
			}
			if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
				t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
			}
		})
	}
}

// A cancelled context must abandon the batch rather than run it to completion.
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
	s, _ := newTestStore(t)
	for _, k := range []string{"asura:a", "asura:b", "asura:c"} {
		seedForCheck(t, s, k, "https://asurascans.com/comics/"+k, 0)
	}

	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	f := &fakeFetcher{body: "", status: 200}
	newTestPoller(t, s, f, time.UnixMilli(5_000_000)).runOnce(ctx)

	if got := f.callCount(); got != 0 {
		t.Fatalf("fetched %d series with a cancelled context, want 0", got)
	}
}

func TestFetchableSeriesURL(t *testing.T) {
	tests := []struct {
		name      string
		site      string
		seriesURL string
		want      bool
	}{
		{"asura https", "asura", "https://asurascans.com/comics/x-aabbccdd", true},
		{"demonic https", "demonic", "https://demonicscans.org/manga/X", true},
		{"comix https", "comix", "https://comix.to/title/n8we-dungeons-and-crayons", true},
		{"kagane on its own host", "kagane", "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b", true},
		// The browser fetcher runs JavaScript and carries cookies, so a
		// client-supplied series_url must not be able to aim it anywhere else.
		{"kagane on a foreign host", "kagane", "https://evil.example/series/x", false},
		{"kagane on a lookalike host", "kagane", "https://kagane.to.evil.example/series/x", false},
		{"unknown site", "mangadex", "https://mangadex.org/title/x", false},
		{"non-https", "comix", "http://comix.to/title/x", false},
		{"no host", "comix", "https:///title/x", false},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
				t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
					tt.site, tt.seriesURL, got, tt.want)
			}
		})
	}
}

// A kagane row must not be handed to the plain TLS fetcher: it would only ever
// receive a challenge page, and the browser fetcher is the whole reason kagane
// is pollable at all.
func TestKaganeSkippedWhenNoBrowserFetcher(t *testing.T) {
	s, _ := newTestStore(t)
	if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
		Key:       "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
		Site:      "kagane",
		SeriesID:  "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
		SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
		UpdatedAt: 1000,
	}); err != nil {
		t.Fatalf("seed: %v", err)
	}

	f := &fakeFetcher{body: kaganeAPIFixture, status: 200}
	p := &Poller{
		Store: s, Fetch: f,
		Now:      func() time.Time { return time.UnixMilli(5_000_000) },
		Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
	}
	p.runOnce(context.Background())

	if len(f.calls) != 0 {
		t.Errorf("TLS fetcher was called for kagane: %v", f.calls)
	}
}

// With a browser fetcher wired up, kagane goes to it and not to the TLS one.
func TestKaganeUsesBrowserFetcher(t *testing.T) {
	s, _ := newTestStore(t)
	key := "kagane:019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
	if _, err := s.Upsert(s.OwnerID(), store.Bookmark{
		Key:       key,
		Site:      "kagane",
		SeriesID:  "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
		SeriesURL: "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b",
		UpdatedAt: 1000,
	}); err != nil {
		t.Fatalf("seed: %v", err)
	}

	tlsF := &fakeFetcher{body: "", status: 200}
	browserF := &fakeFetcher{body: kaganeAPIFixture, status: 200}
	p := &Poller{
		Store: s, Fetch: tlsF, BrowserFetch: browserF,
		Now:      func() time.Time { return time.UnixMilli(5_000_000) },
		Cooldown: time.Hour, Interval: time.Hour, Batch: 10,
	}
	p.runOnce(context.Background())

	if len(tlsF.calls) != 0 {
		t.Errorf("TLS fetcher was called for kagane: %v", tlsF.calls)
	}
	if len(browserF.calls) != 1 {
		t.Fatalf("browser fetcher calls = %v, want 1", browserF.calls)
	}
	got, found, err := s.Get(s.OwnerID(), key)
	if err != nil || !found {
		t.Fatalf("Get: %v found=%v", err, found)
	}
	if got.LatestChapterNum == nil || *got.LatestChapterNum != 41 {
		t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
	}
}

func TestFetcherForRoutesNovelSites(t *testing.T) {
	tls := &fakeFetcher{}
	browser := &fakeFetcher{}
	p := &Poller{Fetch: tls, BrowserFetch: browser}

	cases := []struct {
		site string
		want Fetcher
	}{
		{"asura", tls},
		{"lightnovelworld", tls},
		{"kagane", browser},
		{"novelfull", browser},
	}
	for _, tc := range cases {
		t.Run(tc.site, func(t *testing.T) {
			if got := p.fetcherFor(tc.site); got != tc.want {
				t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
			}
		})
	}
}

func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
	cases := []struct {
		name string
		site string
		url  string
		want bool
	}{
		{"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true},
		{"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false},
		{"novelfull over http", "novelfull", "http://novelfull.com/x.html", false},
		{"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true},
		{"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false},
		{"unknown site", "webnovel", "https://webnovel.com/x", false},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
				t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
			}
		})
	}
}