feat: one Poll Lane per Site, replacing the shared pace (#100)

Each Site runs its own poll goroutine, paced by Rest and Gap from the
registry (sites.go) instead of the shared cooldown/interval/stagger/batch
config. Rest is enforced by the due query's WHERE clause; the Lane sleeps
its effective gap between fetches (hour/eligible, floored at 1s).

Lane-local failures: two refusals stop that Site for 15m, a lost browser
stops only the round's remaining browser Lanes, and cover heals moved to
background goroutines so a slow CDN cannot consume a Lane's gap. The five
LATEST_CHAPTER_POLL_* pace env vars are gone; only the kill switch
remains.
This commit is contained in:
2026-08-16 12:51:01 +07:00
parent ddbd57070d
commit 90ce14aec6
22 changed files with 3645 additions and 1884 deletions
+47 -12
View File
@@ -298,7 +298,7 @@ func TestDueForLatestCheck(t *testing.T) {
s := newTestStore(t)
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
due, err := s.DueForLatestCheck("asura", now-hour)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -309,22 +309,25 @@ func TestDueForLatestCheck(t *testing.T) {
}
}
func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
func TestDueForLatestCheckOldestFirstAndScopedToSite(t *testing.T) {
s := newTestStore(t)
// Insert newest-checked first so a correct ORDER BY has to reverse it.
seedForCheck(t, s, "asura:c", "https://asurascans.com/comics/c", 300)
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
// A second Site's due series must not appear in asura's list: each Lane
// asks for one Site, and no Lane may see another's queue.
seedForCheck(t, s, "demonic:z", "https://demonicscans.org/manga/z", 0)
due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
due, err := s.DueForLatestCheck("asura", 1000)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 2 {
t.Fatalf("got %d rows, want 2 (limit)", len(due))
if len(due) != 3 {
t.Fatalf("got %d rows, want 3 (all of asura's, none of demonic's)", len(due))
}
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" {
t.Fatalf("got %q,%q; want asura:a,asura:b (oldest first)", due[0].Key(), due[1].Key())
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" || due[2].Key() != "asura:c" {
t.Fatalf("got %q,%q,%q; want asura:a,asura:b,asura:c (oldest first)", due[0].Key(), due[1].Key(), due[2].Key())
}
}
@@ -496,7 +499,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
}
}
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
due, err := store.DueForLatestCheck("asura", time.Now().UnixMilli())
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -512,6 +515,38 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
}
}
// The gap's denominator counts every Series the Lane will ever Poll: a
// finished Series must not make the Lane faster than it needs to be, and
// another Site's Series must not leak into this Site's count.
func TestEligibleSeriesCount(t *testing.T) {
store := newTestStore(t)
seedForCheck(t, store, "asura:reading", "https://asurascans.com/comics/reading", 0)
seedForCheck(t, store, "asura:archived", "https://asurascans.com/comics/archived", 0)
if _, err := store.Upsert(store.OwnerID(), Bookmark{
Key: "asura:finished", Site: "asura", SeriesID: "finished",
SeriesURL: "https://asurascans.com/comics/finished",
Status: StatusFinished, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed finished: %v", err)
}
seedForCheck(t, store, "demonic:z", "https://demonicscans.org/manga/z", 0)
n, err := store.EligibleSeriesCount("asura")
if err != nil {
t.Fatalf("EligibleSeriesCount: %v", err)
}
if n != 2 {
t.Fatalf("eligible = %d, want 2 (finished excluded, demonic excluded)", n)
}
n, err = store.EligibleSeriesCount("demonic")
if err != nil {
t.Fatalf("EligibleSeriesCount(demonic): %v", err)
}
if n != 1 {
t.Fatalf("eligible(demonic) = %d, want 1", n)
}
}
func TestDisplayChapter(t *testing.T) {
cases := []struct {
name string
@@ -970,7 +1005,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
due, err := s.DueForLatestCheck("asura", 1000)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -996,7 +1031,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
t.Fatalf("seed orphan series: %v", err)
}
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
due, err := s.DueForLatestCheck("asura", 1000)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -1334,7 +1369,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
}
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
due, err := s.DueForLatestCheck("asura", time.Now().UnixMilli())
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
@@ -1350,7 +1385,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
}
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
due, err = s.DueForLatestCheck("asura", time.Now().UnixMilli())
if err != nil {
t.Fatalf("DueForLatestCheck after delete: %v", err)
}