feat: one Poll Lane per Site, replacing the shared pace (#100)
Each Site runs its own poll goroutine, paced by Rest and Gap from the registry (sites.go) instead of the shared cooldown/interval/stagger/batch config. Rest is enforced by the due query's WHERE clause; the Lane sleeps its effective gap between fetches (hour/eligible, floored at 1s). Lane-local failures: two refusals stop that Site for 15m, a lost browser stops only the round's remaining browser Lanes, and cover heals moved to background goroutines so a slow CDN cannot consume a Lane's gap. The five LATEST_CHAPTER_POLL_* pace env vars are gone; only the kill switch remains.
This commit is contained in:
@@ -298,7 +298,7 @@ func TestDueForLatestCheck(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
|
||||
|
||||
due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", now-hour)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -309,22 +309,25 @@ func TestDueForLatestCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
|
||||
func TestDueForLatestCheckOldestFirstAndScopedToSite(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
// Insert newest-checked first so a correct ORDER BY has to reverse it.
|
||||
seedForCheck(t, s, "asura:c", "https://asurascans.com/comics/c", 300)
|
||||
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
|
||||
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
|
||||
// A second Site's due series must not appear in asura's list: each Lane
|
||||
// asks for one Site, and no Lane may see another's queue.
|
||||
seedForCheck(t, s, "demonic:z", "https://demonicscans.org/manga/z", 0)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
|
||||
due, err := s.DueForLatestCheck("asura", 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 2 {
|
||||
t.Fatalf("got %d rows, want 2 (limit)", len(due))
|
||||
if len(due) != 3 {
|
||||
t.Fatalf("got %d rows, want 3 (all of asura's, none of demonic's)", len(due))
|
||||
}
|
||||
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" {
|
||||
t.Fatalf("got %q,%q; want asura:a,asura:b (oldest first)", due[0].Key(), due[1].Key())
|
||||
if due[0].Key() != "asura:a" || due[1].Key() != "asura:b" || due[2].Key() != "asura:c" {
|
||||
t.Fatalf("got %q,%q,%q; want asura:a,asura:b,asura:c (oldest first)", due[0].Key(), due[1].Key(), due[2].Key())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,7 +499,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
due, err := store.DueForLatestCheck("asura", time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -512,6 +515,38 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The gap's denominator counts every Series the Lane will ever Poll: a
|
||||
// finished Series must not make the Lane faster than it needs to be, and
|
||||
// another Site's Series must not leak into this Site's count.
|
||||
func TestEligibleSeriesCount(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
seedForCheck(t, store, "asura:reading", "https://asurascans.com/comics/reading", 0)
|
||||
seedForCheck(t, store, "asura:archived", "https://asurascans.com/comics/archived", 0)
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:finished", Site: "asura", SeriesID: "finished",
|
||||
SeriesURL: "https://asurascans.com/comics/finished",
|
||||
Status: StatusFinished, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed finished: %v", err)
|
||||
}
|
||||
seedForCheck(t, store, "demonic:z", "https://demonicscans.org/manga/z", 0)
|
||||
|
||||
n, err := store.EligibleSeriesCount("asura")
|
||||
if err != nil {
|
||||
t.Fatalf("EligibleSeriesCount: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("eligible = %d, want 2 (finished excluded, demonic excluded)", n)
|
||||
}
|
||||
n, err = store.EligibleSeriesCount("demonic")
|
||||
if err != nil {
|
||||
t.Fatalf("EligibleSeriesCount(demonic): %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("eligible(demonic) = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisplayChapter(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -970,7 +1005,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
|
||||
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -996,7 +1031,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
|
||||
t.Fatalf("seed orphan series: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -1334,7 +1369,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
due, err := s.DueForLatestCheck("asura", time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -1350,7 +1385,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
|
||||
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
due, err = s.DueForLatestCheck("asura", time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck after delete: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user