feat(latest): split browser poll cooldown
This commit is contained in:
@@ -12,6 +12,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
@@ -689,16 +690,17 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
}
|
||||
|
||||
// DueForLatestCheck returns series whose server-side latest-chapter check has
|
||||
// aged past cutoffMs, ordered by how many bookmarks reference them (descending)
|
||||
// then least-recently-checked first, at most limit of them.
|
||||
// aged past the appropriate cutoff, ordered by how many bookmarks reference
|
||||
// them (descending) then least-recently-checked first, at most limit of them.
|
||||
// Browser-backed sites use browserCutoffMs; every other site uses cutoffMs.
|
||||
//
|
||||
// The reader_count ordering is the point of the split (ADR-0003): a series
|
||||
// shared by several readers is fetched once per due cycle, and the popular
|
||||
// ones stay freshest while the long tail absorbs any shortfall. Within one
|
||||
// reader count, oldest-first keeps the poll fair when the backlog outgrows its
|
||||
// reader count, oldest-first keeps the poll fair when the backlog outgrows
|
||||
// throughput: the most neglected series is always next, so a large collection
|
||||
// refreshes uniformly slower rather than leaving a tail that never refreshes at
|
||||
// all. The userscript sorts its own queue the same way (L453).
|
||||
// refreshes uniformly slower rather than leaving a tail that never refreshes
|
||||
// at all. The userscript sorts its own queue the same way (L453).
|
||||
//
|
||||
// Series with no series_url are skipped — there is nothing to fetch, which is
|
||||
// the same filter the userscript applies at L452. Series whose only bookmarks
|
||||
@@ -706,17 +708,20 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
// burns requests. Archived bookmarks still count — knowing what a shelved
|
||||
// series is up to is the whole reason for archiving instead of deleting.
|
||||
// A series with no bookmarks at all never appears: the join excludes it.
|
||||
func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Series, error) {
|
||||
func (s *Store) DueForLatestCheck(cutoffMs, browserCutoffMs int64, browserSites []string, limit int) ([]Series, error) {
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
|
||||
FROM series s
|
||||
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
WHERE s.series_url <> ''
|
||||
AND s.latest_checked_at <= $1
|
||||
AND s.latest_checked_at <= CASE
|
||||
WHEN s.site = ANY($3::text[]) THEN $2::bigint
|
||||
ELSE $1::bigint
|
||||
END
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
|
||||
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
ORDER BY reader_count DESC, s.latest_checked_at ASC
|
||||
LIMIT $2`, cutoffMs, limit)
|
||||
LIMIT $4`, cutoffMs, browserCutoffMs, pgtype.FlatArray[string](browserSites), limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query due series: %w", err)
|
||||
}
|
||||
|
||||
@@ -291,7 +291,7 @@ func TestDueForLatestCheck(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt)
|
||||
|
||||
due, err := s.DueForLatestCheck(now-hour, 10)
|
||||
due, err := s.DueForLatestCheck(now-hour, now-hour, nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -309,7 +309,7 @@ func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) {
|
||||
seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200)
|
||||
seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 2)
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -489,7 +489,7 @@ func TestDueForLatestCheckSkipsFinishedKeepsArchived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), 10)
|
||||
due, err := store.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -844,7 +844,7 @@ func TestDueForLatestCheckOrdersByReaderCountThenAge(t *testing.T) {
|
||||
seedSecondReader(t, s, "asura:pop:2", "asura", "pop", 1001)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 100)
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 10)
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -870,7 +870,7 @@ func TestDueForLatestCheckExcludesOrphanSeries(t *testing.T) {
|
||||
t.Fatalf("seed orphan series: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(1000, 10)
|
||||
due, err := s.DueForLatestCheck(1000, 1000, nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -1207,7 +1207,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
t.Fatalf("series rows = %d, want 1 shared row for two bookmarks", series)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), 10)
|
||||
due, err := s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
@@ -1223,7 +1223,7 @@ func TestTwoReadersShareOneSeriesWithIndependentProgress(t *testing.T) {
|
||||
if b, ok, err := s.Get(s.OwnerID(), "asura:solo"); err != nil || !ok || b.LastChapterNum != 200 {
|
||||
t.Fatalf("owner's bookmark after the other's delete = %+v ok=%v err=%v, want it intact", b, ok, err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), 10)
|
||||
due, err = s.DueForLatestCheck(time.Now().UnixMilli(), time.Now().UnixMilli(), nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck after delete: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user