feat(web,store,latest): forced poll — Check now stamps a Series for the next Lane pass (#146)

The control writes series.force_poll_at (column landed in migration 0014)
and never commands the poller: pending is derived as force_poll_at >
latest_checked_at and self-clears because the check stamp is written before
the fetch. The due query's forced flag overrides the rest cutoff, the
Sighting-deferral and finished-only clauses, jumps the queue, and wakes a
sleeping browser Lane; it never overrides an empty series_url, the Bookmarks
join, the refusal backoff, the sidecar-down skip or the Lane gap.

ADRs: 0013-commands-through-the-database.
This commit is contained in:
2026-08-22 00:53:51 +07:00
parent 6f9109c885
commit fe2cd12049
14 changed files with 818 additions and 44 deletions
+9 -4
View File
@@ -41,7 +41,7 @@ type SeriesFilter struct {
// must never leave the store package — so the projection does not select it,
// and only the anonymous boolean in raisedByReaderAnswer crosses it.
const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at`
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at`
// raisedByReaderAnswer answers "did a Reader's report set this number" without
// naming which Reader. Kept apart from adminSeriesColumns so the column list —
@@ -68,8 +68,12 @@ type AdminSeries struct {
LatestChapter string
LatestChapterNum *float64 // nil until first captured
LatestCheckedAt int64
ReaderCount int
RaisedByReader bool // a Reader's report set LatestChapterNum
// ForcePollAt is the owner's "check now" request stamp (issue #146), zero
// meaning never asked. Pending is derived, never stored: a request is
// pending while ForcePollAt is newer than LatestCheckedAt.
ForcePollAt int64
ReaderCount int
RaisedByReader bool // a Reader's report set LatestChapterNum
}
// SeriesPage is one page of the owner's filtered Series list plus the count
@@ -185,7 +189,7 @@ func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
`+where+`
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
s.latest_raised_by
s.force_poll_at, s.latest_raised_by
`+having+`
ORDER BY s.latest_checked_at, s.site, s.series_id
LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
@@ -260,6 +264,7 @@ func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
if err := scan(
&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
&a.ForcePollAt,
&a.RaisedByReader, &a.ReaderCount, &total,
); err != nil {
return AdminSeries{}, 0, err
+46
View File
@@ -348,3 +348,49 @@ func TestAdminFilterUnknownNameRejected(t *testing.T) {
}
}
}
// ForceSeriesPoll is the idempotent stamp write: a second press overwrites
// the request time, and touching a missing series is not an error.
func TestForceSeriesPollStampsIdempotently(t *testing.T) {
s := newTestStore(t)
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 9000, bookmarks: 1})
if err := s.ForceSeriesPoll("asura", "x", 42); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
if err := s.ForceSeriesPoll("asura", "x", 99); err != nil {
t.Fatalf("ForceSeriesPoll re-stamp: %v", err)
}
// Touching a missing series is not an error: the row may have been
// orphaned, and the caller's read decides what exists.
if err := s.ForceSeriesPoll("asura", "ghost", 99); err != nil {
t.Fatalf("ForceSeriesPoll missing: %v", err)
}
var got int64
if err := s.db.QueryRow(
`SELECT force_poll_at FROM series WHERE site = 'asura' AND series_id = 'x'`).Scan(&got); err != nil {
t.Fatalf("read force_poll_at: %v", err)
}
if got != 99 {
t.Fatalf("force_poll_at = %d, want 99 (the later press wins)", got)
}
}
// The admin projection carries the force stamp so the web layer can derive
// the pending flag without a second read.
func TestAdminSeriesCarriesForcePollAt(t *testing.T) {
s := newTestStore(t)
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 1000, bookmarks: 1})
if err := s.ForceSeriesPoll("asura", "x", 5000); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
page, err := s.SeriesPage(SeriesFilter{})
if err != nil {
t.Fatalf("SeriesPage: %v", err)
}
if len(page.Rows) != 1 || page.Rows[0].ForcePollAt != 5000 {
t.Fatalf("row = %+v, want ForcePollAt 5000", page.Rows)
}
}
+48 -9
View File
@@ -88,6 +88,11 @@ type Series struct {
// readerCount is the number of bookmarks referencing this series, filled
// only by the due-queue query that orders on it.
readerCount int
// Forced is whether the owner asked for a check now (issue #146): the
// request stamp is newer than the check stamp. Derived in the due query,
// never stored, and the flag that jumps the queue and opens the browser
// wake gate.
Forced bool
}
// LanePass is one Poll Lane's durable pass snapshot. Pause and refusal stamps
@@ -621,8 +626,9 @@ func (s *Store) scanBookmark(scan func(...any) error) (Bookmark, error) {
}
// scanSeries reads one row in seriesColumns order, plus the due query's
// reader_count column. latest_chapter_num and latest_raised_by are both
// nullable, same as latest_chapter_num on the bookmark read path.
// forced flag and reader_count columns. latest_chapter_num and
// latest_raised_by are both nullable, same as latest_chapter_num on the
// bookmark read path.
func scanSeries(scan func(...any) error) (Series, error) {
var (
sr Series
@@ -632,7 +638,7 @@ func scanSeries(scan func(...any) error) (Series, error) {
if err := scan(
&sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress,
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy,
&sr.readerCount,
&sr.Forced, &sr.readerCount,
); err != nil {
return Series{}, err
}
@@ -1164,6 +1170,13 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro
// limit — the Lane's own gap paces the fetches, and the batch size that used
// to cap this query is gone with the shared pace.
//
// A forced Series (force_poll_at newer than latest_checked_at, issue #146)
// overrides exactly three gates: the rest cutoff, the Sighting-deferral
// clause and the finished-only bucket. It never overrides an empty
// series_url or the Bookmarks join — nothing to fetch, and no consumer for
// the result — so those stay unconditional. Forced rows sort to the front of
// the queue; the reader-count-then-age ordering among the rest is ADR-0003.
//
// The reader_count ordering is the point of the split (ADR-0003): a series
// shared by several readers is fetched once per due cycle, and the popular
// ones stay freshest while the long tail absorbs any shortfall. Within one
@@ -1192,19 +1205,26 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro
// allowed to defer at all was settled when the Sighting was recorded — see
// RecordSighting.
func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Series, error) {
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
rows, err := s.db.Query(`SELECT `+seriesColumns+`,
(s.force_poll_at > s.latest_checked_at) AS forced,
COUNT(*) AS reader_count
FROM series s
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
WHERE s.site = $1
AND s.series_url <> ''
AND s.latest_checked_at <= $2::bigint
AND (s.latest_checked_at <= $2::bigint
OR s.force_poll_at > s.latest_checked_at)
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
s.force_poll_at
HAVING (COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
OR s.force_poll_at > s.latest_checked_at)
AND (COUNT(*) > 1
OR s.latest_sighted_at <= $2::bigint
OR s.latest_checked_at <= $3::bigint)
ORDER BY reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs)
OR s.latest_checked_at <= $3::bigint
OR s.force_poll_at > s.latest_checked_at)
ORDER BY (s.force_poll_at > s.latest_checked_at) DESC,
reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs)
if err != nil {
return nil, fmt.Errorf("query due series: %w", err)
}
@@ -1264,6 +1284,25 @@ func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
return nil
}
// ForceSeriesPoll stamps a Series with the owner's "check now" request
// (issue #146): a fact about the Series the Lane's next pass reads through
// DueForLatestCheck, never a command to the poller — so the request survives
// a restart. Writing again overwrites the request time; the write is
// idempotent. Touching a missing series is not an error: the row may have
// been orphaned, and the caller's read decides what exists. The stamp never
// expires by itself — an unanswered request keeps ageing — and pending is
// derived as force_poll_at > latest_checked_at, which is why the poller's
// check stamp is written before the fetch: the first attempt ends the
// pending state whatever it returns.
func (s *Store) ForceSeriesPoll(site, seriesID string, at int64) error {
if _, err := s.db.Exec(
`UPDATE series SET force_poll_at = $1 WHERE site = $2 AND series_id = $3`,
at, site, seriesID); err != nil {
return fmt.Errorf("force poll %s:%s: %w", site, seriesID, err)
}
return nil
}
// LatestCheckedAt reads the column MarkLatestChecked writes. It exists for
// tests outside this package (the poller's own tests assert on rest
// bookkeeping) — see MarkLatestChecked for why the field stays off the
+152 -1
View File
@@ -758,7 +758,7 @@ func TestMigration0008DropsLegacyCoverRows(t *testing.T) {
func readSeries(t *testing.T, s *Store, site, seriesID string) Series {
t.Helper()
sr, err := scanSeries(s.db.QueryRow(
`SELECT `+seriesColumns+`, 0 AS reader_count FROM series s
`SELECT `+seriesColumns+`, false AS forced, 0 AS reader_count FROM series s
WHERE s.site = $1 AND s.series_id = $2`, site, seriesID).Scan)
if err != nil {
t.Fatalf("read series %s:%s: %v", site, seriesID, err)
@@ -1748,3 +1748,154 @@ func TestLaneStatePauseResumeAndRefusal(t *testing.T) {
t.Fatalf("lane state rows after resume = %d, want 1", rows)
}
}
// A forced Series is due ahead of the rest cutoff: the request overrides the
// rest gate so the Lane's next pass picks it up however recently it was
// checked. An unforced series under the rest stays out.
func TestDueForLatestCheckForcedOverridesRestCutoff(t *testing.T) {
s := newTestStore(t)
const now = int64(10 * 3600_000)
seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", now-30*60_000)
seedForCheck(t, s, "asura:fresh", "https://asurascans.com/comics/fresh", now-30*60_000)
if err := s.ForceSeriesPoll("asura", "forced", now); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
due, err := s.DueForLatestCheck("asura", now-3600_000, noCeiling)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 1 || due[0].Key() != "asura:forced" {
t.Fatalf("due = %v, want only the forced series", due)
}
}
// The Sighting-deferral clause holds a solitary series back after a recent
// Reader report; a forced request overrides it and puts the series back on
// the Lane's list.
func TestDueForLatestCheckForcedOverridesSightingDeferral(t *testing.T) {
s := newTestStore(t)
const now = int64(10 * 3600_000)
// One bookmark (so deferral can apply), sighted and checked 10m ago:
// inside the deferral window and under the ceiling.
seedForCheck(t, s, "asura:deferred", "https://asurascans.com/comics/deferred", now-10*60_000)
if _, err := s.db.Exec(
`UPDATE series SET latest_sighted_at = $1 WHERE site = 'asura' AND series_id = 'deferred'`,
now-10*60_000); err != nil {
t.Fatalf("seed sighting: %v", err)
}
// Unforced: deferred, and under the rest anyway.
due, err := s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 0 {
t.Fatalf("unforced deferred series is due: %v", due)
}
// Forced: the request overrides the deferral.
if err := s.ForceSeriesPoll("asura", "deferred", now); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
due, err = s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 1 || due[0].Key() != "asura:deferred" {
t.Fatalf("forced deferred series not due: %v", due)
}
}
// The finished-only bucket excludes a series whose only bookmarks are
// finished; a forced request overrides it — the owner asked, so the Lane
// looks.
func TestDueForLatestCheckForcedOverridesFinishedBucket(t *testing.T) {
s := newTestStore(t)
seedForCheck(t, s, "asura:reading", "https://asurascans.com/comics/reading", 0)
if _, err := s.Upsert(s.OwnerID(), Bookmark{
Key: "asura:finished", Site: "asura", SeriesID: "finished",
SeriesURL: "https://asurascans.com/comics/finished",
Status: StatusFinished, UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed finished: %v", err)
}
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
for _, sr := range due {
if sr.Key() == "asura:finished" {
t.Fatalf("unforced finished series is due: %v", due)
}
}
if err := s.ForceSeriesPoll("asura", "finished", 5000); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
due, err = s.DueForLatestCheck("asura", 1000, noCeiling)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
got := map[string]bool{}
for _, sr := range due {
got[sr.Key()] = true
}
if !got["asura:reading"] || !got["asura:finished"] {
t.Fatalf("forced finished series not due: %v", due)
}
}
// A forced Series jumps the queue: it sorts ahead of a more-read series that
// is due by rest, without disturbing the reader-count-then-age tie-break
// among the unforced rows (ADR-0003).
func TestDueForLatestCheckForcedSortsFirst(t *testing.T) {
s := newTestStore(t)
// "popular" has two readers and is long overdue; "forced" has one reader
// and a fresh check stamp. The forced row must come first.
seedForCheck(t, s, "asura:popular", "https://asurascans.com/comics/popular", 100)
seedSecondReader(t, s, "asura:popular:2", "asura", "popular", 1001)
seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", 900)
if err := s.ForceSeriesPoll("asura", "forced", 5000); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 2 {
t.Fatalf("due = %d rows, want 2", len(due))
}
if due[0].Key() != "asura:forced" || due[1].Key() != "asura:popular" {
t.Fatalf("due order = %q, %q; want forced first, then popular", due[0].Key(), due[1].Key())
}
}
// A forced Series with no series URL is still not fetched — nothing to fetch —
// and one with no Bookmarks is still excluded by the join. The force flag
// opens the three gates it is allowed to, not the whole query.
func TestDueForLatestCheckForcedDoesNotOverrideURLOrJoin(t *testing.T) {
s := newTestStore(t)
seedForCheck(t, s, "asura:nourl", "", 0)
if err := s.ForceSeriesPoll("asura", "nourl", 5000); err != nil {
t.Fatalf("ForceSeriesPoll: %v", err)
}
if _, err := s.db.Exec(`
INSERT INTO series (site, series_id, title, series_url, cover, kind,
latest_chapter, latest_chapter_num, latest_checked_at,
force_poll_at)
VALUES ('asura', 'orphan', 'Orphan', 'https://asurascans.com/comics/orphan',
'', 'manga', '', NULL, 0, 5000)`); err != nil {
t.Fatalf("seed orphan: %v", err)
}
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
if err != nil {
t.Fatalf("DueForLatestCheck: %v", err)
}
if len(due) != 0 {
t.Fatalf("due = %v, want neither the URL-less nor the orphan series", due)
}
}