Spec #134, all ten tickets. Closes #134. ## What ships The admin surface becomes four bookmarkable addresses behind one nav row, and Lane observability stops dying with the process. - **#138** `/admin` splits into Overview, Lanes, Readers, Series, each a real route with the active tab underlined. - **#139** `poll_passes` and `poll_lanes` land as durable tables with their store surface. - **#140** cross-Series admin read model, with the privacy boundary in the projection: the Reader id that raised a Latest Chapter never leaves the store package. - **#141** the poller records exactly one pass row per exit, with a skip reason and outcome counts. - **#142** Series list: eight hygiene filters, Site and Library narrowing, paging — all of it in the query string, so a filtered list is a bookmark. - **#143** Overview: a three-state verdict line and a stats block where every non-zero figure links to the list that counts it. - **#144** per-Series detail page, keyed by the `site:series_id` composite the rest of the system already uses. - **#145** the Lanes page reads the database; the in-memory Lane state, `web.LaneReporter` and `latest.Status` are deleted. - **#146** Forced Poll: *Check now* stamps `series.force_poll_at` and never commands the poller. - **#147** pause and resume one Site's Lane, with a mandatory 1h/6h/24h expiry. ## Shape of the design Two decisions carry the rest. **Commands go through the database, never at the poller**: both *Check now* and a Lane pause write a row the next pass reads, so they survive a restart and the whole surface stays testable with no poller running. And **pending is derived, never stored** — the request stamp being newer than the check stamp — which self-clears on the check stamp with no second write and no sweeper, because the check stamp is written before the fetch. ADRs: `docs/adr/0012-persisted-lane-state.md`, `docs/adr/0013-commands-through-the-database.md`. ## Verification `go test ./...` green on the merged base (`264839e`), all packages, Docker-backed. `gofmt -l` and `go vet` clean. Every ticket was reviewed on both axes (`cr-spec` + `cr-standards`) before merge. ## Known, non-blocking - **#143** the verdict ignores never-reported Lanes when other Lanes have reported, and the per-Site table lists Sites that have Series rather than the whole registry. The ticket prose asks for eight hygiene figures per Site; the design mock and the landed `.tbl.sites` grid both say six columns, and the mock won. - **#146** two `SeriesPage` scans per press instead of a keyed read — `ponytail:`-commented in-tree with the upgrade path. - **#147** a paused Site with no pass row yet renders no row and so no control, since the Lanes page lists Sites that have passed. - **#141** a sibling browser Lane declining at the top of a pass records as `sidecar-down`. Specified deliberately; the later spec in this series settles it. Reviewed-on: #148 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #148.
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Series filter names (issue #140), ordered permanent-then-fixable — the
|
||||
// repairs nothing will ever undo first, the ones a Poll can make right after.
|
||||
// A name is the repair a row needs, not the SQL that finds it; the values are
|
||||
// the wire form the Series list URL carries (#142). "all" is the absent and
|
||||
// unknown case: every Series.
|
||||
const (
|
||||
SeriesFilterAll = "all"
|
||||
SeriesFilterNoURL = "no_series_url"
|
||||
SeriesFilterNoChapter = "never_read_a_chapter"
|
||||
SeriesFilterNoReaders = "no_readers"
|
||||
SeriesFilterNeverChecked = "never_checked"
|
||||
SeriesFilterStale = "stale"
|
||||
SeriesFilterNoCover = "no_cover"
|
||||
SeriesFilterReaderReport = "reader_report"
|
||||
)
|
||||
|
||||
// SeriesFilter is one named hygiene predicate over the whole library. Site
|
||||
// and Kind narrow the row read; Name picks the predicate; Cutoff is the
|
||||
// staleness boundary the "stale" filter compares against, supplied by the
|
||||
// caller's clock — the store has no clock; Page is 1-based.
|
||||
type SeriesFilter struct {
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both library buckets' series
|
||||
Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll
|
||||
Cutoff int64 // unix ms; "stale" reads it, the store never does
|
||||
Page int // 1-based page of the row read; default 1
|
||||
}
|
||||
|
||||
// adminSeriesColumns is the owner's library-wide Series projection in
|
||||
// scanAdminSeries order. It is the privacy boundary: a Series' row carries
|
||||
// the Reader id that raised its Latest Chapter (latest_raised_by), and that id
|
||||
// must never leave the store package — so the projection does not select it,
|
||||
// and only the anonymous boolean in raisedByReaderAnswer crosses it.
|
||||
const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at`
|
||||
|
||||
// raisedByReaderAnswer answers "did a Reader's report set this number" without
|
||||
// naming which Reader. Kept apart from adminSeriesColumns so the column list —
|
||||
// the shape scanAdminSeries is fed — stays free of the Sighting-raiser
|
||||
// identity, and the owner learns which rows to act on and nothing about the
|
||||
// Reader behind them.
|
||||
const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader`
|
||||
|
||||
// seriesPageSize is the row read's page length. The tie-break in the query's
|
||||
// ORDER BY is what makes this a stable page boundary — see SeriesPage.
|
||||
const seriesPageSize = 50
|
||||
|
||||
// AdminSeries is one Series as the owner's library-wide view sees it: a
|
||||
// Series-level fact plus an anonymous Reader count. ReaderCount being zero is
|
||||
// the orphan marker. RaisedByReader is the only trace of the Sighting
|
||||
// mechanism here; the Reader id behind it never reaches this type.
|
||||
type AdminSeries struct {
|
||||
Site string
|
||||
SeriesID string
|
||||
Title string
|
||||
SeriesURL string
|
||||
CoverAddress string // "" = no Cover yet
|
||||
Kind string
|
||||
LatestChapter string
|
||||
LatestChapterNum *float64 // nil until first captured
|
||||
LatestCheckedAt int64
|
||||
// ForcePollAt is the owner's "check now" request stamp (issue #146), zero
|
||||
// meaning never asked. Pending is derived, never stored: a request is
|
||||
// pending while ForcePollAt is newer than LatestCheckedAt.
|
||||
ForcePollAt int64
|
||||
ReaderCount int
|
||||
RaisedByReader bool // a Reader's report set LatestChapterNum
|
||||
}
|
||||
|
||||
// SeriesPage is one page of the owner's filtered Series list plus the count
|
||||
// of every Series matching the same filter — a window number, not the page's
|
||||
// len, so the landing page's figure and the list heading come from one query.
|
||||
type SeriesPage struct {
|
||||
Rows []AdminSeries
|
||||
Total int
|
||||
}
|
||||
|
||||
// SiteSeriesShape is one Site's share of the Series matching a filter: how
|
||||
// many, and the manga/novel split. One grouped pass, then library-wide totals
|
||||
// are summed in Go over the rows — the landing page's per-Site table reads
|
||||
// this and never pays for the rows the list discards.
|
||||
type SiteSeriesShape struct {
|
||||
Site string
|
||||
Total int
|
||||
Manga int
|
||||
Novel int
|
||||
}
|
||||
|
||||
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>").
|
||||
func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
|
||||
|
||||
// adminFilter maps a filter's named predicate to its compile-time WHERE and
|
||||
// HAVING clauses and their bound parameters — the name never reaches query
|
||||
// text, and Site and Kind bind as parameters. Shared by the row read and the
|
||||
// per-Site aggregate so the two cannot disagree on what a filter means.
|
||||
//
|
||||
// The WHERE set is: no URL (an empty URL only — the host-failing-the-fetch-
|
||||
// gate case is invisible to SQL, needs the Site registry in Go, and belongs to
|
||||
// a later repair), never-read-a-chapter and never-checked as disjoint halves
|
||||
// (non-zero versus zero check stamp), stale, no cover, and Reader-report.
|
||||
// no_readers is the one HAVING predicate: it is the orphan test, an aggregate
|
||||
// over the LEFT JOIN, where a bare WHERE has no row to test.
|
||||
//
|
||||
// stale is the checked-but-old half of the stamp partition — because the
|
||||
// verdict line wants "not checked in twelve hours" as one figure, and a never
|
||||
// checked Series is already counted on its own "waiting"/never-checked
|
||||
// filter, folding it in would double-report it. The landing page computes the
|
||||
// inclusive number as stale + never-checked.
|
||||
func adminFilter(f SeriesFilter) (where, having string, args []any, err error) {
|
||||
var clauses []string
|
||||
if f.Kind != "" {
|
||||
args = append(args, f.Kind)
|
||||
clauses = append(clauses, "s.kind = $"+strconv.Itoa(len(args)))
|
||||
}
|
||||
switch f.Name {
|
||||
case "", SeriesFilterAll:
|
||||
case SeriesFilterNoURL:
|
||||
clauses = append(clauses, `s.series_url = ''`)
|
||||
case SeriesFilterNoChapter:
|
||||
clauses = append(clauses, `s.latest_checked_at <> 0 AND s.latest_chapter_num IS NULL`)
|
||||
case SeriesFilterNeverChecked:
|
||||
clauses = append(clauses, `s.latest_checked_at = 0`)
|
||||
case SeriesFilterStale:
|
||||
clauses = append(clauses, `s.latest_checked_at > 0 AND s.latest_checked_at < $`+strconv.Itoa(len(args)+1))
|
||||
args = append(args, f.Cutoff)
|
||||
case SeriesFilterNoCover:
|
||||
clauses = append(clauses, `s.cover_address = ''`)
|
||||
case SeriesFilterReaderReport:
|
||||
clauses = append(clauses, `s.latest_raised_by IS NOT NULL`)
|
||||
case SeriesFilterNoReaders:
|
||||
having = `HAVING COUNT(b.reader_id) = 0`
|
||||
default:
|
||||
return "", "", nil, fmt.Errorf("unknown series filter %q", f.Name)
|
||||
}
|
||||
if len(clauses) > 0 {
|
||||
where = "WHERE " + strings.Join(clauses, " AND ")
|
||||
}
|
||||
return where, having, args, nil
|
||||
}
|
||||
|
||||
// SeriesPage returns one page of the Series matching the filter, least
|
||||
// recently checked first. The LEFT JOIN to Bookmarks is what surfaces the
|
||||
// orphans that hygiene has to find — an inner join would hide them, exactly
|
||||
// as the Lane's join does. ReaderCount is a plain count of every Bookmark on
|
||||
// the Series, which knowingly disagrees with the two Lane queries for as long
|
||||
// as the finished lifecycle bucket exists (#140).
|
||||
//
|
||||
// The tie-break is mandatory, not decorative: every unpollable Series shares a
|
||||
// zero check stamp, so ordering on that column alone gives no stable page
|
||||
// boundary and rows would repeat or vanish across pages. (site, series_id) is
|
||||
// the primary key, hence total. The filtered total is a window count in the
|
||||
// same query — window functions run after grouping and before the limit, so
|
||||
// one where-clause cannot disagree with a second copy of itself.
|
||||
func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return SeriesPage{}, err
|
||||
}
|
||||
if f.Page < 1 {
|
||||
f.Page = 1
|
||||
}
|
||||
// Site narrowing is the row read's own; the aggregate must see every Site.
|
||||
if f.Site != "" {
|
||||
args = append(args, f.Site)
|
||||
clause := "s.site = $" + strconv.Itoa(len(args))
|
||||
if where == "" {
|
||||
where = "WHERE " + clause
|
||||
} else {
|
||||
where += " AND " + clause
|
||||
}
|
||||
}
|
||||
base := len(args)
|
||||
args = append(args, seriesPageSize, seriesPageSize*(f.Page-1))
|
||||
rows, err := s.db.Query(`
|
||||
SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`,
|
||||
COUNT(b.reader_id) AS reader_count,
|
||||
COUNT(*) OVER () AS filtered_total
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at, s.latest_raised_by
|
||||
`+having+`
|
||||
ORDER BY s.latest_checked_at, s.site, s.series_id
|
||||
LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("query series page: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := SeriesPage{}
|
||||
for rows.Next() {
|
||||
a, total, err := scanAdminSeries(rows.Scan)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("scan series page: %w", err)
|
||||
}
|
||||
out.Rows = append(out.Rows, a)
|
||||
out.Total = total
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SeriesShapes returns each Site's share of the Series matching the filter,
|
||||
// one grouped pass. Site and Page are row-read concerns and are ignored; the
|
||||
// Landing page reads this per Site and sums the totals in Go for the
|
||||
// library-wide figure.
|
||||
func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE kind = 'manga') AS manga,
|
||||
COUNT(*) FILTER (WHERE kind = 'novel') AS novel
|
||||
FROM (
|
||||
SELECT s.site, s.kind
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.kind
|
||||
`+having+`
|
||||
) shape
|
||||
GROUP BY site
|
||||
ORDER BY site`, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query series shapes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []SiteSeriesShape{}
|
||||
for rows.Next() {
|
||||
var sh SiteSeriesShape
|
||||
if err := rows.Scan(&sh.Site, &sh.Total, &sh.Manga, &sh.Novel); err != nil {
|
||||
return nil, fmt.Errorf("scan series shape: %w", err)
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer
|
||||
// order, plus the query's reader_count and filtered_total columns, and returns
|
||||
// the window total alongside the row. latest_chapter_num is NULL until first
|
||||
// captured — the "never read a chapter" state. The Sighting-raiser column is
|
||||
// never among the scanned columns.
|
||||
func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
|
||||
var (
|
||||
a AdminSeries
|
||||
latestChapterNum sql.NullFloat64
|
||||
total int
|
||||
)
|
||||
if err := scan(
|
||||
&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
|
||||
&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
|
||||
&a.ForcePollAt,
|
||||
&a.RaisedByReader, &a.ReaderCount, &total,
|
||||
); err != nil {
|
||||
return AdminSeries{}, 0, err
|
||||
}
|
||||
if latestChapterNum.Valid {
|
||||
a.LatestChapterNum = &latestChapterNum.Float64
|
||||
}
|
||||
return a, total, nil
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seriesSeed describes one Series (and optionally its bookmarks) to stand up
|
||||
// for an admin filter test. Direct SQL, because the filters separate rows the
|
||||
// Upsert path could not produce together: an orphan has no bookmark, and a
|
||||
// Reader-raised Latest Chapter needs a Sighting the store does not create.
|
||||
type seriesSeed struct {
|
||||
key string
|
||||
kind string
|
||||
url string
|
||||
cover string // cover_address
|
||||
checkedAt int64
|
||||
latestNum *float64
|
||||
bookmarks int // readers that hold it; 0 = orphan
|
||||
raisedBy bool // a Reader's report is attributed as the raiser
|
||||
}
|
||||
|
||||
// seedAdminSeries inserts one series row and its bookmarks (owner first, then
|
||||
// fresh readers), with the exact admin-relevant facts a test needs.
|
||||
func seedAdminSeries(t *testing.T, s *Store, seed seriesSeed) {
|
||||
t.Helper()
|
||||
site, seriesID, ok := strings.Cut(seed.key, ":")
|
||||
if !ok {
|
||||
t.Fatalf("key %q: no ':' separator", seed.key)
|
||||
}
|
||||
if seed.kind == "" {
|
||||
seed.kind = "manga"
|
||||
}
|
||||
var latestChapter any = ""
|
||||
if seed.latestNum != nil {
|
||||
latestChapter = "Chapter " + strconv.FormatFloat(*seed.latestNum, 'f', -1, 64)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO series (site, series_id, title, kind, series_url, cover_address,
|
||||
latest_checked_at, latest_chapter, latest_chapter_num)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||
site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover,
|
||||
seed.checkedAt, latestChapter, seed.latestNum); err != nil {
|
||||
t.Fatalf("seed series %q: %v", seed.key, err)
|
||||
}
|
||||
for i := range seed.bookmarks {
|
||||
var readerID int64 = s.OwnerID()
|
||||
if i > 0 {
|
||||
readerID = secondReader(t, s)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO bookmarks (reader_id, site, series_id,
|
||||
last_chapter, last_chapter_num, last_chapter_url,
|
||||
favorite, status, updated_at)
|
||||
VALUES ($1, $2, $3, '', 0, '', false, 'reading', $4)`,
|
||||
readerID, site, seriesID, seed.checkedAt); err != nil {
|
||||
t.Fatalf("seed bookmark %q: %v", seed.key, err)
|
||||
}
|
||||
}
|
||||
if seed.raisedBy {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET latest_raised_by = $1 WHERE site = $2 AND series_id = $3`,
|
||||
s.OwnerID(), site, seriesID); err != nil {
|
||||
t.Fatalf("seed raised-by %q: %v", seed.key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pageKeys(t *testing.T, s *Store, f SeriesFilter) map[string]bool {
|
||||
t.Helper()
|
||||
page, err := s.SeriesPage(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(%+v): %v", f, err)
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, a := range page.Rows {
|
||||
keys[a.Key()] = true
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Each filter must return the rows it names and no others, over one shared
|
||||
// seeded mix where every healthy neighbour is present to be wrongly returned.
|
||||
// The stale cutoff is 5000: a Series checked at 9000 is current, at 2000 stale.
|
||||
func TestAdminSeriesFilters(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "https://asurascans.com/comics/healthy", cover: "aaa", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nourl", url: "", cover: "bbb", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "https://asurascans.com/comics/nochapter", cover: "ccc", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "https://asurascans.com/comics/neverchecked", cover: "ddd", checkedAt: 0, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:orphan", url: "https://asurascans.com/comics/orphan", cover: "eee", checkedAt: 9000, latestNum: new(7.0), bookmarks: 0})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:stale", url: "https://asurascans.com/comics/stale", cover: "fff", checkedAt: 2000, latestNum: new(4.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nocover", url: "https://asurascans.com/comics/nocover", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:report", url: "https://asurascans.com/comics/report", cover: "ggg", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
f SeriesFilter
|
||||
want []string
|
||||
}{
|
||||
{"all", SeriesFilter{}, []string{"asura:healthy", "asura:nourl", "asura:nochapter", "asura:neverchecked", "asura:orphan", "asura:stale", "asura:nocover", "asura:report"}},
|
||||
{"no series url", SeriesFilter{Name: SeriesFilterNoURL}, []string{"asura:nourl"}},
|
||||
{"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}},
|
||||
{"never checked", SeriesFilter{Name: SeriesFilterNeverChecked}, []string{"asura:neverchecked"}},
|
||||
{"no readers", SeriesFilter{Name: SeriesFilterNoReaders}, []string{"asura:orphan"}},
|
||||
{"stale", SeriesFilter{Name: SeriesFilterStale, Cutoff: 5000}, []string{"asura:stale"}},
|
||||
{"no cover", SeriesFilter{Name: SeriesFilterNoCover}, []string{"asura:nocover"}},
|
||||
{"reader report", SeriesFilter{Name: SeriesFilterReaderReport}, []string{"asura:report"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := pageKeys(t, s, tc.f)
|
||||
want := map[string]bool{}
|
||||
for _, k := range tc.want {
|
||||
want[k] = true
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want)
|
||||
}
|
||||
for k := range want {
|
||||
if !got[k] {
|
||||
t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// "Never read a chapter" and "never checked" are disjoint by construction:
|
||||
// the first requires a non-zero check stamp, the second a zero one. Over a
|
||||
// mix that should satisfy both, no row may be counted twice.
|
||||
func TestAdminNeverChapterAndNeverCheckedAreDisjoint(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
// A zero-stamp, no-chapter row is never-checked only: if never-read-a-
|
||||
// chapter ever lost its non-zero-stamp guard, it would claim this row too
|
||||
// and the two counts would double-report it.
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:both", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter})
|
||||
neverChecked := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNeverChecked})
|
||||
for k := range noChapter {
|
||||
if neverChecked[k] {
|
||||
t.Fatalf("row %q matches both never-read-a-chapter and never-checked", k)
|
||||
}
|
||||
}
|
||||
if !noChapter["asura:nochapter"] || !neverChecked["asura:neverchecked"] {
|
||||
t.Fatalf("disjoint split lost its own rows: no-chapter=%v never-checked=%v", noChapter, neverChecked)
|
||||
}
|
||||
}
|
||||
|
||||
// Several rows share a zero check stamp, so ordering on latest_checked_at
|
||||
// alone gives no stable page boundary. The (site, series_id) tie-break must
|
||||
// make page 2 a strict continuation of page 1: no repeat, no vanishing row.
|
||||
func TestAdminSeriesPageTieBreakIsStable(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const total = 53 // > one page, < two (page size 50)
|
||||
for i := range total {
|
||||
id := "tie" + strconv.Itoa(i)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:" + id, url: "u", checkedAt: 0, bookmarks: 1})
|
||||
}
|
||||
// A second Site's zero-stamp row is part of the same all-filter list, and
|
||||
// must land on a valid page boundary rather than duplicating or dropping
|
||||
// one of asura's rows: the tie-break is global (site, series_id).
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:z", url: "u", checkedAt: 0, bookmarks: 1})
|
||||
wantTotal := total + 1
|
||||
|
||||
p1, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage page 1: %v", err)
|
||||
}
|
||||
p2, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage page 2: %v", err)
|
||||
}
|
||||
if len(p1.Rows) != seriesPageSize {
|
||||
t.Fatalf("page 1 has %d rows, want %d", len(p1.Rows), seriesPageSize)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range append(append([]AdminSeries{}, p1.Rows...), p2.Rows...) {
|
||||
if seen[a.Key()] {
|
||||
t.Fatalf("row %q repeats across pages", a.Key())
|
||||
}
|
||||
seen[a.Key()] = true
|
||||
}
|
||||
if len(seen) != wantTotal {
|
||||
t.Fatalf("%d distinct rows across pages, want %d (a row vanished)", len(seen), wantTotal)
|
||||
}
|
||||
if p1.Total != wantTotal {
|
||||
t.Fatalf("page total = %d, want %d (the window count must span pages)", p1.Total, wantTotal)
|
||||
}
|
||||
// A page beyond the end is empty, not an error (the list re-reads page 1).
|
||||
// The window count runs over the rows present in the result set, so an
|
||||
// overflow page has no rows and therefore no total — the caller must not
|
||||
// render it, which is exactly why the list re-reads page 1.
|
||||
pFinal, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 99})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage beyond end: %v", err)
|
||||
}
|
||||
if len(pFinal.Rows) != 0 {
|
||||
t.Fatalf("page beyond end = %d rows, want 0", len(pFinal.Rows))
|
||||
}
|
||||
}
|
||||
|
||||
// The filtered total is the window number over the same filter the rows use,
|
||||
// and the per-Site aggregate sums to the same figure — so the landing page's
|
||||
// count and the list's heading can never disagree, whoever computes them.
|
||||
func TestAdminTotalAgreesWithRowsAndShapes(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", cover: "a", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:b", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:c", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(2.0), raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:d", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
filters := []SeriesFilter{
|
||||
{},
|
||||
{Name: SeriesFilterNoCover},
|
||||
{Name: SeriesFilterReaderReport},
|
||||
{Name: SeriesFilterNoChapter},
|
||||
}
|
||||
for _, f := range filters {
|
||||
page, err := s.SeriesPage(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(%+v): %v", f, err)
|
||||
}
|
||||
want := len(page.Rows)
|
||||
if f.Page == 0 && want == seriesPageSize {
|
||||
t.Fatalf("seed produced a full page; bump the seed or drop page size in the test")
|
||||
}
|
||||
if page.Total != want {
|
||||
t.Fatalf("%+v total = %d, want %d (window count disagrees with row count)", f, page.Total, want)
|
||||
}
|
||||
shapes, err := s.SeriesShapes(f)
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes(%+v): %v", f, err)
|
||||
}
|
||||
sum := 0
|
||||
for _, sh := range shapes {
|
||||
sum += sh.Total
|
||||
}
|
||||
if sum != want {
|
||||
t.Fatalf("%+v aggregate sum = %d, want %d (aggregate disagrees with row query)", f, sum, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The default filter's aggregate carries the library shape: per-Site
|
||||
// totals and the manga/novel split, summed in Go for library wide.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes default: %v", err)
|
||||
}
|
||||
if len(shapes) != 2 || shapes[0].Site != "asura" || shapes[1].Site != "demonic" {
|
||||
t.Fatalf("shapes = %+v, want asura then demonic", shapes)
|
||||
}
|
||||
if shapes[0].Total != 3 || shapes[0].Manga != 3 || shapes[0].Novel != 0 {
|
||||
t.Fatalf("asura shape = %+v, want 3 manga, 0 novel", shapes[0])
|
||||
}
|
||||
if shapes[1].Total != 1 || shapes[1].Manga != 0 || shapes[1].Novel != 1 {
|
||||
t.Fatalf("demonic shape = %+v, want 1 novel", shapes[1])
|
||||
}
|
||||
}
|
||||
|
||||
// Site and Library narrowing stack on a named filter without changing what
|
||||
// the filter means.
|
||||
func TestAdminFilterSiteAndKindNarrow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:aa", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(1.0)})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:ab", url: "", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:aa", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "demonic:ab", url: "", kind: "novel", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
got := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Site: "asura"})
|
||||
if len(got) != 1 || !got["asura:ab"] {
|
||||
t.Fatalf("site+nourl = %v, want only asura:ab", got)
|
||||
}
|
||||
got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"})
|
||||
if len(got) != 1 || !got["demonic:ab"] {
|
||||
t.Fatalf("kind+nourl = %v, want only demonic:ab", got)
|
||||
}
|
||||
got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterAll, Site: "demonic", Kind: "novel"})
|
||||
if len(got) != 2 || !got["demonic:aa"] || !got["demonic:ab"] {
|
||||
t.Fatalf("site+kind+all = %v, want both demonic rows", got)
|
||||
}
|
||||
|
||||
// The aggregate ignores the Site narrowing (it is per-Site by shape), but
|
||||
// honours the Library narrowing: asura's missing-URL row is manga, so the
|
||||
// novel no-URL list is demonic alone.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes: %v", err)
|
||||
}
|
||||
if len(shapes) != 1 || shapes[0].Site != "demonic" ||
|
||||
shapes[0].Total != 1 || shapes[0].Novel != 1 {
|
||||
t.Fatalf("novel no-URL aggregate = %+v, want demonic {Total:1 Novel:1}", shapes)
|
||||
}
|
||||
}
|
||||
|
||||
// The projection is the privacy boundary: a Series whose Latest Chapter was
|
||||
// raised by a Reader's report reads back with the anonymous boolean set, not
|
||||
// with the Reader's id, and no Reader id travels in any returned row.
|
||||
func TestAdminSeriesReportsAnonymously(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:raised", url: "u", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1, raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1})
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
byKey := map[string]AdminSeries{}
|
||||
for _, a := range page.Rows {
|
||||
byKey[a.Key()] = a
|
||||
}
|
||||
if !byKey["asura:raised"].RaisedByReader {
|
||||
t.Fatal("Reader-raised Series read back RaisedByReader=false")
|
||||
}
|
||||
if byKey["asura:polled"].RaisedByReader {
|
||||
t.Fatal("Poll-raised Series read back RaisedByReader=true")
|
||||
}
|
||||
}
|
||||
|
||||
// The privacy test that cannot rot into a template-only guarantee: assert the
|
||||
// admin column constant does not carry the Sighting-raiser column and that the
|
||||
// admin row type has no field for it, modelled on the guard on the Bookmark
|
||||
// column list.
|
||||
func TestAdminProjectionHidesSightingRaiser(t *testing.T) {
|
||||
if strings.Contains(adminSeriesColumns, "latest_raised_by") {
|
||||
t.Fatal("admin column list carries latest_raised_by: the Sighting-raiser id would reach the owner")
|
||||
}
|
||||
if _, ok := reflect.TypeOf(AdminSeries{}).FieldByName("LatestRaisedBy"); ok {
|
||||
t.Fatal("AdminSeries carries a field for the Sighting-raiser id")
|
||||
}
|
||||
}
|
||||
|
||||
// An unknown filter name is rejected rather than silently meaning "all" —
|
||||
// otherwise a mistyped URL would present an empty page as the whole library.
|
||||
func TestAdminFilterUnknownNameRejected(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
for name, call := range map[string]func() error{
|
||||
"page": func() error { _, err := s.SeriesPage(SeriesFilter{Name: "bogus"}); return err },
|
||||
"shape": func() error { _, err := s.SeriesShapes(SeriesFilter{Name: "bogus"}); return err },
|
||||
} {
|
||||
if err := call(); err == nil || !strings.Contains(err.Error(), "unknown series filter") {
|
||||
t.Fatalf("%s with bogus filter = %v, want unknown-filter error", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ForceSeriesPoll is the idempotent stamp write: a second press overwrites
|
||||
// the request time, and touching a missing series is not an error.
|
||||
func TestForceSeriesPollStampsIdempotently(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 9000, bookmarks: 1})
|
||||
|
||||
if err := s.ForceSeriesPoll("asura", "x", 42); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
if err := s.ForceSeriesPoll("asura", "x", 99); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll re-stamp: %v", err)
|
||||
}
|
||||
// Touching a missing series is not an error: the row may have been
|
||||
// orphaned, and the caller's read decides what exists.
|
||||
if err := s.ForceSeriesPoll("asura", "ghost", 99); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll missing: %v", err)
|
||||
}
|
||||
|
||||
var got int64
|
||||
if err := s.db.QueryRow(
|
||||
`SELECT force_poll_at FROM series WHERE site = 'asura' AND series_id = 'x'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read force_poll_at: %v", err)
|
||||
}
|
||||
if got != 99 {
|
||||
t.Fatalf("force_poll_at = %d, want 99 (the later press wins)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The admin projection carries the force stamp so the web layer can derive
|
||||
// the pending flag without a second read.
|
||||
func TestAdminSeriesCarriesForcePollAt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 1000, bookmarks: 1})
|
||||
if err := s.ForceSeriesPoll("asura", "x", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
if len(page.Rows) != 1 || page.Rows[0].ForcePollAt != 5000 {
|
||||
t.Fatalf("row = %+v, want ForcePollAt 5000", page.Rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
-- One durable state row per Poll Lane. Zero means no pause or refusal is set.
|
||||
CREATE TABLE poll_lanes (
|
||||
site text NOT NULL PRIMARY KEY,
|
||||
paused_until bigint NOT NULL DEFAULT 0,
|
||||
refuse_until bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Append-only Lane Pass log. Timestamps are unix milliseconds from the poller's clock.
|
||||
CREATE TABLE poll_passes (
|
||||
site text NOT NULL,
|
||||
ran_at bigint NOT NULL,
|
||||
skip text NOT NULL,
|
||||
due integer NOT NULL,
|
||||
checked integer NOT NULL,
|
||||
gap_ms bigint NOT NULL,
|
||||
clamped boolean NOT NULL,
|
||||
refused integer NOT NULL,
|
||||
unreachable integer NOT NULL,
|
||||
no_chapter integer NOT NULL,
|
||||
unfetchable integer NOT NULL,
|
||||
errors integer NOT NULL,
|
||||
PRIMARY KEY (site, ran_at)
|
||||
);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Admin read-model foundation (#140). The Series list's default order is
|
||||
-- least-recently-checked first, so the table — which has only its primary key
|
||||
-- today — gets an index that can serve it. A grouped query over a join may
|
||||
-- ignore the index, so this is a judgement, not a measurement: re-time on real
|
||||
-- data before adding a second.
|
||||
CREATE INDEX series_latest_checked_at_idx ON series (latest_checked_at);
|
||||
|
||||
-- force_poll_at is the "ask for one Series to be checked now" stamp (#146).
|
||||
-- Zero means never forced; nothing reads the column before that ticket wires
|
||||
-- it, so it lands here unused.
|
||||
ALTER TABLE series ADD COLUMN force_poll_at bigint NOT NULL DEFAULT 0;
|
||||
+273
-12
@@ -88,6 +88,39 @@ type Series struct {
|
||||
// readerCount is the number of bookmarks referencing this series, filled
|
||||
// only by the due-queue query that orders on it.
|
||||
readerCount int
|
||||
// Forced is whether the owner asked for a check now (issue #146): the
|
||||
// request stamp is newer than the check stamp. Derived in the due query,
|
||||
// never stored, and the flag that jumps the queue and opens the browser
|
||||
// wake gate.
|
||||
Forced bool
|
||||
}
|
||||
|
||||
// LanePass is one Poll Lane's durable pass snapshot. Pause and refusal stamps
|
||||
// are joined from poll_lanes on read; they are not pass facts.
|
||||
type LanePass struct {
|
||||
Site string
|
||||
RanAt int64
|
||||
Skip string
|
||||
Due, Checked int
|
||||
GapMS int64
|
||||
Clamped bool
|
||||
Refused, Unreachable, NoChapter int
|
||||
Unfetchable, Errors int
|
||||
PausedUntil, RefuseUntil int64
|
||||
}
|
||||
|
||||
// SiteOutcomes is one Site's summed Lane Pass outcomes over a caller-supplied
|
||||
// window.
|
||||
type SiteOutcomes struct {
|
||||
Site string
|
||||
Refused, Unreachable, NoChapter int
|
||||
Unfetchable, Errors int
|
||||
}
|
||||
|
||||
// LanePause is one persisted Lane pause stamp.
|
||||
type LanePause struct {
|
||||
Site string
|
||||
PausedUntil int64
|
||||
}
|
||||
|
||||
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>"),
|
||||
@@ -188,9 +221,9 @@ const (
|
||||
//go:embed migrations/*.sql
|
||||
var migrations embed.FS
|
||||
|
||||
// bookmarkColumns is the only value ever concatenated into query text. It is a
|
||||
// compile-time constant; every request value is bound as a parameter. The
|
||||
// series-owned fields are joined in from the series table, in scanBookmark
|
||||
// These column lists are the only values ever concatenated into query text.
|
||||
// They are compile-time constants; every request value is bound as a parameter.
|
||||
// The series-owned fields are joined in from the series table, in scanBookmark
|
||||
// order, so the flat Bookmark reads back whole despite the split (ADR-0004).
|
||||
const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_address,
|
||||
b.last_chapter, b.last_chapter_num, b.last_chapter_url,
|
||||
@@ -202,6 +235,10 @@ const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_add
|
||||
const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by`
|
||||
|
||||
const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped,
|
||||
p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors,
|
||||
COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)`
|
||||
|
||||
// Owner is the person running the service: the first Reader, seeded at startup
|
||||
// so a fresh deployment has a library before anyone logs in. The seed makes
|
||||
// sure exactly one readers row matches their Discord ID, carrying the SHA-256
|
||||
@@ -589,8 +626,9 @@ func (s *Store) scanBookmark(scan func(...any) error) (Bookmark, error) {
|
||||
}
|
||||
|
||||
// scanSeries reads one row in seriesColumns order, plus the due query's
|
||||
// reader_count column. latest_chapter_num and latest_raised_by are both
|
||||
// nullable, same as latest_chapter_num on the bookmark read path.
|
||||
// forced flag and reader_count columns. latest_chapter_num and
|
||||
// latest_raised_by are both nullable, same as latest_chapter_num on the
|
||||
// bookmark read path.
|
||||
func scanSeries(scan func(...any) error) (Series, error) {
|
||||
var (
|
||||
sr Series
|
||||
@@ -600,7 +638,7 @@ func scanSeries(scan func(...any) error) (Series, error) {
|
||||
if err := scan(
|
||||
&sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress,
|
||||
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy,
|
||||
&sr.readerCount,
|
||||
&sr.Forced, &sr.readerCount,
|
||||
); err != nil {
|
||||
return Series{}, err
|
||||
}
|
||||
@@ -613,6 +651,18 @@ func scanSeries(scan func(...any) error) (Series, error) {
|
||||
return sr, nil
|
||||
}
|
||||
|
||||
func scanLanePass(scan func(...any) error) (LanePass, error) {
|
||||
var p LanePass
|
||||
if err := scan(
|
||||
&p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped,
|
||||
&p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors,
|
||||
&p.PausedUntil, &p.RefuseUntil,
|
||||
); err != nil {
|
||||
return LanePass{}, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying database handle.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
@@ -934,6 +984,184 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordLanePass appends one pass and prunes every older row in the same
|
||||
// transaction. retainBefore is supplied by the poller's clock.
|
||||
func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin lane pass %s: %w", p.Site, err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO poll_passes
|
||||
(site, ran_at, skip, due, checked, gap_ms, clamped,
|
||||
refused, unreachable, no_chapter, unfetchable, errors)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
||||
p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped,
|
||||
p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil {
|
||||
return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil {
|
||||
return fmt.Errorf("prune lane passes before %d: %w", retainBefore, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit lane pass %s at %d: %w", p.Site, p.RanAt, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LatestLanePass returns the newest pass for one Site, with its current Lane
|
||||
// state joined on. A Site without a pass has no durable snapshot yet.
|
||||
func (s *Store) LatestLanePass(site string) (LanePass, bool, error) {
|
||||
p, err := scanLanePass(s.db.QueryRow(`SELECT `+lanePassColumns+`
|
||||
FROM poll_passes p
|
||||
LEFT JOIN poll_lanes l ON l.site = p.site
|
||||
WHERE p.site = $1
|
||||
ORDER BY p.ran_at DESC
|
||||
LIMIT 1`, site).Scan)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return LanePass{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return LanePass{}, false, fmt.Errorf("latest lane pass %s: %w", site, err)
|
||||
}
|
||||
return p, true, nil
|
||||
}
|
||||
|
||||
// LatestLanePasses returns the newest pass for each Site, with current Lane
|
||||
// state joined on. Sites without a pass have no row yet.
|
||||
func (s *Store) LatestLanePasses() ([]LanePass, error) {
|
||||
rows, err := s.db.Query(`SELECT ` + lanePassColumns + `
|
||||
FROM (
|
||||
SELECT DISTINCT ON (site)
|
||||
site, ran_at, skip, due, checked, gap_ms, clamped,
|
||||
refused, unreachable, no_chapter, unfetchable, errors
|
||||
FROM poll_passes
|
||||
ORDER BY site, ran_at DESC
|
||||
) p
|
||||
LEFT JOIN poll_lanes l ON l.site = p.site
|
||||
ORDER BY p.site`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query latest lane passes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []LanePass{}
|
||||
for rows.Next() {
|
||||
p, err := scanLanePass(rows.Scan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan latest lane pass: %w", err)
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LanePassOutcomes sums the named outcomes for each Site at or after since.
|
||||
// The window boundary is supplied by the caller; the store has no clock.
|
||||
func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter),
|
||||
SUM(unfetchable), SUM(errors)
|
||||
FROM poll_passes
|
||||
WHERE ran_at >= $1
|
||||
GROUP BY site
|
||||
ORDER BY site`, since)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query lane pass outcomes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []SiteOutcomes{}
|
||||
for rows.Next() {
|
||||
var outcomes SiteOutcomes
|
||||
if err := rows.Scan(
|
||||
&outcomes.Site, &outcomes.Refused, &outcomes.Unreachable,
|
||||
&outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan lane pass outcomes: %w", err)
|
||||
}
|
||||
out = append(out, outcomes)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetLaneRefusal persists a Site's refusal backoff stamp without touching its
|
||||
// pause. until is supplied by the caller's clock.
|
||||
func (s *Store) SetLaneRefusal(site string, until int64) error {
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO poll_lanes (site, refuse_until) VALUES ($1, $2)
|
||||
ON CONFLICT (site) DO UPDATE SET refuse_until = EXCLUDED.refuse_until`, site, until); err != nil {
|
||||
return fmt.Errorf("set lane refusal %s: %w", site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PauseLane persists a bounded pause. The caller must ensure until is after
|
||||
// its current timestamp; the store has no clock and rejects only the invalid
|
||||
// zero and negative sentinels.
|
||||
func (s *Store) PauseLane(site string, until int64) error {
|
||||
if until <= 0 {
|
||||
return fmt.Errorf("pause lane %s: expiry must be positive", site)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO poll_lanes (site, paused_until) VALUES ($1, $2)
|
||||
ON CONFLICT (site) DO UPDATE SET paused_until = EXCLUDED.paused_until`, site, until); err != nil {
|
||||
return fmt.Errorf("pause lane %s: %w", site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResumeLane clears only the pause stamp and keeps the Lane state row, along
|
||||
// with any refusal stamp already persisted on it.
|
||||
func (s *Store) ResumeLane(site string) error {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE poll_lanes SET paused_until = 0 WHERE site = $1`, site); err != nil {
|
||||
return fmt.Errorf("resume lane %s: %w", site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PausedLanes returns Lane rows with a nonzero pause stamp. Expiry comparison
|
||||
// stays with the caller because the store is deliberately clockless.
|
||||
func (s *Store) PausedLanes() ([]LanePause, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site, paused_until
|
||||
FROM poll_lanes
|
||||
WHERE paused_until > 0
|
||||
ORDER BY site`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query paused lanes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []LanePause{}
|
||||
for rows.Next() {
|
||||
var pause LanePause
|
||||
if err := rows.Scan(&pause.Site, &pause.PausedUntil); err != nil {
|
||||
return nil, fmt.Errorf("scan paused lane: %w", err)
|
||||
}
|
||||
out = append(out, pause)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LaneGates reads a Site's pause and refusal stamps in one row read — the
|
||||
// top-of-pass gate the poller uses (issue #141). A missing state row is the
|
||||
// default: unpaused and not refusing.
|
||||
func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err error) {
|
||||
err = s.db.QueryRow(
|
||||
`SELECT paused_until, refuse_until FROM poll_lanes WHERE site = $1`, site).
|
||||
Scan(&pausedUntil, &refuseUntil)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("lane gates %s: %w", site, err)
|
||||
}
|
||||
return pausedUntil, refuseUntil, nil
|
||||
}
|
||||
|
||||
// DueForLatestCheck returns one Site's series whose server-side
|
||||
// latest-chapter check has aged past cutoffMs, ordered by how many bookmarks
|
||||
// reference them (descending) then least-recently-checked first. One Site per
|
||||
@@ -942,6 +1170,13 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
// limit — the Lane's own gap paces the fetches, and the batch size that used
|
||||
// to cap this query is gone with the shared pace.
|
||||
//
|
||||
// A forced Series (force_poll_at newer than latest_checked_at, issue #146)
|
||||
// overrides exactly three gates: the rest cutoff, the Sighting-deferral
|
||||
// clause and the finished-only bucket. It never overrides an empty
|
||||
// series_url or the Bookmarks join — nothing to fetch, and no consumer for
|
||||
// the result — so those stay unconditional. Forced rows sort to the front of
|
||||
// the queue; the reader-count-then-age ordering among the rest is ADR-0003.
|
||||
//
|
||||
// The reader_count ordering is the point of the split (ADR-0003): a series
|
||||
// shared by several readers is fetched once per due cycle, and the popular
|
||||
// ones stay freshest while the long tail absorbs any shortfall. Within one
|
||||
@@ -970,19 +1205,26 @@ func (s *Store) Delete(readerID int64, key string) error {
|
||||
// allowed to defer at all was settled when the Sighting was recorded — see
|
||||
// RecordSighting.
|
||||
func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Series, error) {
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count
|
||||
rows, err := s.db.Query(`SELECT `+seriesColumns+`,
|
||||
(s.force_poll_at > s.latest_checked_at) AS forced,
|
||||
COUNT(*) AS reader_count
|
||||
FROM series s
|
||||
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
WHERE s.site = $1
|
||||
AND s.series_url <> ''
|
||||
AND s.latest_checked_at <= $2::bigint
|
||||
AND (s.latest_checked_at <= $2::bigint
|
||||
OR s.force_poll_at > s.latest_checked_at)
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
|
||||
HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at
|
||||
HAVING (COUNT(*) FILTER (WHERE b.status <> 'finished') > 0
|
||||
OR s.force_poll_at > s.latest_checked_at)
|
||||
AND (COUNT(*) > 1
|
||||
OR s.latest_sighted_at <= $2::bigint
|
||||
OR s.latest_checked_at <= $3::bigint)
|
||||
ORDER BY reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs)
|
||||
OR s.latest_checked_at <= $3::bigint
|
||||
OR s.force_poll_at > s.latest_checked_at)
|
||||
ORDER BY (s.force_poll_at > s.latest_checked_at) DESC,
|
||||
reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query due series: %w", err)
|
||||
}
|
||||
@@ -1042,6 +1284,25 @@ func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ForceSeriesPoll stamps a Series with the owner's "check now" request
|
||||
// (issue #146): a fact about the Series the Lane's next pass reads through
|
||||
// DueForLatestCheck, never a command to the poller — so the request survives
|
||||
// a restart. Writing again overwrites the request time; the write is
|
||||
// idempotent. Touching a missing series is not an error: the row may have
|
||||
// been orphaned, and the caller's read decides what exists. The stamp never
|
||||
// expires by itself — an unanswered request keeps ageing — and pending is
|
||||
// derived as force_poll_at > latest_checked_at, which is why the poller's
|
||||
// check stamp is written before the fetch: the first attempt ends the
|
||||
// pending state whatever it returns.
|
||||
func (s *Store) ForceSeriesPoll(site, seriesID string, at int64) error {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET force_poll_at = $1 WHERE site = $2 AND series_id = $3`,
|
||||
at, site, seriesID); err != nil {
|
||||
return fmt.Errorf("force poll %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LatestCheckedAt reads the column MarkLatestChecked writes. It exists for
|
||||
// tests outside this package (the poller's own tests assert on rest
|
||||
// bookkeeping) — see MarkLatestChecked for why the field stays off the
|
||||
|
||||
@@ -758,7 +758,7 @@ func TestMigration0008DropsLegacyCoverRows(t *testing.T) {
|
||||
func readSeries(t *testing.T, s *Store, site, seriesID string) Series {
|
||||
t.Helper()
|
||||
sr, err := scanSeries(s.db.QueryRow(
|
||||
`SELECT `+seriesColumns+`, 0 AS reader_count FROM series s
|
||||
`SELECT `+seriesColumns+`, false AS forced, 0 AS reader_count FROM series s
|
||||
WHERE s.site = $1 AND s.series_id = $2`, site, seriesID).Scan)
|
||||
if err != nil {
|
||||
t.Fatalf("read series %s:%s: %v", site, seriesID, err)
|
||||
@@ -1584,3 +1584,318 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
|
||||
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
|
||||
}
|
||||
}
|
||||
func TestRecordLanePassPrunesBeforeInsertCutoff(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, pass := range []LanePass{
|
||||
{Site: "asura", RanAt: 99},
|
||||
{Site: "asura", RanAt: 100},
|
||||
} {
|
||||
if err := s.RecordLanePass(pass, 100); err != nil {
|
||||
t.Fatalf("RecordLanePass(%d): %v", pass.RanAt, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 200}, 100); err != nil {
|
||||
t.Fatalf("RecordLanePass(200): %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&count); err != nil {
|
||||
t.Fatalf("count passes: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Fatalf("retained passes = %d, want 2", count)
|
||||
}
|
||||
if _, ok, err := s.LatestLanePass("asura"); err != nil || !ok {
|
||||
t.Fatalf("LatestLanePass = ok %v, err %v; want latest row", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestLanePassesKeepsNewestPerSiteAndJoinsState(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, pass := range []LanePass{
|
||||
{Site: "asura", RanAt: 100, Due: 1},
|
||||
{Site: "asura", RanAt: 200, Skip: "due-query", Due: 2, Checked: 3, GapMS: 4000, Clamped: true},
|
||||
{Site: "demonic", RanAt: 150, Due: 4},
|
||||
} {
|
||||
if err := s.RecordLanePass(pass, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass(%s/%d): %v", pass.Site, pass.RanAt, err)
|
||||
}
|
||||
}
|
||||
if err := s.PauseLane("asura", 1234); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
if err := s.SetLaneRefusal("asura", 5678); err != nil {
|
||||
t.Fatalf("SetLaneRefusal: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.LatestLanePasses()
|
||||
if err != nil {
|
||||
t.Fatalf("LatestLanePasses: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("latest passes = %d, want one per Site", len(got))
|
||||
}
|
||||
bySite := map[string]LanePass{}
|
||||
for _, pass := range got {
|
||||
bySite[pass.Site] = pass
|
||||
}
|
||||
asura := bySite["asura"]
|
||||
if asura.RanAt != 200 || asura.Skip != "due-query" || asura.Due != 2 || asura.Checked != 3 || asura.GapMS != 4000 || !asura.Clamped ||
|
||||
asura.PausedUntil != 1234 || asura.RefuseUntil != 5678 {
|
||||
t.Fatalf("asura latest pass = %+v, want newest pass and joined state", asura)
|
||||
}
|
||||
if demonic := bySite["demonic"]; demonic.RanAt != 150 || demonic.Due != 4 {
|
||||
t.Fatalf("demonic latest pass = %+v, want its only pass", demonic)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLanePassOutcomesSumsWindow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, pass := range []LanePass{
|
||||
{Site: "asura", RanAt: 99, Refused: 1, Unreachable: 2, NoChapter: 3, Unfetchable: 4, Errors: 5},
|
||||
{Site: "asura", RanAt: 100, Refused: 2, Unreachable: 3, NoChapter: 4, Unfetchable: 5, Errors: 6},
|
||||
{Site: "asura", RanAt: 200, Refused: 3, Unreachable: 4, NoChapter: 5, Unfetchable: 6, Errors: 7},
|
||||
{Site: "demonic", RanAt: 150, Refused: 8, Unreachable: 9, NoChapter: 10, Unfetchable: 11, Errors: 12},
|
||||
} {
|
||||
if err := s.RecordLanePass(pass, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass(%s/%d): %v", pass.Site, pass.RanAt, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := s.LanePassOutcomes(100)
|
||||
if err != nil {
|
||||
t.Fatalf("LanePassOutcomes: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("outcome Sites = %d, want 2", len(got))
|
||||
}
|
||||
bySite := map[string]SiteOutcomes{}
|
||||
for _, outcomes := range got {
|
||||
bySite[outcomes.Site] = outcomes
|
||||
}
|
||||
if want := (SiteOutcomes{Site: "asura", Refused: 5, Unreachable: 7, NoChapter: 9, Unfetchable: 11, Errors: 13}); bySite["asura"] != want {
|
||||
t.Fatalf("asura outcomes = %+v, want %+v", bySite["asura"], want)
|
||||
}
|
||||
if want := (SiteOutcomes{Site: "demonic", Refused: 8, Unreachable: 9, NoChapter: 10, Unfetchable: 11, Errors: 12}); bySite["demonic"] != want {
|
||||
t.Fatalf("demonic outcomes = %+v, want %+v", bySite["demonic"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLaneGatesReadsOneRow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
paused, refused, err := s.LaneGates("asura")
|
||||
if err != nil || paused != 0 || refused != 0 {
|
||||
t.Fatalf("LaneGates on a missing state row = (%d, %d, %v), want (0, 0, nil)", paused, refused, err)
|
||||
}
|
||||
if err := s.PauseLane("asura", 2000); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
if err := s.SetLaneRefusal("asura", 3000); err != nil {
|
||||
t.Fatalf("SetLaneRefusal: %v", err)
|
||||
}
|
||||
paused, refused, err = s.LaneGates("asura")
|
||||
if err != nil || paused != 2000 || refused != 3000 {
|
||||
t.Fatalf("LaneGates = (%d, %d, %v), want (2000, 3000, nil)", paused, refused, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLaneStatePauseResumeAndRefusal(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for _, until := range []int64{0, -1} {
|
||||
if err := s.PauseLane("asura", until); err == nil {
|
||||
t.Fatalf("PauseLane(%d) accepted a non-future expiry", until)
|
||||
}
|
||||
}
|
||||
if err := s.PauseLane("asura", 2000); err != nil {
|
||||
t.Fatalf("PauseLane: %v", err)
|
||||
}
|
||||
if err := s.SetLaneRefusal("asura", 3000); err != nil {
|
||||
t.Fatalf("SetLaneRefusal: %v", err)
|
||||
}
|
||||
if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 1}, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass: %v", err)
|
||||
}
|
||||
pausedUntil, refuseUntil, err := s.LaneGates("asura")
|
||||
if err != nil || pausedUntil != 2000 || refuseUntil != 3000 {
|
||||
t.Fatalf("LaneGates = %d/%d, %v; want 2000/3000", pausedUntil, refuseUntil, err)
|
||||
}
|
||||
paused, err := s.PausedLanes()
|
||||
if err != nil {
|
||||
t.Fatalf("PausedLanes: %v", err)
|
||||
}
|
||||
if len(paused) != 1 || paused[0] != (LanePause{Site: "asura", PausedUntil: 2000}) {
|
||||
t.Fatalf("PausedLanes = %+v, want asura/2000", paused)
|
||||
}
|
||||
|
||||
if err := s.ResumeLane("asura"); err != nil {
|
||||
t.Fatalf("ResumeLane: %v", err)
|
||||
}
|
||||
latest, ok, err := s.LatestLanePass("asura")
|
||||
if err != nil || !ok || latest.RefuseUntil != 3000 {
|
||||
t.Fatalf("latest refusal after resume = %+v, ok=%v, err=%v; want 3000 preserved", latest, ok, err)
|
||||
}
|
||||
if got, got2, err := s.LaneGates("asura"); err != nil || got != 0 || got2 != 3000 {
|
||||
t.Fatalf("LaneGates after resume = %d/%d, %v; want 0/3000", got, got2, err)
|
||||
}
|
||||
if paused, err := s.PausedLanes(); err != nil || len(paused) != 0 {
|
||||
t.Fatalf("PausedLanes after resume = %+v, %v; want empty", paused, err)
|
||||
}
|
||||
var rows int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM poll_lanes WHERE site = $1`, "asura").Scan(&rows); err != nil {
|
||||
t.Fatalf("count lane state: %v", err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("lane state rows after resume = %d, want 1", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A forced Series is due ahead of the rest cutoff: the request overrides the
|
||||
// rest gate so the Lane's next pass picks it up however recently it was
|
||||
// checked. An unforced series under the rest stays out.
|
||||
func TestDueForLatestCheckForcedOverridesRestCutoff(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const now = int64(10 * 3600_000)
|
||||
seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", now-30*60_000)
|
||||
seedForCheck(t, s, "asura:fresh", "https://asurascans.com/comics/fresh", now-30*60_000)
|
||||
if err := s.ForceSeriesPoll("asura", "forced", now); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", now-3600_000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 || due[0].Key() != "asura:forced" {
|
||||
t.Fatalf("due = %v, want only the forced series", due)
|
||||
}
|
||||
}
|
||||
|
||||
// The Sighting-deferral clause holds a solitary series back after a recent
|
||||
// Reader report; a forced request overrides it and puts the series back on
|
||||
// the Lane's list.
|
||||
func TestDueForLatestCheckForcedOverridesSightingDeferral(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const now = int64(10 * 3600_000)
|
||||
// One bookmark (so deferral can apply), sighted and checked 10m ago:
|
||||
// inside the deferral window and under the ceiling.
|
||||
seedForCheck(t, s, "asura:deferred", "https://asurascans.com/comics/deferred", now-10*60_000)
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET latest_sighted_at = $1 WHERE site = 'asura' AND series_id = 'deferred'`,
|
||||
now-10*60_000); err != nil {
|
||||
t.Fatalf("seed sighting: %v", err)
|
||||
}
|
||||
|
||||
// Unforced: deferred, and under the rest anyway.
|
||||
due, err := s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 0 {
|
||||
t.Fatalf("unforced deferred series is due: %v", due)
|
||||
}
|
||||
|
||||
// Forced: the request overrides the deferral.
|
||||
if err := s.ForceSeriesPoll("asura", "deferred", now); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 1 || due[0].Key() != "asura:deferred" {
|
||||
t.Fatalf("forced deferred series not due: %v", due)
|
||||
}
|
||||
}
|
||||
|
||||
// The finished-only bucket excludes a series whose only bookmarks are
|
||||
// finished; a forced request overrides it — the owner asked, so the Lane
|
||||
// looks.
|
||||
func TestDueForLatestCheckForcedOverridesFinishedBucket(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:reading", "https://asurascans.com/comics/reading", 0)
|
||||
if _, err := s.Upsert(s.OwnerID(), Bookmark{
|
||||
Key: "asura:finished", Site: "asura", SeriesID: "finished",
|
||||
SeriesURL: "https://asurascans.com/comics/finished",
|
||||
Status: StatusFinished, UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed finished: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
for _, sr := range due {
|
||||
if sr.Key() == "asura:finished" {
|
||||
t.Fatalf("unforced finished series is due: %v", due)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.ForceSeriesPoll("asura", "finished", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, sr := range due {
|
||||
got[sr.Key()] = true
|
||||
}
|
||||
if !got["asura:reading"] || !got["asura:finished"] {
|
||||
t.Fatalf("forced finished series not due: %v", due)
|
||||
}
|
||||
}
|
||||
|
||||
// A forced Series jumps the queue: it sorts ahead of a more-read series that
|
||||
// is due by rest, without disturbing the reader-count-then-age tie-break
|
||||
// among the unforced rows (ADR-0003).
|
||||
func TestDueForLatestCheckForcedSortsFirst(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
// "popular" has two readers and is long overdue; "forced" has one reader
|
||||
// and a fresh check stamp. The forced row must come first.
|
||||
seedForCheck(t, s, "asura:popular", "https://asurascans.com/comics/popular", 100)
|
||||
seedSecondReader(t, s, "asura:popular:2", "asura", "popular", 1001)
|
||||
seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", 900)
|
||||
if err := s.ForceSeriesPoll("asura", "forced", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 2 {
|
||||
t.Fatalf("due = %d rows, want 2", len(due))
|
||||
}
|
||||
if due[0].Key() != "asura:forced" || due[1].Key() != "asura:popular" {
|
||||
t.Fatalf("due order = %q, %q; want forced first, then popular", due[0].Key(), due[1].Key())
|
||||
}
|
||||
}
|
||||
|
||||
// A forced Series with no series URL is still not fetched — nothing to fetch —
|
||||
// and one with no Bookmarks is still excluded by the join. The force flag
|
||||
// opens the three gates it is allowed to, not the whole query.
|
||||
func TestDueForLatestCheckForcedDoesNotOverrideURLOrJoin(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:nourl", "", 0)
|
||||
if err := s.ForceSeriesPoll("asura", "nourl", 5000); err != nil {
|
||||
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO series (site, series_id, title, series_url, cover, kind,
|
||||
latest_chapter, latest_chapter_num, latest_checked_at,
|
||||
force_poll_at)
|
||||
VALUES ('asura', 'orphan', 'Orphan', 'https://asurascans.com/comics/orphan',
|
||||
'', 'manga', '', NULL, 0, 5000)`); err != nil {
|
||||
t.Fatalf("seed orphan: %v", err)
|
||||
}
|
||||
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
if len(due) != 0 {
|
||||
t.Fatalf("due = %v, want neither the URL-less nor the orphan series", due)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user