Implements spec #137 (spec 4 of 4 from wayfinder map #114). Closes #137. Tickets: #164, #165, #166, #167, #168, #169, #170, #171, #172, #173 — all closed, landed on this branch. ## Summary - #164/#168: sixth outcome word `not_found`; per-Site completed marker predicate. - #165/#169: `poll_failures` row is the failure state; the pass remembers a Site-reported completion. - #166/#171: two new admin filters (`failing`, `unverified`); outbound owner notification + stall condition. - #167/#170: a failure names itself on the Series page; the completion hint reaches the owner and decides nothing. - #172: the other three fault conditions (no-browser-route, sidecar-down, adapter-broken) feeding the notifier. - #173: the landing verdict line shares the same `latest.FaultsFrom` judgement the notifier uses, so the page and the push cannot disagree. `cd backend && go test ./...` green on the merged branch (8 packages). Reviewed-on: #174 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #174.
This commit is contained in:
@@ -9,32 +9,38 @@ import (
|
||||
|
||||
// Series filter names (issue #140): the seven repair filters are ordered
|
||||
// permanent-then-fixable — the repairs nothing will ever undo first, the
|
||||
// ones a Poll can make right after. SeriesFilterFinished is not part of
|
||||
// that ordering: a finished Series is a deliberate state, not a repair, so
|
||||
// it sits last, informational. A name is the repair a row needs, not the
|
||||
// SQL that finds it; the values are the wire form the Series list URL
|
||||
// carries (#142). "all" is the absent and unknown case: every Series.
|
||||
// ones a Poll can make right after. SeriesFilterFinished and
|
||||
// SeriesFilterSiteCompleted are not part of that ordering: a finished
|
||||
// Series is a deliberate state and a Site-completed one is the Site's own
|
||||
// marker, not repairs, so the pair rides the tail, informational. A name is
|
||||
// the repair a row needs, not the SQL that finds it; the values are the
|
||||
// wire form the Series list URL carries (#142). "all" is the absent and
|
||||
// unknown case: every Series.
|
||||
const (
|
||||
SeriesFilterAll = "all"
|
||||
SeriesFilterNoURL = "no_series_url"
|
||||
SeriesFilterNoChapter = "never_read_a_chapter"
|
||||
SeriesFilterNoReaders = "no_readers"
|
||||
SeriesFilterNeverChecked = "never_checked"
|
||||
SeriesFilterStale = "stale"
|
||||
SeriesFilterNoCover = "no_cover"
|
||||
SeriesFilterReaderReport = "reader_report"
|
||||
SeriesFilterFinished = "finished"
|
||||
SeriesFilterAll = "all"
|
||||
SeriesFilterNoURL = "no_series_url"
|
||||
SeriesFilterNoChapter = "never_read_a_chapter"
|
||||
SeriesFilterNoReaders = "no_readers"
|
||||
SeriesFilterNeverChecked = "never_checked"
|
||||
SeriesFilterStale = "stale"
|
||||
SeriesFilterNoCover = "no_cover"
|
||||
SeriesFilterReaderReport = "reader_report"
|
||||
SeriesFilterFinished = "finished"
|
||||
SeriesFilterSiteCompleted = "site_completed"
|
||||
SeriesFilterFailing = "failing"
|
||||
SeriesFilterUnverified = "unverified"
|
||||
)
|
||||
|
||||
// SeriesFilter is one named filter predicate over the whole library. Site
|
||||
// and Kind narrow the row read; Name picks the predicate; Cutoff is the
|
||||
// staleness boundary the "stale" filter compares against, supplied by the
|
||||
// caller's clock — the store has no clock; Page is 1-based.
|
||||
// staleness boundary the age-based filters — "stale" and the failing pair —
|
||||
// compare against, supplied by the caller's clock — the store has no clock;
|
||||
// Page is 1-based.
|
||||
type SeriesFilter struct {
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both library buckets' series
|
||||
Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll
|
||||
Cutoff int64 // unix ms; "stale" reads it, the store never does
|
||||
Cutoff int64 // unix ms; the age-based filters read it, the store never does
|
||||
Page int // 1-based page of the row read; default 1
|
||||
}
|
||||
|
||||
@@ -45,7 +51,7 @@ type SeriesFilter struct {
|
||||
// and only the anonymous boolean in raisedByReaderAnswer crosses it.
|
||||
const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at,
|
||||
s.latest_corrected_at, s.finished_at`
|
||||
s.latest_corrected_at, s.finished_at, s.site_completed_at`
|
||||
|
||||
// raisedByReaderAnswer answers "did a Reader's report set this number" without
|
||||
// naming which Reader. Kept apart from adminSeriesColumns so the column list —
|
||||
@@ -54,6 +60,12 @@ const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_
|
||||
// Reader behind them.
|
||||
const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader`
|
||||
|
||||
// failureAnswer is the poll-failures row's answer (issue #165), kept apart
|
||||
// from adminSeriesColumns like raisedByReaderAnswer: outcome and failing_since
|
||||
// are not Series columns, and the COALESCE keeps the row scannable when the
|
||||
// LEFT JOIN finds no failure row.
|
||||
const failureAnswer = `COALESCE(f.outcome, ''), COALESCE(f.failing_since, 0)`
|
||||
|
||||
// seriesPageSize is the row read's page length. The tie-break in the query's
|
||||
// ORDER BY is what makes this a stable page boundary — see SeriesPage.
|
||||
const seriesPageSize = 50
|
||||
@@ -83,9 +95,19 @@ type AdminSeries struct {
|
||||
LatestCorrectedAt int64
|
||||
ReaderCount int
|
||||
RaisedByReader bool // a Reader's report set LatestChapterNum
|
||||
// FailureOutcome is the outcome word of the Series' standing failure, ""
|
||||
// when no failure row stands. FailingSince is when the run of failures
|
||||
// began, zero with no row. Both come from the LEFT JOIN, not the Series
|
||||
// row (issue #165: the row's existence is the state).
|
||||
FailureOutcome string
|
||||
FailingSince int64
|
||||
// FinishedAt is the owner's finish stamp: unix ms, zero while the Series is
|
||||
// not finished — the same shape as the Correction stamp, and its own undo.
|
||||
FinishedAt int64
|
||||
// SiteCompletedAt is when the last successful Poll read saw the Site's own
|
||||
// completed value, zero meaning it did not (issue #168). The Site's marker,
|
||||
// never a Lifecycle decision: the owner's Finish is the only retirement.
|
||||
SiteCompletedAt int64
|
||||
}
|
||||
|
||||
// SeriesPage is one page of the owner's filtered Series list plus the count
|
||||
@@ -119,17 +141,26 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
|
||||
// gate case is invisible to SQL, needs the Site registry in Go, and belongs to
|
||||
// a later repair), never-read-a-chapter and never-checked as disjoint halves
|
||||
// (non-zero versus zero check stamp), stale, no cover, finished (the
|
||||
// retirement stamp, read directly), and Reader-report.
|
||||
// retirement stamp, read directly), site completed (the Site's marker, read
|
||||
// directly), Reader-report, and the failing pair — failing (the failure row
|
||||
// exists, a chapter exists, and failing_since is past the cutoff) and
|
||||
// unverified (the Reader-attributed subset of failing).
|
||||
// failing's chapter IS NOT NULL is the exact complement of never-read-a-
|
||||
// chapter's IS NULL half, so the two are disjoint by construction.
|
||||
// no_readers is the one HAVING predicate: it is the orphan test, an aggregate
|
||||
// over the LEFT JOIN, where a bare WHERE has no row to test.
|
||||
//
|
||||
// The clock-versus-outcome split decides which predicates exclude finished
|
||||
// Series (`s.finished_at = 0` in each of the four): the clock-driven one —
|
||||
// Series (`s.finished_at = 0` in each of the five): the clock-driven ones —
|
||||
// never-checked, stale, no-chapter, no-cover — keep ticking after the last
|
||||
// Poll, so they would report a retired row as a problem no Poll is coming to
|
||||
// fix; the three outcome-driven ones — no-URL, no-readers, Reader-report —
|
||||
// read stored facts that simply stop arriving, so a finished Series needing a
|
||||
// genuine repair still shows up under them.
|
||||
// fix; site-completed's stamp is the Site's, and it keeps standing after the
|
||||
// owner retires the row, so a finished Series would be reported as work
|
||||
// nobody is going to do. The outcome-driven ones — no-URL, no-readers,
|
||||
// Reader-report, failing, unverified — read stored facts that simply stop
|
||||
// arriving, so a finished Series needing a genuine repair still shows up
|
||||
// under them. The failing pair carries no finished guard for exactly that
|
||||
// contrast: a failure row is a stored outcome, not a ticking clock.
|
||||
//
|
||||
// stale is the checked-but-old half of the stamp partition — because the
|
||||
// verdict line wants "not checked in twelve hours" as one figure, and a never
|
||||
@@ -157,8 +188,16 @@ func adminFilter(f SeriesFilter) (where, having string, args []any, err error) {
|
||||
clauses = append(clauses, `s.cover_address = '' AND s.finished_at = 0`)
|
||||
case SeriesFilterReaderReport:
|
||||
clauses = append(clauses, `s.latest_raised_by IS NOT NULL`)
|
||||
case SeriesFilterFailing:
|
||||
clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1))
|
||||
args = append(args, f.Cutoff)
|
||||
case SeriesFilterUnverified:
|
||||
clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1)+` AND s.latest_raised_by IS NOT NULL`)
|
||||
args = append(args, f.Cutoff)
|
||||
case SeriesFilterFinished:
|
||||
clauses = append(clauses, `s.finished_at > 0`)
|
||||
case SeriesFilterSiteCompleted:
|
||||
clauses = append(clauses, `s.site_completed_at > 0 AND s.finished_at = 0`)
|
||||
case SeriesFilterNoReaders:
|
||||
having = `HAVING COUNT(b.reader_id) = 0`
|
||||
default:
|
||||
@@ -204,15 +243,18 @@ func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
|
||||
base := len(args)
|
||||
args = append(args, seriesPageSize, seriesPageSize*(f.Page-1))
|
||||
rows, err := s.db.Query(`
|
||||
SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`,
|
||||
SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, `+failureAnswer+`,
|
||||
COUNT(b.reader_id) AS reader_count,
|
||||
COUNT(*) OVER () AS filtered_total
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by
|
||||
s.force_poll_at, s.latest_corrected_at, s.finished_at, s.site_completed_at,
|
||||
s.latest_raised_by,
|
||||
f.outcome, f.failing_since
|
||||
`+having+`
|
||||
ORDER BY s.latest_checked_at, s.site, s.series_id
|
||||
LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
|
||||
@@ -251,6 +293,7 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
|
||||
SELECT s.site, s.kind
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.kind
|
||||
`+having+`
|
||||
@@ -273,11 +316,11 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer
|
||||
// order, plus the query's reader_count and filtered_total columns, and returns
|
||||
// the window total alongside the row. latest_chapter_num is NULL until first
|
||||
// captured — the "never read a chapter" state. The Sighting-raiser column is
|
||||
// never among the scanned columns.
|
||||
// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer +
|
||||
// failureAnswer order, plus the query's reader_count and filtered_total
|
||||
// columns, and returns the window total alongside the row. latest_chapter_num
|
||||
// is NULL until first captured — the "never read a chapter" state. The
|
||||
// Sighting-raiser column is never among the scanned columns.
|
||||
func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
|
||||
var (
|
||||
a AdminSeries
|
||||
@@ -287,8 +330,8 @@ func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
|
||||
if err := scan(
|
||||
&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
|
||||
&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
|
||||
&a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt,
|
||||
&a.RaisedByReader, &a.ReaderCount, &total,
|
||||
&a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt, &a.SiteCompletedAt,
|
||||
&a.RaisedByReader, &a.FailureOutcome, &a.FailingSince, &a.ReaderCount, &total,
|
||||
); err != nil {
|
||||
return AdminSeries{}, 0, err
|
||||
}
|
||||
|
||||
@@ -12,14 +12,15 @@ import (
|
||||
// Upsert path could not produce together: an orphan has no bookmark, and a
|
||||
// Reader-raised Latest Chapter needs a Sighting the store does not create.
|
||||
type seriesSeed struct {
|
||||
key string
|
||||
kind string
|
||||
url string
|
||||
cover string // cover_address
|
||||
checkedAt int64
|
||||
latestNum *float64
|
||||
bookmarks int // readers that hold it; 0 = orphan
|
||||
raisedBy bool // a Reader's report is attributed as the raiser
|
||||
key string
|
||||
kind string
|
||||
url string
|
||||
cover string // cover_address
|
||||
checkedAt int64
|
||||
latestNum *float64
|
||||
bookmarks int // readers that hold it; 0 = orphan
|
||||
raisedBy bool // a Reader's report is attributed as the raiser
|
||||
siteCompletedAt int64 // the Site's own marker (issue #168), 0 = not set
|
||||
}
|
||||
|
||||
// seedAdminSeries inserts one series row and its bookmarks (owner first, then
|
||||
@@ -39,10 +40,10 @@ func seedAdminSeries(t *testing.T, s *Store, seed seriesSeed) {
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO series (site, series_id, title, kind, series_url, cover_address,
|
||||
latest_checked_at, latest_chapter, latest_chapter_num)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||
latest_checked_at, latest_chapter, latest_chapter_num, site_completed_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
|
||||
site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover,
|
||||
seed.checkedAt, latestChapter, seed.latestNum); err != nil {
|
||||
seed.checkedAt, latestChapter, seed.latestNum, seed.siteCompletedAt); err != nil {
|
||||
t.Fatalf("seed series %q: %v", seed.key, err)
|
||||
}
|
||||
for i := range seed.bookmarks {
|
||||
@@ -527,3 +528,209 @@ func TestAdminSeriesCarriesFinishedAt(t *testing.T) {
|
||||
t.Fatalf("row = %+v, want FinishedAt 5000", page.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The failing pair reads the failure row's age, not the Series row (issue
|
||||
// #165): failing requires the joined row, a Latest Chapter, and failing_since
|
||||
// past the cutoff — the same constant stale reads; unverified is the
|
||||
// Reader-attributed subset of the same test. A failure inside the window is
|
||||
// in neither — one bad fetch is not a fault to correct — and a Series that
|
||||
// never captured a chapter is never failing, the exact complement of
|
||||
// never-read-a-chapter's IS NULL half, so the two filters are disjoint by
|
||||
// construction. A finished failing Series still appears: this pair reads
|
||||
// stored outcomes, which simply stop arriving, and carries no finished guard.
|
||||
func TestAdminFailingAndUnverifiedFilters(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const cutoff = 5000
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:recent", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", cover: "c", checkedAt: 9000, bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", cover: "c", checkedAt: 9000, latestNum: new(7.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:fin-failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(6.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "u", cover: "c", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1})
|
||||
|
||||
// Failure rows seed the run's start stamp; the cutoff decides the age.
|
||||
for _, f := range []struct {
|
||||
key, word string
|
||||
since int64
|
||||
}{
|
||||
{"asura:failing", "not_found", 2000},
|
||||
{"asura:recent", "not_found", 9000},
|
||||
{"asura:nochapter", "not_found", 2000},
|
||||
{"asura:reader", "not_found", 2000},
|
||||
{"asura:polled", "errors", 2000},
|
||||
{"asura:fin-failing", "not_found", 2000},
|
||||
} {
|
||||
site, id, _ := strings.Cut(f.key, ":")
|
||||
if err := s.RecordSeriesFailure(site, id, f.word, f.since); err != nil {
|
||||
t.Fatalf("seed failure %s: %v", f.key, err)
|
||||
}
|
||||
}
|
||||
if err := s.SetSeriesFinished("asura", "fin-failing", 1000); err != nil {
|
||||
t.Fatalf("finish asura:fin-failing: %v", err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
f SeriesFilter
|
||||
want []string
|
||||
}{
|
||||
{"failing", SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff}, []string{"asura:failing", "asura:reader", "asura:polled", "asura:fin-failing"}},
|
||||
{"unverified", SeriesFilter{Name: SeriesFilterUnverified, Cutoff: cutoff}, []string{"asura:reader"}},
|
||||
{"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := pageKeys(t, s, tc.f)
|
||||
want := map[string]bool{}
|
||||
for _, k := range tc.want {
|
||||
want[k] = true
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want)
|
||||
}
|
||||
for k := range want {
|
||||
if !got[k] {
|
||||
t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Disjointness: the failing pair and never-read-a-chapter are disjoint by
|
||||
// the chapter column — IS NOT NULL here, IS NULL there — so no row may be
|
||||
// counted under both.
|
||||
failing := pageKeys(t, s, SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff})
|
||||
noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter})
|
||||
for k := range failing {
|
||||
if noChapter[k] {
|
||||
t.Fatalf("row %q matches both failing and never-read-a-chapter", k)
|
||||
}
|
||||
}
|
||||
if failing["asura:nochapter"] {
|
||||
t.Fatal("a Series with no chapter ever captured appears in failing")
|
||||
}
|
||||
if !noChapter["asura:nochapter"] {
|
||||
t.Fatal("the no-chapter Series vanished from never-read-a-chapter")
|
||||
}
|
||||
|
||||
// The aggregates count the same rows the lists do: the landing page's
|
||||
// figures and the select's options come from these two passes.
|
||||
for _, name := range []string{SeriesFilterFailing, SeriesFilterUnverified} {
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{Name: name, Cutoff: cutoff})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes(%s): %v", name, err)
|
||||
}
|
||||
sum := 0
|
||||
for _, sh := range shapes {
|
||||
sum += sh.Total
|
||||
}
|
||||
want := len(pageKeys(t, s, SeriesFilter{Name: name, Cutoff: cutoff}))
|
||||
if sum != want {
|
||||
t.Fatalf("%s aggregate sum = %d, want %d (aggregate disagrees with row query)", name, sum, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The projection carries the failure facts from the LEFT JOIN, not the
|
||||
// Series row: a failing Series reads back its outcome word and the stamp its
|
||||
// run began at; a Series with no failure row reads empty and zero. Nothing
|
||||
// new is projected from the Series row itself.
|
||||
func TestAdminFailureProjection(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:fine", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1})
|
||||
if err := s.RecordSeriesFailure("asura", "broken", "not_found", 2000); err != nil {
|
||||
t.Fatalf("seed failure row: %v", err)
|
||||
}
|
||||
|
||||
page, err := s.SeriesPage(SeriesFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage: %v", err)
|
||||
}
|
||||
byKey := map[string]AdminSeries{}
|
||||
for _, a := range page.Rows {
|
||||
byKey[a.Key()] = a
|
||||
}
|
||||
if byKey["asura:broken"].FailureOutcome != "not_found" || byKey["asura:broken"].FailingSince != 2000 {
|
||||
t.Fatalf("broken row = %+v, want FailureOutcome not_found, FailingSince 2000", byKey["asura:broken"])
|
||||
}
|
||||
if byKey["asura:fine"].FailureOutcome != "" || byKey["asura:fine"].FailingSince != 0 {
|
||||
t.Fatalf("fine row = %+v, want empty outcome and zero stamp", byKey["asura:fine"])
|
||||
}
|
||||
}
|
||||
|
||||
// The site-completed filter (issue #170) lists the Site's own marker as work
|
||||
// to work through, carrying the same finished guard the clock-driven
|
||||
// predicates gained: the Site's stamp keeps standing after the owner retires
|
||||
// the row, so a finished Series would be reported as work nobody is going to
|
||||
// do. A zero stamp never appears. Un-finishing puts the Series back in the
|
||||
// Poll query — the finished_at gate is #157's own, asserted through
|
||||
// DueForLatestCheck rather than re-derived here.
|
||||
func TestAdminSiteCompletedFilter(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:done", url: "u", cover: "c", checkedAt: 9000, latestNum: new(4.0), bookmarks: 1, siteCompletedAt: 5000})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:never", url: "u", cover: "c", checkedAt: 9000, latestNum: new(4.0), bookmarks: 1})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:retired", url: "u", cover: "c", checkedAt: 0, latestNum: new(4.0), bookmarks: 1, siteCompletedAt: 5000})
|
||||
seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "u", cover: "c", checkedAt: 9000, latestNum: new(4.0), bookmarks: 1})
|
||||
if err := s.SetSeriesFinished("asura", "retired", 1000); err != nil {
|
||||
t.Fatalf("finish asura:retired: %v", err)
|
||||
}
|
||||
|
||||
got := pageKeys(t, s, SeriesFilter{Name: SeriesFilterSiteCompleted})
|
||||
if len(got) != 1 || !got["asura:done"] {
|
||||
t.Fatalf("site-completed returned %v, want only asura:done", got)
|
||||
}
|
||||
|
||||
// The projection carries the stamp so the detail page can age it.
|
||||
page, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterSiteCompleted})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesPage(site_completed): %v", err)
|
||||
}
|
||||
if len(page.Rows) != 1 || page.Rows[0].SiteCompletedAt != 5000 {
|
||||
t.Fatalf("row = %+v, want SiteCompletedAt 5000", page.Rows)
|
||||
}
|
||||
|
||||
// The aggregate counts the same row: the landing figure and the select's
|
||||
// option come from this pass, so they cannot disagree with the list.
|
||||
shapes, err := s.SeriesShapes(SeriesFilter{Name: SeriesFilterSiteCompleted})
|
||||
if err != nil {
|
||||
t.Fatalf("SeriesShapes(site_completed): %v", err)
|
||||
}
|
||||
sum := 0
|
||||
for _, sh := range shapes {
|
||||
sum += sh.Total
|
||||
}
|
||||
if sum != 1 {
|
||||
t.Fatalf("site-completed aggregate = %d, want 1", sum)
|
||||
}
|
||||
|
||||
// Un-finishing puts the Series back in the Poll query: the due read's
|
||||
// finished_at gate (issue #157) admits it again — asserted through the
|
||||
// Lane's own read, not re-derived here.
|
||||
due, err := s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck: %v", err)
|
||||
}
|
||||
for _, sr := range due {
|
||||
if sr.Key() == "asura:retired" {
|
||||
t.Fatalf("a finished Series is still due for a Poll:\n%+v", due)
|
||||
}
|
||||
}
|
||||
if err := s.SetSeriesFinished("asura", "retired", 0); err != nil {
|
||||
t.Fatalf("un-finish asura:retired: %v", err)
|
||||
}
|
||||
due, err = s.DueForLatestCheck("asura", 1000, noCeiling)
|
||||
if err != nil {
|
||||
t.Fatalf("DueForLatestCheck after un-finish: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, sr := range due {
|
||||
if sr.Key() == "asura:retired" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("un-finished Series is not due for a Poll:\n%+v", due)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- A 4xx other than the 403 refusal is the page being gone, not the Site being
|
||||
-- unwell; the pass log counts it separately so the Lanes page can say "not
|
||||
-- found" (#164). DEFAULT 0 keeps pre-existing rows readable.
|
||||
ALTER TABLE poll_passes ADD COLUMN not_found integer NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,13 @@
|
||||
-- One row per Series that is failing right now (ADR-0016): the row's
|
||||
-- existence is the failure state, failing_since ages the run of failures,
|
||||
-- and a correct read deletes the row. Keyed by the same (site, series_id)
|
||||
-- composite the rest of the system uses, with the cascade so deleting a
|
||||
-- Series takes its failure row and orphan removal stays a single statement.
|
||||
CREATE TABLE poll_failures (
|
||||
site text NOT NULL,
|
||||
series_id text NOT NULL,
|
||||
outcome text NOT NULL,
|
||||
failing_since bigint NOT NULL,
|
||||
PRIMARY KEY (site, series_id),
|
||||
FOREIGN KEY (site, series_id) REFERENCES series (site, series_id) ON DELETE CASCADE
|
||||
);
|
||||
@@ -0,0 +1,4 @@
|
||||
-- When the last successful Poll read saw the Site's own completed value
|
||||
-- (#168): epoch-ms, zero meaning it did not. DEFAULT 0 keeps pre-existing
|
||||
-- rows readable.
|
||||
ALTER TABLE series ADD COLUMN site_completed_at bigint NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Owner notices (issue #171): one row per episode, remembered only as "the
|
||||
-- owner was told". The row's presence is the whole state — the poller checks
|
||||
-- it before sending, writes it after a successful send, and clears it when
|
||||
-- the condition no longer holds. site is '' for a fault that is not one
|
||||
-- Site's; the composite primary key is what makes the row a lock against a
|
||||
-- second message for the same episode.
|
||||
CREATE TABLE owner_notices (
|
||||
condition text NOT NULL,
|
||||
site text NOT NULL,
|
||||
notified_at bigint NOT NULL,
|
||||
PRIMARY KEY (condition, site)
|
||||
);
|
||||
+195
-13
@@ -84,6 +84,10 @@ type Series struct {
|
||||
LatestChapter string
|
||||
LatestChapterNum *float64 // nil until first captured
|
||||
LatestCheckedAt int64 // unix ms; see MarkLatestChecked
|
||||
// SiteCompletedAt is when the last successful Poll read saw the Site's
|
||||
// own completed value, zero meaning it did not (issue #168). A poller
|
||||
// fact, not reading progress: Upsert never touches it.
|
||||
SiteCompletedAt int64
|
||||
// LatestRaisedBy is the Reader whose Sighting last raised LatestChapter,
|
||||
// and nil when the stored value is a Poll's own finding. It is what lets a
|
||||
// Poll that contradicts the value downwards name a Reader instead of
|
||||
@@ -110,7 +114,7 @@ type LanePass struct {
|
||||
GapMS int64
|
||||
Clamped bool
|
||||
Refused, Unreachable, NoChapter int
|
||||
Unfetchable, Errors int
|
||||
Unfetchable, Errors, NotFound int
|
||||
PausedUntil, RefuseUntil int64
|
||||
}
|
||||
|
||||
@@ -119,7 +123,7 @@ type LanePass struct {
|
||||
type SiteOutcomes struct {
|
||||
Site string
|
||||
Refused, Unreachable, NoChapter int
|
||||
Unfetchable, Errors int
|
||||
Unfetchable, Errors, NotFound int
|
||||
}
|
||||
|
||||
// LanePause is one persisted Lane pause stamp.
|
||||
@@ -240,10 +244,11 @@ const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_add
|
||||
// due query. latest_checked_at lives only on series — see MarkLatestChecked
|
||||
// for why it stays off every client-visible write.
|
||||
const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by`
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by,
|
||||
s.site_completed_at`
|
||||
|
||||
const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped,
|
||||
p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors,
|
||||
p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, p.not_found,
|
||||
COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)`
|
||||
|
||||
// Owner is the person running the service: the first Reader, seeded at startup
|
||||
@@ -646,7 +651,7 @@ func scanSeries(scan func(...any) error) (Series, error) {
|
||||
if err := scan(
|
||||
&sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress,
|
||||
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy,
|
||||
&sr.Forced, &sr.readerCount,
|
||||
&sr.SiteCompletedAt, &sr.Forced, &sr.readerCount,
|
||||
); err != nil {
|
||||
return Series{}, err
|
||||
}
|
||||
@@ -663,7 +668,7 @@ func scanLanePass(scan func(...any) error) (LanePass, error) {
|
||||
var p LanePass
|
||||
if err := scan(
|
||||
&p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped,
|
||||
&p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors,
|
||||
&p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, &p.NotFound,
|
||||
&p.PausedUntil, &p.RefuseUntil,
|
||||
); err != nil {
|
||||
return LanePass{}, err
|
||||
@@ -1131,10 +1136,10 @@ func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error {
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO poll_passes
|
||||
(site, ran_at, skip, due, checked, gap_ms, clamped,
|
||||
refused, unreachable, no_chapter, unfetchable, errors)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
||||
refused, unreachable, no_chapter, unfetchable, errors, not_found)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`,
|
||||
p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped,
|
||||
p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil {
|
||||
p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors, p.NotFound); err != nil {
|
||||
return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil {
|
||||
@@ -1171,7 +1176,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) {
|
||||
FROM (
|
||||
SELECT DISTINCT ON (site)
|
||||
site, ran_at, skip, due, checked, gap_ms, clamped,
|
||||
refused, unreachable, no_chapter, unfetchable, errors
|
||||
refused, unreachable, no_chapter, unfetchable, errors, not_found
|
||||
FROM poll_passes
|
||||
ORDER BY site, ran_at DESC
|
||||
) p
|
||||
@@ -1198,7 +1203,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) {
|
||||
func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter),
|
||||
SUM(unfetchable), SUM(errors)
|
||||
SUM(unfetchable), SUM(errors), SUM(not_found)
|
||||
FROM poll_passes
|
||||
WHERE ran_at >= $1
|
||||
GROUP BY site
|
||||
@@ -1213,7 +1218,7 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
|
||||
var outcomes SiteOutcomes
|
||||
if err := rows.Scan(
|
||||
&outcomes.Site, &outcomes.Refused, &outcomes.Unreachable,
|
||||
&outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors,
|
||||
&outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, &outcomes.NotFound,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan lane pass outcomes: %w", err)
|
||||
}
|
||||
@@ -1222,6 +1227,101 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RefusingSince reports, per Site, when that Site's current unbroken run of
|
||||
// refusing passes began: a pass refuses when the Lane gate returned early
|
||||
// (skip = 'refusing') or the pass loop counted refusals (refused > 0), so a
|
||||
// Site parked in refusal backoff keeps its run unbroken. A Site whose
|
||||
// latest pass did not refuse is absent. The run is bounded by the pass
|
||||
// log's retention — a run older than the log answers with the oldest row
|
||||
// present — and nothing after now counts: the cutoff is the caller's clock.
|
||||
func (s *Store) RefusingSince(now int64) (map[string]int64, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site, MIN(ran_at)
|
||||
FROM poll_passes p
|
||||
WHERE ran_at <= $1
|
||||
AND (refused > 0 OR skip = 'refusing')
|
||||
AND ran_at > COALESCE((
|
||||
SELECT MAX(q.ran_at) FROM poll_passes q
|
||||
WHERE q.site = p.site AND q.ran_at <= $1
|
||||
AND NOT (q.refused > 0 OR q.skip = 'refusing')
|
||||
), 0)
|
||||
GROUP BY site`, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query refusing since: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]int64{}
|
||||
for rows.Next() {
|
||||
var site string
|
||||
var since int64
|
||||
if err := rows.Scan(&site, &since); err != nil {
|
||||
return nil, fmt.Errorf("scan refusing since: %w", err)
|
||||
}
|
||||
out[site] = since
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SidecarOK reports, per Site in sites, the unix ms of that Site's most
|
||||
// recent pass that actually reached the sidecar: the pass ran its loop
|
||||
// (skip = '') and no Series read lost Chrome (unreachable = 0) — a
|
||||
// challenge answer still reached the sidecar, a lost sidecar did not. A
|
||||
// Site with no such pass is absent: an asleep Lane never reached it, so it
|
||||
// is absent too and cannot age into a sidecar-down alarm by itself.
|
||||
func (s *Store) SidecarOK(sites []string) (map[string]int64, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT DISTINCT ON (site) site, ran_at
|
||||
FROM poll_passes
|
||||
WHERE site = ANY($1) AND skip = '' AND unreachable = 0
|
||||
ORDER BY site, ran_at DESC`, sites)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query sidecar ok: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]int64{}
|
||||
for rows.Next() {
|
||||
var site string
|
||||
var ranAt int64
|
||||
if err := rows.Scan(&site, &ranAt); err != nil {
|
||||
return nil, fmt.Errorf("scan sidecar ok: %w", err)
|
||||
}
|
||||
out[site] = ranAt
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// NoChapterShare reports, per Site, the share of that Site's Series holding
|
||||
// a no-chapter failure row older than cutoff: old rows over the Site's
|
||||
// whole Series count, so a four-Series Site and a 200-Series Site are
|
||||
// judged on the same scale. A Site with no Series has no share and is
|
||||
// absent.
|
||||
func (s *Store) NoChapterShare(cutoff int64) (map[string]float64, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT s.site,
|
||||
(COUNT(*) FILTER (WHERE f.outcome = 'no_chapter' AND f.failing_since < $1))::float8
|
||||
/ (COUNT(*)::float8)
|
||||
FROM series s
|
||||
LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id
|
||||
GROUP BY s.site
|
||||
ORDER BY s.site`, cutoff)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query no-chapter share: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]float64{}
|
||||
for rows.Next() {
|
||||
var site string
|
||||
var share float64
|
||||
if err := rows.Scan(&site, &share); err != nil {
|
||||
return nil, fmt.Errorf("scan no-chapter share: %w", err)
|
||||
}
|
||||
out[site] = share
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
// SetLaneRefusal persists a Site's refusal backoff stamp without touching its
|
||||
// pause. until is supplied by the caller's clock.
|
||||
func (s *Store) SetLaneRefusal(site string, until int64) error {
|
||||
@@ -1298,6 +1398,72 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro
|
||||
return pausedUntil, refuseUntil, nil
|
||||
}
|
||||
|
||||
// RecordSeriesFailure upserts one Series' failure row: the outcome word is
|
||||
// updated when it changes, failing_since is never overwritten, and an
|
||||
// unchanged word writes nothing. One statement, so the identical-word no-op
|
||||
// and the keep-the-stamp rule are the same guarantee: the SET arm fires only
|
||||
// when the word differs, and failing_since is simply absent from it
|
||||
// (ADR-0016).
|
||||
func (s *Store) RecordSeriesFailure(site, seriesID, outcome string, now int64) error {
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO poll_failures (site, series_id, outcome, failing_since) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome
|
||||
WHERE poll_failures.outcome <> EXCLUDED.outcome`, site, seriesID, outcome, now); err != nil {
|
||||
return fmt.Errorf("record series failure %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearSeriesFailure deletes one Series' failure row. A row that is not there
|
||||
// is not an error.
|
||||
func (s *Store) ClearSeriesFailure(site, seriesID string) error {
|
||||
if _, err := s.db.Exec(
|
||||
`DELETE FROM poll_failures WHERE site = $1 AND series_id = $2`, site, seriesID); err != nil {
|
||||
return fmt.Errorf("clear series failure %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NoticeSent reports whether the owner has already been told about this
|
||||
// episode (issue #171). The row's presence is the whole state, so a missing
|
||||
// row reads false without error.
|
||||
func (s *Store) NoticeSent(condition, site string) (bool, error) {
|
||||
var at int64
|
||||
err := s.db.QueryRow(
|
||||
`SELECT notified_at FROM owner_notices WHERE condition = $1 AND site = $2`,
|
||||
condition, site).Scan(&at)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("notice sent %s/%s: %w", condition, site, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// MarkNoticeSent records that the owner was told. Called only after a
|
||||
// successful send. Re-marking the same episode just moves the stamp: the row
|
||||
// is a lock against a second message, not a log.
|
||||
func (s *Store) MarkNoticeSent(condition, site string, at int64) error {
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO owner_notices (condition, site, notified_at) VALUES ($1, $2, $3)
|
||||
ON CONFLICT (condition, site) DO UPDATE SET notified_at = EXCLUDED.notified_at`,
|
||||
condition, site, at); err != nil {
|
||||
return fmt.Errorf("mark notice sent %s/%s: %w", condition, site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearNotice forgets an episode, so the condition's next occurrence sends
|
||||
// again. A row that is not there is not an error.
|
||||
func (s *Store) ClearNotice(condition, site string) error {
|
||||
if _, err := s.db.Exec(
|
||||
`DELETE FROM owner_notices WHERE condition = $1 AND site = $2`, condition, site); err != nil {
|
||||
return fmt.Errorf("clear notice %s/%s: %w", condition, site, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DueForLatestCheck returns one Site's series whose server-side
|
||||
// latest-chapter check has aged past cutoffMs, ordered by how many bookmarks
|
||||
// reference them (descending) then least-recently-checked first. One Site per
|
||||
@@ -1356,7 +1522,7 @@ func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Ser
|
||||
AND (s.finished_at = 0 OR s.force_poll_at > s.latest_checked_at)
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at, s.finished_at
|
||||
s.site_completed_at, s.force_poll_at, s.finished_at
|
||||
HAVING (COUNT(*) > 1
|
||||
OR s.latest_sighted_at <= $2::bigint
|
||||
OR s.latest_checked_at <= $3::bigint
|
||||
@@ -1491,6 +1657,22 @@ func (s *Store) SetLatestChapter(site, seriesID, label string, num float64) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSiteCompletedAt stores when the last successful read saw the Site's
|
||||
// completed value, zero meaning it did not. Series-level, like the Latest
|
||||
// Chapter: it is a fact about the work, not about one Reader's bookmark, and
|
||||
// the bookmark's updated_at is never touched — this is not reading progress
|
||||
// and must not reorder any Reader's list, the same rule SetLatestChapter's
|
||||
// comment states. Touching a missing series is not an error: the row may
|
||||
// have been orphaned, and the caller's read decides what exists.
|
||||
func (s *Store) SetSiteCompletedAt(site, seriesID string, at int64) error {
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE series SET site_completed_at = $1 WHERE site = $2 AND series_id = $3`,
|
||||
at, site, seriesID); err != nil {
|
||||
return fmt.Errorf("set site completed %s:%s: %w", site, seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSeriesURL stores the owner's repair for a Series' source address
|
||||
// (issue #151): the one write that lifts the write-once rule documented on
|
||||
// Series.SeriesURL. It is a store, not a verification — the caller has
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -2611,3 +2612,288 @@ func TestRemoveSeriesMissingKeyIsCleanNoOp(t *testing.T) {
|
||||
t.Fatalf("RemoveSeries on a missing key = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
// The failure row's existence is the failure state (ADR-0016):
|
||||
// RecordSeriesFailure upserts the outcome word, keeps the original
|
||||
// failing_since through a word change, and an unchanged word writes nothing
|
||||
// at all — a broken Series costs the same as a healthy one every hour.
|
||||
func TestRecordSeriesFailureUpsertKeepsFailingSince(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0)
|
||||
|
||||
if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil {
|
||||
t.Fatalf("record: %v", err)
|
||||
}
|
||||
word, since, found := failureRow(t, s, "asura", "solo")
|
||||
if !found || word != "not_found" || since != 1000 {
|
||||
t.Fatalf("row after create = (%q, %d, %v), want (not_found, 1000, true)", word, since, found)
|
||||
}
|
||||
|
||||
// A changed word updates the outcome and never touches failing_since.
|
||||
if err := s.RecordSeriesFailure("asura", "solo", "errors", 2000); err != nil {
|
||||
t.Fatalf("record changed word: %v", err)
|
||||
}
|
||||
word, since, found = failureRow(t, s, "asura", "solo")
|
||||
if !found || word != "errors" || since != 1000 {
|
||||
t.Fatalf("row after word change = (%q, %d, %v), want (errors, 1000, true)", word, since, found)
|
||||
}
|
||||
|
||||
// An identical word writes nothing: the stamp survives a later now.
|
||||
if err := s.RecordSeriesFailure("asura", "solo", "errors", 3000); err != nil {
|
||||
t.Fatalf("record identical word: %v", err)
|
||||
}
|
||||
word, since, found = failureRow(t, s, "asura", "solo")
|
||||
if !found || word != "errors" || since != 1000 {
|
||||
t.Fatalf("row after identical word = (%q, %d, %v), want the earlier (errors, 1000, true)", word, since, found)
|
||||
}
|
||||
}
|
||||
|
||||
// ClearSeriesFailure deletes the row, and a row that is not there is not an
|
||||
// error — the poller clears on every successful read, so most clears find
|
||||
// nothing.
|
||||
func TestClearSeriesFailure(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0)
|
||||
|
||||
if err := s.ClearSeriesFailure("asura", "solo"); err != nil {
|
||||
t.Fatalf("clear a missing row: %v, want nil", err)
|
||||
}
|
||||
if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil {
|
||||
t.Fatalf("record: %v", err)
|
||||
}
|
||||
if err := s.ClearSeriesFailure("asura", "solo"); err != nil {
|
||||
t.Fatalf("clear: %v", err)
|
||||
}
|
||||
if _, _, found := failureRow(t, s, "asura", "solo"); found {
|
||||
t.Fatal("row after clear: still present")
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting a Series takes its failure row with it (the composite FK's
|
||||
// cascade), so orphan removal stays a single statement.
|
||||
func TestRemoveSeriesCascadesToFailureRow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0)
|
||||
if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil {
|
||||
t.Fatalf("record: %v", err)
|
||||
}
|
||||
if err := s.Delete(s.OwnerID(), "asura:solo"); err != nil {
|
||||
t.Fatalf("delete bookmark: %v", err)
|
||||
}
|
||||
if err := s.RemoveSeries("asura", "solo"); err != nil {
|
||||
t.Fatalf("RemoveSeries: %v", err)
|
||||
}
|
||||
if _, _, found := failureRow(t, s, "asura", "solo"); found {
|
||||
t.Fatal("failure row after series delete: still present (no cascade)")
|
||||
}
|
||||
}
|
||||
|
||||
// failureRow reads one Series' failure row as stored.
|
||||
func failureRow(t *testing.T, s *Store, site, seriesID string) (word string, since int64, found bool) {
|
||||
t.Helper()
|
||||
err := s.db.QueryRow(
|
||||
`SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`,
|
||||
site, seriesID).Scan(&word, &since)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", 0, false
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read failure row %s:%s: %v", site, seriesID, err)
|
||||
}
|
||||
return word, since, true
|
||||
}
|
||||
|
||||
// Owner notices are one row per episode: the row's presence is the whole
|
||||
// state. MarkNoticeSent stamps it, NoticeSent reads it, and ClearNotice
|
||||
// forgets it — including a row that was never there, which is not an error.
|
||||
func TestOwnerNoticeRows(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
if sent, err := s.NoticeSent("stall", "comix"); err != nil || sent {
|
||||
t.Fatalf("NoticeSent on a fresh table = %v, %v; want false, nil", sent, err)
|
||||
}
|
||||
|
||||
if err := s.MarkNoticeSent("stall", "comix", 4242); err != nil {
|
||||
t.Fatalf("MarkNoticeSent: %v", err)
|
||||
}
|
||||
if sent, err := s.NoticeSent("stall", "comix"); err != nil || !sent {
|
||||
t.Fatalf("NoticeSent after mark = %v, %v; want true, nil", sent, err)
|
||||
}
|
||||
|
||||
// Re-marking the same episode just moves the stamp: the row is a lock,
|
||||
// not a log. A different site is its own episode — the key is
|
||||
// (condition, site).
|
||||
if err := s.MarkNoticeSent("stall", "comix", 4343); err != nil {
|
||||
t.Fatalf("re-mark: %v", err)
|
||||
}
|
||||
if sent, err := s.NoticeSent("stall", "kagane"); err != nil || sent {
|
||||
t.Fatalf("NoticeSent on another site = %v, %v; want false, nil", sent, err)
|
||||
}
|
||||
|
||||
if err := s.ClearNotice("stall", "comix"); err != nil {
|
||||
t.Fatalf("ClearNotice: %v", err)
|
||||
}
|
||||
if sent, err := s.NoticeSent("stall", "comix"); err != nil || sent {
|
||||
t.Fatalf("NoticeSent after clear = %v, %v; want false, nil", sent, err)
|
||||
}
|
||||
if err := s.ClearNotice("stall", "comix"); err != nil {
|
||||
t.Fatalf("ClearNotice on a missing row: %v", err)
|
||||
}
|
||||
}
|
||||
// RefusingSince reads one Site's current unbroken run of refusing passes
|
||||
// (issue #172). A refusing pass is one the Lane gate returned early from
|
||||
// (skip 'refusing') or one whose loop counted refusals (refused > 0) — the
|
||||
// two shapes a persistently-challenged Site alternates between, so the run
|
||||
// must not break when the gate row follows the refusal row. A Site whose
|
||||
// latest pass did not refuse is absent, nothing after the caller's clock
|
||||
// counts, and a run older than the log answers with the oldest row present.
|
||||
func TestRefusingSince(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seed := func(site string, ranAt int64, skip string, refused int) {
|
||||
t.Helper()
|
||||
if err := s.RecordLanePass(LanePass{Site: site, RanAt: ranAt, Skip: skip, Refused: refused}, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass(%s/%d): %v", site, ranAt, err)
|
||||
}
|
||||
}
|
||||
// asura: a refusing run broken by a healthy pass, then resumed; the
|
||||
// current run began at the pass after the break.
|
||||
seed("asura", 100, "", 2)
|
||||
seed("asura", 200, "", 0)
|
||||
seed("asura", 300, "refusing", 0)
|
||||
seed("asura", 400, "", 2)
|
||||
// demonic: the latest pass is healthy — no run, absent.
|
||||
seed("demonic", 100, "", 2)
|
||||
seed("demonic", 200, "", 0)
|
||||
// novelfull: a healthy pass after now must not break the run — the run
|
||||
// is judged as of the caller's clock.
|
||||
seed("novelfull", 100, "", 2)
|
||||
seed("novelfull", 600, "", 0)
|
||||
// comix: an unbroken run reaches back to the oldest row present.
|
||||
seed("comix", 100, "", 2)
|
||||
seed("comix", 200, "refusing", 0)
|
||||
|
||||
got, err := s.RefusingSince(450)
|
||||
if err != nil {
|
||||
t.Fatalf("RefusingSince: %v", err)
|
||||
}
|
||||
want := map[string]int64{"asura": 300, "novelfull": 100, "comix": 100}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("RefusingSince = %v, want %v", got, want)
|
||||
}
|
||||
for site, since := range want {
|
||||
if got[site] != since {
|
||||
t.Fatalf("RefusingSince[%s] = %d, want %d (got %v)", site, got[site], since, got)
|
||||
}
|
||||
}
|
||||
if _, ok := got["demonic"]; ok {
|
||||
t.Fatalf("RefusingSince names demonic, whose latest pass did not refuse: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// SidecarOK reads, per Site, the most recent pass that actually reached the
|
||||
// sidecar (issue #172): the pass ran its loop (skip '') and no read lost
|
||||
// Chrome (unreachable 0). A challenge answer still reached the sidecar, a
|
||||
// lost sidecar and every skip value did not, and a Site with no qualifying
|
||||
// pass — an asleep Lane included — is absent.
|
||||
func TestSidecarOK(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seed := func(site string, ranAt int64, skip string, unreachable int) {
|
||||
t.Helper()
|
||||
if err := s.RecordLanePass(LanePass{Site: site, RanAt: ranAt, Skip: skip, Unreachable: unreachable}, -1); err != nil {
|
||||
t.Fatalf("RecordLanePass(%s/%d): %v", site, ranAt, err)
|
||||
}
|
||||
}
|
||||
// comix: only the skip='' unreachable=0 pass reached the sidecar; the
|
||||
// mid-loop browser-loss row (skip '', unreachable > 0) and the skip
|
||||
// values never did.
|
||||
seed("comix", 100, "", 1)
|
||||
seed("comix", 200, "sidecar-down", 0)
|
||||
seed("comix", 300, "", 0)
|
||||
seed("comix", 400, "refusing", 0)
|
||||
// kagane: two passes reached the sidecar; the newest wins.
|
||||
seed("kagane", 150, "", 0)
|
||||
seed("kagane", 250, "", 0)
|
||||
// novelfull: an asleep Lane never reached it.
|
||||
seed("novelfull", 120, "asleep", 0)
|
||||
|
||||
got, err := s.SidecarOK([]string{"comix", "kagane", "novelfull", "lightnovelworld"})
|
||||
if err != nil {
|
||||
t.Fatalf("SidecarOK: %v", err)
|
||||
}
|
||||
want := map[string]int64{"comix": 300, "kagane": 250}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("SidecarOK = %v, want %v", got, want)
|
||||
}
|
||||
for site, ranAt := range want {
|
||||
if got[site] != ranAt {
|
||||
t.Fatalf("SidecarOK[%s] = %d, want %d (got %v)", site, got[site], ranAt, got)
|
||||
}
|
||||
}
|
||||
for _, site := range []string{"novelfull", "lightnovelworld"} {
|
||||
if _, ok := got[site]; ok {
|
||||
t.Fatalf("SidecarOK names %s, which never reached the sidecar: %v", site, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NoChapterShare reads, per Site, the share of its Series holding a
|
||||
// no-chapter failure row older than the cutoff (issue #172): old rows over
|
||||
// the Site's whole Series count, so a four-Series Site and a 200-Series
|
||||
// Site are judged on the same scale. A fresh no-chapter row and a row of
|
||||
// any other outcome do not count.
|
||||
func TestNoChapterShare(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
for i := 1; i <= 4; i++ {
|
||||
seedForCheck(t, s, fmt.Sprintf("asura:a%d", i), fmt.Sprintf("https://asurascans.com/comics/a%d", i), 0)
|
||||
}
|
||||
seedForCheck(t, s, "comix:c1", "https://comix.to/title/c1", 0)
|
||||
seedForCheck(t, s, "comix:c2", "https://comix.to/title/c2", 0)
|
||||
seedForCheck(t, s, "demonic:d1", "https://demonicscans.org/series/d1", 0)
|
||||
for i := 1; i <= 200; i++ {
|
||||
seedForCheck(t, s, fmt.Sprintf("novelfull:n%d", i), fmt.Sprintf("https://novelfull.com/n%d.html", i), 0)
|
||||
}
|
||||
|
||||
const cutoff = 1000
|
||||
oldNoChapter := func(site, seriesID string) {
|
||||
t.Helper()
|
||||
if err := s.RecordSeriesFailure(site, seriesID, "no_chapter", 100); err != nil {
|
||||
t.Fatalf("seed no-chapter failure %s:%s: %v", site, seriesID, err)
|
||||
}
|
||||
}
|
||||
// asura: three of four hold an old no-chapter row; the fourth's
|
||||
// no-chapter row is fresh — it is not old, so it does not count.
|
||||
oldNoChapter("asura", "a1")
|
||||
oldNoChapter("asura", "a2")
|
||||
oldNoChapter("asura", "a3")
|
||||
if err := s.RecordSeriesFailure("asura", "a4", "no_chapter", 2000); err != nil {
|
||||
t.Fatalf("seed fresh no-chapter failure: %v", err)
|
||||
}
|
||||
// comix: an old no-chapter row and an old errors row — the errors row
|
||||
// is not a no-chapter row, so the share is 1/2, the exact boundary.
|
||||
oldNoChapter("comix", "c1")
|
||||
if err := s.RecordSeriesFailure("comix", "c2", "errors", 100); err != nil {
|
||||
t.Fatalf("seed errors failure: %v", err)
|
||||
}
|
||||
// demonic and novelfull: one old no-chapter row each, against sites of
|
||||
// one and two hundred Series.
|
||||
oldNoChapter("demonic", "d1")
|
||||
oldNoChapter("novelfull", "n1")
|
||||
|
||||
got, err := s.NoChapterShare(cutoff)
|
||||
if err != nil {
|
||||
t.Fatalf("NoChapterShare: %v", err)
|
||||
}
|
||||
checks := []struct {
|
||||
site string
|
||||
want float64
|
||||
}{
|
||||
{"asura", 3.0 / 4.0},
|
||||
{"comix", 1.0 / 2.0},
|
||||
{"demonic", 1.0 / 1.0},
|
||||
{"novelfull", 1.0 / 200.0},
|
||||
}
|
||||
for _, c := range checks {
|
||||
if got[c.site] != c.want {
|
||||
t.Fatalf("NoChapterShare[%s] = %v, want %v", c.site, got[c.site], c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user