Two new Series filters: failing, and the unverified Reader subset (#166)

backend/internal/store/admin.go

@@ -24,6 +24,8 @@ const (
 	SeriesFilterNoCover      = "no_cover"
 	SeriesFilterReaderReport = "reader_report"
 	SeriesFilterFinished     = "finished"
+	SeriesFilterFailing      = "failing"
+	SeriesFilterUnverified   = "unverified"
 )
 
 // SeriesFilter is one named filter predicate over the whole library. Site
@@ -54,6 +56,12 @@ const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_
 // Reader behind them.
 const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader`
 
+// failureAnswer is the poll-failures row's answer (issue #165), kept apart
+// from adminSeriesColumns like raisedByReaderAnswer: outcome and failing_since
+// are not Series columns, and the COALESCE keeps the row scannable when the
+// LEFT JOIN finds no failure row.
+const failureAnswer = `COALESCE(f.outcome, ''), COALESCE(f.failing_since, 0)`
+
 // seriesPageSize is the row read's page length. The tie-break in the query's
 // ORDER BY is what makes this a stable page boundary — see SeriesPage.
 const seriesPageSize = 50
@@ -83,6 +91,12 @@ type AdminSeries struct {
 	LatestCorrectedAt int64
 	ReaderCount       int
 	RaisedByReader    bool // a Reader's report set LatestChapterNum
+	// FailureOutcome is the outcome word of the Series' standing failure, ""
+	// when no failure row stands. FailingSince is when the run of failures
+	// began, zero with no row. Both come from the LEFT JOIN, not the Series
+	// row (issue #165: the row's existence is the state).
+	FailureOutcome string
+	FailingSince   int64
 	// FinishedAt is the owner's finish stamp: unix ms, zero while the Series is
 	// not finished — the same shape as the Correction stamp, and its own undo.
 	FinishedAt int64
@@ -119,7 +133,11 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
 // gate case is invisible to SQL, needs the Site registry in Go, and belongs to
 // a later repair), never-read-a-chapter and never-checked as disjoint halves
 // (non-zero versus zero check stamp), stale, no cover, finished (the
-// retirement stamp, read directly), and Reader-report.
+// retirement stamp, read directly), Reader-report, and the failing pair —
+// failing (the failure row exists, a chapter exists, and failing_since is
+// past the cutoff) and unverified (the Reader-attributed subset of failing).
+// failing's chapter IS NOT NULL is the exact complement of never-read-a-
+// chapter's IS NULL half, so the two are disjoint by construction.
 // no_readers is the one HAVING predicate: it is the orphan test, an aggregate
 // over the LEFT JOIN, where a bare WHERE has no row to test.
 //
@@ -127,9 +145,11 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
 // Series (`s.finished_at = 0` in each of the four): the clock-driven one —
 // never-checked, stale, no-chapter, no-cover — keep ticking after the last
 // Poll, so they would report a retired row as a problem no Poll is coming to
-// fix; the three outcome-driven ones — no-URL, no-readers, Reader-report —
+// fix; the outcome-driven ones — no-URL, no-readers, Reader-report, failing,
-// read stored facts that simply stop arriving, so a finished Series needing a
+// unverified — read stored facts that simply stop arriving, so a finished
-// genuine repair still shows up under them.
+// Series needing a genuine repair still shows up under them. The failing pair
+// carries no finished guard for exactly that contrast: a failure row is a
+// stored outcome, not a ticking clock.
 //
 // stale is the checked-but-old half of the stamp partition — because the
 // verdict line wants "not checked in twelve hours" as one figure, and a never
@@ -157,6 +177,12 @@ func adminFilter(f SeriesFilter) (where, having string, args []any, err error) {
 		clauses = append(clauses, `s.cover_address = '' AND s.finished_at = 0`)
 	case SeriesFilterReaderReport:
 		clauses = append(clauses, `s.latest_raised_by IS NOT NULL`)
+	case SeriesFilterFailing:
+		clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1))
+		args = append(args, f.Cutoff)
+	case SeriesFilterUnverified:
+		clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1)+` AND s.latest_raised_by IS NOT NULL`)
+		args = append(args, f.Cutoff)
 	case SeriesFilterFinished:
 		clauses = append(clauses, `s.finished_at > 0`)
 	case SeriesFilterNoReaders:
@@ -204,15 +230,17 @@ func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
 	base := len(args)
 	args = append(args, seriesPageSize, seriesPageSize*(f.Page-1))
 	rows, err := s.db.Query(`
-		SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`,
+		SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, `+failureAnswer+`,
 		       COUNT(b.reader_id) AS reader_count,
 		       COUNT(*) OVER () AS filtered_total
 		FROM series s
 		LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
+		LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id
 		`+where+`
 		GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
 		         s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
-		         s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by
+		         s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by,
+		         f.outcome, f.failing_since
 		`+having+`
 		ORDER BY s.latest_checked_at, s.site, s.series_id
 		LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
@@ -251,6 +279,7 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
 			SELECT s.site, s.kind
 			FROM series s
 			LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
+			LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id
 			`+where+`
 			GROUP BY s.site, s.series_id, s.kind
 			`+having+`
@@ -273,11 +302,11 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
 	return out, rows.Err()
 }
 
-// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer
+// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer +
-// order, plus the query's reader_count and filtered_total columns, and returns
+// failureAnswer order, plus the query's reader_count and filtered_total
-// the window total alongside the row. latest_chapter_num is NULL until first
+// columns, and returns the window total alongside the row. latest_chapter_num
-// captured — the "never read a chapter" state. The Sighting-raiser column is
+// is NULL until first captured — the "never read a chapter" state. The
-// never among the scanned columns.
+// Sighting-raiser column is never among the scanned columns.
 func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
 	var (
 		a                AdminSeries
@@ -288,7 +317,7 @@ func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
 		&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
 		&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
 		&a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt,
-		&a.RaisedByReader, &a.ReaderCount, &total,
+		&a.RaisedByReader, &a.FailureOutcome, &a.FailingSince, &a.ReaderCount, &total,
 	); err != nil {
 		return AdminSeries{}, 0, err
 	}

backend/internal/store/admin_test.go

@@ -527,3 +527,134 @@ func TestAdminSeriesCarriesFinishedAt(t *testing.T) {
 		t.Fatalf("row = %+v, want FinishedAt 5000", page.Rows)
 	}
 }
+
+// The failing pair reads the failure row's age, not the Series row (issue
+// #165): failing requires the joined row, a Latest Chapter, and failing_since
+// past the cutoff — the same constant stale reads; unverified is the
+// Reader-attributed subset of the same test. A failure inside the window is
+// in neither — one bad fetch is not a fault to correct — and a Series that
+// never captured a chapter is never failing, the exact complement of
+// never-read-a-chapter's IS NULL half, so the two filters are disjoint by
+// construction. A finished failing Series still appears: this pair reads
+// stored outcomes, which simply stop arriving, and carries no finished guard.
+func TestAdminFailingAndUnverifiedFilters(t *testing.T) {
+	s := newTestStore(t)
+	const cutoff = 5000
+	seedAdminSeries(t, s, seriesSeed{key: "asura:failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:recent", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", cover: "c", checkedAt: 9000, bookmarks: 1})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", cover: "c", checkedAt: 9000, latestNum: new(7.0), bookmarks: 1})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:fin-failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(6.0), bookmarks: 1})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "u", cover: "c", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1})
+
+	// Failure rows seed the run's start stamp; the cutoff decides the age.
+	for _, f := range []struct {
+		key, word string
+		since     int64
+	}{
+		{"asura:failing", "not_found", 2000},
+		{"asura:recent", "not_found", 9000},
+		{"asura:nochapter", "not_found", 2000},
+		{"asura:reader", "not_found", 2000},
+		{"asura:polled", "errors", 2000},
+		{"asura:fin-failing", "not_found", 2000},
+	} {
+		site, id, _ := strings.Cut(f.key, ":")
+		if err := s.RecordSeriesFailure(site, id, f.word, f.since); err != nil {
+			t.Fatalf("seed failure %s: %v", f.key, err)
+		}
+	}
+	if err := s.SetSeriesFinished("asura", "fin-failing", 1000); err != nil {
+		t.Fatalf("finish asura:fin-failing: %v", err)
+	}
+
+	cases := []struct {
+		name string
+		f    SeriesFilter
+		want []string
+	}{
+		{"failing", SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff}, []string{"asura:failing", "asura:reader", "asura:polled", "asura:fin-failing"}},
+		{"unverified", SeriesFilter{Name: SeriesFilterUnverified, Cutoff: cutoff}, []string{"asura:reader"}},
+		{"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := pageKeys(t, s, tc.f)
+			want := map[string]bool{}
+			for _, k := range tc.want {
+				want[k] = true
+			}
+			if len(got) != len(want) {
+				t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want)
+			}
+			for k := range want {
+				if !got[k] {
+					t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got)
+				}
+			}
+		})
+	}
+
+	// Disjointness: the failing pair and never-read-a-chapter are disjoint by
+	// the chapter column — IS NOT NULL here, IS NULL there — so no row may be
+	// counted under both.
+	failing := pageKeys(t, s, SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff})
+	noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter})
+	for k := range failing {
+		if noChapter[k] {
+			t.Fatalf("row %q matches both failing and never-read-a-chapter", k)
+		}
+	}
+	if failing["asura:nochapter"] {
+		t.Fatal("a Series with no chapter ever captured appears in failing")
+	}
+	if !noChapter["asura:nochapter"] {
+		t.Fatal("the no-chapter Series vanished from never-read-a-chapter")
+	}
+
+	// The aggregates count the same rows the lists do: the landing page's
+	// figures and the select's options come from these two passes.
+	for _, name := range []string{SeriesFilterFailing, SeriesFilterUnverified} {
+		shapes, err := s.SeriesShapes(SeriesFilter{Name: name, Cutoff: cutoff})
+		if err != nil {
+			t.Fatalf("SeriesShapes(%s): %v", name, err)
+		}
+		sum := 0
+		for _, sh := range shapes {
+			sum += sh.Total
+		}
+		want := len(pageKeys(t, s, SeriesFilter{Name: name, Cutoff: cutoff}))
+		if sum != want {
+			t.Fatalf("%s aggregate sum = %d, want %d (aggregate disagrees with row query)", name, sum, want)
+		}
+	}
+}
+
+// The projection carries the failure facts from the LEFT JOIN, not the
+// Series row: a failing Series reads back its outcome word and the stamp its
+// run began at; a Series with no failure row reads empty and zero. Nothing
+// new is projected from the Series row itself.
+func TestAdminFailureProjection(t *testing.T) {
+	s := newTestStore(t)
+	seedAdminSeries(t, s, seriesSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1})
+	seedAdminSeries(t, s, seriesSeed{key: "asura:fine", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1})
+	if err := s.RecordSeriesFailure("asura", "broken", "not_found", 2000); err != nil {
+		t.Fatalf("seed failure row: %v", err)
+	}
+
+	page, err := s.SeriesPage(SeriesFilter{})
+	if err != nil {
+		t.Fatalf("SeriesPage: %v", err)
+	}
+	byKey := map[string]AdminSeries{}
+	for _, a := range page.Rows {
+		byKey[a.Key()] = a
+	}
+	if byKey["asura:broken"].FailureOutcome != "not_found" || byKey["asura:broken"].FailingSince != 2000 {
+		t.Fatalf("broken row = %+v, want FailureOutcome not_found, FailingSince 2000", byKey["asura:broken"])
+	}
+	if byKey["asura:fine"].FailureOutcome != "" || byKey["asura:fine"].FailingSince != 0 {
+		t.Fatalf("fine row = %+v, want empty outcome and zero stamp", byKey["asura:fine"])
+	}
+}

backend/internal/web/admin_series.go

@@ -33,6 +33,8 @@ var seriesFilterLabels = map[string]string{
 	store.SeriesFilterStale:        "Not checked in 12h",
 	store.SeriesFilterNoCover:      "No cover",
 	store.SeriesFilterReaderReport: "Latest from a Reader",
+	store.SeriesFilterFailing:      "Failing over 12h",
+	store.SeriesFilterUnverified:   "Unverified Reader number",
 	store.SeriesFilterFinished:     "Finished",
 }
 
@@ -49,6 +51,8 @@ var seriesFilterOrder = []string{
 	store.SeriesFilterStale,
 	store.SeriesFilterNoCover,
 	store.SeriesFilterReaderReport,
+	store.SeriesFilterFailing,
+	store.SeriesFilterUnverified,
 	store.SeriesFilterFinished,
 }
 

backend/web_test.go

@@ -3065,8 +3065,8 @@ func TestOverviewFinishedFigureRidesLast(t *testing.T) {
 		}
 		labels = append(labels, m[1])
 	}
-	if len(labels) != 8 {
+	if len(labels) != 10 {
-		t.Fatalf("hygiene block renders %d figures, want 8 (seven problems + finished):\n%s", len(labels), body)
+		t.Fatalf("hygiene block renders %d figures, want 10 (nine problems + finished):\n%s", len(labels), body)
 	}
 	if labels[len(labels)-1] != "Finished" {
 		t.Errorf("finished figure does not ride last; hygiene order = %v:\n%s", labels, body)
@@ -4045,3 +4045,49 @@ func TestRemoveRejectsBadKeysAndCapsBody(t *testing.T) {
 		t.Errorf("an oversized body still removed the row:\n%s", list)
 	}
 }
+
+// The failing pair render in the stats block from the same aggregate the
+// select is numbered from: failing over 12h counts every Series with a
+// failure row older than the cutoff, unverified the Reader-attributed
+// subset, and a measured zero stays a muted digit — never a door.
+func TestOverviewFailingAndUnverifiedFigures(t *testing.T) {
+	st, dsn := newTestStoreURL(t)
+	db, err := sql.Open("pgx", dsn)
+	if err != nil {
+		t.Fatalf("open %s: %v", dsn, err)
+	}
+	defer db.Close()
+	now := time.Now().UnixMilli()
+	seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: now, latestNum: floatPtr(1), bookmarks: 1})
+	seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: now, latestNum: floatPtr(2), bookmarks: 1, raisedBy: true})
+	if err := st.RecordSeriesFailure("asura", "broken", "not_found", now-24*3600*1000); err != nil {
+		t.Fatalf("seed failure row on broken: %v", err)
+	}
+	if err := st.RecordSeriesFailure("asura", "reader", "not_found", now-24*3600*1000); err != nil {
+		t.Fatalf("seed failure row on reader: %v", err)
+	}
+	srv := newRouter(st, testConfig())
+	body := overviewBody(t, srv, st)
+
+	if !strings.Contains(body, `href="/admin/series?filter=failing">2</a>`) {
+		t.Errorf("the failing figure lacks its count and door:\n%s", body)
+	}
+	if !strings.Contains(body, `href="/admin/series?filter=unverified">1</a>`) {
+		t.Errorf("the unverified figure lacks its count and door:\n%s", body)
+	}
+
+	empty, emptySt := newWebTestServer(t, testConfig())
+	body = overviewBody(t, empty, emptySt)
+	if !strings.Contains(body, `Failing over 12h</span><span class="fig zero">0</span>`) {
+		t.Errorf("a zero failing figure does not render as a muted digit:\n%s", body)
+	}
+	if !strings.Contains(body, `Unverified Reader number</span><span class="fig zero">0</span>`) {
+		t.Errorf("a zero unverified figure does not render as a muted digit:\n%s", body)
+	}
+	if strings.Contains(body, `href="/admin/series?filter=failing"`) {
+		t.Errorf("a zero failing figure is still a link:\n%s", body)
+	}
+	if strings.Contains(body, `href="/admin/series?filter=unverified"`) {
+		t.Errorf("a zero unverified figure is still a link:\n%s", body)
+	}
+}