From 8a569075582936b563cead65c9dbb4a12ca00840 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:43:47 +0700 Subject: [PATCH] Two new Series filters: failing, and the unverified Reader subset (#166) --- backend/internal/store/admin.go | 53 ++++++++--- backend/internal/store/admin_test.go | 131 +++++++++++++++++++++++++++ backend/internal/web/admin_series.go | 4 + backend/web_test.go | 50 +++++++++- 4 files changed, 224 insertions(+), 14 deletions(-) diff --git a/backend/internal/store/admin.go b/backend/internal/store/admin.go index 0be59de..059216b 100644 --- a/backend/internal/store/admin.go +++ b/backend/internal/store/admin.go @@ -24,6 +24,8 @@ const ( SeriesFilterNoCover = "no_cover" SeriesFilterReaderReport = "reader_report" SeriesFilterFinished = "finished" + SeriesFilterFailing = "failing" + SeriesFilterUnverified = "unverified" ) // SeriesFilter is one named filter predicate over the whole library. Site @@ -54,6 +56,12 @@ const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_ // Reader behind them. const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader` +// failureAnswer is the poll-failures row's answer (issue #165), kept apart +// from adminSeriesColumns like raisedByReaderAnswer: outcome and failing_since +// are not Series columns, and the COALESCE keeps the row scannable when the +// LEFT JOIN finds no failure row. +const failureAnswer = `COALESCE(f.outcome, ''), COALESCE(f.failing_since, 0)` + // seriesPageSize is the row read's page length. The tie-break in the query's // ORDER BY is what makes this a stable page boundary — see SeriesPage. const seriesPageSize = 50 @@ -83,6 +91,12 @@ type AdminSeries struct { LatestCorrectedAt int64 ReaderCount int RaisedByReader bool // a Reader's report set LatestChapterNum + // FailureOutcome is the outcome word of the Series' standing failure, "" + // when no failure row stands. FailingSince is when the run of failures + // began, zero with no row. Both come from the LEFT JOIN, not the Series + // row (issue #165: the row's existence is the state). + FailureOutcome string + FailingSince int64 // FinishedAt is the owner's finish stamp: unix ms, zero while the Series is // not finished — the same shape as the Correction stamp, and its own undo. FinishedAt int64 @@ -119,7 +133,11 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID } // gate case is invisible to SQL, needs the Site registry in Go, and belongs to // a later repair), never-read-a-chapter and never-checked as disjoint halves // (non-zero versus zero check stamp), stale, no cover, finished (the -// retirement stamp, read directly), and Reader-report. +// retirement stamp, read directly), Reader-report, and the failing pair — +// failing (the failure row exists, a chapter exists, and failing_since is +// past the cutoff) and unverified (the Reader-attributed subset of failing). +// failing's chapter IS NOT NULL is the exact complement of never-read-a- +// chapter's IS NULL half, so the two are disjoint by construction. // no_readers is the one HAVING predicate: it is the orphan test, an aggregate // over the LEFT JOIN, where a bare WHERE has no row to test. // @@ -127,9 +145,11 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID } // Series (`s.finished_at = 0` in each of the four): the clock-driven one — // never-checked, stale, no-chapter, no-cover — keep ticking after the last // Poll, so they would report a retired row as a problem no Poll is coming to -// fix; the three outcome-driven ones — no-URL, no-readers, Reader-report — -// read stored facts that simply stop arriving, so a finished Series needing a -// genuine repair still shows up under them. +// fix; the outcome-driven ones — no-URL, no-readers, Reader-report, failing, +// unverified — read stored facts that simply stop arriving, so a finished +// Series needing a genuine repair still shows up under them. The failing pair +// carries no finished guard for exactly that contrast: a failure row is a +// stored outcome, not a ticking clock. // // stale is the checked-but-old half of the stamp partition — because the // verdict line wants "not checked in twelve hours" as one figure, and a never @@ -157,6 +177,12 @@ func adminFilter(f SeriesFilter) (where, having string, args []any, err error) { clauses = append(clauses, `s.cover_address = '' AND s.finished_at = 0`) case SeriesFilterReaderReport: clauses = append(clauses, `s.latest_raised_by IS NOT NULL`) + case SeriesFilterFailing: + clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1)) + args = append(args, f.Cutoff) + case SeriesFilterUnverified: + clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1)+` AND s.latest_raised_by IS NOT NULL`) + args = append(args, f.Cutoff) case SeriesFilterFinished: clauses = append(clauses, `s.finished_at > 0`) case SeriesFilterNoReaders: @@ -204,15 +230,17 @@ func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) { base := len(args) args = append(args, seriesPageSize, seriesPageSize*(f.Page-1)) rows, err := s.db.Query(` - SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, + SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, `+failureAnswer+`, COUNT(b.reader_id) AS reader_count, COUNT(*) OVER () AS filtered_total FROM series s LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id + LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id `+where+` GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address, s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, - s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by + s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by, + f.outcome, f.failing_since `+having+` ORDER BY s.latest_checked_at, s.site, s.series_id LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...) @@ -251,6 +279,7 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) { SELECT s.site, s.kind FROM series s LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id + LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id `+where+` GROUP BY s.site, s.series_id, s.kind `+having+` @@ -273,11 +302,11 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) { return out, rows.Err() } -// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer -// order, plus the query's reader_count and filtered_total columns, and returns -// the window total alongside the row. latest_chapter_num is NULL until first -// captured — the "never read a chapter" state. The Sighting-raiser column is -// never among the scanned columns. +// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer + +// failureAnswer order, plus the query's reader_count and filtered_total +// columns, and returns the window total alongside the row. latest_chapter_num +// is NULL until first captured — the "never read a chapter" state. The +// Sighting-raiser column is never among the scanned columns. func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) { var ( a AdminSeries @@ -288,7 +317,7 @@ func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) { &a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress, &a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt, &a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt, - &a.RaisedByReader, &a.ReaderCount, &total, + &a.RaisedByReader, &a.FailureOutcome, &a.FailingSince, &a.ReaderCount, &total, ); err != nil { return AdminSeries{}, 0, err } diff --git a/backend/internal/store/admin_test.go b/backend/internal/store/admin_test.go index 5f4077f..fd5c28b 100644 --- a/backend/internal/store/admin_test.go +++ b/backend/internal/store/admin_test.go @@ -527,3 +527,134 @@ func TestAdminSeriesCarriesFinishedAt(t *testing.T) { t.Fatalf("row = %+v, want FinishedAt 5000", page.Rows) } } + +// The failing pair reads the failure row's age, not the Series row (issue +// #165): failing requires the joined row, a Latest Chapter, and failing_since +// past the cutoff — the same constant stale reads; unverified is the +// Reader-attributed subset of the same test. A failure inside the window is +// in neither — one bad fetch is not a fault to correct — and a Series that +// never captured a chapter is never failing, the exact complement of +// never-read-a-chapter's IS NULL half, so the two filters are disjoint by +// construction. A finished failing Series still appears: this pair reads +// stored outcomes, which simply stop arriving, and carries no finished guard. +func TestAdminFailingAndUnverifiedFilters(t *testing.T) { + s := newTestStore(t) + const cutoff = 5000 + seedAdminSeries(t, s, seriesSeed{key: "asura:failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:recent", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", cover: "c", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true}) + seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", cover: "c", checkedAt: 9000, latestNum: new(7.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:fin-failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(6.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "u", cover: "c", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1}) + + // Failure rows seed the run's start stamp; the cutoff decides the age. + for _, f := range []struct { + key, word string + since int64 + }{ + {"asura:failing", "not_found", 2000}, + {"asura:recent", "not_found", 9000}, + {"asura:nochapter", "not_found", 2000}, + {"asura:reader", "not_found", 2000}, + {"asura:polled", "errors", 2000}, + {"asura:fin-failing", "not_found", 2000}, + } { + site, id, _ := strings.Cut(f.key, ":") + if err := s.RecordSeriesFailure(site, id, f.word, f.since); err != nil { + t.Fatalf("seed failure %s: %v", f.key, err) + } + } + if err := s.SetSeriesFinished("asura", "fin-failing", 1000); err != nil { + t.Fatalf("finish asura:fin-failing: %v", err) + } + + cases := []struct { + name string + f SeriesFilter + want []string + }{ + {"failing", SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff}, []string{"asura:failing", "asura:reader", "asura:polled", "asura:fin-failing"}}, + {"unverified", SeriesFilter{Name: SeriesFilterUnverified, Cutoff: cutoff}, []string{"asura:reader"}}, + {"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := pageKeys(t, s, tc.f) + want := map[string]bool{} + for _, k := range tc.want { + want[k] = true + } + if len(got) != len(want) { + t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want) + } + for k := range want { + if !got[k] { + t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got) + } + } + }) + } + + // Disjointness: the failing pair and never-read-a-chapter are disjoint by + // the chapter column — IS NOT NULL here, IS NULL there — so no row may be + // counted under both. + failing := pageKeys(t, s, SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff}) + noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter}) + for k := range failing { + if noChapter[k] { + t.Fatalf("row %q matches both failing and never-read-a-chapter", k) + } + } + if failing["asura:nochapter"] { + t.Fatal("a Series with no chapter ever captured appears in failing") + } + if !noChapter["asura:nochapter"] { + t.Fatal("the no-chapter Series vanished from never-read-a-chapter") + } + + // The aggregates count the same rows the lists do: the landing page's + // figures and the select's options come from these two passes. + for _, name := range []string{SeriesFilterFailing, SeriesFilterUnverified} { + shapes, err := s.SeriesShapes(SeriesFilter{Name: name, Cutoff: cutoff}) + if err != nil { + t.Fatalf("SeriesShapes(%s): %v", name, err) + } + sum := 0 + for _, sh := range shapes { + sum += sh.Total + } + want := len(pageKeys(t, s, SeriesFilter{Name: name, Cutoff: cutoff})) + if sum != want { + t.Fatalf("%s aggregate sum = %d, want %d (aggregate disagrees with row query)", name, sum, want) + } + } +} + +// The projection carries the failure facts from the LEFT JOIN, not the +// Series row: a failing Series reads back its outcome word and the stamp its +// run began at; a Series with no failure row reads empty and zero. Nothing +// new is projected from the Series row itself. +func TestAdminFailureProjection(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:fine", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1}) + if err := s.RecordSeriesFailure("asura", "broken", "not_found", 2000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + + page, err := s.SeriesPage(SeriesFilter{}) + if err != nil { + t.Fatalf("SeriesPage: %v", err) + } + byKey := map[string]AdminSeries{} + for _, a := range page.Rows { + byKey[a.Key()] = a + } + if byKey["asura:broken"].FailureOutcome != "not_found" || byKey["asura:broken"].FailingSince != 2000 { + t.Fatalf("broken row = %+v, want FailureOutcome not_found, FailingSince 2000", byKey["asura:broken"]) + } + if byKey["asura:fine"].FailureOutcome != "" || byKey["asura:fine"].FailingSince != 0 { + t.Fatalf("fine row = %+v, want empty outcome and zero stamp", byKey["asura:fine"]) + } +} diff --git a/backend/internal/web/admin_series.go b/backend/internal/web/admin_series.go index 20298ad..32ab04d 100644 --- a/backend/internal/web/admin_series.go +++ b/backend/internal/web/admin_series.go @@ -33,6 +33,8 @@ var seriesFilterLabels = map[string]string{ store.SeriesFilterStale: "Not checked in 12h", store.SeriesFilterNoCover: "No cover", store.SeriesFilterReaderReport: "Latest from a Reader", + store.SeriesFilterFailing: "Failing over 12h", + store.SeriesFilterUnverified: "Unverified Reader number", store.SeriesFilterFinished: "Finished", } @@ -49,6 +51,8 @@ var seriesFilterOrder = []string{ store.SeriesFilterStale, store.SeriesFilterNoCover, store.SeriesFilterReaderReport, + store.SeriesFilterFailing, + store.SeriesFilterUnverified, store.SeriesFilterFinished, } diff --git a/backend/web_test.go b/backend/web_test.go index 6a59c47..08d191a 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -3065,8 +3065,8 @@ func TestOverviewFinishedFigureRidesLast(t *testing.T) { } labels = append(labels, m[1]) } - if len(labels) != 8 { - t.Fatalf("hygiene block renders %d figures, want 8 (seven problems + finished):\n%s", len(labels), body) + if len(labels) != 10 { + t.Fatalf("hygiene block renders %d figures, want 10 (nine problems + finished):\n%s", len(labels), body) } if labels[len(labels)-1] != "Finished" { t.Errorf("finished figure does not ride last; hygiene order = %v:\n%s", labels, body) @@ -4045,3 +4045,49 @@ func TestRemoveRejectsBadKeysAndCapsBody(t *testing.T) { t.Errorf("an oversized body still removed the row:\n%s", list) } } + +// The failing pair render in the stats block from the same aggregate the +// select is numbered from: failing over 12h counts every Series with a +// failure row older than the cutoff, unverified the Reader-attributed +// subset, and a measured zero stays a muted digit — never a door. +func TestOverviewFailingAndUnverifiedFigures(t *testing.T) { + st, dsn := newTestStoreURL(t) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + now := time.Now().UnixMilli() + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: now, latestNum: floatPtr(1), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: now, latestNum: floatPtr(2), bookmarks: 1, raisedBy: true}) + if err := st.RecordSeriesFailure("asura", "broken", "not_found", now-24*3600*1000); err != nil { + t.Fatalf("seed failure row on broken: %v", err) + } + if err := st.RecordSeriesFailure("asura", "reader", "not_found", now-24*3600*1000); err != nil { + t.Fatalf("seed failure row on reader: %v", err) + } + srv := newRouter(st, testConfig()) + body := overviewBody(t, srv, st) + + if !strings.Contains(body, `href="/admin/series?filter=failing">2`) { + t.Errorf("the failing figure lacks its count and door:\n%s", body) + } + if !strings.Contains(body, `href="/admin/series?filter=unverified">1`) { + t.Errorf("the unverified figure lacks its count and door:\n%s", body) + } + + empty, emptySt := newWebTestServer(t, testConfig()) + body = overviewBody(t, empty, emptySt) + if !strings.Contains(body, `Failing over 12h0`) { + t.Errorf("a zero failing figure does not render as a muted digit:\n%s", body) + } + if !strings.Contains(body, `Unverified Reader number0`) { + t.Errorf("a zero unverified figure does not render as a muted digit:\n%s", body) + } + if strings.Contains(body, `href="/admin/series?filter=failing"`) { + t.Errorf("a zero failing figure is still a link:\n%s", body) + } + if strings.Contains(body, `href="/admin/series?filter=unverified"`) { + t.Errorf("a zero unverified figure is still a link:\n%s", body) + } +}