From 417f809d7511046f8809126ef82f00c10d9ab33d Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:22:57 +0700 Subject: [PATCH] Record poll failures as rows: the row is the failure state (#165) --- backend/internal/latest/poller.go | 41 ++++ backend/internal/latest/poller_test.go | 215 ++++++++++++++++++ .../store/migrations/0018_poll_failures.sql | 13 ++ backend/internal/store/store.go | 25 ++ backend/internal/store/store_test.go | 89 ++++++++ docs/adr/0016-the-failure-row-is-the-state.md | 147 ++++++++++++ 6 files changed, 530 insertions(+) create mode 100644 backend/internal/store/migrations/0018_poll_failures.sql create mode 100644 docs/adr/0016-the-failure-row-is-the-state.md diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index e5125c8..d2d2422 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -287,6 +287,24 @@ const ( outcomeError outcomeNotFound ) +// word returns the wire spelling this outcome stores in poll_failures — the +// same strings the pass log's columns use (C1, issue #164). Success, refusal +// and browser loss return "" so recordFailure's "no statement" case is one +// return: a challenge or a lost sidecar is no evidence about any particular +// Series (ADR-0016). +func (o readOutcome) word() string { + switch o { + case outcomeNotFound: + return "not_found" + case outcomeNoChapter: + return "no_chapter" + case outcomeUnfetchable: + return "unfetchable" + case outcomeError: + return "errors" + } + return "" +} // outcomeCounts are the six named outcome counts of one pass. A success // count is derived, never stored: checked minus the five named failures, @@ -456,6 +474,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. } } outcome := p.checkOne(ctx, sr) + p.recordFailure(sr, outcome) if outcome == outcomeUnreachable { // The mid-loop browser loss writes an empty skip on purpose: the // pass returns before the checked counter increments, so its row @@ -613,6 +632,28 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D } return oldest } +// recordFailure keeps one Series' failure row in step with its read +// (ADR-0016): a failure word is upserted, a successful read deletes the row, +// and refused or unreachable issue no statement at all. Called for every +// outcome from the pass loop, so the four failure words and the success path +// share one write point, and a forced Poll that reads the page clears through +// the ordinary success path — no branch of its own. Log a store failure and +// carry on: this is best-effort, and no single bad Series may stall a Lane. +func (p *Poller) recordFailure(sr store.Series, outcome readOutcome) { + if outcome == outcomeSuccess { + if err := p.Store.ClearSeriesFailure(sr.Site, sr.SeriesID); err != nil { + log.Printf("latest poll %q: clear failure: %v", sr.Key(), err) + } + return + } + word := outcome.word() + if word == "" { + return + } + if err := p.Store.RecordSeriesFailure(sr.Site, sr.SeriesID, word, p.Now().UnixMilli()); err != nil { + log.Printf("latest poll %q: record failure: %v", sr.Key(), err) + } +} // checkOne re-checks one series. Every failure path here is "log and move on": // the poller is a best-effort enhancement, and no single bad series may stall a diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 39576da..0fbacfa 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -2725,3 +2725,218 @@ func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) { t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want) } } +// failureRow reads one Series' failure row as stored, for the poller tests +// that must see the table the store API writes (ADR-0016). +func failureRow(t *testing.T, dbURL, site, seriesID string) (word string, since int64, found bool) { + t.Helper() + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + err = db.QueryRow( + `SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`, + site, seriesID).Scan(&word, &since) + if errors.Is(err, sql.ErrNoRows) { + return "", 0, false + } + if err != nil { + t.Fatalf("read failure row %s:%s: %v", site, seriesID, err) + } + return word, since, true +} + +// The failure row follows the read (ADR-0016): a 404 writes a row with the +// outcome word and the pass's time; a second 404 an hour later leaves +// failing_since alone — the age is the age of the run of failures, not of +// the current word; a good read deletes the row. +func TestRunLanePassFailureRowFollowsTheRead(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seedForCheck(t, s, key, seriesURL, 0) + + f := &fakeFetcher{perURL: map[string]fakeResponse{seriesURL: {status: 404}}} + p := newTestPoller(t, s, f, now) + p.runLanePass(context.Background(), "asura", false) + + word, since, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af") + if !found || word != "not_found" || since != now.UnixMilli() { + t.Fatalf("row after 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli()) + } + + // A repeated failure an hour later: the word is unchanged and the stamp + // is untouched, so the age keeps meaning the run of failures. + p.Now = func() time.Time { return now.Add(2 * time.Hour) } + p.runLanePass(context.Background(), "asura", false) + word, since, found = failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af") + if !found || word != "not_found" || since != now.UnixMilli() { + t.Fatalf("row after repeated 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli()) + } + + // A good read ends the failure: the row is gone. + f.perURL[seriesURL] = fakeResponse{body: asuraSeriesFixture, status: 200} + p.Now = func() time.Time { return now.Add(4 * time.Hour) } + p.runLanePass(context.Background(), "asura", false) + if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found { + t.Fatal("failure row after a good read: still present") + } +} + +// Every failure word the pass counts lands in the table under the wire +// spelling the worklist left-joins on (C1): no_chapter, unfetchable, errors, +// not_found — and a success writes no row. +func TestRunLanePassStoresEachFailureWord(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + seeds := map[string]string{ + "asura:no-chapter": "https://asurascans.com/comics/no-chapter", + "asura:unfetchable": "https://evil.example/x", + "asura:transport": "https://asurascans.com/series/transport", + "asura:gone": "https://asurascans.com/series/gone", + "asura:healthy": "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", + } + for key, url := range seeds { + seedForCheck(t, s, key, url, 0) + } + f := &fakeFetcher{perURL: map[string]fakeResponse{ + seeds["asura:no-chapter"]: {body: "", status: 200}, + seeds["asura:transport"]: {err: errors.New("dial tcp: refused")}, + seeds["asura:gone"]: {status: 404}, + seeds["asura:healthy"]: {body: asuraSeriesFixture, status: 200}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + want := map[string]string{ + "no-chapter": "no_chapter", + "unfetchable": "unfetchable", + "transport": "errors", + "gone": "not_found", + } + for seriesID, word := range want { + got, _, found := failureRow(t, dbURL, "asura", seriesID) + if !found || got != word { + t.Fatalf("failure word for %s = (%q, %v), want %q", seriesID, got, found, word) + } + } + if _, _, found := failureRow(t, dbURL, "asura", "healthy"); found { + t.Fatal("success Series gained a failure row") + } +} + +// refused and unreachable issue no statement at all (ADR-0016): a challenge +// or a lost sidecar is no evidence about any particular Series, so a +// pre-seeded row survives both untouched and a Series with no row gains none. +func TestRunLanePassRefusalAndUnreachableWriteNothing(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + + t.Run("refused", func(t *testing.T) { + const ( + seeded = "asura:held" + neverFail = "asura:other" + ) + seedForCheck(t, s, seeded, "https://asurascans.com/series/held", 0) + seedForCheck(t, s, neverFail, "https://asurascans.com/series/other", 0) + if err := s.RecordSeriesFailure("asura", "held", "not_found", 7_000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + f := &fakeFetcher{perURL: map[string]fakeResponse{ + "https://asurascans.com/series/held": {status: 403}, + "https://asurascans.com/series/other": {status: 403}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + word, since, found := failureRow(t, dbURL, "asura", "held") + if !found || word != "not_found" || since != 7_000 { + t.Fatalf("pre-seeded row after refusal = (%q, %d, %v), want (not_found, 7000, true)", word, since, found) + } + if _, _, found := failureRow(t, dbURL, "asura", "other"); found { + t.Fatal("row appeared for a refused Series with none: refusal must write nothing") + } + }) + + t.Run("unreachable mid-loop", func(t *testing.T) { + const ( + seeded = "comix:c" + neverFail = "comix:d" + ) + seedForCheck(t, s, seeded, "https://comix.to/title/c", 0) + seedForCheck(t, s, neverFail, "https://comix.to/title/d", 0) + if err := s.RecordSeriesFailure("comix", "c", "errors", 7_000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + browser := &fakeFetcher{status: 200, err: interrupted} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + p.runLanePass(context.Background(), "comix", false) + + word, since, found := failureRow(t, dbURL, "comix", "c") + if !found || word != "errors" || since != 7_000 { + t.Fatalf("pre-seeded row after unreachable = (%q, %d, %v), want (errors, 7000, true)", word, since, found) + } + if _, _, found := failureRow(t, dbURL, "comix", "d"); found { + t.Fatal("row appeared for an unreachable Series with none: unreachable must write nothing") + } + }) +} + +// A Forced Poll request clears nothing — it only stamps the request — and a +// forced Poll that then reads the page clears the row through the ordinary +// success path, with no forced branch in the code. +func TestRunLanePassForcedPollClearsThroughSuccess(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seedForCheck(t, s, key, seriesURL, 0) + if err := s.RecordSeriesFailure("asura", "chronicles-of-the-demon-faction-f886a8af", "not_found", 7_000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + if err := s.ForceSeriesPoll("asura", "chronicles-of-the-demon-faction-f886a8af", now.Add(time.Hour).UnixMilli()); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + // The request alone cleared nothing. + if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); !found { + t.Fatal("failure row gone after only a Forced Poll request") + } + + newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now). + runLanePass(context.Background(), "asura", false) + if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found { + t.Fatal("failure row after a forced Poll that read the page: still present") + } +} + +// The due query does not join the failure table (AC7): a failing Series is +// polled at the same pace as any other, so its failure age keeps meaning what +// the worklist reads it as. +func TestPollFailureDoesNotChangePacing(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seedForCheck(t, s, "asura:failing", "https://asurascans.com/series/failing", 0) + seedForCheck(t, s, "asura:healthy", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + f := &fakeFetcher{perURL: map[string]fakeResponse{ + "https://asurascans.com/series/failing": {status: 404}, + "https://asurascans.com/series/healthy": {body: asuraSeriesFixture, status: 200}, + }} + p := newTestPoller(t, s, f, now) + p.runLanePass(context.Background(), "asura", false) + if got := f.callCount(); got != 2 { + t.Fatalf("first pass fetched %d series, want both (the failure row must not exclude the failing one)", got) + } + + // After the rest both are due again: the row the first pass wrote changed + // nothing about when the failing Series is polled. + p.Now = func() time.Time { return now.Add(2 * time.Hour) } + p.runLanePass(context.Background(), "asura", false) + if got := f.callCount(); got != 4 { + t.Fatalf("second pass fetched %d series, want both again", got) + } +} diff --git a/backend/internal/store/migrations/0018_poll_failures.sql b/backend/internal/store/migrations/0018_poll_failures.sql new file mode 100644 index 0000000..5aa2854 --- /dev/null +++ b/backend/internal/store/migrations/0018_poll_failures.sql @@ -0,0 +1,13 @@ +-- One row per Series that is failing right now (ADR-0016): the row's +-- existence is the failure state, failing_since ages the run of failures, +-- and a correct read deletes the row. Keyed by the same (site, series_id) +-- composite the rest of the system uses, with the cascade so deleting a +-- Series takes its failure row and orphan removal stays a single statement. +CREATE TABLE poll_failures ( + site text NOT NULL, + series_id text NOT NULL, + outcome text NOT NULL, + failing_since bigint NOT NULL, + PRIMARY KEY (site, series_id), + FOREIGN KEY (site, series_id) REFERENCES series (site, series_id) ON DELETE CASCADE +); diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 153471d..56e6970 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1297,6 +1297,31 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro } return pausedUntil, refuseUntil, nil } +// RecordSeriesFailure upserts one Series' failure row: the outcome word is +// updated when it changes, failing_since is never overwritten, and an +// unchanged word writes nothing. One statement, so the identical-word no-op +// and the keep-the-stamp rule are the same guarantee: the SET arm fires only +// when the word differs, and failing_since is simply absent from it +// (ADR-0016). +func (s *Store) RecordSeriesFailure(site, seriesID, outcome string, now int64) error { + if _, err := s.db.Exec(` + INSERT INTO poll_failures (site, series_id, outcome, failing_since) VALUES ($1, $2, $3, $4) + ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome + WHERE poll_failures.outcome <> EXCLUDED.outcome`, site, seriesID, outcome, now); err != nil { + return fmt.Errorf("record series failure %s:%s: %w", site, seriesID, err) + } + return nil +} + +// ClearSeriesFailure deletes one Series' failure row. A row that is not there +// is not an error. +func (s *Store) ClearSeriesFailure(site, seriesID string) error { + if _, err := s.db.Exec( + `DELETE FROM poll_failures WHERE site = $1 AND series_id = $2`, site, seriesID); err != nil { + return fmt.Errorf("clear series failure %s:%s: %w", site, seriesID, err) + } + return nil +} // DueForLatestCheck returns one Site's series whose server-side // latest-chapter check has aged past cutoffMs, ordered by how many bookmarks diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index 8480fec..286c212 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -2611,3 +2611,92 @@ func TestRemoveSeriesMissingKeyIsCleanNoOp(t *testing.T) { t.Fatalf("RemoveSeries on a missing key = %v, want nil", err) } } +// The failure row's existence is the failure state (ADR-0016): +// RecordSeriesFailure upserts the outcome word, keeps the original +// failing_since through a word change, and an unchanged word writes nothing +// at all — a broken Series costs the same as a healthy one every hour. +func TestRecordSeriesFailureUpsertKeepsFailingSince(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0) + + if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil { + t.Fatalf("record: %v", err) + } + word, since, found := failureRow(t, s, "asura", "solo") + if !found || word != "not_found" || since != 1000 { + t.Fatalf("row after create = (%q, %d, %v), want (not_found, 1000, true)", word, since, found) + } + + // A changed word updates the outcome and never touches failing_since. + if err := s.RecordSeriesFailure("asura", "solo", "errors", 2000); err != nil { + t.Fatalf("record changed word: %v", err) + } + word, since, found = failureRow(t, s, "asura", "solo") + if !found || word != "errors" || since != 1000 { + t.Fatalf("row after word change = (%q, %d, %v), want (errors, 1000, true)", word, since, found) + } + + // An identical word writes nothing: the stamp survives a later now. + if err := s.RecordSeriesFailure("asura", "solo", "errors", 3000); err != nil { + t.Fatalf("record identical word: %v", err) + } + word, since, found = failureRow(t, s, "asura", "solo") + if !found || word != "errors" || since != 1000 { + t.Fatalf("row after identical word = (%q, %d, %v), want the earlier (errors, 1000, true)", word, since, found) + } +} + +// ClearSeriesFailure deletes the row, and a row that is not there is not an +// error — the poller clears on every successful read, so most clears find +// nothing. +func TestClearSeriesFailure(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0) + + if err := s.ClearSeriesFailure("asura", "solo"); err != nil { + t.Fatalf("clear a missing row: %v, want nil", err) + } + if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil { + t.Fatalf("record: %v", err) + } + if err := s.ClearSeriesFailure("asura", "solo"); err != nil { + t.Fatalf("clear: %v", err) + } + if _, _, found := failureRow(t, s, "asura", "solo"); found { + t.Fatal("row after clear: still present") + } +} + +// Deleting a Series takes its failure row with it (the composite FK's +// cascade), so orphan removal stays a single statement. +func TestRemoveSeriesCascadesToFailureRow(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0) + if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil { + t.Fatalf("record: %v", err) + } + if err := s.Delete(s.OwnerID(), "asura:solo"); err != nil { + t.Fatalf("delete bookmark: %v", err) + } + if err := s.RemoveSeries("asura", "solo"); err != nil { + t.Fatalf("RemoveSeries: %v", err) + } + if _, _, found := failureRow(t, s, "asura", "solo"); found { + t.Fatal("failure row after series delete: still present (no cascade)") + } +} + +// failureRow reads one Series' failure row as stored. +func failureRow(t *testing.T, s *Store, site, seriesID string) (word string, since int64, found bool) { + t.Helper() + err := s.db.QueryRow( + `SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`, + site, seriesID).Scan(&word, &since) + if errors.Is(err, sql.ErrNoRows) { + return "", 0, false + } + if err != nil { + t.Fatalf("read failure row %s:%s: %v", site, seriesID, err) + } + return word, since, true +} diff --git a/docs/adr/0016-the-failure-row-is-the-state.md b/docs/adr/0016-the-failure-row-is-the-state.md new file mode 100644 index 0000000..f09b73a --- /dev/null +++ b/docs/adr/0016-the-failure-row-is-the-state.md @@ -0,0 +1,147 @@ +# ADR-0016: The failure row is the state + +Date: 2026-08-22 +Status: accepted + +## Decision + +A Series that used to work and has stopped is recorded in one table, +`poll_failures`, keyed by the same `(site, series_id)` composite the rest of +the system uses. One row per failing Series, carrying the outcome word and a +`failing_since` stamp. The row's **existence** is the failure state: there is +no success sentinel, no counter, no history, and nothing added to the Series +row. A correct read deletes the row, and a repeated identical failure writes +nothing — the stamp ages the run of failures, not the current word. + +The write is one upsert, from the poll pass loop: + +```sql +INSERT INTO poll_failures (site, series_id, outcome, failing_since) +VALUES ($1, $2, $3, $4) +ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome +WHERE poll_failures.outcome <> EXCLUDED.outcome +``` + +`failing_since` is simply absent from the `SET` arm, so it is never +overwritten, and the `WHERE` makes an unchanged word write nothing at all — +one statement, no read-then-write race. The clear is a plain `DELETE`; a row +that is not there is not an error, because the poller clears on every +successful read and most clears find nothing. The composite foreign key +cascades: deleting a Series takes its failure row, so orphan removal stays a +single statement. + +## Why a future reader will find this surprising + +The failure state lives in a table of its own, not on the Series row and not +in the pass log. A flag on `series` would be the familiar shape, and the +pass log already records outcomes per pass — why a third place? + +Because the failure must outlive the pass that observed it and mean +something the pass row cannot say. The pass log is a per-Lane stream: it +records that a Site returned N `errors` and M `not_found` on a given run, +but "this exact Series has been failing for three months" is not a question +any single pass row answers — it is a question across rows, and the Lanes +page is a live view of the last 14 days, not a Series index. A Series-level +fact needs Series-level storage, and a flag on the Series row is the wrong +shape too: the failure is *transient by definition* (a correct read ends it) +and *repeatable* (the same Series can fail again next year), so the honest +record of "failing since" is a stamp that moves, not a column that flips. +A row that is created and deleted by the read's outcome is that stamp, and +nothing else: the moment a read succeeds the row is gone, so "is this Series +failing?" is answered by one indexed existence check — no success sentinel +to keep consistent with the failure, no counter to reset, no history to +prune. The state cannot drift out of step with the reads that maintain it, +because the reads *are* the maintenance. + +The two no-write outcomes are the second surprise. `refused` (the Site is +holding a challenge) and `unreachable` (the browser sidecar was lost +mid-loop) issue **no statement at all** — neither an upsert nor a delete. +The direction matters, and both directions are wrong to touch: + +- **Writing would condemn a whole library for one Site's bad day.** A + refusal is the Site's mood, not a fact about any particular Series: when + Cloudflare turns a zone's JS detection on, every Series on that Site reads + refused in the same pass. Writing those rows would stamp every Series in + the library as failing on the evidence of one Site's configuration, and + #166's worklist would present a Site-wide outage as thousands of broken + Series. +- **Deleting would claim a recovery nothing read.** A lost sidecar tells us + nothing about the page — the read never happened. Deleting the row would + report the Series healthy, and worse, it would reset `failing_since`: the + age that makes a three-month failure findable would start over on a + browser restart, erasing evidence no page read contradicted. + +So a refusal or an unreachable pass leaves the table exactly as it found +it — the pass neither adds rows that would condemn nor deletes rows that +would claim recovery. The four outcomes that are real evidence about the +page (`not_found`, `no_chapter`, `unfetchable`, `errors`) write; the two +that are not write nothing; success deletes. There is no third case. + +## Considered options + +**A `failing_since` column on the Series row, alongside the failure word.** +Rejected: it is two more columns on a table every due query and every +bookmark join already touches, for a state that is transient and repeatable. +The column would need its own "clear on success" writer anyway — the same +maintenance the row has — while permanently widening the hottest table in +the system for a value that is usually absent. And the worklist's join +would have to distinguish "never failed" from "currently healthy", which +is exactly the sentinel problem the row avoids: an empty column means both. + +**A counter or last-outcome timestamp instead of (or beside) the row.** +Rejected: nothing in the system consumes a failure *count* or a +last-outcome time — the worklist needs "failing and since when". A counter +invites "three failures = something" thresholds that the ticket explicitly +keeps out of scope, and a last-outcome timestamp conflates "the word +changed" with "the run restarted", which the age is deliberately defined +against. The stamp is the only number that means anything, and the row +carries exactly that. + +**Write into `poll_passes` and derive the failure state from the pass +stream.** +Rejected: a pass row is per-Lane and per-run; deriving "this Series is +failing since" means scanning 14 days of outcome counts per Series and +guessing at continuity across retention boundaries. The pass log answers +"what did this Site's lane do recently"; the failure table answers "which +Series are broken right now". Two questions, two tables — the pass log is +already pruned on a fixed cutoff, which would silently reset every +failure's age the moment the evidence aged out. + +**Refused/unreachable write nothing, but the delete happens anyway (or the +upsert happens, with no delete).** +Rejected in both directions, above: writing condemns a library for one +Site's mood; deleting claims a recovery nothing read and resets the age +that makes a long failure findable. The asymmetry is the point — the two +outcomes are evidence about the *environment*, never about a page. + +## Consequences + +- The poller writes from the pass loop, one call after `checkOne`, and + clears through the ordinary success path: a Forced Poll that reads the + page deletes the row with no forced branch of its own, and a Forced Poll + *request* — which only stamps the request — clears nothing. +- `poll_failures` is a poll-write table like `poll_lanes` and + `poll_passes`: the store's two methods live next to the Lane-pass + writers, and neither runs on a request path. A store failure is logged + and the poll continues — a failure row is best-effort, and no single bad + Series may stall a Lane. +- The due query does not join the table. A failing Series is polled at the + same pace as any other, because the moment the query started pacing by + failure state, the age would stop meaning what the worklist reads it as. +- Orphan removal stays a single statement: the composite foreign key's + `ON DELETE CASCADE` is what takes the failure row with the Series. +- The table starts empty at deploy, so nothing is findable for the first + twelve hours after deploy and a pre-existing breakage reads as new. + Accepted; the alternative is inventing history. + +## Cost of reversing + +The failure state is derived, not stored: a rollback drops the table and +every in-progress failure age with it, leaving the pass log's outcome +counts as the only trace — which is exactly the "log line nobody reads" +the ticket set out to replace. Recreating the table later starts the ages +over, so a reversal that is later reversed loses the evidence of the +intervening failures. The failure state itself, however, is the one thing +that is *not* lost by reversing: it is re-derived from the next pass, in +the same direction the original design derives it — the row is +recreated by the next failing read and deleted by the next good one.