Spec #137: per-Series poll failure state, completion hint, and outbound owner notification #174
@@ -287,6 +287,24 @@ const (
|
|||||||
outcomeError
|
outcomeError
|
||||||
outcomeNotFound
|
outcomeNotFound
|
||||||
)
|
)
|
||||||
|
// word returns the wire spelling this outcome stores in poll_failures — the
|
||||||
|
// same strings the pass log's columns use (C1, issue #164). Success, refusal
|
||||||
|
// and browser loss return "" so recordFailure's "no statement" case is one
|
||||||
|
// return: a challenge or a lost sidecar is no evidence about any particular
|
||||||
|
// Series (ADR-0016).
|
||||||
|
func (o readOutcome) word() string {
|
||||||
|
switch o {
|
||||||
|
case outcomeNotFound:
|
||||||
|
return "not_found"
|
||||||
|
case outcomeNoChapter:
|
||||||
|
return "no_chapter"
|
||||||
|
case outcomeUnfetchable:
|
||||||
|
return "unfetchable"
|
||||||
|
case outcomeError:
|
||||||
|
return "errors"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// outcomeCounts are the six named outcome counts of one pass. A success
|
// outcomeCounts are the six named outcome counts of one pass. A success
|
||||||
// count is derived, never stored: checked minus the five named failures,
|
// count is derived, never stored: checked minus the five named failures,
|
||||||
@@ -456,6 +474,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
outcome := p.checkOne(ctx, sr)
|
outcome := p.checkOne(ctx, sr)
|
||||||
|
p.recordFailure(sr, outcome)
|
||||||
if outcome == outcomeUnreachable {
|
if outcome == outcomeUnreachable {
|
||||||
// The mid-loop browser loss writes an empty skip on purpose: the
|
// The mid-loop browser loss writes an empty skip on purpose: the
|
||||||
// pass returns before the checked counter increments, so its row
|
// pass returns before the checked counter increments, so its row
|
||||||
@@ -613,6 +632,28 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D
|
|||||||
}
|
}
|
||||||
return oldest
|
return oldest
|
||||||
}
|
}
|
||||||
|
// recordFailure keeps one Series' failure row in step with its read
|
||||||
|
// (ADR-0016): a failure word is upserted, a successful read deletes the row,
|
||||||
|
// and refused or unreachable issue no statement at all. Called for every
|
||||||
|
// outcome from the pass loop, so the four failure words and the success path
|
||||||
|
// share one write point, and a forced Poll that reads the page clears through
|
||||||
|
// the ordinary success path — no branch of its own. Log a store failure and
|
||||||
|
// carry on: this is best-effort, and no single bad Series may stall a Lane.
|
||||||
|
func (p *Poller) recordFailure(sr store.Series, outcome readOutcome) {
|
||||||
|
if outcome == outcomeSuccess {
|
||||||
|
if err := p.Store.ClearSeriesFailure(sr.Site, sr.SeriesID); err != nil {
|
||||||
|
log.Printf("latest poll %q: clear failure: %v", sr.Key(), err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
word := outcome.word()
|
||||||
|
if word == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := p.Store.RecordSeriesFailure(sr.Site, sr.SeriesID, word, p.Now().UnixMilli()); err != nil {
|
||||||
|
log.Printf("latest poll %q: record failure: %v", sr.Key(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// checkOne re-checks one series. Every failure path here is "log and move on":
|
// checkOne re-checks one series. Every failure path here is "log and move on":
|
||||||
// the poller is a best-effort enhancement, and no single bad series may stall a
|
// the poller is a best-effort enhancement, and no single bad series may stall a
|
||||||
|
|||||||
@@ -2725,3 +2725,218 @@ func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) {
|
|||||||
t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// failureRow reads one Series' failure row as stored, for the poller tests
|
||||||
|
// that must see the table the store API writes (ADR-0016).
|
||||||
|
func failureRow(t *testing.T, dbURL, site, seriesID string) (word string, since int64, found bool) {
|
||||||
|
t.Helper()
|
||||||
|
db, err := sql.Open("pgx", dbURL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open %s: %v", dbURL, err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
err = db.QueryRow(
|
||||||
|
`SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`,
|
||||||
|
site, seriesID).Scan(&word, &since)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return "", 0, false
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read failure row %s:%s: %v", site, seriesID, err)
|
||||||
|
}
|
||||||
|
return word, since, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// The failure row follows the read (ADR-0016): a 404 writes a row with the
|
||||||
|
// outcome word and the pass's time; a second 404 an hour later leaves
|
||||||
|
// failing_since alone — the age is the age of the run of failures, not of
|
||||||
|
// the current word; a good read deletes the row.
|
||||||
|
func TestRunLanePassFailureRowFollowsTheRead(t *testing.T) {
|
||||||
|
s, dbURL := newTestStore(t)
|
||||||
|
now := time.UnixMilli(5_000_000)
|
||||||
|
const (
|
||||||
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||||
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||||
|
)
|
||||||
|
seedForCheck(t, s, key, seriesURL, 0)
|
||||||
|
|
||||||
|
f := &fakeFetcher{perURL: map[string]fakeResponse{seriesURL: {status: 404}}}
|
||||||
|
p := newTestPoller(t, s, f, now)
|
||||||
|
p.runLanePass(context.Background(), "asura", false)
|
||||||
|
|
||||||
|
word, since, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af")
|
||||||
|
if !found || word != "not_found" || since != now.UnixMilli() {
|
||||||
|
t.Fatalf("row after 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A repeated failure an hour later: the word is unchanged and the stamp
|
||||||
|
// is untouched, so the age keeps meaning the run of failures.
|
||||||
|
p.Now = func() time.Time { return now.Add(2 * time.Hour) }
|
||||||
|
p.runLanePass(context.Background(), "asura", false)
|
||||||
|
word, since, found = failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af")
|
||||||
|
if !found || word != "not_found" || since != now.UnixMilli() {
|
||||||
|
t.Fatalf("row after repeated 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A good read ends the failure: the row is gone.
|
||||||
|
f.perURL[seriesURL] = fakeResponse{body: asuraSeriesFixture, status: 200}
|
||||||
|
p.Now = func() time.Time { return now.Add(4 * time.Hour) }
|
||||||
|
p.runLanePass(context.Background(), "asura", false)
|
||||||
|
if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found {
|
||||||
|
t.Fatal("failure row after a good read: still present")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every failure word the pass counts lands in the table under the wire
|
||||||
|
// spelling the worklist left-joins on (C1): no_chapter, unfetchable, errors,
|
||||||
|
// not_found — and a success writes no row.
|
||||||
|
func TestRunLanePassStoresEachFailureWord(t *testing.T) {
|
||||||
|
s, dbURL := newTestStore(t)
|
||||||
|
now := time.UnixMilli(5_000_000)
|
||||||
|
seeds := map[string]string{
|
||||||
|
"asura:no-chapter": "https://asurascans.com/comics/no-chapter",
|
||||||
|
"asura:unfetchable": "https://evil.example/x",
|
||||||
|
"asura:transport": "https://asurascans.com/series/transport",
|
||||||
|
"asura:gone": "https://asurascans.com/series/gone",
|
||||||
|
"asura:healthy": "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
||||||
|
}
|
||||||
|
for key, url := range seeds {
|
||||||
|
seedForCheck(t, s, key, url, 0)
|
||||||
|
}
|
||||||
|
f := &fakeFetcher{perURL: map[string]fakeResponse{
|
||||||
|
seeds["asura:no-chapter"]: {body: "<html></html>", status: 200},
|
||||||
|
seeds["asura:transport"]: {err: errors.New("dial tcp: refused")},
|
||||||
|
seeds["asura:gone"]: {status: 404},
|
||||||
|
seeds["asura:healthy"]: {body: asuraSeriesFixture, status: 200},
|
||||||
|
}}
|
||||||
|
newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false)
|
||||||
|
|
||||||
|
want := map[string]string{
|
||||||
|
"no-chapter": "no_chapter",
|
||||||
|
"unfetchable": "unfetchable",
|
||||||
|
"transport": "errors",
|
||||||
|
"gone": "not_found",
|
||||||
|
}
|
||||||
|
for seriesID, word := range want {
|
||||||
|
got, _, found := failureRow(t, dbURL, "asura", seriesID)
|
||||||
|
if !found || got != word {
|
||||||
|
t.Fatalf("failure word for %s = (%q, %v), want %q", seriesID, got, found, word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, found := failureRow(t, dbURL, "asura", "healthy"); found {
|
||||||
|
t.Fatal("success Series gained a failure row")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refused and unreachable issue no statement at all (ADR-0016): a challenge
|
||||||
|
// or a lost sidecar is no evidence about any particular Series, so a
|
||||||
|
// pre-seeded row survives both untouched and a Series with no row gains none.
|
||||||
|
func TestRunLanePassRefusalAndUnreachableWriteNothing(t *testing.T) {
|
||||||
|
s, dbURL := newTestStore(t)
|
||||||
|
now := time.UnixMilli(5_000_000)
|
||||||
|
|
||||||
|
t.Run("refused", func(t *testing.T) {
|
||||||
|
const (
|
||||||
|
seeded = "asura:held"
|
||||||
|
neverFail = "asura:other"
|
||||||
|
)
|
||||||
|
seedForCheck(t, s, seeded, "https://asurascans.com/series/held", 0)
|
||||||
|
seedForCheck(t, s, neverFail, "https://asurascans.com/series/other", 0)
|
||||||
|
if err := s.RecordSeriesFailure("asura", "held", "not_found", 7_000); err != nil {
|
||||||
|
t.Fatalf("seed failure row: %v", err)
|
||||||
|
}
|
||||||
|
f := &fakeFetcher{perURL: map[string]fakeResponse{
|
||||||
|
"https://asurascans.com/series/held": {status: 403},
|
||||||
|
"https://asurascans.com/series/other": {status: 403},
|
||||||
|
}}
|
||||||
|
newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false)
|
||||||
|
|
||||||
|
word, since, found := failureRow(t, dbURL, "asura", "held")
|
||||||
|
if !found || word != "not_found" || since != 7_000 {
|
||||||
|
t.Fatalf("pre-seeded row after refusal = (%q, %d, %v), want (not_found, 7000, true)", word, since, found)
|
||||||
|
}
|
||||||
|
if _, _, found := failureRow(t, dbURL, "asura", "other"); found {
|
||||||
|
t.Fatal("row appeared for a refused Series with none: refusal must write nothing")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unreachable mid-loop", func(t *testing.T) {
|
||||||
|
const (
|
||||||
|
seeded = "comix:c"
|
||||||
|
neverFail = "comix:d"
|
||||||
|
)
|
||||||
|
seedForCheck(t, s, seeded, "https://comix.to/title/c", 0)
|
||||||
|
seedForCheck(t, s, neverFail, "https://comix.to/title/d", 0)
|
||||||
|
if err := s.RecordSeriesFailure("comix", "c", "errors", 7_000); err != nil {
|
||||||
|
t.Fatalf("seed failure row: %v", err)
|
||||||
|
}
|
||||||
|
interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart"))
|
||||||
|
browser := &fakeFetcher{status: 200, err: interrupted}
|
||||||
|
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
|
||||||
|
p.BrowserFetch = browser
|
||||||
|
p.runLanePass(context.Background(), "comix", false)
|
||||||
|
|
||||||
|
word, since, found := failureRow(t, dbURL, "comix", "c")
|
||||||
|
if !found || word != "errors" || since != 7_000 {
|
||||||
|
t.Fatalf("pre-seeded row after unreachable = (%q, %d, %v), want (errors, 7000, true)", word, since, found)
|
||||||
|
}
|
||||||
|
if _, _, found := failureRow(t, dbURL, "comix", "d"); found {
|
||||||
|
t.Fatal("row appeared for an unreachable Series with none: unreachable must write nothing")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Forced Poll request clears nothing — it only stamps the request — and a
|
||||||
|
// forced Poll that then reads the page clears the row through the ordinary
|
||||||
|
// success path, with no forced branch in the code.
|
||||||
|
func TestRunLanePassForcedPollClearsThroughSuccess(t *testing.T) {
|
||||||
|
s, dbURL := newTestStore(t)
|
||||||
|
now := time.UnixMilli(5_000_000)
|
||||||
|
const (
|
||||||
|
key = "asura:chronicles-of-the-demon-faction-f886a8af"
|
||||||
|
seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
|
||||||
|
)
|
||||||
|
seedForCheck(t, s, key, seriesURL, 0)
|
||||||
|
if err := s.RecordSeriesFailure("asura", "chronicles-of-the-demon-faction-f886a8af", "not_found", 7_000); err != nil {
|
||||||
|
t.Fatalf("seed failure row: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.ForceSeriesPoll("asura", "chronicles-of-the-demon-faction-f886a8af", now.Add(time.Hour).UnixMilli()); err != nil {
|
||||||
|
t.Fatalf("ForceSeriesPoll: %v", err)
|
||||||
|
}
|
||||||
|
// The request alone cleared nothing.
|
||||||
|
if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); !found {
|
||||||
|
t.Fatal("failure row gone after only a Forced Poll request")
|
||||||
|
}
|
||||||
|
|
||||||
|
newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now).
|
||||||
|
runLanePass(context.Background(), "asura", false)
|
||||||
|
if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found {
|
||||||
|
t.Fatal("failure row after a forced Poll that read the page: still present")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The due query does not join the failure table (AC7): a failing Series is
|
||||||
|
// polled at the same pace as any other, so its failure age keeps meaning what
|
||||||
|
// the worklist reads it as.
|
||||||
|
func TestPollFailureDoesNotChangePacing(t *testing.T) {
|
||||||
|
s, _ := newTestStore(t)
|
||||||
|
now := time.UnixMilli(5_000_000)
|
||||||
|
seedForCheck(t, s, "asura:failing", "https://asurascans.com/series/failing", 0)
|
||||||
|
seedForCheck(t, s, "asura:healthy", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
|
||||||
|
f := &fakeFetcher{perURL: map[string]fakeResponse{
|
||||||
|
"https://asurascans.com/series/failing": {status: 404},
|
||||||
|
"https://asurascans.com/series/healthy": {body: asuraSeriesFixture, status: 200},
|
||||||
|
}}
|
||||||
|
p := newTestPoller(t, s, f, now)
|
||||||
|
p.runLanePass(context.Background(), "asura", false)
|
||||||
|
if got := f.callCount(); got != 2 {
|
||||||
|
t.Fatalf("first pass fetched %d series, want both (the failure row must not exclude the failing one)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// After the rest both are due again: the row the first pass wrote changed
|
||||||
|
// nothing about when the failing Series is polled.
|
||||||
|
p.Now = func() time.Time { return now.Add(2 * time.Hour) }
|
||||||
|
p.runLanePass(context.Background(), "asura", false)
|
||||||
|
if got := f.callCount(); got != 4 {
|
||||||
|
t.Fatalf("second pass fetched %d series, want both again", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
-- One row per Series that is failing right now (ADR-0016): the row's
|
||||||
|
-- existence is the failure state, failing_since ages the run of failures,
|
||||||
|
-- and a correct read deletes the row. Keyed by the same (site, series_id)
|
||||||
|
-- composite the rest of the system uses, with the cascade so deleting a
|
||||||
|
-- Series takes its failure row and orphan removal stays a single statement.
|
||||||
|
CREATE TABLE poll_failures (
|
||||||
|
site text NOT NULL,
|
||||||
|
series_id text NOT NULL,
|
||||||
|
outcome text NOT NULL,
|
||||||
|
failing_since bigint NOT NULL,
|
||||||
|
PRIMARY KEY (site, series_id),
|
||||||
|
FOREIGN KEY (site, series_id) REFERENCES series (site, series_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
@@ -1298,6 +1298,32 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro
|
|||||||
return pausedUntil, refuseUntil, nil
|
return pausedUntil, refuseUntil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecordSeriesFailure upserts one Series' failure row: the outcome word is
|
||||||
|
// updated when it changes, failing_since is never overwritten, and an
|
||||||
|
// unchanged word writes nothing. One statement, so the identical-word no-op
|
||||||
|
// and the keep-the-stamp rule are the same guarantee: the SET arm fires only
|
||||||
|
// when the word differs, and failing_since is simply absent from it
|
||||||
|
// (ADR-0016).
|
||||||
|
func (s *Store) RecordSeriesFailure(site, seriesID, outcome string, now int64) error {
|
||||||
|
if _, err := s.db.Exec(`
|
||||||
|
INSERT INTO poll_failures (site, series_id, outcome, failing_since) VALUES ($1, $2, $3, $4)
|
||||||
|
ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome
|
||||||
|
WHERE poll_failures.outcome <> EXCLUDED.outcome`, site, seriesID, outcome, now); err != nil {
|
||||||
|
return fmt.Errorf("record series failure %s:%s: %w", site, seriesID, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearSeriesFailure deletes one Series' failure row. A row that is not there
|
||||||
|
// is not an error.
|
||||||
|
func (s *Store) ClearSeriesFailure(site, seriesID string) error {
|
||||||
|
if _, err := s.db.Exec(
|
||||||
|
`DELETE FROM poll_failures WHERE site = $1 AND series_id = $2`, site, seriesID); err != nil {
|
||||||
|
return fmt.Errorf("clear series failure %s:%s: %w", site, seriesID, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// DueForLatestCheck returns one Site's series whose server-side
|
// DueForLatestCheck returns one Site's series whose server-side
|
||||||
// latest-chapter check has aged past cutoffMs, ordered by how many bookmarks
|
// latest-chapter check has aged past cutoffMs, ordered by how many bookmarks
|
||||||
// reference them (descending) then least-recently-checked first. One Site per
|
// reference them (descending) then least-recently-checked first. One Site per
|
||||||
|
|||||||
@@ -2611,3 +2611,92 @@ func TestRemoveSeriesMissingKeyIsCleanNoOp(t *testing.T) {
|
|||||||
t.Fatalf("RemoveSeries on a missing key = %v, want nil", err)
|
t.Fatalf("RemoveSeries on a missing key = %v, want nil", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The failure row's existence is the failure state (ADR-0016):
|
||||||
|
// RecordSeriesFailure upserts the outcome word, keeps the original
|
||||||
|
// failing_since through a word change, and an unchanged word writes nothing
|
||||||
|
// at all — a broken Series costs the same as a healthy one every hour.
|
||||||
|
func TestRecordSeriesFailureUpsertKeepsFailingSince(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0)
|
||||||
|
|
||||||
|
if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil {
|
||||||
|
t.Fatalf("record: %v", err)
|
||||||
|
}
|
||||||
|
word, since, found := failureRow(t, s, "asura", "solo")
|
||||||
|
if !found || word != "not_found" || since != 1000 {
|
||||||
|
t.Fatalf("row after create = (%q, %d, %v), want (not_found, 1000, true)", word, since, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A changed word updates the outcome and never touches failing_since.
|
||||||
|
if err := s.RecordSeriesFailure("asura", "solo", "errors", 2000); err != nil {
|
||||||
|
t.Fatalf("record changed word: %v", err)
|
||||||
|
}
|
||||||
|
word, since, found = failureRow(t, s, "asura", "solo")
|
||||||
|
if !found || word != "errors" || since != 1000 {
|
||||||
|
t.Fatalf("row after word change = (%q, %d, %v), want (errors, 1000, true)", word, since, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An identical word writes nothing: the stamp survives a later now.
|
||||||
|
if err := s.RecordSeriesFailure("asura", "solo", "errors", 3000); err != nil {
|
||||||
|
t.Fatalf("record identical word: %v", err)
|
||||||
|
}
|
||||||
|
word, since, found = failureRow(t, s, "asura", "solo")
|
||||||
|
if !found || word != "errors" || since != 1000 {
|
||||||
|
t.Fatalf("row after identical word = (%q, %d, %v), want the earlier (errors, 1000, true)", word, since, found)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearSeriesFailure deletes the row, and a row that is not there is not an
|
||||||
|
// error — the poller clears on every successful read, so most clears find
|
||||||
|
// nothing.
|
||||||
|
func TestClearSeriesFailure(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0)
|
||||||
|
|
||||||
|
if err := s.ClearSeriesFailure("asura", "solo"); err != nil {
|
||||||
|
t.Fatalf("clear a missing row: %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil {
|
||||||
|
t.Fatalf("record: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.ClearSeriesFailure("asura", "solo"); err != nil {
|
||||||
|
t.Fatalf("clear: %v", err)
|
||||||
|
}
|
||||||
|
if _, _, found := failureRow(t, s, "asura", "solo"); found {
|
||||||
|
t.Fatal("row after clear: still present")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deleting a Series takes its failure row with it (the composite FK's
|
||||||
|
// cascade), so orphan removal stays a single statement.
|
||||||
|
func TestRemoveSeriesCascadesToFailureRow(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0)
|
||||||
|
if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil {
|
||||||
|
t.Fatalf("record: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.Delete(s.OwnerID(), "asura:solo"); err != nil {
|
||||||
|
t.Fatalf("delete bookmark: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.RemoveSeries("asura", "solo"); err != nil {
|
||||||
|
t.Fatalf("RemoveSeries: %v", err)
|
||||||
|
}
|
||||||
|
if _, _, found := failureRow(t, s, "asura", "solo"); found {
|
||||||
|
t.Fatal("failure row after series delete: still present (no cascade)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// failureRow reads one Series' failure row as stored.
|
||||||
|
func failureRow(t *testing.T, s *Store, site, seriesID string) (word string, since int64, found bool) {
|
||||||
|
t.Helper()
|
||||||
|
err := s.db.QueryRow(
|
||||||
|
`SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`,
|
||||||
|
site, seriesID).Scan(&word, &since)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return "", 0, false
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read failure row %s:%s: %v", site, seriesID, err)
|
||||||
|
}
|
||||||
|
return word, since, true
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
# ADR-0016: The failure row is the state
|
||||||
|
|
||||||
|
Date: 2026-08-22
|
||||||
|
Status: accepted
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
A Series that used to work and has stopped is recorded in one table,
|
||||||
|
`poll_failures`, keyed by the same `(site, series_id)` composite the rest of
|
||||||
|
the system uses. One row per failing Series, carrying the outcome word and a
|
||||||
|
`failing_since` stamp. The row's **existence** is the failure state: there is
|
||||||
|
no success sentinel, no counter, no history, and nothing added to the Series
|
||||||
|
row. A correct read deletes the row, and a repeated identical failure writes
|
||||||
|
nothing — the stamp ages the run of failures, not the current word.
|
||||||
|
|
||||||
|
The write is one upsert, from the poll pass loop:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
INSERT INTO poll_failures (site, series_id, outcome, failing_since)
|
||||||
|
VALUES ($1, $2, $3, $4)
|
||||||
|
ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome
|
||||||
|
WHERE poll_failures.outcome <> EXCLUDED.outcome
|
||||||
|
```
|
||||||
|
|
||||||
|
`failing_since` is simply absent from the `SET` arm, so it is never
|
||||||
|
overwritten, and the `WHERE` makes an unchanged word write nothing at all —
|
||||||
|
one statement, no read-then-write race. The clear is a plain `DELETE`; a row
|
||||||
|
that is not there is not an error, because the poller clears on every
|
||||||
|
successful read and most clears find nothing. The composite foreign key
|
||||||
|
cascades: deleting a Series takes its failure row, so orphan removal stays a
|
||||||
|
single statement.
|
||||||
|
|
||||||
|
## Why a future reader will find this surprising
|
||||||
|
|
||||||
|
The failure state lives in a table of its own, not on the Series row and not
|
||||||
|
in the pass log. A flag on `series` would be the familiar shape, and the
|
||||||
|
pass log already records outcomes per pass — why a third place?
|
||||||
|
|
||||||
|
Because the failure must outlive the pass that observed it and mean
|
||||||
|
something the pass row cannot say. The pass log is a per-Lane stream: it
|
||||||
|
records that a Site returned N `errors` and M `not_found` on a given run,
|
||||||
|
but "this exact Series has been failing for three months" is not a question
|
||||||
|
any single pass row answers — it is a question across rows, and the Lanes
|
||||||
|
page is a live view of the last 14 days, not a Series index. A Series-level
|
||||||
|
fact needs Series-level storage, and a flag on the Series row is the wrong
|
||||||
|
shape too: the failure is *transient by definition* (a correct read ends it)
|
||||||
|
and *repeatable* (the same Series can fail again next year), so the honest
|
||||||
|
record of "failing since" is a stamp that moves, not a column that flips.
|
||||||
|
A row that is created and deleted by the read's outcome is that stamp, and
|
||||||
|
nothing else: the moment a read succeeds the row is gone, so "is this Series
|
||||||
|
failing?" is answered by one indexed existence check — no success sentinel
|
||||||
|
to keep consistent with the failure, no counter to reset, no history to
|
||||||
|
prune. The state cannot drift out of step with the reads that maintain it,
|
||||||
|
because the reads *are* the maintenance.
|
||||||
|
|
||||||
|
The two no-write outcomes are the second surprise. `refused` (the Site is
|
||||||
|
holding a challenge) and `unreachable` (the browser sidecar was lost
|
||||||
|
mid-loop) issue **no statement at all** — neither an upsert nor a delete.
|
||||||
|
The direction matters, and both directions are wrong to touch:
|
||||||
|
|
||||||
|
- **Writing would condemn a whole library for one Site's bad day.** A
|
||||||
|
refusal is the Site's mood, not a fact about any particular Series: when
|
||||||
|
Cloudflare turns a zone's JS detection on, every Series on that Site reads
|
||||||
|
refused in the same pass. Writing those rows would stamp every Series in
|
||||||
|
the library as failing on the evidence of one Site's configuration, and
|
||||||
|
#166's worklist would present a Site-wide outage as thousands of broken
|
||||||
|
Series.
|
||||||
|
- **Deleting would claim a recovery nothing read.** A lost sidecar tells us
|
||||||
|
nothing about the page — the read never happened. Deleting the row would
|
||||||
|
report the Series healthy, and worse, it would reset `failing_since`: the
|
||||||
|
age that makes a three-month failure findable would start over on a
|
||||||
|
browser restart, erasing evidence no page read contradicted.
|
||||||
|
|
||||||
|
So a refusal or an unreachable pass leaves the table exactly as it found
|
||||||
|
it — the pass neither adds rows that would condemn nor deletes rows that
|
||||||
|
would claim recovery. The four outcomes that are real evidence about the
|
||||||
|
page (`not_found`, `no_chapter`, `unfetchable`, `errors`) write; the two
|
||||||
|
that are not write nothing; success deletes. There is no third case.
|
||||||
|
|
||||||
|
## Considered options
|
||||||
|
|
||||||
|
**A `failing_since` column on the Series row, alongside the failure word.**
|
||||||
|
Rejected: it is two more columns on a table every due query and every
|
||||||
|
bookmark join already touches, for a state that is transient and repeatable.
|
||||||
|
The column would need its own "clear on success" writer anyway — the same
|
||||||
|
maintenance the row has — while permanently widening the hottest table in
|
||||||
|
the system for a value that is usually absent. And the worklist's join
|
||||||
|
would have to distinguish "never failed" from "currently healthy", which
|
||||||
|
is exactly the sentinel problem the row avoids: an empty column means both.
|
||||||
|
|
||||||
|
**A counter or last-outcome timestamp instead of (or beside) the row.**
|
||||||
|
Rejected: nothing in the system consumes a failure *count* or a
|
||||||
|
last-outcome time — the worklist needs "failing and since when". A counter
|
||||||
|
invites "three failures = something" thresholds that the ticket explicitly
|
||||||
|
keeps out of scope, and a last-outcome timestamp conflates "the word
|
||||||
|
changed" with "the run restarted", which the age is deliberately defined
|
||||||
|
against. The stamp is the only number that means anything, and the row
|
||||||
|
carries exactly that.
|
||||||
|
|
||||||
|
**Write into `poll_passes` and derive the failure state from the pass
|
||||||
|
stream.**
|
||||||
|
Rejected: a pass row is per-Lane and per-run; deriving "this Series is
|
||||||
|
failing since" means scanning 14 days of outcome counts per Series and
|
||||||
|
guessing at continuity across retention boundaries. The pass log answers
|
||||||
|
"what did this Site's lane do recently"; the failure table answers "which
|
||||||
|
Series are broken right now". Two questions, two tables — the pass log is
|
||||||
|
already pruned on a fixed cutoff, which would silently reset every
|
||||||
|
failure's age the moment the evidence aged out.
|
||||||
|
|
||||||
|
**Refused/unreachable write nothing, but the delete happens anyway (or the
|
||||||
|
upsert happens, with no delete).**
|
||||||
|
Rejected in both directions, above: writing condemns a library for one
|
||||||
|
Site's mood; deleting claims a recovery nothing read and resets the age
|
||||||
|
that makes a long failure findable. The asymmetry is the point — the two
|
||||||
|
outcomes are evidence about the *environment*, never about a page.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- The poller writes from the pass loop, one call after `checkOne`, and
|
||||||
|
clears through the ordinary success path: a Forced Poll that reads the
|
||||||
|
page deletes the row with no forced branch of its own, and a Forced Poll
|
||||||
|
*request* — which only stamps the request — clears nothing.
|
||||||
|
- `poll_failures` is a poll-write table like `poll_lanes` and
|
||||||
|
`poll_passes`: the store's two methods live next to the Lane-pass
|
||||||
|
writers, and neither runs on a request path. A store failure is logged
|
||||||
|
and the poll continues — a failure row is best-effort, and no single bad
|
||||||
|
Series may stall a Lane.
|
||||||
|
- The due query does not join the table. A failing Series is polled at the
|
||||||
|
same pace as any other, because the moment the query started pacing by
|
||||||
|
failure state, the age would stop meaning what the worklist reads it as.
|
||||||
|
- Orphan removal stays a single statement: the composite foreign key's
|
||||||
|
`ON DELETE CASCADE` is what takes the failure row with the Series.
|
||||||
|
- The table starts empty at deploy, so nothing is findable for the first
|
||||||
|
twelve hours after deploy and a pre-existing breakage reads as new.
|
||||||
|
Accepted; the alternative is inventing history.
|
||||||
|
|
||||||
|
## Cost of reversing
|
||||||
|
|
||||||
|
The failure state is derived, not stored: a rollback drops the table and
|
||||||
|
every in-progress failure age with it, leaving the pass log's outcome
|
||||||
|
counts as the only trace — which is exactly the "log line nobody reads"
|
||||||
|
the ticket set out to replace. Recreating the table later starts the ages
|
||||||
|
over, so a reversal that is later reversed loses the evidence of the
|
||||||
|
intervening failures. The failure state itself, however, is the one thing
|
||||||
|
that is *not* lost by reversing: it is re-derived from the next pass, in
|
||||||
|
the same direction the original design derives it — the row is
|
||||||
|
recreated by the next failing read and deleted by the next good one.
|
||||||
Reference in New Issue
Block a user