From 62c2cea74e402a390b6a5183ea3f01ee36d8132a Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 18:19:23 +0700 Subject: [PATCH] Spec #164: A sixth outcome word: not_found splits out of errors --- backend/internal/latest/poller.go | 25 ++++++++----- backend/internal/latest/poller_test.go | 36 +++++++++++++++++-- backend/internal/latest/read.go | 6 ++++ .../migrations/0017_poll_passes_not_found.sql | 4 +++ backend/internal/store/store.go | 20 +++++------ backend/internal/web/admin_lanes.go | 1 + backend/web_test.go | 22 ++++++++---- 7 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 backend/internal/store/migrations/0017_poll_passes_not_found.sql diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 1c6ae7c..4cfc2f1 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -273,7 +273,8 @@ const ( // (issue #141). The classification the read already makes is counted, never a // second taxonomy: refused is the Site holding a challenge, unreachable the // browser interrupting, noChapter a 200 with real HTML but no chapter links, -// unfetchable the host pin or a missing fetcher, and errors everything else. +// unfetchable the host pin or a missing fetcher, notFound a 4xx other than +// the 403 refusal — the page is gone — and errors everything else. type readOutcome int const ( @@ -283,14 +284,15 @@ const ( outcomeNoChapter outcomeUnfetchable outcomeError + outcomeNotFound ) -// outcomeCounts are the five named outcome counts of one pass. A success -// count is derived, never stored: checked minus the four, with unreachable -// excluded because the sidecar-loss path returns before the checked counter -// increments (issue #141). +// outcomeCounts are the six named outcome counts of one pass. A success +// count is derived, never stored: checked minus the five named failures, +// with unreachable excluded because the sidecar-loss path returns before the +// checked counter increments (issue #141). type outcomeCounts struct { - refused, unreachable, noChapter, unfetchable, errors int + refused, unreachable, noChapter, unfetchable, errors, notFound int } func (c *outcomeCounts) add(o readOutcome) { @@ -303,6 +305,8 @@ func (c *outcomeCounts) add(o readOutcome) { c.noChapter++ case outcomeUnfetchable: c.unfetchable++ + case outcomeNotFound: + c.notFound++ case outcomeError: c.errors++ } @@ -517,6 +521,7 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) { Unreachable: rec.counts.unreachable, NoChapter: rec.counts.noChapter, Unfetchable: rec.counts.unfetchable, + NotFound: rec.counts.notFound, Errors: rec.counts.errors, } if row.GapMS == 0 { @@ -612,8 +617,8 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D // the poller is a best-effort enhancement, and no single bad series may stall a // Lane or take down the process. The returned outcome classifies the read for // the pass row (issue #141), so the Lane can count a refusal, a lost browser, -// a chapter-less page, an unfetchable address or a transport error without -// re-deriving the taxonomy. +// a chapter-less page, an unfetchable address, a missing page or a transport +// error without re-deriving the taxonomy. func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) { defer func() { if r := recover(); r != nil { @@ -657,6 +662,10 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut if errors.Is(err, errBrowserInterrupted) { return outcomeUnreachable } + if errors.Is(err, errNotFound) { + log.Printf("latest poll %q: page not found: %v", sr.Key(), err) + return outcomeNotFound + } return outcomeError } // A legacy cover source is healed independently of the page read. diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 301225b..39576da 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -1947,9 +1947,9 @@ func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) { }) } -// The pass row's five outcome counts are the classification the Series read +// The pass row's six outcome counts are the classification the Series read // already makes — never a second taxonomy (issue #141). Success is derived, -// never stored: checked minus the four named counts, unreachable excluded +// never stored: checked minus the five named failures, unreachable excluded // because its exit returns before the checked counter increments. func TestRunLanePassCountsOutcomes(t *testing.T) { s, _ := newTestStore(t) @@ -1986,7 +1986,7 @@ func TestRunLanePassCountsOutcomes(t *testing.T) { t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1", pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors) } - if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 { + if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 1 { t.Fatalf("derived success = %d, want 1", success) } // The one genuine read went through: the success Series carries the @@ -2009,6 +2009,36 @@ func TestRunLanePassCountsOutcomes(t *testing.T) { } } +// A 4xx other than the 403 refusal means the page is gone — a sixth outcome +// word (issue #164) — while a 5xx stays errors. Both land in the durable row, +// so the owner can tell "correct the address" from "the Site is unwell" +// without opening a log. +func TestRunLanePassSplitsNotFoundFromErrors(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seeds := map[string]string{ + "asura:gone": "https://asurascans.com/series/gone", + "asura:unwell": "https://asurascans.com/series/unwell", + } + for key, url := range seeds { + seedForCheck(t, s, key, url, 0) + } + + f := &fakeFetcher{perURL: map[string]fakeResponse{ + seeds["asura:gone"]: {status: 404}, + seeds["asura:unwell"]: {status: 503}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + pass := latestPassFor(t, s, "asura") + if pass.NotFound != 1 || pass.Errors != 1 { + t.Fatalf("not_found=%d errors=%d, want 1 1", pass.NotFound, pass.Errors) + } + if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 0 { + t.Fatalf("derived success = %d, want 0 (both reads failed)", success) + } +} + // A refusal is the Site's mood and outlives our process: the durable stamp a // pass writes is honoured by a freshly constructed poller, which must not // re-probe the Site inside its backoff (issue #141). diff --git a/backend/internal/latest/read.go b/backend/internal/latest/read.go index 134223d..8a9c611 100644 --- a/backend/internal/latest/read.go +++ b/backend/internal/latest/read.go @@ -25,9 +25,12 @@ type seriesRead struct { // errChallengeHeld (browser.go) is the outcome of a Site that answered with // its interstitial — status 403 (cf-mitigated) or a challenge page body — and // is how a Lane tells a refusal from an ordinary failure (issue #100). +// errNotFound marks any other 4xx: the page is gone — a fact the owner can act +// on — as distinct from a Site or database that is merely unwell (issue #164). var ( errNotFetchable = errors.New("series url not fetchable") errNoFetcher = errors.New("no fetcher for site") + errNotFound = errors.New("series page not found") ) // readSeriesPage performs the series-page read the poll and the acquirer have @@ -59,6 +62,9 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status) } if status != 200 { + if status >= 400 && status < 500 { + return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errNotFound, seriesURL, status) + } return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status) } if isInterstitial(body) { diff --git a/backend/internal/store/migrations/0017_poll_passes_not_found.sql b/backend/internal/store/migrations/0017_poll_passes_not_found.sql new file mode 100644 index 0000000..8f93b7d --- /dev/null +++ b/backend/internal/store/migrations/0017_poll_passes_not_found.sql @@ -0,0 +1,4 @@ +-- A 4xx other than the 403 refusal is the page being gone, not the Site being +-- unwell; the pass log counts it separately so the Lanes page can say "not +-- found" (#164). DEFAULT 0 keeps pre-existing rows readable. +ALTER TABLE poll_passes ADD COLUMN not_found integer NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 2552fa9..153471d 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -110,7 +110,7 @@ type LanePass struct { GapMS int64 Clamped bool Refused, Unreachable, NoChapter int - Unfetchable, Errors int + Unfetchable, Errors, NotFound int PausedUntil, RefuseUntil int64 } @@ -119,7 +119,7 @@ type LanePass struct { type SiteOutcomes struct { Site string Refused, Unreachable, NoChapter int - Unfetchable, Errors int + Unfetchable, Errors, NotFound int } // LanePause is one persisted Lane pause stamp. @@ -243,7 +243,7 @@ const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.co s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by` const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped, - p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, + p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, p.not_found, COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)` // Owner is the person running the service: the first Reader, seeded at startup @@ -663,7 +663,7 @@ func scanLanePass(scan func(...any) error) (LanePass, error) { var p LanePass if err := scan( &p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped, - &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, + &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, &p.NotFound, &p.PausedUntil, &p.RefuseUntil, ); err != nil { return LanePass{}, err @@ -1131,10 +1131,10 @@ func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error { if _, err := tx.Exec(` INSERT INTO poll_passes (site, ran_at, skip, due, checked, gap_ms, clamped, - refused, unreachable, no_chapter, unfetchable, errors) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + refused, unreachable, no_chapter, unfetchable, errors, not_found) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped, - p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil { + p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors, p.NotFound); err != nil { return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err) } if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil { @@ -1171,7 +1171,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) { FROM ( SELECT DISTINCT ON (site) site, ran_at, skip, due, checked, gap_ms, clamped, - refused, unreachable, no_chapter, unfetchable, errors + refused, unreachable, no_chapter, unfetchable, errors, not_found FROM poll_passes ORDER BY site, ran_at DESC ) p @@ -1198,7 +1198,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) { func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) { rows, err := s.db.Query(` SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter), - SUM(unfetchable), SUM(errors) + SUM(unfetchable), SUM(errors), SUM(not_found) FROM poll_passes WHERE ran_at >= $1 GROUP BY site @@ -1213,7 +1213,7 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) { var outcomes SiteOutcomes if err := rows.Scan( &outcomes.Site, &outcomes.Refused, &outcomes.Unreachable, - &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, + &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, &outcomes.NotFound, ); err != nil { return nil, fmt.Errorf("scan lane pass outcomes: %w", err) } diff --git a/backend/internal/web/admin_lanes.go b/backend/internal/web/admin_lanes.go index 599bb64..c70b972 100644 --- a/backend/internal/web/admin_lanes.go +++ b/backend/internal/web/admin_lanes.go @@ -242,6 +242,7 @@ func outcomeChips(o store.SiteOutcomes) []chip { {"unreachable", o.Unreachable}, {"no chapter", o.NoChapter}, {"unfetchable", o.Unfetchable}, + {"not found", o.NotFound}, {"errors", o.Errors}, } var out []chip diff --git a/backend/web_test.go b/backend/web_test.go index bdfe09c..6a59c47 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -953,22 +953,32 @@ func TestSkipReasonIsNotAStall(t *testing.T) { } } -// The five outcome counts render named — the page never prints the word -// "failures" — and a Site with none observed says so rather than drawing a -// blank cell. +// The six outcome counts render named — the page never prints the word +// "failures" — in the taxonomy's fixed order, not found beside the word it +// split out of. A Site with none observed says so rather than drawing a +// blank cell, and a zero not_found renders no chip of its own. func TestNamedOutcomeChips(t *testing.T) { router, st := newWebTestServer(t, lanesConfig()) now := time.Now() - seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, NoChapter: 2, Errors: 1}) - seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 3, Unfetchable: 4}) + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, Unreachable: 3, NoChapter: 2, Unfetchable: 4, NotFound: 6, Errors: 1}) + seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 7, Unfetchable: 8}) seedPass(t, st, store.LanePass{Site: "comix", RanAt: now.UnixMilli()}) body := lanesBody(t, router, st) - for _, want := range []string{"refused 5", "no chapter 2", "errors 1", "unreachable 3", "unfetchable 4"} { + for _, want := range []string{"refused 5", "unreachable 3", "no chapter 2", "unfetchable 4", "not found 6", "errors 1"} { if !strings.Contains(body, want) { t.Errorf("lane chips lack %q:\n%s", want, body) } } + wantSeq := `refused 5 · unreachable 3 · no chapter 2 · unfetchable 4 · not found 6 · errors 1` + if !strings.Contains(body, wantSeq) { + t.Errorf("chips do not render in the fixed order (… unfetchable, not found, errors …):\n%s", body) + } + // demonic has outcomes but no not_found: exactly one "not found" in the + // whole fragment — asura's — so a zero count renders none. + if got := strings.Count(body, "not found"); got != 1 { + t.Errorf("the word \"not found\" appears %d times, want exactly the one seeded chip", got) + } if strings.Contains(body, "failures") { t.Errorf("the page prints the forbidden word \"failures\":\n%s", body) }