diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go
index 1c6ae7c..e5125c8 100644
--- a/backend/internal/latest/poller.go
+++ b/backend/internal/latest/poller.go
@@ -273,7 +273,9 @@ const (
// (issue #141). The classification the read already makes is counted, never a
// second taxonomy: refused is the Site holding a challenge, unreachable the
// browser interrupting, noChapter a 200 with real HTML but no chapter links,
-// unfetchable the host pin or a missing fetcher, and errors everything else.
+// unfetchable the host pin or a missing fetcher, notFound a 4xx other than
+// the 403 refusal — the Site answered with a client status — and errors
+// everything else.
type readOutcome int
const (
@@ -283,14 +285,15 @@ const (
outcomeNoChapter
outcomeUnfetchable
outcomeError
+ outcomeNotFound
)
-// outcomeCounts are the five named outcome counts of one pass. A success
-// count is derived, never stored: checked minus the four, with unreachable
-// excluded because the sidecar-loss path returns before the checked counter
-// increments (issue #141).
+// outcomeCounts are the six named outcome counts of one pass. A success
+// count is derived, never stored: checked minus the five named failures,
+// with unreachable excluded because the sidecar-loss path returns before the
+// checked counter increments (issue #141).
type outcomeCounts struct {
- refused, unreachable, noChapter, unfetchable, errors int
+ refused, unreachable, noChapter, unfetchable, errors, notFound int
}
func (c *outcomeCounts) add(o readOutcome) {
@@ -303,6 +306,8 @@ func (c *outcomeCounts) add(o readOutcome) {
c.noChapter++
case outcomeUnfetchable:
c.unfetchable++
+ case outcomeNotFound:
+ c.notFound++
case outcomeError:
c.errors++
}
@@ -517,6 +522,7 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) {
Unreachable: rec.counts.unreachable,
NoChapter: rec.counts.noChapter,
Unfetchable: rec.counts.unfetchable,
+ NotFound: rec.counts.notFound,
Errors: rec.counts.errors,
}
if row.GapMS == 0 {
@@ -612,8 +618,8 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D
// the poller is a best-effort enhancement, and no single bad series may stall a
// Lane or take down the process. The returned outcome classifies the read for
// the pass row (issue #141), so the Lane can count a refusal, a lost browser,
-// a chapter-less page, an unfetchable address or a transport error without
-// re-deriving the taxonomy.
+// a chapter-less page, an unfetchable address, a missing page or a transport
+// error without re-deriving the taxonomy.
func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) {
defer func() {
if r := recover(); r != nil {
@@ -657,6 +663,9 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut
if errors.Is(err, errBrowserInterrupted) {
return outcomeUnreachable
}
+ if errors.Is(err, errNotFound) {
+ return outcomeNotFound
+ }
return outcomeError
}
// A legacy cover source is healed independently of the page read.
diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go
index 301225b..39576da 100644
--- a/backend/internal/latest/poller_test.go
+++ b/backend/internal/latest/poller_test.go
@@ -1947,9 +1947,9 @@ func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) {
})
}
-// The pass row's five outcome counts are the classification the Series read
+// The pass row's six outcome counts are the classification the Series read
// already makes — never a second taxonomy (issue #141). Success is derived,
-// never stored: checked minus the four named counts, unreachable excluded
+// never stored: checked minus the five named failures, unreachable excluded
// because its exit returns before the checked counter increments.
func TestRunLanePassCountsOutcomes(t *testing.T) {
s, _ := newTestStore(t)
@@ -1986,7 +1986,7 @@ func TestRunLanePassCountsOutcomes(t *testing.T) {
t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1",
pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors)
}
- if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 {
+ if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 1 {
t.Fatalf("derived success = %d, want 1", success)
}
// The one genuine read went through: the success Series carries the
@@ -2009,6 +2009,36 @@ func TestRunLanePassCountsOutcomes(t *testing.T) {
}
}
+// A 4xx other than the 403 refusal means the page is gone — a sixth outcome
+// word (issue #164) — while a 5xx stays errors. Both land in the durable row,
+// so the owner can tell "correct the address" from "the Site is unwell"
+// without opening a log.
+func TestRunLanePassSplitsNotFoundFromErrors(t *testing.T) {
+ s, _ := newTestStore(t)
+ now := time.UnixMilli(5_000_000)
+ seeds := map[string]string{
+ "asura:gone": "https://asurascans.com/series/gone",
+ "asura:unwell": "https://asurascans.com/series/unwell",
+ }
+ for key, url := range seeds {
+ seedForCheck(t, s, key, url, 0)
+ }
+
+ f := &fakeFetcher{perURL: map[string]fakeResponse{
+ seeds["asura:gone"]: {status: 404},
+ seeds["asura:unwell"]: {status: 503},
+ }}
+ newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false)
+
+ pass := latestPassFor(t, s, "asura")
+ if pass.NotFound != 1 || pass.Errors != 1 {
+ t.Fatalf("not_found=%d errors=%d, want 1 1", pass.NotFound, pass.Errors)
+ }
+ if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 0 {
+ t.Fatalf("derived success = %d, want 0 (both reads failed)", success)
+ }
+}
+
// A refusal is the Site's mood and outlives our process: the durable stamp a
// pass writes is honoured by a freshly constructed poller, which must not
// re-probe the Site inside its backoff (issue #141).
diff --git a/backend/internal/latest/read.go b/backend/internal/latest/read.go
index 134223d..8a9c611 100644
--- a/backend/internal/latest/read.go
+++ b/backend/internal/latest/read.go
@@ -25,9 +25,12 @@ type seriesRead struct {
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
// is how a Lane tells a refusal from an ordinary failure (issue #100).
+// errNotFound marks any other 4xx: the page is gone — a fact the owner can act
+// on — as distinct from a Site or database that is merely unwell (issue #164).
var (
errNotFetchable = errors.New("series url not fetchable")
errNoFetcher = errors.New("no fetcher for site")
+ errNotFound = errors.New("series page not found")
)
// readSeriesPage performs the series-page read the poll and the acquirer have
@@ -59,6 +62,9 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
}
if status != 200 {
+ if status >= 400 && status < 500 {
+ return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errNotFound, seriesURL, status)
+ }
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
}
if isInterstitial(body) {
diff --git a/backend/internal/store/migrations/0017_poll_passes_not_found.sql b/backend/internal/store/migrations/0017_poll_passes_not_found.sql
new file mode 100644
index 0000000..8f93b7d
--- /dev/null
+++ b/backend/internal/store/migrations/0017_poll_passes_not_found.sql
@@ -0,0 +1,4 @@
+-- A 4xx other than the 403 refusal is the page being gone, not the Site being
+-- unwell; the pass log counts it separately so the Lanes page can say "not
+-- found" (#164). DEFAULT 0 keeps pre-existing rows readable.
+ALTER TABLE poll_passes ADD COLUMN not_found integer NOT NULL DEFAULT 0;
\ No newline at end of file
diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go
index 2552fa9..153471d 100644
--- a/backend/internal/store/store.go
+++ b/backend/internal/store/store.go
@@ -110,7 +110,7 @@ type LanePass struct {
GapMS int64
Clamped bool
Refused, Unreachable, NoChapter int
- Unfetchable, Errors int
+ Unfetchable, Errors, NotFound int
PausedUntil, RefuseUntil int64
}
@@ -119,7 +119,7 @@ type LanePass struct {
type SiteOutcomes struct {
Site string
Refused, Unreachable, NoChapter int
- Unfetchable, Errors int
+ Unfetchable, Errors, NotFound int
}
// LanePause is one persisted Lane pause stamp.
@@ -243,7 +243,7 @@ const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.co
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by`
const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped,
- p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors,
+ p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, p.not_found,
COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)`
// Owner is the person running the service: the first Reader, seeded at startup
@@ -663,7 +663,7 @@ func scanLanePass(scan func(...any) error) (LanePass, error) {
var p LanePass
if err := scan(
&p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped,
- &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors,
+ &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, &p.NotFound,
&p.PausedUntil, &p.RefuseUntil,
); err != nil {
return LanePass{}, err
@@ -1131,10 +1131,10 @@ func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error {
if _, err := tx.Exec(`
INSERT INTO poll_passes
(site, ran_at, skip, due, checked, gap_ms, clamped,
- refused, unreachable, no_chapter, unfetchable, errors)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
+ refused, unreachable, no_chapter, unfetchable, errors, not_found)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`,
p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped,
- p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil {
+ p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors, p.NotFound); err != nil {
return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err)
}
if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil {
@@ -1171,7 +1171,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) {
FROM (
SELECT DISTINCT ON (site)
site, ran_at, skip, due, checked, gap_ms, clamped,
- refused, unreachable, no_chapter, unfetchable, errors
+ refused, unreachable, no_chapter, unfetchable, errors, not_found
FROM poll_passes
ORDER BY site, ran_at DESC
) p
@@ -1198,7 +1198,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) {
func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
rows, err := s.db.Query(`
SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter),
- SUM(unfetchable), SUM(errors)
+ SUM(unfetchable), SUM(errors), SUM(not_found)
FROM poll_passes
WHERE ran_at >= $1
GROUP BY site
@@ -1213,7 +1213,7 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
var outcomes SiteOutcomes
if err := rows.Scan(
&outcomes.Site, &outcomes.Refused, &outcomes.Unreachable,
- &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors,
+ &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, &outcomes.NotFound,
); err != nil {
return nil, fmt.Errorf("scan lane pass outcomes: %w", err)
}
diff --git a/backend/internal/web/admin_lanes.go b/backend/internal/web/admin_lanes.go
index 599bb64..c70b972 100644
--- a/backend/internal/web/admin_lanes.go
+++ b/backend/internal/web/admin_lanes.go
@@ -242,6 +242,7 @@ func outcomeChips(o store.SiteOutcomes) []chip {
{"unreachable", o.Unreachable},
{"no chapter", o.NoChapter},
{"unfetchable", o.Unfetchable},
+ {"not found", o.NotFound},
{"errors", o.Errors},
}
var out []chip
diff --git a/backend/web_test.go b/backend/web_test.go
index bdfe09c..6a59c47 100644
--- a/backend/web_test.go
+++ b/backend/web_test.go
@@ -953,22 +953,32 @@ func TestSkipReasonIsNotAStall(t *testing.T) {
}
}
-// The five outcome counts render named — the page never prints the word
-// "failures" — and a Site with none observed says so rather than drawing a
-// blank cell.
+// The six outcome counts render named — the page never prints the word
+// "failures" — in the taxonomy's fixed order, not found beside the word it
+// split out of. A Site with none observed says so rather than drawing a
+// blank cell, and a zero not_found renders no chip of its own.
func TestNamedOutcomeChips(t *testing.T) {
router, st := newWebTestServer(t, lanesConfig())
now := time.Now()
- seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, NoChapter: 2, Errors: 1})
- seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 3, Unfetchable: 4})
+ seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, Unreachable: 3, NoChapter: 2, Unfetchable: 4, NotFound: 6, Errors: 1})
+ seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 7, Unfetchable: 8})
seedPass(t, st, store.LanePass{Site: "comix", RanAt: now.UnixMilli()})
body := lanesBody(t, router, st)
- for _, want := range []string{"refused 5", "no chapter 2", "errors 1", "unreachable 3", "unfetchable 4"} {
+ for _, want := range []string{"refused 5", "unreachable 3", "no chapter 2", "unfetchable 4", "not found 6", "errors 1"} {
if !strings.Contains(body, want) {
t.Errorf("lane chips lack %q:\n%s", want, body)
}
}
+ wantSeq := `refused 5 · unreachable 3 · no chapter 2 · unfetchable 4 · not found 6 · errors 1`
+ if !strings.Contains(body, wantSeq) {
+ t.Errorf("chips do not render in the fixed order (… unfetchable, not found, errors …):\n%s", body)
+ }
+ // demonic has outcomes but no not_found: exactly one "not found" in the
+ // whole fragment — asura's — so a zero count renders none.
+ if got := strings.Count(body, "not found"); got != 1 {
+ t.Errorf("the word \"not found\" appears %d times, want exactly the one seeded chip", got)
+ }
if strings.Contains(body, "failures") {
t.Errorf("the page prints the forbidden word \"failures\":\n%s", body)
}