sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Spec #164: A sixth outcome word: not_found splits out of errors

Browse Source
This commit is contained in:
Sulthan Zaki
2026-08-22 18:19:23 +07:00
parent 8e4fa6448e
commit 62c2cea74e
7 changed files with 87 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/poller.go
+17 -8
View File
@@ -273,7 +273,8 @@ const (
// (issue #141). The classification the read already makes is counted, never a
// second taxonomy: refused is the Site holding a challenge, unreachable the
// browser interrupting, noChapter a 200 with real HTML but no chapter links,
// unfetchable the host pin or a missing fetcher, and errors everything else.
// unfetchable the host pin or a missing fetcher, notFound a 4xx other than
// the 403 refusal — the page is gone — and errors everything else.
type readOutcome int
const (
@@ -283,14 +284,15 @@ const (
outcomeNoChapter
outcomeUnfetchable
outcomeError
outcomeNotFound
)
// outcomeCounts are the five named outcome counts of one pass. A success
// count is derived, never stored: checked minus the four, with unreachable
// excluded because the sidecar-loss path returns before the checked counter
// increments (issue #141).
// outcomeCounts are the six named outcome counts of one pass. A success
// count is derived, never stored: checked minus the five named failures,
// with unreachable excluded because the sidecar-loss path returns before the
// checked counter increments (issue #141).
type outcomeCounts struct {
refused, unreachable, noChapter, unfetchable, errors int
refused, unreachable, noChapter, unfetchable, errors, notFound int
}
func (c *outcomeCounts) add(o readOutcome) {
@@ -303,6 +305,8 @@ func (c *outcomeCounts) add(o readOutcome) {
c.noChapter++
case outcomeUnfetchable:
c.unfetchable++
case outcomeNotFound:
c.notFound++
case outcomeError:
c.errors++
}
@@ -517,6 +521,7 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) {
Unreachable: rec.counts.unreachable,
NoChapter: rec.counts.noChapter,
Unfetchable: rec.counts.unfetchable,
NotFound: rec.counts.notFound,
Errors: rec.counts.errors,
}
if row.GapMS == 0 {
@@ -612,8 +617,8 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D
// the poller is a best-effort enhancement, and no single bad series may stall a
// Lane or take down the process. The returned outcome classifies the read for
// the pass row (issue #141), so the Lane can count a refusal, a lost browser,
// a chapter-less page, an unfetchable address or a transport error without
// re-deriving the taxonomy.
// a chapter-less page, an unfetchable address, a missing page or a transport
// error without re-deriving the taxonomy.
func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) {
defer func() {
if r := recover(); r != nil {
@@ -657,6 +662,10 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut
if errors.Is(err, errBrowserInterrupted) {
return outcomeUnreachable
}
if errors.Is(err, errNotFound) {
log.Printf("latest poll %q: page not found: %v", sr.Key(), err)
return outcomeNotFound
}
return outcomeError
}
// A legacy cover source is healed independently of the page read.
backend/internal/latest/poller_test.go
+33 -3
View File
@@ -1947,9 +1947,9 @@ func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) {
})
}
// The pass row's five outcome counts are the classification the Series read
// The pass row's six outcome counts are the classification the Series read
// already makes — never a second taxonomy (issue #141). Success is derived,
// never stored: checked minus the four named counts, unreachable excluded
// never stored: checked minus the five named failures, unreachable excluded
// because its exit returns before the checked counter increments.
func TestRunLanePassCountsOutcomes(t *testing.T) {
s, _ := newTestStore(t)
@@ -1986,7 +1986,7 @@ func TestRunLanePassCountsOutcomes(t *testing.T) {
t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1",
pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors)
}
if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 {
if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 1 {
t.Fatalf("derived success = %d, want 1", success)
}
// The one genuine read went through: the success Series carries the
@@ -2009,6 +2009,36 @@ func TestRunLanePassCountsOutcomes(t *testing.T) {
}
}
// A 4xx other than the 403 refusal means the page is gone — a sixth outcome
// word (issue #164) — while a 5xx stays errors. Both land in the durable row,
// so the owner can tell "correct the address" from "the Site is unwell"
// without opening a log.
func TestRunLanePassSplitsNotFoundFromErrors(t *testing.T) {
s, _ := newTestStore(t)
now := time.UnixMilli(5_000_000)
seeds := map[string]string{
"asura:gone": "https://asurascans.com/series/gone",
"asura:unwell": "https://asurascans.com/series/unwell",
}
for key, url := range seeds {
seedForCheck(t, s, key, url, 0)
}
f := &fakeFetcher{perURL: map[string]fakeResponse{
seeds["asura:gone"]: {status: 404},
seeds["asura:unwell"]: {status: 503},
}}
newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false)
pass := latestPassFor(t, s, "asura")
if pass.NotFound != 1 || pass.Errors != 1 {
t.Fatalf("not_found=%d errors=%d, want 1 1", pass.NotFound, pass.Errors)
}
if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 0 {
t.Fatalf("derived success = %d, want 0 (both reads failed)", success)
}
}
// A refusal is the Site's mood and outlives our process: the durable stamp a
// pass writes is honoured by a freshly constructed poller, which must not
// re-probe the Site inside its backoff (issue #141).
backend/internal/latest/read.go
+6
View File
@@ -25,9 +25,12 @@ type seriesRead struct {
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
// is how a Lane tells a refusal from an ordinary failure (issue #100).
// errNotFound marks any other 4xx: the page is gone — a fact the owner can act
// on — as distinct from a Site or database that is merely unwell (issue #164).
var (
errNotFetchable = errors.New("series url not fetchable")
errNoFetcher = errors.New("no fetcher for site")
errNotFound = errors.New("series page not found")
)
// readSeriesPage performs the series-page read the poll and the acquirer have
@@ -59,6 +62,9 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
}
if status != 200 {
if status >= 400 && status < 500 {
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errNotFound, seriesURL, status)
}
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
}
if isInterstitial(body) {
backend/internal/store/migrations/0017_poll_passes_not_found.sql
+4
View File
@@ -0,0 +1,4 @@
-- A 4xx other than the 403 refusal is the page being gone, not the Site being
-- unwell; the pass log counts it separately so the Lanes page can say "not
-- found" (#164). DEFAULT 0 keeps pre-existing rows readable.
ALTER TABLE poll_passes ADD COLUMN not_found integer NOT NULL DEFAULT 0;
backend/internal/store/store.go
+10 -10
View File
@@ -110,7 +110,7 @@ type LanePass struct {
GapMS int64
Clamped bool
Refused, Unreachable, NoChapter int
Unfetchable, Errors int
Unfetchable, Errors, NotFound int
PausedUntil, RefuseUntil int64
}
@@ -119,7 +119,7 @@ type LanePass struct {
type SiteOutcomes struct {
Site string
Refused, Unreachable, NoChapter int
Unfetchable, Errors int
Unfetchable, Errors, NotFound int
}
// LanePause is one persisted Lane pause stamp.
@@ -243,7 +243,7 @@ const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.co
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by`
const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped,
p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors,
p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, p.not_found,
COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)`
// Owner is the person running the service: the first Reader, seeded at startup
@@ -663,7 +663,7 @@ func scanLanePass(scan func(...any) error) (LanePass, error) {
var p LanePass
if err := scan(
&p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped,
&p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors,
&p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, &p.NotFound,
&p.PausedUntil, &p.RefuseUntil,
); err != nil {
return LanePass{}, err
@@ -1131,10 +1131,10 @@ func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error {
if _, err := tx.Exec(`
INSERT INTO poll_passes
(site, ran_at, skip, due, checked, gap_ms, clamped,
refused, unreachable, no_chapter, unfetchable, errors)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
refused, unreachable, no_chapter, unfetchable, errors, not_found)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`,
p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped,
p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil {
p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors, p.NotFound); err != nil {
return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err)
}
if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil {
@@ -1171,7 +1171,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) {
FROM (
SELECT DISTINCT ON (site)
site, ran_at, skip, due, checked, gap_ms, clamped,
refused, unreachable, no_chapter, unfetchable, errors
refused, unreachable, no_chapter, unfetchable, errors, not_found
FROM poll_passes
ORDER BY site, ran_at DESC
) p
@@ -1198,7 +1198,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) {
func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
rows, err := s.db.Query(`
SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter),
SUM(unfetchable), SUM(errors)
SUM(unfetchable), SUM(errors), SUM(not_found)
FROM poll_passes
WHERE ran_at >= $1
GROUP BY site
@@ -1213,7 +1213,7 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
var outcomes SiteOutcomes
if err := rows.Scan(
&outcomes.Site, &outcomes.Refused, &outcomes.Unreachable,
&outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors,
&outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, &outcomes.NotFound,
); err != nil {
return nil, fmt.Errorf("scan lane pass outcomes: %w", err)
}
backend/internal/web/admin_lanes.go
+1
View File
@@ -242,6 +242,7 @@ func outcomeChips(o store.SiteOutcomes) []chip {
{"unreachable", o.Unreachable},
{"no chapter", o.NoChapter},
{"unfetchable", o.Unfetchable},
{"not found", o.NotFound},
{"errors", o.Errors},
}
var out []chip
backend/web_test.go
+16 -6
View File
@@ -953,22 +953,32 @@ func TestSkipReasonIsNotAStall(t *testing.T) {
}
}
// The five outcome counts render named — the page never prints the word
// "failures" — and a Site with none observed says so rather than drawing a
// blank cell.
// The six outcome counts render named — the page never prints the word
// "failures" — in the taxonomy's fixed order, not found beside the word it
// split out of. A Site with none observed says so rather than drawing a
// blank cell, and a zero not_found renders no chip of its own.
func TestNamedOutcomeChips(t *testing.T) {
router, st := newWebTestServer(t, lanesConfig())
now := time.Now()
seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, NoChapter: 2, Errors: 1})
seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 3, Unfetchable: 4})
seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, Unreachable: 3, NoChapter: 2, Unfetchable: 4, NotFound: 6, Errors: 1})
seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 7, Unfetchable: 8})
seedPass(t, st, store.LanePass{Site: "comix", RanAt: now.UnixMilli()})
body := lanesBody(t, router, st)
for _, want := range []string{"refused 5", "no chapter 2", "errors 1", "unreachable 3", "unfetchable 4"} {
for _, want := range []string{"refused 5", "unreachable 3", "no chapter 2", "unfetchable 4", "not found 6", "errors 1"} {
if !strings.Contains(body, want) {
t.Errorf("lane chips lack %q:\n%s", want, body)
}
}
wantSeq := `<span class="mark">refused 5</span><span class="mark-faint"> · </span><span class="mark">unreachable 3</span><span class="mark-faint"> · </span><span class="mark">no chapter 2</span><span class="mark-faint"> · </span><span class="mark">unfetchable 4</span><span class="mark-faint"> · </span><span class="mark">not found 6</span><span class="mark-faint"> · </span><span class="mark">errors 1</span>`
if !strings.Contains(body, wantSeq) {
t.Errorf("chips do not render in the fixed order (… unfetchable, not found, errors …):\n%s", body)
}
// demonic has outcomes but no not_found: exactly one "not found" in the
// whole fragment — asura's — so a zero count renders none.
if got := strings.Count(body, "not found"); got != 1 {
t.Errorf("the word \"not found\" appears %d times, want exactly the one seeded chip", got)
}
if strings.Contains(body, "failures") {
t.Errorf("the page prints the forbidden word \"failures\":\n%s", body)
}