From 62c2cea74e402a390b6a5183ea3f01ee36d8132a Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 18:19:23 +0700 Subject: [PATCH 01/20] Spec #164: A sixth outcome word: not_found splits out of errors --- backend/internal/latest/poller.go | 25 ++++++++----- backend/internal/latest/poller_test.go | 36 +++++++++++++++++-- backend/internal/latest/read.go | 6 ++++ .../migrations/0017_poll_passes_not_found.sql | 4 +++ backend/internal/store/store.go | 20 +++++------ backend/internal/web/admin_lanes.go | 1 + backend/web_test.go | 22 ++++++++---- 7 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 backend/internal/store/migrations/0017_poll_passes_not_found.sql diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 1c6ae7c..4cfc2f1 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -273,7 +273,8 @@ const ( // (issue #141). The classification the read already makes is counted, never a // second taxonomy: refused is the Site holding a challenge, unreachable the // browser interrupting, noChapter a 200 with real HTML but no chapter links, -// unfetchable the host pin or a missing fetcher, and errors everything else. +// unfetchable the host pin or a missing fetcher, notFound a 4xx other than +// the 403 refusal — the page is gone — and errors everything else. type readOutcome int const ( @@ -283,14 +284,15 @@ const ( outcomeNoChapter outcomeUnfetchable outcomeError + outcomeNotFound ) -// outcomeCounts are the five named outcome counts of one pass. A success -// count is derived, never stored: checked minus the four, with unreachable -// excluded because the sidecar-loss path returns before the checked counter -// increments (issue #141). +// outcomeCounts are the six named outcome counts of one pass. A success +// count is derived, never stored: checked minus the five named failures, +// with unreachable excluded because the sidecar-loss path returns before the +// checked counter increments (issue #141). type outcomeCounts struct { - refused, unreachable, noChapter, unfetchable, errors int + refused, unreachable, noChapter, unfetchable, errors, notFound int } func (c *outcomeCounts) add(o readOutcome) { @@ -303,6 +305,8 @@ func (c *outcomeCounts) add(o readOutcome) { c.noChapter++ case outcomeUnfetchable: c.unfetchable++ + case outcomeNotFound: + c.notFound++ case outcomeError: c.errors++ } @@ -517,6 +521,7 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) { Unreachable: rec.counts.unreachable, NoChapter: rec.counts.noChapter, Unfetchable: rec.counts.unfetchable, + NotFound: rec.counts.notFound, Errors: rec.counts.errors, } if row.GapMS == 0 { @@ -612,8 +617,8 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D // the poller is a best-effort enhancement, and no single bad series may stall a // Lane or take down the process. The returned outcome classifies the read for // the pass row (issue #141), so the Lane can count a refusal, a lost browser, -// a chapter-less page, an unfetchable address or a transport error without -// re-deriving the taxonomy. +// a chapter-less page, an unfetchable address, a missing page or a transport +// error without re-deriving the taxonomy. func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) { defer func() { if r := recover(); r != nil { @@ -657,6 +662,10 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut if errors.Is(err, errBrowserInterrupted) { return outcomeUnreachable } + if errors.Is(err, errNotFound) { + log.Printf("latest poll %q: page not found: %v", sr.Key(), err) + return outcomeNotFound + } return outcomeError } // A legacy cover source is healed independently of the page read. diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 301225b..39576da 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -1947,9 +1947,9 @@ func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) { }) } -// The pass row's five outcome counts are the classification the Series read +// The pass row's six outcome counts are the classification the Series read // already makes — never a second taxonomy (issue #141). Success is derived, -// never stored: checked minus the four named counts, unreachable excluded +// never stored: checked minus the five named failures, unreachable excluded // because its exit returns before the checked counter increments. func TestRunLanePassCountsOutcomes(t *testing.T) { s, _ := newTestStore(t) @@ -1986,7 +1986,7 @@ func TestRunLanePassCountsOutcomes(t *testing.T) { t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1", pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors) } - if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 { + if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 1 { t.Fatalf("derived success = %d, want 1", success) } // The one genuine read went through: the success Series carries the @@ -2009,6 +2009,36 @@ func TestRunLanePassCountsOutcomes(t *testing.T) { } } +// A 4xx other than the 403 refusal means the page is gone — a sixth outcome +// word (issue #164) — while a 5xx stays errors. Both land in the durable row, +// so the owner can tell "correct the address" from "the Site is unwell" +// without opening a log. +func TestRunLanePassSplitsNotFoundFromErrors(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seeds := map[string]string{ + "asura:gone": "https://asurascans.com/series/gone", + "asura:unwell": "https://asurascans.com/series/unwell", + } + for key, url := range seeds { + seedForCheck(t, s, key, url, 0) + } + + f := &fakeFetcher{perURL: map[string]fakeResponse{ + seeds["asura:gone"]: {status: 404}, + seeds["asura:unwell"]: {status: 503}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + pass := latestPassFor(t, s, "asura") + if pass.NotFound != 1 || pass.Errors != 1 { + t.Fatalf("not_found=%d errors=%d, want 1 1", pass.NotFound, pass.Errors) + } + if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.NotFound + pass.Errors); success != 0 { + t.Fatalf("derived success = %d, want 0 (both reads failed)", success) + } +} + // A refusal is the Site's mood and outlives our process: the durable stamp a // pass writes is honoured by a freshly constructed poller, which must not // re-probe the Site inside its backoff (issue #141). diff --git a/backend/internal/latest/read.go b/backend/internal/latest/read.go index 134223d..8a9c611 100644 --- a/backend/internal/latest/read.go +++ b/backend/internal/latest/read.go @@ -25,9 +25,12 @@ type seriesRead struct { // errChallengeHeld (browser.go) is the outcome of a Site that answered with // its interstitial — status 403 (cf-mitigated) or a challenge page body — and // is how a Lane tells a refusal from an ordinary failure (issue #100). +// errNotFound marks any other 4xx: the page is gone — a fact the owner can act +// on — as distinct from a Site or database that is merely unwell (issue #164). var ( errNotFetchable = errors.New("series url not fetchable") errNoFetcher = errors.New("no fetcher for site") + errNotFound = errors.New("series page not found") ) // readSeriesPage performs the series-page read the poll and the acquirer have @@ -59,6 +62,9 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status) } if status != 200 { + if status >= 400 && status < 500 { + return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errNotFound, seriesURL, status) + } return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status) } if isInterstitial(body) { diff --git a/backend/internal/store/migrations/0017_poll_passes_not_found.sql b/backend/internal/store/migrations/0017_poll_passes_not_found.sql new file mode 100644 index 0000000..8f93b7d --- /dev/null +++ b/backend/internal/store/migrations/0017_poll_passes_not_found.sql @@ -0,0 +1,4 @@ +-- A 4xx other than the 403 refusal is the page being gone, not the Site being +-- unwell; the pass log counts it separately so the Lanes page can say "not +-- found" (#164). DEFAULT 0 keeps pre-existing rows readable. +ALTER TABLE poll_passes ADD COLUMN not_found integer NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 2552fa9..153471d 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -110,7 +110,7 @@ type LanePass struct { GapMS int64 Clamped bool Refused, Unreachable, NoChapter int - Unfetchable, Errors int + Unfetchable, Errors, NotFound int PausedUntil, RefuseUntil int64 } @@ -119,7 +119,7 @@ type LanePass struct { type SiteOutcomes struct { Site string Refused, Unreachable, NoChapter int - Unfetchable, Errors int + Unfetchable, Errors, NotFound int } // LanePause is one persisted Lane pause stamp. @@ -243,7 +243,7 @@ const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.co s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by` const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped, - p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, + p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, p.not_found, COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)` // Owner is the person running the service: the first Reader, seeded at startup @@ -663,7 +663,7 @@ func scanLanePass(scan func(...any) error) (LanePass, error) { var p LanePass if err := scan( &p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped, - &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, + &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, &p.NotFound, &p.PausedUntil, &p.RefuseUntil, ); err != nil { return LanePass{}, err @@ -1131,10 +1131,10 @@ func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error { if _, err := tx.Exec(` INSERT INTO poll_passes (site, ran_at, skip, due, checked, gap_ms, clamped, - refused, unreachable, no_chapter, unfetchable, errors) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + refused, unreachable, no_chapter, unfetchable, errors, not_found) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped, - p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil { + p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors, p.NotFound); err != nil { return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err) } if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil { @@ -1171,7 +1171,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) { FROM ( SELECT DISTINCT ON (site) site, ran_at, skip, due, checked, gap_ms, clamped, - refused, unreachable, no_chapter, unfetchable, errors + refused, unreachable, no_chapter, unfetchable, errors, not_found FROM poll_passes ORDER BY site, ran_at DESC ) p @@ -1198,7 +1198,7 @@ func (s *Store) LatestLanePasses() ([]LanePass, error) { func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) { rows, err := s.db.Query(` SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter), - SUM(unfetchable), SUM(errors) + SUM(unfetchable), SUM(errors), SUM(not_found) FROM poll_passes WHERE ran_at >= $1 GROUP BY site @@ -1213,7 +1213,7 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) { var outcomes SiteOutcomes if err := rows.Scan( &outcomes.Site, &outcomes.Refused, &outcomes.Unreachable, - &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, + &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, &outcomes.NotFound, ); err != nil { return nil, fmt.Errorf("scan lane pass outcomes: %w", err) } diff --git a/backend/internal/web/admin_lanes.go b/backend/internal/web/admin_lanes.go index 599bb64..c70b972 100644 --- a/backend/internal/web/admin_lanes.go +++ b/backend/internal/web/admin_lanes.go @@ -242,6 +242,7 @@ func outcomeChips(o store.SiteOutcomes) []chip { {"unreachable", o.Unreachable}, {"no chapter", o.NoChapter}, {"unfetchable", o.Unfetchable}, + {"not found", o.NotFound}, {"errors", o.Errors}, } var out []chip diff --git a/backend/web_test.go b/backend/web_test.go index bdfe09c..6a59c47 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -953,22 +953,32 @@ func TestSkipReasonIsNotAStall(t *testing.T) { } } -// The five outcome counts render named — the page never prints the word -// "failures" — and a Site with none observed says so rather than drawing a -// blank cell. +// The six outcome counts render named — the page never prints the word +// "failures" — in the taxonomy's fixed order, not found beside the word it +// split out of. A Site with none observed says so rather than drawing a +// blank cell, and a zero not_found renders no chip of its own. func TestNamedOutcomeChips(t *testing.T) { router, st := newWebTestServer(t, lanesConfig()) now := time.Now() - seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, NoChapter: 2, Errors: 1}) - seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 3, Unfetchable: 4}) + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, Unreachable: 3, NoChapter: 2, Unfetchable: 4, NotFound: 6, Errors: 1}) + seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 7, Unfetchable: 8}) seedPass(t, st, store.LanePass{Site: "comix", RanAt: now.UnixMilli()}) body := lanesBody(t, router, st) - for _, want := range []string{"refused 5", "no chapter 2", "errors 1", "unreachable 3", "unfetchable 4"} { + for _, want := range []string{"refused 5", "unreachable 3", "no chapter 2", "unfetchable 4", "not found 6", "errors 1"} { if !strings.Contains(body, want) { t.Errorf("lane chips lack %q:\n%s", want, body) } } + wantSeq := `refused 5 · unreachable 3 · no chapter 2 · unfetchable 4 · not found 6 · errors 1` + if !strings.Contains(body, wantSeq) { + t.Errorf("chips do not render in the fixed order (… unfetchable, not found, errors …):\n%s", body) + } + // demonic has outcomes but no not_found: exactly one "not found" in the + // whole fragment — asura's — so a zero count renders none. + if got := strings.Count(body, "not found"); got != 1 { + t.Errorf("the word \"not found\" appears %d times, want exactly the one seeded chip", got) + } if strings.Contains(body, "failures") { t.Errorf("the page prints the forbidden word \"failures\":\n%s", body) } -- 2.52.0 From f4e4055a1354212f67ccc4b6889dc34dfc384f4c Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 18:22:03 +0700 Subject: [PATCH 02/20] Review: drop duplicate 404 log, scope comment wording to 4xx (Spec #164) --- backend/internal/latest/poller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 4cfc2f1..e5125c8 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -274,7 +274,8 @@ const ( // second taxonomy: refused is the Site holding a challenge, unreachable the // browser interrupting, noChapter a 200 with real HTML but no chapter links, // unfetchable the host pin or a missing fetcher, notFound a 4xx other than -// the 403 refusal — the page is gone — and errors everything else. +// the 403 refusal — the Site answered with a client status — and errors +// everything else. type readOutcome int const ( @@ -663,7 +664,6 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut return outcomeUnreachable } if errors.Is(err, errNotFound) { - log.Printf("latest poll %q: page not found: %v", sr.Key(), err) return outcomeNotFound } return outcomeError -- 2.52.0 From e69e513be4f13d0a1ddcdbf7a46ec091fa23f25e Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 18:23:42 +0700 Subject: [PATCH 03/20] Add completed markers for all six sites (#168) site gains a Completed predicate answering whether a fetched body carries the Site's own completed value: asura escaped props status, demonic info-block Status pair, comix scoped detail entry (shared comixDetailQuery helper with the cover read), kagane publication_status only, novelfull status link, lightnovelworld JSON-LD creativeWorkStatus. siteCompletedFrom dispatches it; an absent hint stays false. Fixtures trimmed from live pages fetched 2026-08-22; TestSiteCompletedFrom covers completed, ongoing, selector-removed, and challenge bodies per site, the kagane upload/publication divergence, and an unknown site. --- backend/internal/latest/sites.go | 127 ++++++++++-- backend/internal/latest/sites_test.go | 269 ++++++++++++++++++++++++++ 2 files changed, 382 insertions(+), 14 deletions(-) diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index 52f3824..b23d1b1 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -22,10 +22,10 @@ type latestChapter struct { // site answers the fixed questions every series-page read asks of its Site // (ADR-0009): the host its addresses must carry, how to find the Latest -// Chapter and the Cover address in a body, and — for a Site behind a -// JavaScript challenge — how to read its payload from a cleared tab. One -// entry describes everything about one Site, and nowhere else gets to compare -// the site string. +// Chapter and the Cover address in a body, whether the Site calls the work +// completed, and — for a Site behind a JavaScript challenge — how to read its +// payload from a cleared tab. One entry describes everything about one Site, +// and nowhere else gets to compare the site string. type site struct { // Host is the exact hostname a series_url for this Site must carry. Host string @@ -33,6 +33,10 @@ type site struct { LatestChapter func(seriesURL, body string) (latestChapter, bool) // Cover finds the Cover address in a fetched body. Cover func(seriesURL, body string) (string, bool) + // Completed reports whether this body carries the Site's own completed + // value. False for a body that carries any other value, and false for a + // failed extraction — never an error and never a third state. + Completed func(seriesURL, body string) bool // Rest is how long a Series of this Site rests between Polls. Rest time.Duration // Gap is the Lane's strictest pace: at least one second must pass between @@ -299,34 +303,42 @@ func kaganeCoverURL(body string) string { return "" } -func comixCoverURL(seriesURL, body string) string { +// comixDetailQuery returns the ["manga","detail",""] query entry of +// comix's initial-data JSON, or nil. The cover and completed reads share the +// scoped lookup so a "recommended" strip entry can never contribute either +// answer. +func comixDetailQuery(seriesURL, body string) json.RawMessage { id, ok := comixSeriesID(seriesURL) if !ok { - return "" + return nil } data := comixInitialDataRe.FindStringSubmatch(body) if data == nil { - return "" + return nil } var state struct { Queries map[string]json.RawMessage `json:"queries"` } if err := json.Unmarshal([]byte(data[1]), &state); err != nil { + return nil + } + return state.Queries[`["manga","detail","`+id+`"]`] +} + +func comixCoverURL(seriesURL, body string) string { + detail := comixDetailQuery(seriesURL, body) + if len(detail) == 0 { return "" } - raw := state.Queries[`["manga","detail","`+id+`"]`] - if len(raw) == 0 { - return "" - } - var detail struct { + var entry struct { Poster struct { Medium string `json:"medium"` } `json:"poster"` } - if err := json.Unmarshal(raw, &detail); err != nil { + if err := json.Unmarshal(detail, &entry); err != nil { return "" } - return publishedCoverURL(detail.Poster.Medium) + return publishedCoverURL(entry.Poster.Medium) } // ogImageCover reads the og:image metadata shared by asura, demonic and @@ -360,6 +372,87 @@ func coverFrom(site, seriesURL, body string) (string, bool) { return "", false } +// asuraStatusRe matches the status value inside the escaped astro-island +// props blob, in both the " form the served document carries and the " +// form a decoded copy would. "completed" is the only true value: "dropped" +// is scanlation editorial (the work itself continues elsewhere) and hiatus is +// its own value. +var asuraStatusRe = regexp.MustCompile(`(?:"|")status(?:"|"):\[0,(?:"|")completed(?:"|")\]`) + +func asuraCompleted(_, body string) bool { + return asuraStatusRe.MatchString(body) +} + +// demonicStatusRe matches the info block's status pair: a Status label
  • +// immediately followed by the value
  • . The site's whole status vocabulary +// is {Ongoing, Completed} (its advanced-search status filter), so the literal +// Completed value is the entire signal. +var demonicStatusRe = regexp.MustCompile(`]*>\s*Status\s*
  • \s*]*>\s*Completed\s*`) + +func demonicCompleted(_, body string) bool { + return demonicStatusRe.MatchString(body) +} + +// comixCompleted reads "status" from the scoped detail entry only; +// "finished" is the completed value, and on_hiatus and discontinued are +// distinct values. +func comixCompleted(seriesURL, body string) bool { + detail := comixDetailQuery(seriesURL, body) + if len(detail) == 0 { + return false + } + var entry struct { + Status string `json:"status"` + } + if err := json.Unmarshal(detail, &entry); err != nil { + return false + } + return entry.Status == "finished" +} + +// kaganeCompleted reads publication_status only: upload_status is the +// release's state, and the two provably diverge ('Cause Calypso Can, +// 2026-08-19: publication Ongoing, upload Hiatus), so a Completed upload +// must never read as a Completed work. +func kaganeCompleted(_, body string) bool { + var series struct { + PublicationStatus string `json:"publication_status"` + } + if err := json.Unmarshal([]byte(body), &series); err != nil { + return false + } + return series.PublicationStatus == "Completed" +} + +// novelfullStatusRe matches the info panel's status link. The page's whole +// status vocabulary is {Ongoing, Completed} (the "OnGoing" spelling aliases +// "Ongoing" on the taxonomy), so the Completed href is the signal. +var novelfullStatusRe = regexp.MustCompile(`href="/status/Completed"`) + +func novelfullCompleted(_, body string) bool { + return novelfullStatusRe.MatchString(body) +} + +// lnwCompleted matches creativeWorkStatus in the head's JSON-LD block. The +// whole body is scanned and the comment marker is not required, unlike +// lnwLatestChapter: the status block sits ahead of the visitor-writable +// thread, and hiatus maps to a distinct PotentialActionStatus value. +var lnwCompletedRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`) + +func lnwCompleted(_, body string) bool { + return lnwCompletedRe.MatchString(body) +} + +// siteCompletedFrom reports whether the Site calls this work completed, via +// the Site's registry entry. False for an unknown site, a challenge body and +// a redesign alike: an absent hint, never a claim. +func siteCompletedFrom(site, seriesURL, body string) bool { + if fn := sites[site].Completed; fn != nil { + return fn(seriesURL, body) + } + return false +} + // metaContent returns the content of the first whose attrName is // attrValue. It keeps scanning after an empty match so a later published cover // is not hidden by an empty tag. @@ -449,6 +542,7 @@ var sites = map[string]site{ Host: "asurascans.com", LatestChapter: asuraLatestChapter, Cover: ogImageCover, + Completed: asuraCompleted, Rest: defaultRest, Gap: defaultGap, }, @@ -456,6 +550,7 @@ var sites = map[string]site{ Host: "demonicscans.org", LatestChapter: demonicLatestChapter, Cover: ogImageCover, + Completed: demonicCompleted, Rest: defaultRest, Gap: defaultGap, }, @@ -463,6 +558,7 @@ var sites = map[string]site{ Host: "comix.to", LatestChapter: comixLatestChapter, Cover: comixCoverEntry, + Completed: comixCompleted, Rest: defaultRest, Gap: defaultGap, Browser: &browserRead{ @@ -479,6 +575,7 @@ var sites = map[string]site{ Host: "kagane.to", LatestChapter: kaganeLatestChapter, Cover: kaganeCoverEntry, + Completed: kaganeCompleted, Rest: defaultRest, Gap: defaultGap, Browser: &browserRead{ @@ -493,6 +590,7 @@ var sites = map[string]site{ Host: "novelfull.com", LatestChapter: novelfullLatestChapter, Cover: novelfullCoverEntry, + Completed: novelfullCompleted, Rest: defaultRest, Gap: defaultGap, Browser: &browserRead{ @@ -507,6 +605,7 @@ var sites = map[string]site{ Host: "lightnovelworld.net", LatestChapter: lnwLatestChapter, Cover: ogImageCover, + Completed: lnwCompleted, Rest: defaultRest, Gap: defaultGap, }, diff --git a/backend/internal/latest/sites_test.go b/backend/internal/latest/sites_test.go index 989aeb0..95368c9 100644 --- a/backend/internal/latest/sites_test.go +++ b/backend/internal/latest/sites_test.go @@ -188,6 +188,100 @@ const lnwSeriesFixture = ` ` +// Completed-marker fixtures, trimmed from the live pages fetched 2026-08-22 +// for the verification step below. Selector-removed rows delete the marker +// from the consts and must answer false. + +// Trimmed from https://asurascans.com/comics/solo-leveling (301 to +// /comics/solo-leveling-b60d532c) fetched 2026-08-22. The astro-island props +// are HTML-escaped in the served document, so the quotes arrive as ". +const asuraCompletedFixture = ` +"bookmarkCount":[0,39128],"status":[0,"completed"],"type":[0,"manhwa"] +` + +// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af +// fetched 2026-08-22. +const asuraOngoingFixture = ` +"bookmarkCount":[0,54027],"status":[0,"ongoing"],"type":[0,"manhwa"] +` + +// Trimmed from https://demonicscans.org/manga/Solo-Leveling fetched +// 2026-08-22. The info block is sloppy: bare
  • s inside a
    , label and +// value as a sibling pair. +const demonicCompletedFixture = ` +
    +
  • Status
  • +
  • Completed
  • + +` + +// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer +// fetched 2026-08-22. Same block, ongoing value. +const demonicOngoingFixture = ` +
    +
  • Status
  • +
  • Ongoing
  • +
    +` + +// Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 over a +// cleared Chrome tab (the in-tab fetch of the Series URL, the same +// server-rendered payload the poll reads). The recommended strip on this very +// page carries a finished entry; only the ["manga","detail","q77m"] entry +// counts. +const comixCompletedFixture = `` + +// Trimmed from https://comix.to/title/n8we-dungeons-and-crayons fetched +// 2026-08-22 the same way. The recommended strip includes a finished entry; +// the target detail is releasing, and only the detail read counts. +const comixOngoingFixture = `` + +// Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6 +// on 2026-08-22. +const kaganeCompletedFixture = `{"series_id":"019c29c3-5abd-70e6-9efc-1f1f22d839f6","title":"Real Account 1-17","publication_status":"Completed","upload_status":"Completed"}` + +// Trimmed from GET https://kagane.to/api/v2/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b +// on 2026-08-22. +const kaganeOngoingFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Ongoing"}` + +// Composed, not a single live trim: upload Completed alongside a +// non-Completed publication status is the research note's inferred inverse +// case and did not turn up in the ~1900-series live scan of 2026-08-22 (both +// field vocabularies are live-verified; the note's live divergence is 'Cause +// Calypso Can, publication Ongoing + upload Hiatus). The predicate must read +// publication_status only. +const kaganeDivergentFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Completed"}` + +// Trimmed from https://novelfull.com/reverend-insanity.html fetched +// 2026-08-22 (the challenge served to plain TLS cleared in a real Chrome the +// same day). +const novelfullCompletedFixture = `

    Status:

    Completed
    ` + +// Trimmed from https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html +// fetched 2026-08-22. +const novelfullOngoingFixture = `

    Status:

    Ongoing
    ` + +// Trimmed from the JSON-LD block in the head of +// https://lightnovelworld.net/novel/a-will-eternal/ fetched 2026-08-22. The +// block sits ahead of the wpdiscuz thread, so the predicate reads the whole +// body and needs no comment marker, unlike lnwLatestChapter. +const lnwCompletedFixture = `` + +// Trimmed from +// https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/ +// fetched 2026-08-22. Same block, ongoing value. +const lnwOngoingFixture = `` + // Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af // (redirected to ...-00dcbf97) on 2026-08-10. const asuraCoverFixture = `` @@ -473,3 +567,178 @@ func TestLatestChapterFrom(t *testing.T) { }) } } + +func TestSiteCompletedFrom(t *testing.T) { + const asuraURL = "https://asurascans.com/comics/solo-leveling-b60d532c" + const demonicURL = "https://demonicscans.org/manga/Solo-Leveling" + const comixURL = "https://comix.to/title/q77m-countach" + const kaganeURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + const novelfullURL = "https://novelfull.com/reverend-insanity.html" + const lnwURL = "https://lightnovelworld.net/novel/a-will-eternal/" + + tests := []struct { + name string + site string + seriesURL string + body string + want bool + }{ + { + name: "asura completed via escaped props status", + site: "asura", seriesURL: asuraURL, body: asuraCompletedFixture, + want: true, + }, + { + name: "asura ongoing value is not completed", + site: "asura", + seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", + body: asuraOngoingFixture, + want: false, + }, + { + name: "asura with the status key removed", + site: "asura", seriesURL: asuraURL, + body: strings.ReplaceAll(asuraCompletedFixture, `"status":[0,"completed"]`, ""), + want: false, + }, + { + name: "asura challenge page", + site: "asura", seriesURL: asuraURL, body: challengeFixture, + want: false, + }, + { + name: "demonic completed info-block pair", + site: "demonic", seriesURL: demonicURL, body: demonicCompletedFixture, + want: true, + }, + { + name: "demonic ongoing value is not completed", + site: "demonic", + seriesURL: "https://demonicscans.org/manga/Catastrophic-Necromancer", + body: demonicOngoingFixture, + want: false, + }, + { + name: "demonic with the value li removed", + site: "demonic", seriesURL: demonicURL, + body: strings.ReplaceAll(demonicCompletedFixture, "
  • Completed
  • ", ""), + want: false, + }, + { + name: "comix recommended finished does not count", + site: "comix", + seriesURL: "https://comix.to/title/n8we-dungeons-and-crayons", + body: comixOngoingFixture, + want: false, + }, + { + name: "comix detail finished wins over a finished recommended strip", + site: "comix", + seriesURL: comixURL, + body: comixCompletedFixture, + want: true, + }, + { + name: "comix with the detail status removed", + site: "comix", seriesURL: comixURL, + body: strings.ReplaceAll(comixCompletedFixture, `"status":"finished",`, ""), + want: false, + }, + { + name: "comix challenge page", + site: "comix", seriesURL: comixURL, body: challengeFixture, + want: false, + }, + { + name: "kagane publication Completed", + site: "kagane", + seriesURL: "https://kagane.to/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6", + body: kaganeCompletedFixture, + want: true, + }, + { + name: "kagane upload Completed does not count", + site: "kagane", seriesURL: kaganeURL, body: kaganeDivergentFixture, + want: false, + }, + { + name: "kagane ongoing values are not completed", + site: "kagane", seriesURL: kaganeURL, body: kaganeOngoingFixture, + want: false, + }, + { + name: "kagane with publication_status removed", + site: "kagane", seriesURL: kaganeURL, + body: strings.ReplaceAll(kaganeCompletedFixture, `"publication_status":"Completed",`, ""), + want: false, + }, + { + name: "kagane challenge page", + site: "kagane", seriesURL: kaganeURL, body: challengeFixture, + want: false, + }, + { + name: "novelfull status link Completed", + site: "novelfull", + seriesURL: novelfullURL, + body: novelfullCompletedFixture, + want: true, + }, + { + name: "novelfull ongoing link is not completed", + site: "novelfull", + seriesURL: "https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html", + body: novelfullOngoingFixture, + want: false, + }, + { + name: "novelfull with the status link removed", + site: "novelfull", seriesURL: novelfullURL, + body: strings.ReplaceAll(novelfullCompletedFixture, `Completed`, ""), + want: false, + }, + { + name: "novelfull challenge page", + site: "novelfull", seriesURL: novelfullURL, body: challengeFixture, + want: false, + }, + { + name: "lightnovelworld JSON-LD CompletedActionStatus", + site: "lightnovelworld", + seriesURL: lnwURL, + body: lnwCompletedFixture, + want: true, + }, + { + name: "lightnovelworld ActiveActionStatus is not completed", + site: "lightnovelworld", + seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", + body: lnwOngoingFixture, + want: false, + }, + { + name: "lightnovelworld with creativeWorkStatus removed", + site: "lightnovelworld", seriesURL: lnwURL, + body: strings.ReplaceAll(lnwCompletedFixture, `"creativeWorkStatus": "https://schema.org/CompletedActionStatus"`, ""), + want: false, + }, + { + name: "lightnovelworld challenge page", + site: "lightnovelworld", seriesURL: lnwURL, body: challengeFixture, + want: false, + }, + { + name: "unknown site", + site: "mangadex", seriesURL: "https://mangadex.org/title/x", body: asuraCompletedFixture, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := siteCompletedFrom(tt.site, tt.seriesURL, tt.body); got != tt.want { + t.Errorf("siteCompletedFrom(%q, %q, body) = %v, want %v", tt.site, tt.seriesURL, got, tt.want) + } + }) + } +} -- 2.52.0 From f6782e212a5fbe9bd89470d1d36af88e7861fe55 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 18:24:58 +0700 Subject: [PATCH 04/20] Note comix key-encoding fidelity in fixture comments (#168) --- backend/internal/latest/sites_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/internal/latest/sites_test.go b/backend/internal/latest/sites_test.go index 95368c9..833f82a 100644 --- a/backend/internal/latest/sites_test.go +++ b/backend/internal/latest/sites_test.go @@ -226,14 +226,16 @@ const demonicOngoingFixture = ` // Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 over a // cleared Chrome tab (the in-tab fetch of the Series URL, the same -// server-rendered payload the poll reads). The recommended strip on this very -// page carries a finished entry; only the ["manga","detail","q77m"] entry -// counts. +// server-rendered payload the poll reads). The served page escapes the query +// map keys as \u0022; the fixture carries the decoded form, which parses to +// the same key. The recommended strip on this very page carries a finished +// entry; only the ["manga","detail","q77m"] entry counts. const comixCompletedFixture = `` // Trimmed from https://comix.to/title/n8we-dungeons-and-crayons fetched -// 2026-08-22 the same way. The recommended strip includes a finished entry; -// the target detail is releasing, and only the detail read counts. +// 2026-08-22 the same way (keys in their decoded form, as above). The +// recommended strip includes a finished entry; the target detail is +// releasing, and only the detail read counts. const comixOngoingFixture = `` // Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6 -- 2.52.0 From b38dfe54e8e175dce54cf04892b7e79b66009737 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 18:26:34 +0700 Subject: [PATCH 05/20] Record per-site probe provenance in completed fixtures (#168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec review flagged the live-verification acceptance criterion as unproven in the diff: the brief's adapter rule wants the probe recorded, not just the date. Each fixture comment now names how the live body was fetched on 2026-08-22 — curl probe for asura, demonic and lightnovelworld; cleared Chrome tab (CDP sidecar) for comix, kagane and novelfull, including the challenge/403 fallback story for novelfull. Also renamed lnwCompletedRe to lnwStatusRe for symmetry with the other site-marker vars. --- backend/internal/latest/sites.go | 4 +-- backend/internal/latest/sites_test.go | 42 ++++++++++++++------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index b23d1b1..5289346 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -437,10 +437,10 @@ func novelfullCompleted(_, body string) bool { // whole body is scanned and the comment marker is not required, unlike // lnwLatestChapter: the status block sits ahead of the visitor-writable // thread, and hiatus maps to a distinct PotentialActionStatus value. -var lnwCompletedRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`) +var lnwStatusRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`) func lnwCompleted(_, body string) bool { - return lnwCompletedRe.MatchString(body) + return lnwStatusRe.MatchString(body) } // siteCompletedFrom reports whether the Site calls this work completed, via diff --git a/backend/internal/latest/sites_test.go b/backend/internal/latest/sites_test.go index 833f82a..40d3926 100644 --- a/backend/internal/latest/sites_test.go +++ b/backend/internal/latest/sites_test.go @@ -193,21 +193,22 @@ const lnwSeriesFixture = ` // from the consts and must answer false. // Trimmed from https://asurascans.com/comics/solo-leveling (301 to -// /comics/solo-leveling-b60d532c) fetched 2026-08-22. The astro-island props -// are HTML-escaped in the served document, so the quotes arrive as ". +// /comics/solo-leveling-b60d532c) fetched 2026-08-22 by a curl probe from +// this machine. The astro-island props are HTML-escaped in the served +// document, so the quotes arrive as ". const asuraCompletedFixture = ` "bookmarkCount":[0,39128],"status":[0,"completed"],"type":[0,"manhwa"] ` // Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af -// fetched 2026-08-22. +// fetched 2026-08-22 by a curl probe. const asuraOngoingFixture = ` "bookmarkCount":[0,54027],"status":[0,"ongoing"],"type":[0,"manhwa"] ` // Trimmed from https://demonicscans.org/manga/Solo-Leveling fetched -// 2026-08-22. The info block is sloppy: bare
  • s inside a
    , label and -// value as a sibling pair. +// 2026-08-22 by a curl probe. The info block is sloppy: bare
  • s inside a +//
    , label and value as a sibling pair. const demonicCompletedFixture = `
  • Status
  • @@ -216,7 +217,7 @@ const demonicCompletedFixture = ` ` // Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer -// fetched 2026-08-22. Same block, ongoing value. +// fetched 2026-08-22 by a curl probe. Same block, ongoing value. const demonicOngoingFixture = `
  • Status
  • @@ -224,12 +225,12 @@ const demonicOngoingFixture = `
    ` -// Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 over a -// cleared Chrome tab (the in-tab fetch of the Series URL, the same -// server-rendered payload the poll reads). The served page escapes the query -// map keys as \u0022; the fixture carries the decoded form, which parses to -// the same key. The recommended strip on this very page carries a finished -// entry; only the ["manga","detail","q77m"] entry counts. +// Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 by a +// cleared Chrome tab (the CDP sidecar: the Series URL is fetched in-tab, the +// same server-rendered payload the poll reads). The served page escapes the +// query map keys as \u0022; the fixture carries the decoded form, which +// parses to the same key. The recommended strip on this very page carries a +// finished entry; only the ["manga","detail","q77m"] entry counts. const comixCompletedFixture = `` // Trimmed from https://comix.to/title/n8we-dungeons-and-crayons fetched @@ -238,14 +239,15 @@ const comixCompletedFixture = `` -// Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6 -// on 2026-08-22. -const kaganeCompletedFixture = `{"series_id":"019c29c3-5abd-70e6-9efc-1f1f22d839f6","title":"Real Account 1-17","publication_status":"Completed","upload_status":"Completed"}` - // Trimmed from GET https://kagane.to/api/v2/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b -// on 2026-08-22. +// fetched 2026-08-22 in a cleared Chrome tab (plain TLS serves the Cloudflare +// challenge). const kaganeOngoingFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Ongoing"}` +// Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6 +// fetched 2026-08-22 the same way. +const kaganeCompletedFixture = `{"series_id":"019c29c3-5abd-70e6-9efc-1f1f22d839f6","title":"Real Account 1-17","publication_status":"Completed","upload_status":"Completed"}` + // Composed, not a single live trim: upload Completed alongside a // non-Completed publication status is the research note's inferred inverse // case and did not turn up in the ~1900-series live scan of 2026-08-22 (both @@ -255,12 +257,12 @@ const kaganeOngoingFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b const kaganeDivergentFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Completed"}` // Trimmed from https://novelfull.com/reverend-insanity.html fetched -// 2026-08-22 (the challenge served to plain TLS cleared in a real Chrome the -// same day). +// 2026-08-22 in a cleared Chrome tab after the plain-TLS probe was served the +// challenge (time-varying; plain curl answered 403 the same day). const novelfullCompletedFixture = `

    Status:

    Completed
    ` // Trimmed from https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html -// fetched 2026-08-22. +// fetched 2026-08-22 the same way. const novelfullOngoingFixture = `

    Status:

    Ongoing
    ` // Trimmed from the JSON-LD block in the head of -- 2.52.0 From b517858619e5104f313fe5f513183a424a71862b Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:15:46 +0700 Subject: [PATCH 06/20] Remember when a Site called a work completed (#169) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add series.site_completed_at (epoch-ms, default 0): the durable answer #168's predicates compute. checkOne writes it after the successful read on the zero/non-zero transition only, so a Series still completed keeps its original stamp (the age #170 prints is 'since the Site first said so'), one that stopped is zeroed, one that became completed is stamped. The due query projects the column, so the transition is a field comparison on the snapshot checkOne already holds. Refused, unreachable and errored reads reach nothing (AC4 is placement, not a guard); the write sits before the no-chapter and unchanged-number returns, so a completed page whose chapter did not change still writes. Store failure logs and carries on — the outcome word never changes. Series-level like Latest Chapter: a bookmark's updated_at is never touched. #170 is the surface; nothing reads the column yet. --- backend/internal/latest/poller.go | 19 +++ backend/internal/latest/poller_test.go | 151 ++++++++++++++++++ backend/internal/latest/read.go | 25 ++- .../migrations/0019_series_site_completed.sql | 4 + backend/internal/store/store.go | 27 +++- 5 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 backend/internal/store/migrations/0019_series_site_completed.sql diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index e5125c8..c4c44ca 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -678,6 +678,25 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut } else { p.fillBlankCover(ctx, sr, facts.Cover) } + + // Learned from the successful read: the write sits after the error switch + // (a refused, unreachable or errored read reaches nothing) and before the + // returns below — a completed page whose chapter number did not change + // still has to write. The transition is zero-versus-nonzero, not the + // stamp's value: a Series still completed keeps its original stamp, so the + // age #170 prints is "since the Site first said so"; one that stopped + // being completed is zeroed. + want := int64(0) + if facts.SiteCompleted { + want = p.Now().UnixMilli() + } + if (sr.SiteCompletedAt == 0) != (want == 0) { + if err := p.Store.SetSiteCompletedAt(sr.Site, sr.SeriesID, want); err != nil { + // Best-effort, like every poller write: never change the outcome + // word the pass counts. + log.Printf("latest poll %q: set site completed: %v", sr.Key(), err) + } + } if !facts.HasLatest { // Most likely a challenge page or a layout change. Either way the row is // already stamped, so this waits out a rest instead of hot-looping. diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 39576da..7ea4199 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -2725,3 +2725,154 @@ func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) { t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want) } } + +// readSiteCompletedAt reads the poller's site_completed_at column directly: +// it is a poller fact with no client-visible getter, and #170 is the surface +// that will read it. +func readSiteCompletedAt(t *testing.T, dbURL, site, seriesID string) int64 { + t.Helper() + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + var at int64 + if err := db.QueryRow( + `SELECT site_completed_at FROM series WHERE site = $1 AND series_id = $2`, + site, seriesID).Scan(&at); err != nil { + t.Fatalf("read site_completed_at %s:%s: %v", site, seriesID, err) + } + return at +} + +// seedSiteCompletedAt writes the column directly, for the refused/unreachable +// tests that need a pre-existing stamp. +func seedSiteCompletedAt(t *testing.T, dbURL, site, seriesID string, at int64) { + t.Helper() + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + if _, err := db.Exec( + `UPDATE series SET site_completed_at = $3 WHERE site = $1 AND series_id = $2`, + site, seriesID, at); err != nil { + t.Fatalf("seed site_completed_at %s:%s: %v", site, seriesID, err) + } +} + +// A completed page stamps site_completed_at with the pass's own clock; the +// same page an hour later is due again but must not move the stamp — the age +// #170 prints is "since the Site first said so", not the age of the last +// Poll. The transition is zero-versus-nonzero, so the exact value asserted +// here is load-bearing. +func TestRunOnceSiteCompletedStampsOnce(t *testing.T) { + s, dbURL := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + completed := asuraSeriesFixture + asuraCompletedFixture + at := time.UnixMilli(5_000_000) + seedForCheck(t, s, key, seriesURL, at.Add(-time.Hour).UnixMilli()) + + newTestPoller(t, s, &fakeFetcher{body: completed, status: 200}, at).runOnce(context.Background()) + if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != at.UnixMilli() { + t.Fatalf("site_completed_at after completed poll = %d, want %d", got, at.UnixMilli()) + } + + hourLater := at.Add(time.Hour) + newTestPoller(t, s, &fakeFetcher{body: completed, status: 200}, hourLater).runOnce(context.Background()) + if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != at.UnixMilli() { + t.Fatalf("site_completed_at after a second completed poll = %d, want the original stamp %d", got, at.UnixMilli()) + } +} + +// An ongoing page is the inverse transition: a never-hinted Series stays zero +// (an ordinary Poll of an ongoing Series writes nothing), and a hinted one is +// zeroed — the Site no longer says completed, so the claim must not outlive +// the evidence. +func TestRunOnceOngoingPageClearsOrSkipsSiteCompleted(t *testing.T) { + s, dbURL := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + ongoing := asuraSeriesFixture + asuraOngoingFixture + at := time.UnixMilli(5_000_000) + seedForCheck(t, s, key, seriesURL, at.Add(-time.Hour).UnixMilli()) + + newTestPoller(t, s, &fakeFetcher{body: ongoing, status: 200}, at).runOnce(context.Background()) + if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != 0 { + t.Fatalf("site_completed_at after ongoing poll of a never-hinted series = %d, want 0", got) + } + + seedSiteCompletedAt(t, dbURL, "asura", seriesID, at.UnixMilli()) + hourLater := at.Add(time.Hour) + newTestPoller(t, s, &fakeFetcher{body: ongoing, status: 200}, hourLater).runOnce(context.Background()) + if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != 0 { + t.Fatalf("site_completed_at after ongoing poll of a hinted series = %d, want 0", got) + } +} + +// A refused or unreachable read reaches nothing after checkOne's error switch: +// a challenge is not evidence about the work, so a pre-seeded stamp must +// survive both. +func TestRunOnceRefusedOrUnreachableLeavesSiteCompletedUntouched(t *testing.T) { + now := time.UnixMilli(5_000_000) + + t.Run("refused", func(t *testing.T) { + s, dbURL := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seedForCheck(t, s, key, seriesURL, 0) + seedSiteCompletedAt(t, dbURL, "asura", seriesID, now.UnixMilli()) + newTestPoller(t, s, &fakeFetcher{status: 403}, now).runOnce(context.Background()) + if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != now.UnixMilli() { + t.Fatalf("site_completed_at after refused poll = %d, want the pre-seeded %d", got, now.UnixMilli()) + } + }) + + t.Run("mid-loop unreachable", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + seedSiteCompletedAt(t, dbURL, "comix", "c", now.UnixMilli()) + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + browser := &fakeFetcher{status: 200, err: interrupted} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + p.runLanePass(context.Background(), "comix", false) + if got := readSiteCompletedAt(t, dbURL, "comix", "c"); got != now.UnixMilli() { + t.Fatalf("site_completed_at after unreachable poll = %d, want the pre-seeded %d", got, now.UnixMilli()) + } + }) +} + +// The regression AC2's second sentence exists to prevent: the completed write +// must not ride the chapter setter, so a completed page whose chapter number +// is unchanged — checkOne's equality early return — still stamps the column. +func TestRunOnceSiteCompletedWritesDespiteUnchangedChapter(t *testing.T) { + s, dbURL := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesID = "chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + completed := asuraSeriesFixture + asuraCompletedFixture + at := time.UnixMilli(5_000_000) + seedForCheck(t, s, key, seriesURL, 0) + // The page parses Chapter 181 as the latest; store the same number so the + // equality early return fires and only the completed write can run. + if err := s.SetLatestChapter("asura", seriesID, "Chapter 181", 181); err != nil { + t.Fatalf("seed latest chapter: %v", err) + } + newTestPoller(t, s, &fakeFetcher{body: completed, status: 200}, at).runOnce(context.Background()) + if got := readSiteCompletedAt(t, dbURL, "asura", seriesID); got != at.UnixMilli() { + t.Fatalf("site_completed_at after unchanged-chapter completed poll = %d, want %d", got, at.UnixMilli()) + } +} diff --git a/backend/internal/latest/read.go b/backend/internal/latest/read.go index 8a9c611..dc51b04 100644 --- a/backend/internal/latest/read.go +++ b/backend/internal/latest/read.go @@ -6,15 +6,19 @@ import ( "fmt" ) -// seriesRead carries the two facts the poll and the acquirer both extract -// from a series page. Persistence, stamps and scheduling stay with the -// callers, so the policies that keep the two flows distinct (stamp order, -// rests) are not swallowed by the module. +// seriesRead carries the facts the poll and the acquirer both extract from a +// series page. Persistence, stamps and scheduling stay with the callers, so +// the policies that keep the two flows distinct (stamp order, rests) are not +// swallowed by the module. type seriesRead struct { Latest latestChapter HasLatest bool Cover string HasCover bool + // SiteCompleted is whether the Site's own completed value was on the page. + // A challenge body and a redesign both read false — an absent hint, never + // a claim (issue #168). + SiteCompleted bool // BodyLen is the fetched body's length, surfaced because the no-chapter // log uses it to tell a markup change from a body the size cap cut short. BodyLen int @@ -35,8 +39,8 @@ var ( // readSeriesPage performs the series-page read the poll and the acquirer have // in common: gate the address, choose the route, fetch the page, extract the -// Latest Chapter and the Cover address. It persists nothing and stamps -// nothing. +// Latest Chapter, the Cover address and the Site's completed value. It +// persists nothing and stamps nothing. // // series_url arrives in a client-supplied PUT body (PUT /bookmarks/{key} // accepts any string), so the gate is not an optimisation against burning a @@ -75,5 +79,12 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe } latest, hasLatest := latestChapterFrom(site, seriesURL, body) cover, hasCover := coverFrom(site, seriesURL, body) - return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil + return seriesRead{ + Latest: latest, + HasLatest: hasLatest, + Cover: cover, + HasCover: hasCover, + SiteCompleted: siteCompletedFrom(site, seriesURL, body), + BodyLen: len(body), + }, nil } diff --git a/backend/internal/store/migrations/0019_series_site_completed.sql b/backend/internal/store/migrations/0019_series_site_completed.sql new file mode 100644 index 0000000..ee47164 --- /dev/null +++ b/backend/internal/store/migrations/0019_series_site_completed.sql @@ -0,0 +1,4 @@ +-- When the last successful Poll read saw the Site's own completed value +-- (#168): epoch-ms, zero meaning it did not. DEFAULT 0 keeps pre-existing +-- rows readable; nothing reads it yet — #170 is the surface. +ALTER TABLE series ADD COLUMN site_completed_at bigint NOT NULL DEFAULT 0; diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 153471d..91f5602 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -84,6 +84,10 @@ type Series struct { LatestChapter string LatestChapterNum *float64 // nil until first captured LatestCheckedAt int64 // unix ms; see MarkLatestChecked + // SiteCompletedAt is when the last successful Poll read saw the Site's + // own completed value, zero meaning it did not (issue #168). A poller + // fact, not reading progress: Upsert never touches it. + SiteCompletedAt int64 // LatestRaisedBy is the Reader whose Sighting last raised LatestChapter, // and nil when the stored value is a Poll's own finding. It is what lets a // Poll that contradicts the value downwards name a Reader instead of @@ -240,7 +244,8 @@ const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_add // due query. latest_checked_at lives only on series — see MarkLatestChecked // for why it stays off every client-visible write. const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.cover_address, - s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by` + s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by, + s.site_completed_at` const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped, p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, p.not_found, @@ -646,7 +651,7 @@ func scanSeries(scan func(...any) error) (Series, error) { if err := scan( &sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress, &sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy, - &sr.Forced, &sr.readerCount, + &sr.SiteCompletedAt, &sr.Forced, &sr.readerCount, ); err != nil { return Series{}, err } @@ -1356,7 +1361,7 @@ func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Ser AND (s.finished_at = 0 OR s.force_poll_at > s.latest_checked_at) GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover, s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, - s.force_poll_at, s.finished_at + s.site_completed_at, s.force_poll_at, s.finished_at HAVING (COUNT(*) > 1 OR s.latest_sighted_at <= $2::bigint OR s.latest_checked_at <= $3::bigint @@ -1491,6 +1496,22 @@ func (s *Store) SetLatestChapter(site, seriesID, label string, num float64) erro return nil } +// SetSiteCompletedAt stores when the last successful read saw the Site's +// completed value, zero meaning it did not. Series-level, like the Latest +// Chapter: it is a fact about the work, not about one Reader's bookmark, and +// the bookmark's updated_at is never touched — this is not reading progress +// and must not reorder any Reader's list, the same rule SetLatestChapter's +// comment states. Touching a missing series is not an error: the row may +// have been orphaned, and the caller's read decides what exists. +func (s *Store) SetSiteCompletedAt(site, seriesID string, at int64) error { + if _, err := s.db.Exec( + `UPDATE series SET site_completed_at = $1 WHERE site = $2 AND series_id = $3`, + at, site, seriesID); err != nil { + return fmt.Errorf("set site completed %s:%s: %w", site, seriesID, err) + } + return nil +} + // SetSeriesURL stores the owner's repair for a Series' source address // (issue #151): the one write that lifts the write-once rule documented on // Series.SeriesURL. It is a store, not a verification — the caller has -- 2.52.0 From 86da47850c7bf334c73974c5f2ee812d8d1637ff Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:18:45 +0700 Subject: [PATCH 07/20] Address review nits on #169: stamp name, migration comment (#169) --- backend/internal/latest/poller.go | 8 ++++---- .../store/migrations/0019_series_site_completed.sql | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index c4c44ca..79610d2 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -686,12 +686,12 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut // stamp's value: a Series still completed keeps its original stamp, so the // age #170 prints is "since the Site first said so"; one that stopped // being completed is zeroed. - want := int64(0) + stamp := int64(0) if facts.SiteCompleted { - want = p.Now().UnixMilli() + stamp = p.Now().UnixMilli() } - if (sr.SiteCompletedAt == 0) != (want == 0) { - if err := p.Store.SetSiteCompletedAt(sr.Site, sr.SeriesID, want); err != nil { + if (sr.SiteCompletedAt == 0) != (stamp == 0) { + if err := p.Store.SetSiteCompletedAt(sr.Site, sr.SeriesID, stamp); err != nil { // Best-effort, like every poller write: never change the outcome // word the pass counts. log.Printf("latest poll %q: set site completed: %v", sr.Key(), err) diff --git a/backend/internal/store/migrations/0019_series_site_completed.sql b/backend/internal/store/migrations/0019_series_site_completed.sql index ee47164..75783cb 100644 --- a/backend/internal/store/migrations/0019_series_site_completed.sql +++ b/backend/internal/store/migrations/0019_series_site_completed.sql @@ -1,4 +1,4 @@ -- When the last successful Poll read saw the Site's own completed value -- (#168): epoch-ms, zero meaning it did not. DEFAULT 0 keeps pre-existing --- rows readable; nothing reads it yet — #170 is the surface. +-- rows readable. ALTER TABLE series ADD COLUMN site_completed_at bigint NOT NULL DEFAULT 0; -- 2.52.0 From 417f809d7511046f8809126ef82f00c10d9ab33d Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:22:57 +0700 Subject: [PATCH 08/20] Record poll failures as rows: the row is the failure state (#165) --- backend/internal/latest/poller.go | 41 ++++ backend/internal/latest/poller_test.go | 215 ++++++++++++++++++ .../store/migrations/0018_poll_failures.sql | 13 ++ backend/internal/store/store.go | 25 ++ backend/internal/store/store_test.go | 89 ++++++++ docs/adr/0016-the-failure-row-is-the-state.md | 147 ++++++++++++ 6 files changed, 530 insertions(+) create mode 100644 backend/internal/store/migrations/0018_poll_failures.sql create mode 100644 docs/adr/0016-the-failure-row-is-the-state.md diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index e5125c8..d2d2422 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -287,6 +287,24 @@ const ( outcomeError outcomeNotFound ) +// word returns the wire spelling this outcome stores in poll_failures — the +// same strings the pass log's columns use (C1, issue #164). Success, refusal +// and browser loss return "" so recordFailure's "no statement" case is one +// return: a challenge or a lost sidecar is no evidence about any particular +// Series (ADR-0016). +func (o readOutcome) word() string { + switch o { + case outcomeNotFound: + return "not_found" + case outcomeNoChapter: + return "no_chapter" + case outcomeUnfetchable: + return "unfetchable" + case outcomeError: + return "errors" + } + return "" +} // outcomeCounts are the six named outcome counts of one pass. A success // count is derived, never stored: checked minus the five named failures, @@ -456,6 +474,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. } } outcome := p.checkOne(ctx, sr) + p.recordFailure(sr, outcome) if outcome == outcomeUnreachable { // The mid-loop browser loss writes an empty skip on purpose: the // pass returns before the checked counter increments, so its row @@ -613,6 +632,28 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D } return oldest } +// recordFailure keeps one Series' failure row in step with its read +// (ADR-0016): a failure word is upserted, a successful read deletes the row, +// and refused or unreachable issue no statement at all. Called for every +// outcome from the pass loop, so the four failure words and the success path +// share one write point, and a forced Poll that reads the page clears through +// the ordinary success path — no branch of its own. Log a store failure and +// carry on: this is best-effort, and no single bad Series may stall a Lane. +func (p *Poller) recordFailure(sr store.Series, outcome readOutcome) { + if outcome == outcomeSuccess { + if err := p.Store.ClearSeriesFailure(sr.Site, sr.SeriesID); err != nil { + log.Printf("latest poll %q: clear failure: %v", sr.Key(), err) + } + return + } + word := outcome.word() + if word == "" { + return + } + if err := p.Store.RecordSeriesFailure(sr.Site, sr.SeriesID, word, p.Now().UnixMilli()); err != nil { + log.Printf("latest poll %q: record failure: %v", sr.Key(), err) + } +} // checkOne re-checks one series. Every failure path here is "log and move on": // the poller is a best-effort enhancement, and no single bad series may stall a diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 39576da..0fbacfa 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -2725,3 +2725,218 @@ func TestRunOnceForcedPassReclaimFailureDoesNotFailPoll(t *testing.T) { t.Fatalf("Cover = %q, want the replacement %q", got.Cover, want) } } +// failureRow reads one Series' failure row as stored, for the poller tests +// that must see the table the store API writes (ADR-0016). +func failureRow(t *testing.T, dbURL, site, seriesID string) (word string, since int64, found bool) { + t.Helper() + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + err = db.QueryRow( + `SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`, + site, seriesID).Scan(&word, &since) + if errors.Is(err, sql.ErrNoRows) { + return "", 0, false + } + if err != nil { + t.Fatalf("read failure row %s:%s: %v", site, seriesID, err) + } + return word, since, true +} + +// The failure row follows the read (ADR-0016): a 404 writes a row with the +// outcome word and the pass's time; a second 404 an hour later leaves +// failing_since alone — the age is the age of the run of failures, not of +// the current word; a good read deletes the row. +func TestRunLanePassFailureRowFollowsTheRead(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seedForCheck(t, s, key, seriesURL, 0) + + f := &fakeFetcher{perURL: map[string]fakeResponse{seriesURL: {status: 404}}} + p := newTestPoller(t, s, f, now) + p.runLanePass(context.Background(), "asura", false) + + word, since, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af") + if !found || word != "not_found" || since != now.UnixMilli() { + t.Fatalf("row after 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli()) + } + + // A repeated failure an hour later: the word is unchanged and the stamp + // is untouched, so the age keeps meaning the run of failures. + p.Now = func() time.Time { return now.Add(2 * time.Hour) } + p.runLanePass(context.Background(), "asura", false) + word, since, found = failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af") + if !found || word != "not_found" || since != now.UnixMilli() { + t.Fatalf("row after repeated 404 = (%q, %d, %v), want (not_found, %d, true)", word, since, found, now.UnixMilli()) + } + + // A good read ends the failure: the row is gone. + f.perURL[seriesURL] = fakeResponse{body: asuraSeriesFixture, status: 200} + p.Now = func() time.Time { return now.Add(4 * time.Hour) } + p.runLanePass(context.Background(), "asura", false) + if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found { + t.Fatal("failure row after a good read: still present") + } +} + +// Every failure word the pass counts lands in the table under the wire +// spelling the worklist left-joins on (C1): no_chapter, unfetchable, errors, +// not_found — and a success writes no row. +func TestRunLanePassStoresEachFailureWord(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + seeds := map[string]string{ + "asura:no-chapter": "https://asurascans.com/comics/no-chapter", + "asura:unfetchable": "https://evil.example/x", + "asura:transport": "https://asurascans.com/series/transport", + "asura:gone": "https://asurascans.com/series/gone", + "asura:healthy": "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", + } + for key, url := range seeds { + seedForCheck(t, s, key, url, 0) + } + f := &fakeFetcher{perURL: map[string]fakeResponse{ + seeds["asura:no-chapter"]: {body: "", status: 200}, + seeds["asura:transport"]: {err: errors.New("dial tcp: refused")}, + seeds["asura:gone"]: {status: 404}, + seeds["asura:healthy"]: {body: asuraSeriesFixture, status: 200}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + want := map[string]string{ + "no-chapter": "no_chapter", + "unfetchable": "unfetchable", + "transport": "errors", + "gone": "not_found", + } + for seriesID, word := range want { + got, _, found := failureRow(t, dbURL, "asura", seriesID) + if !found || got != word { + t.Fatalf("failure word for %s = (%q, %v), want %q", seriesID, got, found, word) + } + } + if _, _, found := failureRow(t, dbURL, "asura", "healthy"); found { + t.Fatal("success Series gained a failure row") + } +} + +// refused and unreachable issue no statement at all (ADR-0016): a challenge +// or a lost sidecar is no evidence about any particular Series, so a +// pre-seeded row survives both untouched and a Series with no row gains none. +func TestRunLanePassRefusalAndUnreachableWriteNothing(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + + t.Run("refused", func(t *testing.T) { + const ( + seeded = "asura:held" + neverFail = "asura:other" + ) + seedForCheck(t, s, seeded, "https://asurascans.com/series/held", 0) + seedForCheck(t, s, neverFail, "https://asurascans.com/series/other", 0) + if err := s.RecordSeriesFailure("asura", "held", "not_found", 7_000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + f := &fakeFetcher{perURL: map[string]fakeResponse{ + "https://asurascans.com/series/held": {status: 403}, + "https://asurascans.com/series/other": {status: 403}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + word, since, found := failureRow(t, dbURL, "asura", "held") + if !found || word != "not_found" || since != 7_000 { + t.Fatalf("pre-seeded row after refusal = (%q, %d, %v), want (not_found, 7000, true)", word, since, found) + } + if _, _, found := failureRow(t, dbURL, "asura", "other"); found { + t.Fatal("row appeared for a refused Series with none: refusal must write nothing") + } + }) + + t.Run("unreachable mid-loop", func(t *testing.T) { + const ( + seeded = "comix:c" + neverFail = "comix:d" + ) + seedForCheck(t, s, seeded, "https://comix.to/title/c", 0) + seedForCheck(t, s, neverFail, "https://comix.to/title/d", 0) + if err := s.RecordSeriesFailure("comix", "c", "errors", 7_000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + browser := &fakeFetcher{status: 200, err: interrupted} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + p.runLanePass(context.Background(), "comix", false) + + word, since, found := failureRow(t, dbURL, "comix", "c") + if !found || word != "errors" || since != 7_000 { + t.Fatalf("pre-seeded row after unreachable = (%q, %d, %v), want (errors, 7000, true)", word, since, found) + } + if _, _, found := failureRow(t, dbURL, "comix", "d"); found { + t.Fatal("row appeared for an unreachable Series with none: unreachable must write nothing") + } + }) +} + +// A Forced Poll request clears nothing — it only stamps the request — and a +// forced Poll that then reads the page clears the row through the ordinary +// success path, with no forced branch in the code. +func TestRunLanePassForcedPollClearsThroughSuccess(t *testing.T) { + s, dbURL := newTestStore(t) + now := time.UnixMilli(5_000_000) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seedForCheck(t, s, key, seriesURL, 0) + if err := s.RecordSeriesFailure("asura", "chronicles-of-the-demon-faction-f886a8af", "not_found", 7_000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + if err := s.ForceSeriesPoll("asura", "chronicles-of-the-demon-faction-f886a8af", now.Add(time.Hour).UnixMilli()); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + // The request alone cleared nothing. + if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); !found { + t.Fatal("failure row gone after only a Forced Poll request") + } + + newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now). + runLanePass(context.Background(), "asura", false) + if _, _, found := failureRow(t, dbURL, "asura", "chronicles-of-the-demon-faction-f886a8af"); found { + t.Fatal("failure row after a forced Poll that read the page: still present") + } +} + +// The due query does not join the failure table (AC7): a failing Series is +// polled at the same pace as any other, so its failure age keeps meaning what +// the worklist reads it as. +func TestPollFailureDoesNotChangePacing(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seedForCheck(t, s, "asura:failing", "https://asurascans.com/series/failing", 0) + seedForCheck(t, s, "asura:healthy", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + f := &fakeFetcher{perURL: map[string]fakeResponse{ + "https://asurascans.com/series/failing": {status: 404}, + "https://asurascans.com/series/healthy": {body: asuraSeriesFixture, status: 200}, + }} + p := newTestPoller(t, s, f, now) + p.runLanePass(context.Background(), "asura", false) + if got := f.callCount(); got != 2 { + t.Fatalf("first pass fetched %d series, want both (the failure row must not exclude the failing one)", got) + } + + // After the rest both are due again: the row the first pass wrote changed + // nothing about when the failing Series is polled. + p.Now = func() time.Time { return now.Add(2 * time.Hour) } + p.runLanePass(context.Background(), "asura", false) + if got := f.callCount(); got != 4 { + t.Fatalf("second pass fetched %d series, want both again", got) + } +} diff --git a/backend/internal/store/migrations/0018_poll_failures.sql b/backend/internal/store/migrations/0018_poll_failures.sql new file mode 100644 index 0000000..5aa2854 --- /dev/null +++ b/backend/internal/store/migrations/0018_poll_failures.sql @@ -0,0 +1,13 @@ +-- One row per Series that is failing right now (ADR-0016): the row's +-- existence is the failure state, failing_since ages the run of failures, +-- and a correct read deletes the row. Keyed by the same (site, series_id) +-- composite the rest of the system uses, with the cascade so deleting a +-- Series takes its failure row and orphan removal stays a single statement. +CREATE TABLE poll_failures ( + site text NOT NULL, + series_id text NOT NULL, + outcome text NOT NULL, + failing_since bigint NOT NULL, + PRIMARY KEY (site, series_id), + FOREIGN KEY (site, series_id) REFERENCES series (site, series_id) ON DELETE CASCADE +); diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 153471d..56e6970 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1297,6 +1297,31 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro } return pausedUntil, refuseUntil, nil } +// RecordSeriesFailure upserts one Series' failure row: the outcome word is +// updated when it changes, failing_since is never overwritten, and an +// unchanged word writes nothing. One statement, so the identical-word no-op +// and the keep-the-stamp rule are the same guarantee: the SET arm fires only +// when the word differs, and failing_since is simply absent from it +// (ADR-0016). +func (s *Store) RecordSeriesFailure(site, seriesID, outcome string, now int64) error { + if _, err := s.db.Exec(` + INSERT INTO poll_failures (site, series_id, outcome, failing_since) VALUES ($1, $2, $3, $4) + ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome + WHERE poll_failures.outcome <> EXCLUDED.outcome`, site, seriesID, outcome, now); err != nil { + return fmt.Errorf("record series failure %s:%s: %w", site, seriesID, err) + } + return nil +} + +// ClearSeriesFailure deletes one Series' failure row. A row that is not there +// is not an error. +func (s *Store) ClearSeriesFailure(site, seriesID string) error { + if _, err := s.db.Exec( + `DELETE FROM poll_failures WHERE site = $1 AND series_id = $2`, site, seriesID); err != nil { + return fmt.Errorf("clear series failure %s:%s: %w", site, seriesID, err) + } + return nil +} // DueForLatestCheck returns one Site's series whose server-side // latest-chapter check has aged past cutoffMs, ordered by how many bookmarks diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index 8480fec..286c212 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -2611,3 +2611,92 @@ func TestRemoveSeriesMissingKeyIsCleanNoOp(t *testing.T) { t.Fatalf("RemoveSeries on a missing key = %v, want nil", err) } } +// The failure row's existence is the failure state (ADR-0016): +// RecordSeriesFailure upserts the outcome word, keeps the original +// failing_since through a word change, and an unchanged word writes nothing +// at all — a broken Series costs the same as a healthy one every hour. +func TestRecordSeriesFailureUpsertKeepsFailingSince(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0) + + if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil { + t.Fatalf("record: %v", err) + } + word, since, found := failureRow(t, s, "asura", "solo") + if !found || word != "not_found" || since != 1000 { + t.Fatalf("row after create = (%q, %d, %v), want (not_found, 1000, true)", word, since, found) + } + + // A changed word updates the outcome and never touches failing_since. + if err := s.RecordSeriesFailure("asura", "solo", "errors", 2000); err != nil { + t.Fatalf("record changed word: %v", err) + } + word, since, found = failureRow(t, s, "asura", "solo") + if !found || word != "errors" || since != 1000 { + t.Fatalf("row after word change = (%q, %d, %v), want (errors, 1000, true)", word, since, found) + } + + // An identical word writes nothing: the stamp survives a later now. + if err := s.RecordSeriesFailure("asura", "solo", "errors", 3000); err != nil { + t.Fatalf("record identical word: %v", err) + } + word, since, found = failureRow(t, s, "asura", "solo") + if !found || word != "errors" || since != 1000 { + t.Fatalf("row after identical word = (%q, %d, %v), want the earlier (errors, 1000, true)", word, since, found) + } +} + +// ClearSeriesFailure deletes the row, and a row that is not there is not an +// error — the poller clears on every successful read, so most clears find +// nothing. +func TestClearSeriesFailure(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0) + + if err := s.ClearSeriesFailure("asura", "solo"); err != nil { + t.Fatalf("clear a missing row: %v, want nil", err) + } + if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil { + t.Fatalf("record: %v", err) + } + if err := s.ClearSeriesFailure("asura", "solo"); err != nil { + t.Fatalf("clear: %v", err) + } + if _, _, found := failureRow(t, s, "asura", "solo"); found { + t.Fatal("row after clear: still present") + } +} + +// Deleting a Series takes its failure row with it (the composite FK's +// cascade), so orphan removal stays a single statement. +func TestRemoveSeriesCascadesToFailureRow(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:solo", "https://asurascans.com/comics/solo", 0) + if err := s.RecordSeriesFailure("asura", "solo", "not_found", 1000); err != nil { + t.Fatalf("record: %v", err) + } + if err := s.Delete(s.OwnerID(), "asura:solo"); err != nil { + t.Fatalf("delete bookmark: %v", err) + } + if err := s.RemoveSeries("asura", "solo"); err != nil { + t.Fatalf("RemoveSeries: %v", err) + } + if _, _, found := failureRow(t, s, "asura", "solo"); found { + t.Fatal("failure row after series delete: still present (no cascade)") + } +} + +// failureRow reads one Series' failure row as stored. +func failureRow(t *testing.T, s *Store, site, seriesID string) (word string, since int64, found bool) { + t.Helper() + err := s.db.QueryRow( + `SELECT outcome, failing_since FROM poll_failures WHERE site = $1 AND series_id = $2`, + site, seriesID).Scan(&word, &since) + if errors.Is(err, sql.ErrNoRows) { + return "", 0, false + } + if err != nil { + t.Fatalf("read failure row %s:%s: %v", site, seriesID, err) + } + return word, since, true +} diff --git a/docs/adr/0016-the-failure-row-is-the-state.md b/docs/adr/0016-the-failure-row-is-the-state.md new file mode 100644 index 0000000..f09b73a --- /dev/null +++ b/docs/adr/0016-the-failure-row-is-the-state.md @@ -0,0 +1,147 @@ +# ADR-0016: The failure row is the state + +Date: 2026-08-22 +Status: accepted + +## Decision + +A Series that used to work and has stopped is recorded in one table, +`poll_failures`, keyed by the same `(site, series_id)` composite the rest of +the system uses. One row per failing Series, carrying the outcome word and a +`failing_since` stamp. The row's **existence** is the failure state: there is +no success sentinel, no counter, no history, and nothing added to the Series +row. A correct read deletes the row, and a repeated identical failure writes +nothing — the stamp ages the run of failures, not the current word. + +The write is one upsert, from the poll pass loop: + +```sql +INSERT INTO poll_failures (site, series_id, outcome, failing_since) +VALUES ($1, $2, $3, $4) +ON CONFLICT (site, series_id) DO UPDATE SET outcome = EXCLUDED.outcome +WHERE poll_failures.outcome <> EXCLUDED.outcome +``` + +`failing_since` is simply absent from the `SET` arm, so it is never +overwritten, and the `WHERE` makes an unchanged word write nothing at all — +one statement, no read-then-write race. The clear is a plain `DELETE`; a row +that is not there is not an error, because the poller clears on every +successful read and most clears find nothing. The composite foreign key +cascades: deleting a Series takes its failure row, so orphan removal stays a +single statement. + +## Why a future reader will find this surprising + +The failure state lives in a table of its own, not on the Series row and not +in the pass log. A flag on `series` would be the familiar shape, and the +pass log already records outcomes per pass — why a third place? + +Because the failure must outlive the pass that observed it and mean +something the pass row cannot say. The pass log is a per-Lane stream: it +records that a Site returned N `errors` and M `not_found` on a given run, +but "this exact Series has been failing for three months" is not a question +any single pass row answers — it is a question across rows, and the Lanes +page is a live view of the last 14 days, not a Series index. A Series-level +fact needs Series-level storage, and a flag on the Series row is the wrong +shape too: the failure is *transient by definition* (a correct read ends it) +and *repeatable* (the same Series can fail again next year), so the honest +record of "failing since" is a stamp that moves, not a column that flips. +A row that is created and deleted by the read's outcome is that stamp, and +nothing else: the moment a read succeeds the row is gone, so "is this Series +failing?" is answered by one indexed existence check — no success sentinel +to keep consistent with the failure, no counter to reset, no history to +prune. The state cannot drift out of step with the reads that maintain it, +because the reads *are* the maintenance. + +The two no-write outcomes are the second surprise. `refused` (the Site is +holding a challenge) and `unreachable` (the browser sidecar was lost +mid-loop) issue **no statement at all** — neither an upsert nor a delete. +The direction matters, and both directions are wrong to touch: + +- **Writing would condemn a whole library for one Site's bad day.** A + refusal is the Site's mood, not a fact about any particular Series: when + Cloudflare turns a zone's JS detection on, every Series on that Site reads + refused in the same pass. Writing those rows would stamp every Series in + the library as failing on the evidence of one Site's configuration, and + #166's worklist would present a Site-wide outage as thousands of broken + Series. +- **Deleting would claim a recovery nothing read.** A lost sidecar tells us + nothing about the page — the read never happened. Deleting the row would + report the Series healthy, and worse, it would reset `failing_since`: the + age that makes a three-month failure findable would start over on a + browser restart, erasing evidence no page read contradicted. + +So a refusal or an unreachable pass leaves the table exactly as it found +it — the pass neither adds rows that would condemn nor deletes rows that +would claim recovery. The four outcomes that are real evidence about the +page (`not_found`, `no_chapter`, `unfetchable`, `errors`) write; the two +that are not write nothing; success deletes. There is no third case. + +## Considered options + +**A `failing_since` column on the Series row, alongside the failure word.** +Rejected: it is two more columns on a table every due query and every +bookmark join already touches, for a state that is transient and repeatable. +The column would need its own "clear on success" writer anyway — the same +maintenance the row has — while permanently widening the hottest table in +the system for a value that is usually absent. And the worklist's join +would have to distinguish "never failed" from "currently healthy", which +is exactly the sentinel problem the row avoids: an empty column means both. + +**A counter or last-outcome timestamp instead of (or beside) the row.** +Rejected: nothing in the system consumes a failure *count* or a +last-outcome time — the worklist needs "failing and since when". A counter +invites "three failures = something" thresholds that the ticket explicitly +keeps out of scope, and a last-outcome timestamp conflates "the word +changed" with "the run restarted", which the age is deliberately defined +against. The stamp is the only number that means anything, and the row +carries exactly that. + +**Write into `poll_passes` and derive the failure state from the pass +stream.** +Rejected: a pass row is per-Lane and per-run; deriving "this Series is +failing since" means scanning 14 days of outcome counts per Series and +guessing at continuity across retention boundaries. The pass log answers +"what did this Site's lane do recently"; the failure table answers "which +Series are broken right now". Two questions, two tables — the pass log is +already pruned on a fixed cutoff, which would silently reset every +failure's age the moment the evidence aged out. + +**Refused/unreachable write nothing, but the delete happens anyway (or the +upsert happens, with no delete).** +Rejected in both directions, above: writing condemns a library for one +Site's mood; deleting claims a recovery nothing read and resets the age +that makes a long failure findable. The asymmetry is the point — the two +outcomes are evidence about the *environment*, never about a page. + +## Consequences + +- The poller writes from the pass loop, one call after `checkOne`, and + clears through the ordinary success path: a Forced Poll that reads the + page deletes the row with no forced branch of its own, and a Forced Poll + *request* — which only stamps the request — clears nothing. +- `poll_failures` is a poll-write table like `poll_lanes` and + `poll_passes`: the store's two methods live next to the Lane-pass + writers, and neither runs on a request path. A store failure is logged + and the poll continues — a failure row is best-effort, and no single bad + Series may stall a Lane. +- The due query does not join the table. A failing Series is polled at the + same pace as any other, because the moment the query started pacing by + failure state, the age would stop meaning what the worklist reads it as. +- Orphan removal stays a single statement: the composite foreign key's + `ON DELETE CASCADE` is what takes the failure row with the Series. +- The table starts empty at deploy, so nothing is findable for the first + twelve hours after deploy and a pre-existing breakage reads as new. + Accepted; the alternative is inventing history. + +## Cost of reversing + +The failure state is derived, not stored: a rollback drops the table and +every in-progress failure age with it, leaving the pass log's outcome +counts as the only trace — which is exactly the "log line nobody reads" +the ticket set out to replace. Recreating the table later starts the ages +over, so a reversal that is later reversed loses the evidence of the +intervening failures. The failure state itself, however, is the one thing +that is *not* lost by reversing: it is re-derived from the next pass, in +the same direction the original design derives it — the row is +recreated by the next failing read and deleted by the next good one. -- 2.52.0 From caabdff15b72f360d993fbbf01ce08c48dafe9f8 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:25:26 +0700 Subject: [PATCH 09/20] Tidy blank line before RecordSeriesFailure (#165) --- backend/internal/store/store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 56e6970..bab6b61 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1297,6 +1297,7 @@ func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err erro } return pausedUntil, refuseUntil, nil } + // RecordSeriesFailure upserts one Series' failure row: the outcome word is // updated when it changes, failing_since is never overwritten, and an // unchanged word writes nothing. One statement, so the identical-word no-op -- 2.52.0 From 8a569075582936b563cead65c9dbb4a12ca00840 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:43:47 +0700 Subject: [PATCH 10/20] Two new Series filters: failing, and the unverified Reader subset (#166) --- backend/internal/store/admin.go | 53 ++++++++--- backend/internal/store/admin_test.go | 131 +++++++++++++++++++++++++++ backend/internal/web/admin_series.go | 4 + backend/web_test.go | 50 +++++++++- 4 files changed, 224 insertions(+), 14 deletions(-) diff --git a/backend/internal/store/admin.go b/backend/internal/store/admin.go index 0be59de..059216b 100644 --- a/backend/internal/store/admin.go +++ b/backend/internal/store/admin.go @@ -24,6 +24,8 @@ const ( SeriesFilterNoCover = "no_cover" SeriesFilterReaderReport = "reader_report" SeriesFilterFinished = "finished" + SeriesFilterFailing = "failing" + SeriesFilterUnverified = "unverified" ) // SeriesFilter is one named filter predicate over the whole library. Site @@ -54,6 +56,12 @@ const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_ // Reader behind them. const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader` +// failureAnswer is the poll-failures row's answer (issue #165), kept apart +// from adminSeriesColumns like raisedByReaderAnswer: outcome and failing_since +// are not Series columns, and the COALESCE keeps the row scannable when the +// LEFT JOIN finds no failure row. +const failureAnswer = `COALESCE(f.outcome, ''), COALESCE(f.failing_since, 0)` + // seriesPageSize is the row read's page length. The tie-break in the query's // ORDER BY is what makes this a stable page boundary — see SeriesPage. const seriesPageSize = 50 @@ -83,6 +91,12 @@ type AdminSeries struct { LatestCorrectedAt int64 ReaderCount int RaisedByReader bool // a Reader's report set LatestChapterNum + // FailureOutcome is the outcome word of the Series' standing failure, "" + // when no failure row stands. FailingSince is when the run of failures + // began, zero with no row. Both come from the LEFT JOIN, not the Series + // row (issue #165: the row's existence is the state). + FailureOutcome string + FailingSince int64 // FinishedAt is the owner's finish stamp: unix ms, zero while the Series is // not finished — the same shape as the Correction stamp, and its own undo. FinishedAt int64 @@ -119,7 +133,11 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID } // gate case is invisible to SQL, needs the Site registry in Go, and belongs to // a later repair), never-read-a-chapter and never-checked as disjoint halves // (non-zero versus zero check stamp), stale, no cover, finished (the -// retirement stamp, read directly), and Reader-report. +// retirement stamp, read directly), Reader-report, and the failing pair — +// failing (the failure row exists, a chapter exists, and failing_since is +// past the cutoff) and unverified (the Reader-attributed subset of failing). +// failing's chapter IS NOT NULL is the exact complement of never-read-a- +// chapter's IS NULL half, so the two are disjoint by construction. // no_readers is the one HAVING predicate: it is the orphan test, an aggregate // over the LEFT JOIN, where a bare WHERE has no row to test. // @@ -127,9 +145,11 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID } // Series (`s.finished_at = 0` in each of the four): the clock-driven one — // never-checked, stale, no-chapter, no-cover — keep ticking after the last // Poll, so they would report a retired row as a problem no Poll is coming to -// fix; the three outcome-driven ones — no-URL, no-readers, Reader-report — -// read stored facts that simply stop arriving, so a finished Series needing a -// genuine repair still shows up under them. +// fix; the outcome-driven ones — no-URL, no-readers, Reader-report, failing, +// unverified — read stored facts that simply stop arriving, so a finished +// Series needing a genuine repair still shows up under them. The failing pair +// carries no finished guard for exactly that contrast: a failure row is a +// stored outcome, not a ticking clock. // // stale is the checked-but-old half of the stamp partition — because the // verdict line wants "not checked in twelve hours" as one figure, and a never @@ -157,6 +177,12 @@ func adminFilter(f SeriesFilter) (where, having string, args []any, err error) { clauses = append(clauses, `s.cover_address = '' AND s.finished_at = 0`) case SeriesFilterReaderReport: clauses = append(clauses, `s.latest_raised_by IS NOT NULL`) + case SeriesFilterFailing: + clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1)) + args = append(args, f.Cutoff) + case SeriesFilterUnverified: + clauses = append(clauses, `f.site IS NOT NULL AND s.latest_chapter_num IS NOT NULL AND f.failing_since < $`+strconv.Itoa(len(args)+1)+` AND s.latest_raised_by IS NOT NULL`) + args = append(args, f.Cutoff) case SeriesFilterFinished: clauses = append(clauses, `s.finished_at > 0`) case SeriesFilterNoReaders: @@ -204,15 +230,17 @@ func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) { base := len(args) args = append(args, seriesPageSize, seriesPageSize*(f.Page-1)) rows, err := s.db.Query(` - SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, + SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, `+failureAnswer+`, COUNT(b.reader_id) AS reader_count, COUNT(*) OVER () AS filtered_total FROM series s LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id + LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id `+where+` GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address, s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, - s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by + s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by, + f.outcome, f.failing_since `+having+` ORDER BY s.latest_checked_at, s.site, s.series_id LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...) @@ -251,6 +279,7 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) { SELECT s.site, s.kind FROM series s LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id + LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id `+where+` GROUP BY s.site, s.series_id, s.kind `+having+` @@ -273,11 +302,11 @@ func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) { return out, rows.Err() } -// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer -// order, plus the query's reader_count and filtered_total columns, and returns -// the window total alongside the row. latest_chapter_num is NULL until first -// captured — the "never read a chapter" state. The Sighting-raiser column is -// never among the scanned columns. +// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer + +// failureAnswer order, plus the query's reader_count and filtered_total +// columns, and returns the window total alongside the row. latest_chapter_num +// is NULL until first captured — the "never read a chapter" state. The +// Sighting-raiser column is never among the scanned columns. func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) { var ( a AdminSeries @@ -288,7 +317,7 @@ func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) { &a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress, &a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt, &a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt, - &a.RaisedByReader, &a.ReaderCount, &total, + &a.RaisedByReader, &a.FailureOutcome, &a.FailingSince, &a.ReaderCount, &total, ); err != nil { return AdminSeries{}, 0, err } diff --git a/backend/internal/store/admin_test.go b/backend/internal/store/admin_test.go index 5f4077f..fd5c28b 100644 --- a/backend/internal/store/admin_test.go +++ b/backend/internal/store/admin_test.go @@ -527,3 +527,134 @@ func TestAdminSeriesCarriesFinishedAt(t *testing.T) { t.Fatalf("row = %+v, want FinishedAt 5000", page.Rows) } } + +// The failing pair reads the failure row's age, not the Series row (issue +// #165): failing requires the joined row, a Latest Chapter, and failing_since +// past the cutoff — the same constant stale reads; unverified is the +// Reader-attributed subset of the same test. A failure inside the window is +// in neither — one bad fetch is not a fault to correct — and a Series that +// never captured a chapter is never failing, the exact complement of +// never-read-a-chapter's IS NULL half, so the two filters are disjoint by +// construction. A finished failing Series still appears: this pair reads +// stored outcomes, which simply stop arriving, and carries no finished guard. +func TestAdminFailingAndUnverifiedFilters(t *testing.T) { + s := newTestStore(t) + const cutoff = 5000 + seedAdminSeries(t, s, seriesSeed{key: "asura:failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:recent", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", cover: "c", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true}) + seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", cover: "c", checkedAt: 9000, latestNum: new(7.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:fin-failing", url: "u", cover: "c", checkedAt: 9000, latestNum: new(6.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "u", cover: "c", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1}) + + // Failure rows seed the run's start stamp; the cutoff decides the age. + for _, f := range []struct { + key, word string + since int64 + }{ + {"asura:failing", "not_found", 2000}, + {"asura:recent", "not_found", 9000}, + {"asura:nochapter", "not_found", 2000}, + {"asura:reader", "not_found", 2000}, + {"asura:polled", "errors", 2000}, + {"asura:fin-failing", "not_found", 2000}, + } { + site, id, _ := strings.Cut(f.key, ":") + if err := s.RecordSeriesFailure(site, id, f.word, f.since); err != nil { + t.Fatalf("seed failure %s: %v", f.key, err) + } + } + if err := s.SetSeriesFinished("asura", "fin-failing", 1000); err != nil { + t.Fatalf("finish asura:fin-failing: %v", err) + } + + cases := []struct { + name string + f SeriesFilter + want []string + }{ + {"failing", SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff}, []string{"asura:failing", "asura:reader", "asura:polled", "asura:fin-failing"}}, + {"unverified", SeriesFilter{Name: SeriesFilterUnverified, Cutoff: cutoff}, []string{"asura:reader"}}, + {"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := pageKeys(t, s, tc.f) + want := map[string]bool{} + for _, k := range tc.want { + want[k] = true + } + if len(got) != len(want) { + t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want) + } + for k := range want { + if !got[k] { + t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got) + } + } + }) + } + + // Disjointness: the failing pair and never-read-a-chapter are disjoint by + // the chapter column — IS NOT NULL here, IS NULL there — so no row may be + // counted under both. + failing := pageKeys(t, s, SeriesFilter{Name: SeriesFilterFailing, Cutoff: cutoff}) + noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter}) + for k := range failing { + if noChapter[k] { + t.Fatalf("row %q matches both failing and never-read-a-chapter", k) + } + } + if failing["asura:nochapter"] { + t.Fatal("a Series with no chapter ever captured appears in failing") + } + if !noChapter["asura:nochapter"] { + t.Fatal("the no-chapter Series vanished from never-read-a-chapter") + } + + // The aggregates count the same rows the lists do: the landing page's + // figures and the select's options come from these two passes. + for _, name := range []string{SeriesFilterFailing, SeriesFilterUnverified} { + shapes, err := s.SeriesShapes(SeriesFilter{Name: name, Cutoff: cutoff}) + if err != nil { + t.Fatalf("SeriesShapes(%s): %v", name, err) + } + sum := 0 + for _, sh := range shapes { + sum += sh.Total + } + want := len(pageKeys(t, s, SeriesFilter{Name: name, Cutoff: cutoff})) + if sum != want { + t.Fatalf("%s aggregate sum = %d, want %d (aggregate disagrees with row query)", name, sum, want) + } + } +} + +// The projection carries the failure facts from the LEFT JOIN, not the +// Series row: a failing Series reads back its outcome word and the stamp its +// run began at; a Series with no failure row reads empty and zero. Nothing +// new is projected from the Series row itself. +func TestAdminFailureProjection(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:fine", url: "u", cover: "c", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1}) + if err := s.RecordSeriesFailure("asura", "broken", "not_found", 2000); err != nil { + t.Fatalf("seed failure row: %v", err) + } + + page, err := s.SeriesPage(SeriesFilter{}) + if err != nil { + t.Fatalf("SeriesPage: %v", err) + } + byKey := map[string]AdminSeries{} + for _, a := range page.Rows { + byKey[a.Key()] = a + } + if byKey["asura:broken"].FailureOutcome != "not_found" || byKey["asura:broken"].FailingSince != 2000 { + t.Fatalf("broken row = %+v, want FailureOutcome not_found, FailingSince 2000", byKey["asura:broken"]) + } + if byKey["asura:fine"].FailureOutcome != "" || byKey["asura:fine"].FailingSince != 0 { + t.Fatalf("fine row = %+v, want empty outcome and zero stamp", byKey["asura:fine"]) + } +} diff --git a/backend/internal/web/admin_series.go b/backend/internal/web/admin_series.go index 20298ad..32ab04d 100644 --- a/backend/internal/web/admin_series.go +++ b/backend/internal/web/admin_series.go @@ -33,6 +33,8 @@ var seriesFilterLabels = map[string]string{ store.SeriesFilterStale: "Not checked in 12h", store.SeriesFilterNoCover: "No cover", store.SeriesFilterReaderReport: "Latest from a Reader", + store.SeriesFilterFailing: "Failing over 12h", + store.SeriesFilterUnverified: "Unverified Reader number", store.SeriesFilterFinished: "Finished", } @@ -49,6 +51,8 @@ var seriesFilterOrder = []string{ store.SeriesFilterStale, store.SeriesFilterNoCover, store.SeriesFilterReaderReport, + store.SeriesFilterFailing, + store.SeriesFilterUnverified, store.SeriesFilterFinished, } diff --git a/backend/web_test.go b/backend/web_test.go index 6a59c47..08d191a 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -3065,8 +3065,8 @@ func TestOverviewFinishedFigureRidesLast(t *testing.T) { } labels = append(labels, m[1]) } - if len(labels) != 8 { - t.Fatalf("hygiene block renders %d figures, want 8 (seven problems + finished):\n%s", len(labels), body) + if len(labels) != 10 { + t.Fatalf("hygiene block renders %d figures, want 10 (nine problems + finished):\n%s", len(labels), body) } if labels[len(labels)-1] != "Finished" { t.Errorf("finished figure does not ride last; hygiene order = %v:\n%s", labels, body) @@ -4045,3 +4045,49 @@ func TestRemoveRejectsBadKeysAndCapsBody(t *testing.T) { t.Errorf("an oversized body still removed the row:\n%s", list) } } + +// The failing pair render in the stats block from the same aggregate the +// select is numbered from: failing over 12h counts every Series with a +// failure row older than the cutoff, unverified the Reader-attributed +// subset, and a measured zero stays a muted digit — never a door. +func TestOverviewFailingAndUnverifiedFigures(t *testing.T) { + st, dsn := newTestStoreURL(t) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + now := time.Now().UnixMilli() + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:broken", url: "u", cover: "c", checkedAt: now, latestNum: floatPtr(1), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:reader", url: "u", cover: "c", checkedAt: now, latestNum: floatPtr(2), bookmarks: 1, raisedBy: true}) + if err := st.RecordSeriesFailure("asura", "broken", "not_found", now-24*3600*1000); err != nil { + t.Fatalf("seed failure row on broken: %v", err) + } + if err := st.RecordSeriesFailure("asura", "reader", "not_found", now-24*3600*1000); err != nil { + t.Fatalf("seed failure row on reader: %v", err) + } + srv := newRouter(st, testConfig()) + body := overviewBody(t, srv, st) + + if !strings.Contains(body, `href="/admin/series?filter=failing">2`) { + t.Errorf("the failing figure lacks its count and door:\n%s", body) + } + if !strings.Contains(body, `href="/admin/series?filter=unverified">1`) { + t.Errorf("the unverified figure lacks its count and door:\n%s", body) + } + + empty, emptySt := newWebTestServer(t, testConfig()) + body = overviewBody(t, empty, emptySt) + if !strings.Contains(body, `Failing over 12h0`) { + t.Errorf("a zero failing figure does not render as a muted digit:\n%s", body) + } + if !strings.Contains(body, `Unverified Reader number0`) { + t.Errorf("a zero unverified figure does not render as a muted digit:\n%s", body) + } + if strings.Contains(body, `href="/admin/series?filter=failing"`) { + t.Errorf("a zero failing figure is still a link:\n%s", body) + } + if strings.Contains(body, `href="/admin/series?filter=unverified"`) { + t.Errorf("a zero unverified figure is still a link:\n%s", body) + } +} -- 2.52.0 From a560fc76d3bdf079382f5465b2829b18ec3d0de4 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 23:46:19 +0700 Subject: [PATCH 11/20] Update stale Cutoff comments: the failing pair reads it too (#166) --- backend/internal/store/admin.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/internal/store/admin.go b/backend/internal/store/admin.go index 059216b..bbd0819 100644 --- a/backend/internal/store/admin.go +++ b/backend/internal/store/admin.go @@ -30,13 +30,14 @@ const ( // SeriesFilter is one named filter predicate over the whole library. Site // and Kind narrow the row read; Name picks the predicate; Cutoff is the -// staleness boundary the "stale" filter compares against, supplied by the -// caller's clock — the store has no clock; Page is 1-based. +// staleness boundary the age-based filters — "stale" and the failing pair — +// compare against, supplied by the caller's clock — the store has no clock; +// Page is 1-based. type SeriesFilter struct { Site string // "" = every Site Kind string // "" = both library buckets' series Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll - Cutoff int64 // unix ms; "stale" reads it, the store never does + Cutoff int64 // unix ms; the age-based filters read it, the store never does Page int // 1-based page of the row read; default 1 } -- 2.52.0 From 7b22460f5ef11ec641522d338c0d63b39e75596a Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 00:06:39 +0700 Subject: [PATCH 12/20] Owner notices: the stall, one webhook path from env to Discord (#171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rollout note: the owner_notices table starts empty, so the first pass after deploy sends for conditions already true — correct per one-row-per-episode; say so rather than have it reported as a bug. Prod step: create the webhook, set DISCORD_WEBHOOK_URL on the deployment, redeploy — unset is silent by design, and without that step the feature ships dark. Security invariants preserved: the webhook address is a secret in the class of TOKEN_KEY (never logged, never on a config-printing line), and the owner gate is unchanged. --- .env.example | 8 +- backend/internal/latest/faults.go | 101 ++++++++ backend/internal/latest/poller.go | 70 ++++- backend/internal/latest/poller_test.go | 240 ++++++++++++++++++ backend/internal/notify/notify.go | 122 +++++++++ backend/internal/notify/notify_test.go | 100 ++++++++ .../store/migrations/0020_owner_notices.sql | 12 + backend/internal/store/store.go | 40 +++ backend/internal/store/store_test.go | 38 +++ backend/internal/web/admin.go | 8 +- backend/main.go | 32 ++- backend/main_test.go | 25 +- docker-compose.yml | 7 +- 13 files changed, 785 insertions(+), 18 deletions(-) create mode 100644 backend/internal/latest/faults.go create mode 100644 backend/internal/notify/notify.go create mode 100644 backend/internal/notify/notify_test.go create mode 100644 backend/internal/store/migrations/0020_owner_notices.sql diff --git a/.env.example b/.env.example index 123ac7c..561dd9f 100644 --- a/.env.example +++ b/.env.example @@ -99,7 +99,13 @@ DISCORD_REDIRECT_URI= # request whose Host header isn't an IP or "localhost", which silently breaks # every kagane poll. Left unset here on purpose — a wrong default would poll a # stranger's address, and "no browser" is a safe, self-announcing state. -# BROWSER_WS_URL=ws://100.x.y.z:9222 +# Discord webhook for owner notices (outbound alerting when a poll Lane +# stalls). Unset means the whole path is off — a local stack needs no webhook, +# exactly as the browser URL behaves. The address is a secret in the class of +# TOKEN_KEY: never commit it, never paste it anywhere public. +# DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... + +# Zone the backend stamps its log lines in. Cosmetic only. Nothing else in # Zone the backend stamps its log lines in. Cosmetic only. Nothing else in # the service has a zone: bookmark timestamps are unix ms, and the two real diff --git a/backend/internal/latest/faults.go b/backend/internal/latest/faults.go new file mode 100644 index 0000000..88c810f --- /dev/null +++ b/backend/internal/latest/faults.go @@ -0,0 +1,101 @@ +package latest + +import ( + "context" + "fmt" + "time" + + "bookmarkmanager/backend/internal/store" +) + +// ConditionStall is the owner-notice machine word for a Lane that owed Polls, +// made none, and has nothing to say for it. The word is the message's footer +// and its suppression key; it is wire-stable. #172 declares the other three +// words (no-browser-route, sidecar-down, adapter-broken); this ticket +// declares only the stall. +const ConditionStall = "stall" + +// OwnerWindow is the class-level staleness boundary every owner-notice +// condition measures against — the same twelve hours the Lanes page's +// "not checked in 12h" filter uses (internal/web/admin.go). Declared here +// once so #172's three conditions and the admin filters share one figure. +const OwnerWindow = 12 * time.Hour + +// Fault is one condition the owner is told about, judged from durable rows +// alone. Site is "" for a fault that is not one Site's. +type Fault struct { + Condition string // one of the Condition* words + Site string + Since int64 // unix ms the episode began; the message's age +} + +// FaultInput is everything the judgement reads. A struct so #172's three +// conditions can add inputs without changing either caller. +type FaultInput struct { + Passes []store.LanePass +} + +// FaultsFrom judges the owner-notice conditions from durable rows alone, so +// the poller and the landing page cannot disagree about what a fault is. +// +// A Lane stalls when its latest pass shows due > 0, none checked, no skip +// value and no refusal — exactly the row the mid-loop browser loss writes +// (see the comment at the outcomeUnreachable return in runLanePass), so a +// Lane that owed Polls, made none, and has nothing to say for it is a fault. +// The no-refusal clause is AC6's amendment: the twice-refused break happens +// inside the pass loop, not on an early return, so a newly-gated Site would +// otherwise send two messages. A pause is excluded by Skip == "" (SkipPaused): +// the owner's own act is not reported back (AC10). The episode began at the +// pass that produced the row; the stall test itself has no age clause — the +// class-level twelve hours belongs to #172's conditions. +func FaultsFrom(in FaultInput, now time.Time) []Fault { + var faults []Fault + for _, p := range in.Passes { + if p.Due > 0 && p.Checked == 0 && p.Skip == "" && p.Refused == 0 { + faults = append(faults, Fault{Condition: ConditionStall, Site: p.Site, Since: p.RanAt}) + } + } + return faults +} + +// Notifier delivers one owner notice. The poller neither retries nor queues: +// an error is logged and the suppression row left unwritten, so the next pass +// tries again while the condition holds. +type Notifier interface { + Notify(ctx context.Context, f Fault, sentence, href string) error +} + +// ownerNoticeConditions is every condition this package judges, for the +// clear loop in recordPass: a condition absent from a pass's fault list +// forgets its episode, so the next occurrence sends again. #172 extends the +// list when it adds its conditions. +var ownerNoticeConditions = []string{ConditionStall} + +// noticeFor renders one fault's message: the description sentence — condition, +// age, repair — and the deep link the embed's title points at. Each condition +// provides its own wording; the stall is the only one today (issue #171). +func noticeFor(f Fault, row store.LanePass, now time.Time) (sentence, href string) { + switch f.Condition { + case ConditionStall: + return fmt.Sprintf( + "%s owed %d Polls and made none — %s; check the Lane's browser sidecar and the Site's challenge state", + f.Site, row.Due, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes" + } + return "", "" +} + +// humanAge renders a duration the way an owner reads it in a message: minutes +// under an hour, then hours, then days; a stall that was just born reads +// "just now". +func humanAge(d time.Duration) string { + switch { + case d < time.Minute: + return "just now" + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + default: + return fmt.Sprintf("%dd", int(d.Hours()/24)) + } +} diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index fff24dc..f43d4b3 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -46,7 +46,10 @@ type Poller struct { // CoverBytesFetch is optional; it handles plain-TLS sources through the // same failure-isolated prefetch path. CoverBytesFetch CoverBytesFetcher - Now func() time.Time // injected so tests can freeze it + // Notify delivers owner notices. Nil disables the whole path (issue #171): + // the poller is not the place a missing webhook becomes an error. + Notify Notifier + Now func() time.Time // injected so tests can freeze it // eligibleCount reports how many of a Site's Series are eligible for // polling, defaulting to Store.EligibleSeriesCount. Injected so tests can // fail the count alone: the eligible query shares the due query's tables, @@ -357,7 +360,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. // carries the previous pass's forward inside recordPass. fig := passFigures{} rec := passRecord{site: name, ranAt: now.UnixMilli()} - defer func() { p.recordPass(rec, fig) }() + defer func() { p.recordPass(ctx, rec, fig) }() // One Lane row read at the top of a pass, serving two gates (issue #139). // Both stamps outlive our process, so the gates read the durable row @@ -527,8 +530,9 @@ type passFigures struct { // pass's due, gap, clamped and checked forward rather than stating zeroes it // did not measure; the skip column says why it declined, so the zeroes that // remain (due-query, no-fetcher) read as explanations rather than -// measurements. -func (p *Poller) recordPass(rec passRecord, fig passFigures) { +// measurements. Once the row is durable, the owner-notice judgement runs +// beside it (issue #171). +func (p *Poller) recordPass(ctx context.Context, rec passRecord, fig passFigures) { row := store.LanePass{ Site: rec.site, RanAt: rec.ranAt, @@ -557,7 +561,65 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) { } if err := p.Store.RecordLanePass(row, rec.ranAt-lanePassRetention.Milliseconds()); err != nil { log.Printf("latest poll %s: record lane pass: %v", rec.site, err) + return } + p.ownerNotices(ctx, row) +} + +// ownerNotices judges the owner-notice conditions for the pass just recorded +// and fires (issue #171). It sits in recordPass because that deferred call is +// the one place every return path passes through: two of the four conditions +// occur on early returns and the success path can never see them. Per fault: +// NoticeSent → send → MarkNoticeSent, so a fault lasting a month sends one +// message, not one per pass; a condition absent from this pass's fault list +// forgets its episode, so the next occurrence sends again. Everything here is +// best-effort: a failed send, a failed store read and a failed notice write +// are all logged and never change the pass's outcome counts or its return +// value. The clear runs even when Notify is nil, so a deployment that turns +// the webhook off does not leave stale rows that suppress the first real +// notice after it is turned back on. +func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) { + faults := FaultsFrom(FaultInput{Passes: []store.LanePass{row}}, p.Now()) + for _, f := range faults { + if p.Notify == nil { + continue + } + sent, err := p.Store.NoticeSent(f.Condition, f.Site) + if err != nil { + log.Printf("latest poll %s: notice sent: %v", row.Site, err) + continue + } + if sent { + continue + } + sentence, href := noticeFor(f, row, p.Now()) + if err := p.Notify.Notify(ctx, f, sentence, href); err != nil { + // The stamp stays unset: no queue, no backoff — the condition is + // durable, so the next pass tries again while it holds. + log.Printf("latest poll %s: owner notice %s: %v", row.Site, f.Condition, err) + continue + } + if err := p.Store.MarkNoticeSent(f.Condition, f.Site, p.Now().UnixMilli()); err != nil { + log.Printf("latest poll %s: mark notice sent: %v", row.Site, err) + } + } + for _, cond := range ownerNoticeConditions { + if !hasFault(faults, cond, row.Site) { + if err := p.Store.ClearNotice(cond, row.Site); err != nil { + log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) + } + } + } +} + +// hasFault reports whether faults hold the given condition for the site. +func hasFault(faults []Fault, condition, site string) bool { + for _, f := range faults { + if f.Condition == condition && f.Site == site { + return true + } + } + return false } // countEligible routes the eligible count through the test seam when one is diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index d87b534..f7aff16 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -159,6 +159,35 @@ func (f *fakeBytesCoverFetcher) callCount() int { return len(f.calls) } +// fakeNotifier records the owner notices the poller sends, standing in for +// the Discord webhook the way fakeFetcher stands in for the network. An err +// set after construction makes every subsequent send fail, for the retry +// test. +type fakeNotifier struct { + mu sync.Mutex + calls []Fault + err error +} + +func (f *fakeNotifier) Notify(_ context.Context, fault Fault, _, _ string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, fault) + return f.err +} + +func (f *fakeNotifier) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +func (f *fakeNotifier) fault(i int) Fault { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls[i] +} + // newTestPoller wires a poller with a frozen clock. Rest and gap come from the // Site registry, so tests seed checked_at relative to the one-hour rest; the // round entry point (runOnce) runs every Lane back to back with no real @@ -3092,3 +3121,214 @@ func TestRunOnceSiteCompletedWritesDespiteUnchangedChapter(t *testing.T) { t.Fatalf("site_completed_at after unchanged-chapter completed poll = %d, want %d", got, at.UnixMilli()) } } + +// The stall judgement (issue #171) selects exactly the row the mid-loop +// browser loss writes — due > 0, none checked, no skip value, and no refusal. +// The no-refusal clause is AC6's amendment: the twice-refused break happens +// inside the pass loop, not on an early return, so a newly-gated Site would +// otherwise send two messages. A pause is excluded by Skip == "" (SkipPaused): +// the owner's own act is not reported back (AC10). +func TestFaultsFromStall(t *testing.T) { + now := time.UnixMilli(5_000_000) + tests := []struct { + name string + pass store.LanePass + want int + }{ + { + name: "stall-shaped pass is a stall", + pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 2, Checked: 0, Skip: "", Refused: 0}, + want: 1, + }, + { + name: "nothing due is not a stall", + pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 0, Checked: 0, Skip: "", Refused: 0}, + want: 0, + }, + { + name: "a pass that checked is not a stall", + pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 1, Checked: 1, Skip: "", Refused: 0}, + want: 0, + }, + { + name: "paused is the owner's own act, not a stall", + pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 1, Checked: 0, Skip: SkipPaused, Refused: 0}, + want: 0, + }, + { + name: "refusing has a skip value, not a stall", + pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 1, Checked: 0, Skip: SkipRefusing, Refused: 0}, + want: 0, + }, + { + name: "stalled and refused fires nothing (AC6's collision)", + pass: store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 2, Checked: 0, Skip: "", Refused: 1}, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + faults := FaultsFrom(FaultInput{Passes: []store.LanePass{tt.pass}}, now) + if got := len(faults); got != tt.want { + t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want) + } + for _, f := range faults { + if f.Condition != ConditionStall || f.Site != "comix" || f.Since != now.UnixMilli() { + t.Fatalf("fault = %+v, want stall on comix since the pass time", f) + } + } + }) + } +} + +// The stall fires exactly once while it holds, and again after it lifts and +// returns: the suppression row is the whole state, so the second stall-shaped +// pass is the same episode, a healthy pass in between clears the row, and the +// next stall is a new episode that sends again. +func TestOwnerNoticeStallFiresOncePerEpisode(t *testing.T) { + now := time.UnixMilli(5_000_000) + s, _ := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + notifier := &fakeNotifier{} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} + p.Notify = notifier + + // First stall-shaped pass: the episode begins, the owner is told once. + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after first stall = %d, want 1", got) + } + if f := notifier.fault(0); f.Condition != ConditionStall || f.Site != "comix" || f.Since != now.UnixMilli() { + t.Fatalf("fault = %+v, want stall on comix since the pass time", f) + } + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || !sent { + t.Fatalf("NoticeSent after first stall = %v, %v; want true, nil", sent, err) + } + + // A second stall-shaped pass — the series was stamped, so re-seed it — is + // the same episode: no second message. + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + now = now.Add(RefuseBackoff + time.Minute) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after second stall = %d, want 1 (one per episode)", got) + } + + // A healthy pass in between lifts the stall: the episode ends and the + // suppression row is cleared. + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} + now = now.Add(RefuseBackoff + time.Minute) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after healthy pass = %d, want 1 (a healthy pass sends nothing)", got) + } + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { + t.Fatalf("NoticeSent after healthy pass = %v, %v; want false, nil (row cleared)", sent, err) + } + + // The next stall is a new episode: the owner is told again. + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} + now = now.Add(RefuseBackoff + time.Minute) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 2 { + t.Fatalf("notices after the next stall = %d, want 2 (again after it lifts and returns)", got) + } +} + +// A paused Lane sends nothing: the pause is the owner's own act, and the +// skip value already keeps it out of the stall test (AC10). +func TestOwnerNoticePausedSendsNothing(t *testing.T) { + now := time.UnixMilli(5_000_000) + s, _ := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + if err := s.PauseLane("comix", now.Add(30*time.Minute).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + notifier := &fakeNotifier{} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{status: 200} + p.Notify = notifier + + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 0 { + t.Fatalf("notices while paused = %d, want 0", got) + } + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { + t.Fatalf("NoticeSent while paused = %v, %v; want false, nil", sent, err) + } +} + +// A failed POST is logged and the notified stamp left unset, so the next pass +// retries while the condition holds. No queue, no backoff — the condition is +// durable, and the suppression row is the only state. +func TestOwnerNoticeFailedSendRetriesNextPass(t *testing.T) { + now := time.UnixMilli(5_000_000) + s, _ := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + notifier := &fakeNotifier{err: errors.New("webhook down")} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} + p.Notify = notifier + + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("send attempts = %d, want 1", got) + } + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { + t.Fatalf("NoticeSent after failed send = %v, %v; want false, nil (stamp left unset)", sent, err) + } + + // The webhook recovers: the next stall-shaped pass delivers the one + // message the episode was owed. + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + notifier.err = nil + now = now.Add(RefuseBackoff + time.Minute) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 2 { + t.Fatalf("send attempts after recovery = %d, want 2 (retried next pass)", got) + } + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || !sent { + t.Fatalf("NoticeSent after recovery = %v, %v; want true, nil", sent, err) + } +} + +// Nil notifier means the whole path is off: a stall-shaped pass panics +// nothing and stamps no row, yet the clear still runs, so a deployment that +// turns the webhook off does not leave stale rows that suppress the first +// real notice after it is turned back on. +func TestOwnerNoticeNilNotifierStillClears(t *testing.T) { + now := time.UnixMilli(5_000_000) + s, _ := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + + // A stall-shaped pass with no notifier configured: nothing panics and no + // row is stamped — a missing webhook is a silent, complete off switch. + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.BrowserFetch = &fakeFetcher{status: 200, err: interrupted} + p.runLanePass(context.Background(), "comix", false) + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { + t.Fatalf("NoticeSent with nil notifier = %v, %v; want false, nil", sent, err) + } + + // A stale row from before the webhook was turned off must not survive a + // healthy pass: the clear runs even though no notifier is configured. + if err := s.MarkNoticeSent(ConditionStall, "comix", now.UnixMilli()); err != nil { + t.Fatalf("seed notice row: %v", err) + } + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + now = now.Add(RefuseBackoff + time.Minute) + p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} + p.runLanePass(context.Background(), "comix", false) + if sent, err := s.NoticeSent(ConditionStall, "comix"); err != nil || sent { + t.Fatalf("NoticeSent after healthy pass with nil notifier = %v, %v; want false, nil (cleared)", sent, err) + } +} diff --git a/backend/internal/notify/notify.go b/backend/internal/notify/notify.go new file mode 100644 index 0000000..fedabc8 --- /dev/null +++ b/backend/internal/notify/notify.go @@ -0,0 +1,122 @@ +// Package notify posts owner-notice embeds to a Discord webhook. It is +// deliberately small and stdlib-only: one POST of a JSON body needs no +// Discord library, and the path imports nothing of this repo's session or +// OAuth packages — that independence is why a webhook was chosen over a bot +// (issue #171, AC9). +package notify + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "bookmarkmanager/backend/internal/latest" +) + +// dangerColor is the dark design branch's --danger, #cf5c4d = 13589581. The +// integer is unreadable, so a future edit will otherwise "fix" it — do not: +// --ember means new chapter only, and a fault wearing ember would tell the +// owner a stall is a release. This is the one colour a fault wears. +const dangerColor = 13589581 + +// Client posts owner notices to one Discord webhook. The webhook address is a +// secret in the class of TOKEN_KEY: it is never logged, never rendered, and +// never carried in a returned error, which the poller logs. +type Client struct { + webhookURL string + baseURL string // the deployment's public origin; embed URLs resolve against it + http *http.Client +} + +// New returns a Client posting to webhookURL. baseURL is the deployment's +// public origin (Config.PublicBaseURL); the embed's deep-linked title is +// built from it. +func New(webhookURL, baseURL string) *Client { + return &Client{ + webhookURL: webhookURL, + baseURL: strings.TrimSuffix(baseURL, "/"), + http: &http.Client{Timeout: 10 * time.Second}, + } +} + +// Notify posts one owner notice as a Discord embed: the danger colour, the +// subject as a deep-linked title, the sentence as the description, the pass +// time as the timestamp and the machine word in the footer. The send is +// wrapped in a deadline so a hanging Discord cannot hold a poller Lane. On +// failure the error carries no part of the webhook address (the poller logs +// it), and the caller leaves the suppression row unwritten so the next pass +// retries while the condition holds. +func (c *Client) Notify(ctx context.Context, f latest.Fault, sentence, href string) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + body, err := json.Marshal(c.payload(f, sentence, href)) + if err != nil { + return fmt.Errorf("owner notice: marshal: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.webhookURL, strings.NewReader(string(body))) + if err != nil { + return errors.New("owner notice: build request") + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.http.Do(req) + if err != nil { + // The transport error embeds the webhook address; the address is a + // secret in the class of TOKEN_KEY and the poller logs this error. + return errors.New("owner notice: send failed") + } + defer resp.Body.Close() + // Cap the read: a Discord error page is enough, and draining the body + // lets the connection be reused. + io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return fmt.Errorf("owner notice: webhook status %d", resp.StatusCode) + } + return nil +} + +// payload is the webhook body: one embed and nothing else. No fields, no +// thumbnail, no author block — the wire shape is what Discord reads. +func (c *Client) payload(f latest.Fault, sentence, href string) webhookPayload { + return webhookPayload{Embeds: []embed{{ + Color: dangerColor, + Title: subject(f), + URL: c.baseURL + href, + Description: sentence, + Timestamp: time.UnixMilli(f.Since).UTC().Format(time.RFC3339), + Footer: embedFooter{Text: f.Condition}, + }}} +} + +// subject renders the embed's title: the Site the fault is about, with the +// machine word so the title needs no per-condition wording here — #172's +// conditions pass different sentences, not a different builder. For a fault +// that is not one Site's, the machine word stands alone. +func subject(f latest.Fault) string { + if f.Site == "" { + return f.Condition + } + return f.Site + ": " + f.Condition +} + +type webhookPayload struct { + Embeds []embed `json:"embeds"` +} + +type embed struct { + Color int `json:"color"` + Title string `json:"title"` + URL string `json:"url"` + Description string `json:"description"` + Timestamp string `json:"timestamp"` + Footer embedFooter `json:"footer"` +} + +type embedFooter struct { + Text string `json:"text"` +} diff --git a/backend/internal/notify/notify_test.go b/backend/internal/notify/notify_test.go new file mode 100644 index 0000000..0658636 --- /dev/null +++ b/backend/internal/notify/notify_test.go @@ -0,0 +1,100 @@ +package notify_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "bookmarkmanager/backend/internal/latest" + "bookmarkmanager/backend/internal/notify" +) + +// TestNotifyEmbedShape pins the wire shape Discord actually reads: one embed +// in the danger colour with the subject as a deep-linked title built from the +// base URL, the sentence as the description, the pass time as an RFC3339 +// timestamp, the machine word in the footer, and no fields grid (nor +// thumbnail, nor author block) at all. The webhook is a local server, so no +// test can reach Discord. +func TestNotifyEmbedShape(t *testing.T) { + var body map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if ct := r.Header.Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q, want application/json", ct) + } + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode request body: %v", err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + passTime := time.UnixMilli(5_000_000).UTC() + c := notify.New(srv.URL, "https://bookmarks.test/") + err := c.Notify(context.Background(), + latest.Fault{Condition: latest.ConditionStall, Site: "comix", Since: passTime.UnixMilli()}, + "sentence", "/admin/lanes") + if err != nil { + t.Fatalf("Notify: %v", err) + } + + embeds, ok := body["embeds"].([]any) + if !ok || len(embeds) != 1 { + t.Fatalf("embeds = %#v, want exactly one embed", body["embeds"]) + } + embed, ok := embeds[0].(map[string]any) + if !ok { + t.Fatalf("embed = %#v, want an object", embeds[0]) + } + if color, ok := embed["color"].(float64); !ok || int(color) != 13589581 { + t.Fatalf("color = %#v, want 13589581 (--danger #cf5c4d)", embed["color"]) + } + if url := embed["url"]; url != "https://bookmarks.test/admin/lanes" { + t.Fatalf("url = %#v, want the title deep-linked off the base URL", url) + } + if desc := embed["description"]; desc != "sentence" { + t.Fatalf("description = %#v, want the sentence", desc) + } + footer, ok := embed["footer"].(map[string]any) + if !ok || footer["text"] != latest.ConditionStall { + t.Fatalf("footer = %#v, want the machine word in the footer", embed["footer"]) + } + ts, ok := embed["timestamp"].(string) + if !ok { + t.Fatalf("timestamp = %#v, want an RFC3339 string", embed["timestamp"]) + } + parsed, err := time.Parse(time.RFC3339, ts) + if err != nil || !parsed.Equal(passTime) { + t.Fatalf("timestamp = %q, want %s (the pass time, RFC3339)", ts, passTime.Format(time.RFC3339)) + } + for _, banned := range []string{"fields", "thumbnail", "author"} { + if _, ok := embed[banned]; ok { + t.Fatalf("embed has %q, want it absent (no field grid, no thumbnail, no author block)", banned) + } + } +} + +// A non-2xx answer is an error the caller logs, and the error never carries +// the webhook address — a secret in the class of TOKEN_KEY, and the poller +// logs every notify error. +func TestNotifyNon2xxIsErrorWithoutTheAddress(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + c := notify.New(srv.URL, "https://bookmarks.test") + err := c.Notify(context.Background(), + latest.Fault{Condition: latest.ConditionStall, Site: "comix", Since: 5_000_000}, + "sentence", "/admin/lanes") + if err == nil { + t.Fatal("Notify = nil, want an error for a 500") + } + if strings.Contains(err.Error(), srv.URL) { + t.Fatalf("error %q leaks the webhook address", err) + } +} diff --git a/backend/internal/store/migrations/0020_owner_notices.sql b/backend/internal/store/migrations/0020_owner_notices.sql new file mode 100644 index 0000000..4717ff4 --- /dev/null +++ b/backend/internal/store/migrations/0020_owner_notices.sql @@ -0,0 +1,12 @@ +-- Owner notices (issue #171): one row per episode, remembered only as "the +-- owner was told". The row's presence is the whole state — the poller checks +-- it before sending, writes it after a successful send, and clears it when +-- the condition no longer holds. site is '' for a fault that is not one +-- Site's; the composite primary key is what makes the row a lock against a +-- second message for the same episode. +CREATE TABLE owner_notices ( + condition text NOT NULL, + site text NOT NULL, + notified_at bigint NOT NULL, + PRIMARY KEY (condition, site) +); diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 1f348f3..2f579a3 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1329,6 +1329,46 @@ func (s *Store) ClearSeriesFailure(site, seriesID string) error { return nil } +// NoticeSent reports whether the owner has already been told about this +// episode (issue #171). The row's presence is the whole state, so a missing +// row reads false without error. +func (s *Store) NoticeSent(condition, site string) (bool, error) { + var at int64 + err := s.db.QueryRow( + `SELECT notified_at FROM owner_notices WHERE condition = $1 AND site = $2`, + condition, site).Scan(&at) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("notice sent %s/%s: %w", condition, site, err) + } + return true, nil +} + +// MarkNoticeSent records that the owner was told. Called only after a +// successful send. Re-marking the same episode just moves the stamp: the row +// is a lock against a second message, not a log. +func (s *Store) MarkNoticeSent(condition, site string, at int64) error { + if _, err := s.db.Exec(` + INSERT INTO owner_notices (condition, site, notified_at) VALUES ($1, $2, $3) + ON CONFLICT (condition, site) DO UPDATE SET notified_at = EXCLUDED.notified_at`, + condition, site, at); err != nil { + return fmt.Errorf("mark notice sent %s/%s: %w", condition, site, err) + } + return nil +} + +// ClearNotice forgets an episode, so the condition's next occurrence sends +// again. A row that is not there is not an error. +func (s *Store) ClearNotice(condition, site string) error { + if _, err := s.db.Exec( + `DELETE FROM owner_notices WHERE condition = $1 AND site = $2`, condition, site); err != nil { + return fmt.Errorf("clear notice %s/%s: %w", condition, site, err) + } + return nil +} + // DueForLatestCheck returns one Site's series whose server-side // latest-chapter check has aged past cutoffMs, ordered by how many bookmarks // reference them (descending) then least-recently-checked first. One Site per diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index 286c212..b70093d 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -2700,3 +2700,41 @@ func failureRow(t *testing.T, s *Store, site, seriesID string) (word string, sin } return word, since, true } + +// Owner notices are one row per episode: the row's presence is the whole +// state. MarkNoticeSent stamps it, NoticeSent reads it, and ClearNotice +// forgets it — including a row that was never there, which is not an error. +func TestOwnerNoticeRows(t *testing.T) { + s := newTestStore(t) + + if sent, err := s.NoticeSent("stall", "comix"); err != nil || sent { + t.Fatalf("NoticeSent on a fresh table = %v, %v; want false, nil", sent, err) + } + + if err := s.MarkNoticeSent("stall", "comix", 4242); err != nil { + t.Fatalf("MarkNoticeSent: %v", err) + } + if sent, err := s.NoticeSent("stall", "comix"); err != nil || !sent { + t.Fatalf("NoticeSent after mark = %v, %v; want true, nil", sent, err) + } + + // Re-marking the same episode just moves the stamp: the row is a lock, + // not a log. A different site is its own episode — the key is + // (condition, site). + if err := s.MarkNoticeSent("stall", "comix", 4343); err != nil { + t.Fatalf("re-mark: %v", err) + } + if sent, err := s.NoticeSent("stall", "kagane"); err != nil || sent { + t.Fatalf("NoticeSent on another site = %v, %v; want false, nil", sent, err) + } + + if err := s.ClearNotice("stall", "comix"); err != nil { + t.Fatalf("ClearNotice: %v", err) + } + if sent, err := s.NoticeSent("stall", "comix"); err != nil || sent { + t.Fatalf("NoticeSent after clear = %v, %v; want false, nil", sent, err) + } + if err := s.ClearNotice("stall", "comix"); err != nil { + t.Fatalf("ClearNotice on a missing row: %v", err) + } +} diff --git a/backend/internal/web/admin.go b/backend/internal/web/admin.go index 4ef0dff..887962c 100644 --- a/backend/internal/web/admin.go +++ b/backend/internal/web/admin.go @@ -4,14 +4,16 @@ import ( "log" "net/http" "strconv" - "time" + "bookmarkmanager/backend/internal/latest" "bookmarkmanager/backend/internal/store" ) // ownerWindow is the staleness boundary the Series list's "not checked in -// 12h" filter compares against. Declared once; later admin tickets read it. -const ownerWindow = 12 * time.Hour +// 12h" filter compares against. It reads latest.OwnerWindow — the one place +// the class-level twelve hours lives, shared with the owner-notice +// conditions (issue #171). +const ownerWindow = latest.OwnerWindow // adminView is the shared shell data for an administrative page and the roster // fragment returned after a Reader action. diff --git a/backend/main.go b/backend/main.go index 629e1cf..7d265ae 100644 --- a/backend/main.go +++ b/backend/main.go @@ -14,6 +14,7 @@ import ( "bookmarkmanager/backend/internal/api" "bookmarkmanager/backend/internal/httpmw" "bookmarkmanager/backend/internal/latest" + "bookmarkmanager/backend/internal/notify" "bookmarkmanager/backend/internal/store" "bookmarkmanager/backend/internal/token" "bookmarkmanager/backend/internal/userscript" @@ -59,6 +60,12 @@ type Config struct { // the Lanes page reports the fact and derives reachability from the pass // log rather than asking the poller (issue #145). BrowserWSURL string + // DiscordWebhookURL is the webhook owner notices post to (issue #171). + // Unset means the whole path is off — a local stack needs no webhook, + // exactly as the browser URL behaves. The address is a secret in the + // class of TOKEN_KEY: never logged, and it must not reach any line that + // prints configuration. + DiscordWebhookURL string // LatestPoll configures the background latest-chapter fetcher. LatestPoll LatestPoll } @@ -114,6 +121,7 @@ func loadConfig() Config { UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"), NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"), BrowserWSURL: os.Getenv("BROWSER_WS_URL"), + DiscordWebhookURL: os.Getenv("DISCORD_WEBHOOK_URL"), LatestPoll: loadLatestPoll(), } c.Discord = web.DiscordConfig{ @@ -288,10 +296,22 @@ func main() { } s.OnSeriesCreated = acq.Acquire } + // Owner notices (issue #171): a configured webhook makes the poller tell + // the owner about stalled Lanes. Unset means the whole path is off — a + // local stack needs no webhook, exactly as the browser URL behaves. Only + // the presence is logged; the address itself is a secret in the class of + // TOKEN_KEY. + var notifier latest.Notifier + if u := strings.TrimSpace(cfg.DiscordWebhookURL); u != "" { + notifier = notify.New(u, cfg.PublicBaseURL) + log.Println("owner notices: enabled") + } else { + log.Println("owner notices: disabled (DISCORD_WEBHOOK_URL unset)") + } // The poller's only connection to the web layer is the database now: it is // started for its own sake, and the Lanes page reads the pass rows it // records (issue #145). - startLatestPoller(pollCtx, s, cfg.LatestPoll, browser) + startLatestPoller(pollCtx, s, cfg.LatestPoll, browser, notifier) srv := &http.Server{ Addr: ":" + cfg.Port, @@ -324,7 +344,9 @@ func main() { // newLatestPoller wires the fetcher seams into the poller. Pace is registry // property, not config (issue #100), so there are no knobs to pass through. -func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller { +// notifier is nil when no webhook is configured: a missing webhook is a +// silent off switch, not an error (issue #171). +func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher, notifier latest.Notifier) *latest.Poller { var covers latest.BrowserCoverFetcher if f, ok := browser.(latest.BrowserCoverFetcher); ok { covers = f @@ -335,6 +357,7 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch BrowserFetch: browser, CoverFetch: covers, CoverBytesFetch: latest.NewCoverFetcher(), + Notify: notifier, Now: time.Now, } } @@ -345,7 +368,8 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch // tracking, which is exactly how it behaved before. It returns the running // Poller, or nil when there is none; the caller starts it for its own sake — // the Lanes page reads the pass log, so no return value is wired anywhere. -func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) *latest.Poller { +// notifier is nil when DISCORD_WEBHOOK_URL is unset (issue #171). +func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher, notifier latest.Notifier) *latest.Poller { if !cfg.Enabled { log.Println("latest-chapter poller: disabled by config") return nil @@ -358,7 +382,7 @@ func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, brow // Nil browser: sites behind a JavaScript challenge are simply not polled, // and their latest_chapter comes from the userscript alone — which is how // the service behaved before the sidecar existed. - p := newLatestPoller(s, cfg, f, browser) + p := newLatestPoller(s, cfg, f, browser, notifier) go p.Run(ctx) return p diff --git a/backend/main_test.go b/backend/main_test.go index f2a0146..82836d9 100644 --- a/backend/main_test.go +++ b/backend/main_test.go @@ -21,10 +21,17 @@ func TestLoadLatestPollDefaults(t *testing.T) { } } -func TestLoadConfigReadsCoverDirectory(t *testing.T) { - t.Setenv("COVER_DIR", "/covers") - if got := loadConfig().CoverDir; got != "/covers" { - t.Fatalf("CoverDir = %q, want /covers", got) +func TestLoadConfigReadsDiscordWebhook(t *testing.T) { + // The address is read, never defaulted: unset stays empty (the whole + // path is off), set flows into the Config for the poller's notifier. + const url = "https://discord.com/api/webhooks/000000/secret" + t.Setenv("DISCORD_WEBHOOK_URL", url) + if got := loadConfig().DiscordWebhookURL; got != url { + t.Fatalf("DiscordWebhookURL = %q, want %q", got, url) + } + t.Setenv("DISCORD_WEBHOOK_URL", "") + if got := loadConfig().DiscordWebhookURL; got != "" { + t.Fatalf("DiscordWebhookURL = %q, want empty when unset", got) } } @@ -51,7 +58,8 @@ func TestLoadLatestPollEnabledParsing(t *testing.T) { // nothing here sizes a cooldown any more. func TestNewLatestPollerWiresFetchers(t *testing.T) { tls := &latest.TLSFetcher{} - p := newLatestPoller(nil, LatestPoll{Enabled: true}, tls, nil) + notifier := &stubNotifier{} + p := newLatestPoller(nil, LatestPoll{Enabled: true}, tls, nil, notifier) if p.Fetch != tls { t.Fatalf("Fetch not wired") } @@ -67,8 +75,15 @@ func TestNewLatestPollerWiresFetchers(t *testing.T) { if p.Now == nil { t.Fatalf("Now = nil, want the live clock") } + if p.Notify != notifier { + t.Fatalf("Notify = %v, want the configured notifier", p.Notify) + } } +// stubNotifier satisfies latest.Notifier so newLatestPoller's wiring can be +// asserted; it is never called. +type stubNotifier struct{ latest.Notifier } + func TestPutStatusValidation(t *testing.T) { cases := []struct { name string diff --git a/docker-compose.yml b/docker-compose.yml index de80e68..438ad55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,7 +75,12 @@ services: # Host header is an IP address or "localhost" — confirmed 2026-08-03, # independent of chromedp's own dial logic. The same trap that used to # force a pinned Docker IP now forbids the tailnet name. - BROWSER_WS_URL: ${BROWSER_WS_URL:-} + # Owner-notice webhook (issue #171). Empty default, never a + # required-guard: unset means the whole path is off, so a local stack + # runs exactly as it does today. An env var not listed here never + # reaches the container. + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL:-} + depends_on: depends_on: # The migration runner is the first thing the binary does, so a Postgres # that is still initialising means a crash-loop until it is not. -- 2.52.0 From 6af8859c70ce808860537c021e5e9b7b0c39caba Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 00:10:49 +0700 Subject: [PATCH 13/20] Review fixes: compose duplicate depends_on, .env.example splice, restore COVER_DIR test (#171) --- .env.example | 6 +++--- backend/AGENTS.md | 9 +++++++++ backend/main_test.go | 7 +++++++ docker-compose.yml | 1 - 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 561dd9f..6b9c726 100644 --- a/.env.example +++ b/.env.example @@ -99,14 +99,14 @@ DISCORD_REDIRECT_URI= # request whose Host header isn't an IP or "localhost", which silently breaks # every kagane poll. Left unset here on purpose — a wrong default would poll a # stranger's address, and "no browser" is a safe, self-announcing state. +# BROWSER_WS_URL=ws://100.x.y.z:9222 +# # Discord webhook for owner notices (outbound alerting when a poll Lane # stalls). Unset means the whole path is off — a local stack needs no webhook, # exactly as the browser URL behaves. The address is a secret in the class of # TOKEN_KEY: never commit it, never paste it anywhere public. # DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... - -# Zone the backend stamps its log lines in. Cosmetic only. Nothing else in - +# # Zone the backend stamps its log lines in. Cosmetic only. Nothing else in # the service has a zone: bookmark timestamps are unix ms, and the two real # time columns are timestamptz. Defaults to Asia/Jakarta; set to UTC for the diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 7501da1..247ee7e 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -159,6 +159,15 @@ failures. The flag decays and they probe again. - Browser Lanes wake Chrome only when 5+ Series are due or one has waited 15m, and cover work runs in the background so a slow CDN can't eat a Lane's gap. +### Owner notices — `internal/notify`, `latest.Fault`, `latest.Notifier`, `latest.FaultsFrom` + +The poller's outbound owner-notice path (issue #171): one condition today +(the stall), judged from the durable pass log alone so the poller and any +future reader of the same judgement cannot disagree. The webhook address is a +secret in the class of TOKEN_KEY — never logged, never rendered, never +carried in an error. The `owner_notices` suppression table (one row per +condition + site) is the only state; every threshold is derived, not stored. + ### Covers — `Store.OnSeriesCreated`, `latest.Acquirer`, `latest.CoverBytesFetcher`, `Store.SetSeriesCover` Acquired once when the first Bookmark of a Series is created, then served from diff --git a/backend/main_test.go b/backend/main_test.go index 82836d9..e07917f 100644 --- a/backend/main_test.go +++ b/backend/main_test.go @@ -21,6 +21,13 @@ func TestLoadLatestPollDefaults(t *testing.T) { } } +func TestLoadConfigReadsCoverDirectory(t *testing.T) { + t.Setenv("COVER_DIR", "/covers") + if got := loadConfig().CoverDir; got != "/covers" { + t.Fatalf("CoverDir = %q, want /covers", got) + } +} + func TestLoadConfigReadsDiscordWebhook(t *testing.T) { // The address is read, never defaulted: unset stays empty (the whole // path is off), set flows into the Config for the poller's notifier. diff --git a/docker-compose.yml b/docker-compose.yml index 438ad55..eb77921 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,7 +80,6 @@ services: # runs exactly as it does today. An env var not listed here never # reaches the container. DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL:-} - depends_on: depends_on: # The migration runner is the first thing the binary does, so a Postgres # that is still initialising means a crash-loop until it is not. -- 2.52.0 From a533e4f769024c21ddf8e977615f2b42cf0db903 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 00:31:21 +0700 Subject: [PATCH 14/20] web: site-completed hint reaches the owner, decides nothing (#170) The Site's own completed marker (issue #168) joins the owner's surface as an informational hint, never a decision: - store: SeriesFilterSiteCompleted ('site_completed') sits beside Finished in the filter vocabulary; its predicate carries the same finished_at guard as the clock-driven filters, because the Site's stamp keeps standing after the owner retires the row. The projection and SeriesPage GROUP BY carry site_completed_at so the detail page can age it. - web: the filter joins the Series list select as 'Site says completed' and rides the tail of seriesFilterOrder, so the landing stats figure comes for free and sits after the finished figure. - detail: one hint line beside the Finish control - 'the site says this work is completed (since 3d ago)' - rendered only while the stamp stands and no Finish stands; the Finish control itself is untouched. - tests: filter's finished exclusion and un-finish return to the Poll query (store); figure at zero and nonzero, hint present/absent, and the Finish control's markup byte-identical with a hint present (web). --- backend/internal/store/admin.go | 69 +++--- backend/internal/store/admin_test.go | 98 +++++++- backend/internal/web/admin_overview.go | 18 +- backend/internal/web/admin_series.go | 31 +-- backend/internal/web/admin_series_detail.go | 7 + .../internal/web/templates/series-detail.html | 1 + backend/web_test.go | 220 ++++++++++++++++-- 7 files changed, 363 insertions(+), 81 deletions(-) diff --git a/backend/internal/store/admin.go b/backend/internal/store/admin.go index bbd0819..98419a8 100644 --- a/backend/internal/store/admin.go +++ b/backend/internal/store/admin.go @@ -9,23 +9,26 @@ import ( // Series filter names (issue #140): the seven repair filters are ordered // permanent-then-fixable — the repairs nothing will ever undo first, the -// ones a Poll can make right after. SeriesFilterFinished is not part of -// that ordering: a finished Series is a deliberate state, not a repair, so -// it sits last, informational. A name is the repair a row needs, not the -// SQL that finds it; the values are the wire form the Series list URL -// carries (#142). "all" is the absent and unknown case: every Series. +// ones a Poll can make right after. SeriesFilterFinished and +// SeriesFilterSiteCompleted are not part of that ordering: a finished +// Series is a deliberate state and a Site-completed one is the Site's own +// marker, not repairs, so the pair rides the tail, informational. A name is +// the repair a row needs, not the SQL that finds it; the values are the +// wire form the Series list URL carries (#142). "all" is the absent and +// unknown case: every Series. const ( - SeriesFilterAll = "all" - SeriesFilterNoURL = "no_series_url" - SeriesFilterNoChapter = "never_read_a_chapter" - SeriesFilterNoReaders = "no_readers" - SeriesFilterNeverChecked = "never_checked" - SeriesFilterStale = "stale" - SeriesFilterNoCover = "no_cover" - SeriesFilterReaderReport = "reader_report" - SeriesFilterFinished = "finished" - SeriesFilterFailing = "failing" - SeriesFilterUnverified = "unverified" + SeriesFilterAll = "all" + SeriesFilterNoURL = "no_series_url" + SeriesFilterNoChapter = "never_read_a_chapter" + SeriesFilterNoReaders = "no_readers" + SeriesFilterNeverChecked = "never_checked" + SeriesFilterStale = "stale" + SeriesFilterNoCover = "no_cover" + SeriesFilterReaderReport = "reader_report" + SeriesFilterFinished = "finished" + SeriesFilterSiteCompleted = "site_completed" + SeriesFilterFailing = "failing" + SeriesFilterUnverified = "unverified" ) // SeriesFilter is one named filter predicate over the whole library. Site @@ -48,7 +51,7 @@ type SeriesFilter struct { // and only the anonymous boolean in raisedByReaderAnswer crosses it. const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address, s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at, - s.latest_corrected_at, s.finished_at` + s.latest_corrected_at, s.finished_at, s.site_completed_at` // raisedByReaderAnswer answers "did a Reader's report set this number" without // naming which Reader. Kept apart from adminSeriesColumns so the column list — @@ -101,6 +104,10 @@ type AdminSeries struct { // FinishedAt is the owner's finish stamp: unix ms, zero while the Series is // not finished — the same shape as the Correction stamp, and its own undo. FinishedAt int64 + // SiteCompletedAt is when the last successful Poll read saw the Site's own + // completed value, zero meaning it did not (issue #168). The Site's marker, + // never a Lifecycle decision: the owner's Finish is the only retirement. + SiteCompletedAt int64 } // SeriesPage is one page of the owner's filtered Series list plus the count @@ -134,23 +141,26 @@ func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID } // gate case is invisible to SQL, needs the Site registry in Go, and belongs to // a later repair), never-read-a-chapter and never-checked as disjoint halves // (non-zero versus zero check stamp), stale, no cover, finished (the -// retirement stamp, read directly), Reader-report, and the failing pair — -// failing (the failure row exists, a chapter exists, and failing_since is -// past the cutoff) and unverified (the Reader-attributed subset of failing). +// retirement stamp, read directly), site completed (the Site's marker, read +// directly), Reader-report, and the failing pair — failing (the failure row +// exists, a chapter exists, and failing_since is past the cutoff) and +// unverified (the Reader-attributed subset of failing). // failing's chapter IS NOT NULL is the exact complement of never-read-a- // chapter's IS NULL half, so the two are disjoint by construction. // no_readers is the one HAVING predicate: it is the orphan test, an aggregate // over the LEFT JOIN, where a bare WHERE has no row to test. // // The clock-versus-outcome split decides which predicates exclude finished -// Series (`s.finished_at = 0` in each of the four): the clock-driven one — +// Series (`s.finished_at = 0` in each of the five): the clock-driven ones — // never-checked, stale, no-chapter, no-cover — keep ticking after the last // Poll, so they would report a retired row as a problem no Poll is coming to -// fix; the outcome-driven ones — no-URL, no-readers, Reader-report, failing, -// unverified — read stored facts that simply stop arriving, so a finished -// Series needing a genuine repair still shows up under them. The failing pair -// carries no finished guard for exactly that contrast: a failure row is a -// stored outcome, not a ticking clock. +// fix; site-completed's stamp is the Site's, and it keeps standing after the +// owner retires the row, so a finished Series would be reported as work +// nobody is going to do. The outcome-driven ones — no-URL, no-readers, +// Reader-report, failing, unverified — read stored facts that simply stop +// arriving, so a finished Series needing a genuine repair still shows up +// under them. The failing pair carries no finished guard for exactly that +// contrast: a failure row is a stored outcome, not a ticking clock. // // stale is the checked-but-old half of the stamp partition — because the // verdict line wants "not checked in twelve hours" as one figure, and a never @@ -186,6 +196,8 @@ func adminFilter(f SeriesFilter) (where, having string, args []any, err error) { args = append(args, f.Cutoff) case SeriesFilterFinished: clauses = append(clauses, `s.finished_at > 0`) + case SeriesFilterSiteCompleted: + clauses = append(clauses, `s.site_completed_at > 0 AND s.finished_at = 0`) case SeriesFilterNoReaders: having = `HAVING COUNT(b.reader_id) = 0` default: @@ -240,7 +252,8 @@ func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) { `+where+` GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address, s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, - s.force_poll_at, s.latest_corrected_at, s.finished_at, s.latest_raised_by, + s.force_poll_at, s.latest_corrected_at, s.finished_at, s.site_completed_at, + s.latest_raised_by, f.outcome, f.failing_since `+having+` ORDER BY s.latest_checked_at, s.site, s.series_id @@ -317,7 +330,7 @@ func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) { if err := scan( &a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress, &a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt, - &a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt, + &a.ForcePollAt, &a.LatestCorrectedAt, &a.FinishedAt, &a.SiteCompletedAt, &a.RaisedByReader, &a.FailureOutcome, &a.FailingSince, &a.ReaderCount, &total, ); err != nil { return AdminSeries{}, 0, err diff --git a/backend/internal/store/admin_test.go b/backend/internal/store/admin_test.go index fd5c28b..b80dacf 100644 --- a/backend/internal/store/admin_test.go +++ b/backend/internal/store/admin_test.go @@ -12,14 +12,15 @@ import ( // Upsert path could not produce together: an orphan has no bookmark, and a // Reader-raised Latest Chapter needs a Sighting the store does not create. type seriesSeed struct { - key string - kind string - url string - cover string // cover_address - checkedAt int64 - latestNum *float64 - bookmarks int // readers that hold it; 0 = orphan - raisedBy bool // a Reader's report is attributed as the raiser + key string + kind string + url string + cover string // cover_address + checkedAt int64 + latestNum *float64 + bookmarks int // readers that hold it; 0 = orphan + raisedBy bool // a Reader's report is attributed as the raiser + siteCompletedAt int64 // the Site's own marker (issue #168), 0 = not set } // seedAdminSeries inserts one series row and its bookmarks (owner first, then @@ -39,10 +40,10 @@ func seedAdminSeries(t *testing.T, s *Store, seed seriesSeed) { } if _, err := s.db.Exec(` INSERT INTO series (site, series_id, title, kind, series_url, cover_address, - latest_checked_at, latest_chapter, latest_chapter_num) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + latest_checked_at, latest_chapter, latest_chapter_num, site_completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover, - seed.checkedAt, latestChapter, seed.latestNum); err != nil { + seed.checkedAt, latestChapter, seed.latestNum, seed.siteCompletedAt); err != nil { t.Fatalf("seed series %q: %v", seed.key, err) } for i := range seed.bookmarks { @@ -658,3 +659,78 @@ func TestAdminFailureProjection(t *testing.T) { t.Fatalf("fine row = %+v, want empty outcome and zero stamp", byKey["asura:fine"]) } } + +// The site-completed filter (issue #170) lists the Site's own marker as work +// to work through, carrying the same finished guard the clock-driven +// predicates gained: the Site's stamp keeps standing after the owner retires +// the row, so a finished Series would be reported as work nobody is going to +// do. A zero stamp never appears. Un-finishing puts the Series back in the +// Poll query — the finished_at gate is #157's own, asserted through +// DueForLatestCheck rather than re-derived here. +func TestAdminSiteCompletedFilter(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:done", url: "u", cover: "c", checkedAt: 9000, latestNum: new(4.0), bookmarks: 1, siteCompletedAt: 5000}) + seedAdminSeries(t, s, seriesSeed{key: "asura:never", url: "u", cover: "c", checkedAt: 9000, latestNum: new(4.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:retired", url: "u", cover: "c", checkedAt: 0, latestNum: new(4.0), bookmarks: 1, siteCompletedAt: 5000}) + seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "u", cover: "c", checkedAt: 9000, latestNum: new(4.0), bookmarks: 1}) + if err := s.SetSeriesFinished("asura", "retired", 1000); err != nil { + t.Fatalf("finish asura:retired: %v", err) + } + + got := pageKeys(t, s, SeriesFilter{Name: SeriesFilterSiteCompleted}) + if len(got) != 1 || !got["asura:done"] { + t.Fatalf("site-completed returned %v, want only asura:done", got) + } + + // The projection carries the stamp so the detail page can age it. + page, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterSiteCompleted}) + if err != nil { + t.Fatalf("SeriesPage(site_completed): %v", err) + } + if len(page.Rows) != 1 || page.Rows[0].SiteCompletedAt != 5000 { + t.Fatalf("row = %+v, want SiteCompletedAt 5000", page.Rows) + } + + // The aggregate counts the same row: the landing figure and the select's + // option come from this pass, so they cannot disagree with the list. + shapes, err := s.SeriesShapes(SeriesFilter{Name: SeriesFilterSiteCompleted}) + if err != nil { + t.Fatalf("SeriesShapes(site_completed): %v", err) + } + sum := 0 + for _, sh := range shapes { + sum += sh.Total + } + if sum != 1 { + t.Fatalf("site-completed aggregate = %d, want 1", sum) + } + + // Un-finishing puts the Series back in the Poll query: the due read's + // finished_at gate (issue #157) admits it again — asserted through the + // Lane's own read, not re-derived here. + due, err := s.DueForLatestCheck("asura", 1000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + for _, sr := range due { + if sr.Key() == "asura:retired" { + t.Fatalf("a finished Series is still due for a Poll:\n%+v", due) + } + } + if err := s.SetSeriesFinished("asura", "retired", 0); err != nil { + t.Fatalf("un-finish asura:retired: %v", err) + } + due, err = s.DueForLatestCheck("asura", 1000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck after un-finish: %v", err) + } + found := false + for _, sr := range due { + if sr.Key() == "asura:retired" { + found = true + } + } + if !found { + t.Fatalf("un-finished Series is not due for a Poll:\n%+v", due) + } +} diff --git a/backend/internal/web/admin_overview.go b/backend/internal/web/admin_overview.go index 9857864..aa9db27 100644 --- a/backend/internal/web/admin_overview.go +++ b/backend/internal/web/admin_overview.go @@ -23,10 +23,10 @@ type overviewView struct { // as stale + never_checked: a never-checked Series is already counted on // its own filter, and the verdict wants the inclusive number. Unchecked int - // Hygiene is the seven problem filters in the Series list's own render - // order plus the finished figure riding last (informational); Library is - // the library split plus the roster. Every figure is a door into the list - // that counts it, except a zero. + // Hygiene is the problem filters in the Series list's own render order + // plus the informational tail — finished, then site completed — riding + // last; Library is the library split plus the roster. Every figure is a + // door into the list that counts it, except a zero. Hygiene []fig Library []fig // Sites is the per-Site library shape table, one row per Site with any @@ -60,7 +60,7 @@ type siteRow struct { } // overviewView assembles the landing page from the store's read model: one -// SeriesShapes pass per filter summed in Go (the shipped surface offers nine +// SeriesShapes pass per filter summed in Go (the shipped surface offers ten // grouped passes, not a stats query — #140), the pass log's latest pass per // Site, and the roster. A failure in any read is a 500 with a logged reason, // never a page of silent zeroes. @@ -94,10 +94,10 @@ func (h *Handler) overviewView() (overviewView, error) { view.Unchecked = totals[store.SeriesFilterStale] + totals[store.SeriesFilterNeverChecked] view.Verdict, view.HasCounts = overviewVerdict(passes, now) - // The hygiene figures, in seriesFilterOrder's tail: the seven problem - // filters in permanent-then-fixable order, then the finished figure last — - // informational, not a problem, and last because seriesFilterOrder appends - // it there. The All filter's count belongs to the Library block, not to a + // The hygiene figures, in seriesFilterOrder's tail: the problem filters + // in permanent-then-fixable order, then the informational tail — finished + // and site completed — riding last because seriesFilterOrder appends them + // there. The All filter's count belongs to the Library block, not to a // "hygiene" figure. hygiene := make([]fig, 0, len(seriesFilterOrder)-1) for _, name := range seriesFilterOrder[1:] { diff --git a/backend/internal/web/admin_series.go b/backend/internal/web/admin_series.go index 32ab04d..d6d9cbc 100644 --- a/backend/internal/web/admin_series.go +++ b/backend/internal/web/admin_series.go @@ -25,23 +25,25 @@ const seriesPageSize = 50 // the labels are read by later admin tickets too, so the map and the // constants cannot drift apart. var seriesFilterLabels = map[string]string{ - store.SeriesFilterAll: "All series", - store.SeriesFilterNoURL: "No series URL", - store.SeriesFilterNoChapter: "Never read a chapter", - store.SeriesFilterNoReaders: "No Readers", - store.SeriesFilterNeverChecked: "Never checked", - store.SeriesFilterStale: "Not checked in 12h", - store.SeriesFilterNoCover: "No cover", - store.SeriesFilterReaderReport: "Latest from a Reader", - store.SeriesFilterFailing: "Failing over 12h", - store.SeriesFilterUnverified: "Unverified Reader number", - store.SeriesFilterFinished: "Finished", + store.SeriesFilterAll: "All series", + store.SeriesFilterNoURL: "No series URL", + store.SeriesFilterNoChapter: "Never read a chapter", + store.SeriesFilterNoReaders: "No Readers", + store.SeriesFilterNeverChecked: "Never checked", + store.SeriesFilterStale: "Not checked in 12h", + store.SeriesFilterNoCover: "No cover", + store.SeriesFilterReaderReport: "Latest from a Reader", + store.SeriesFilterFailing: "Failing over 12h", + store.SeriesFilterUnverified: "Unverified Reader number", + store.SeriesFilterFinished: "Finished", + store.SeriesFilterSiteCompleted: "Site says completed", } // seriesFilterOrder is the select's render order: All first, then the -// permanent repairs, then the fixable ones (issue #140). Finished rides the -// tail, last — deliberate, not a repair — and the Overview's stats block -// renders the same tail, which is what sits the finished figure last there. +// permanent repairs, then the fixable ones (issue #140). Finished and the +// site-completed hint ride the tail — deliberate, not repairs — and the +// Overview's stats block renders the same tail, which is what sits the +// finished and site-completed figures last there. var seriesFilterOrder = []string{ store.SeriesFilterAll, store.SeriesFilterNoURL, @@ -54,6 +56,7 @@ var seriesFilterOrder = []string{ store.SeriesFilterFailing, store.SeriesFilterUnverified, store.SeriesFilterFinished, + store.SeriesFilterSiteCompleted, } // seriesListView is the Series list page's data. The template renders strings diff --git a/backend/internal/web/admin_series_detail.go b/backend/internal/web/admin_series_detail.go index 9d0f3d1..dfa8bc7 100644 --- a/backend/internal/web/admin_series_detail.go +++ b/backend/internal/web/admin_series_detail.go @@ -64,6 +64,10 @@ type seriesDetailView struct { // stands. It rides the meta fragment both presses swap, so the answer // itself shows how long the Series has been finished. FinishedSince string + // SiteCompleted is the hint's line, "" while the Site has said nothing or + // the owner has finished the Series: "the site says this work is + // completed (since 3d ago)". A hint, never a control (issue #170). + SiteCompleted string } // adminSeriesDetail renders one Series' page, keyed by the composite @@ -151,6 +155,9 @@ func (h *Handler) seriesDetailView(a store.AdminSeries) seriesDetailView { v.Corrected = correctedAge(time.Now(), a.LatestCorrectedAt) v.Finished = a.FinishedAt != 0 v.FinishedSince = finishedAge(time.Now(), a.FinishedAt) + if a.SiteCompletedAt != 0 && a.FinishedAt == 0 { + v.SiteCompleted = "the site says this work is completed (since " + checkedAge(time.Now(), a.SiteCompletedAt) + ")" + } // Provenance: the actor class behind the current number, evaluated in the // order the classes outrank one another — the owner's stamp, which a diff --git a/backend/internal/web/templates/series-detail.html b/backend/internal/web/templates/series-detail.html index c2204de..044d62f 100644 --- a/backend/internal/web/templates/series-detail.html +++ b/backend/internal/web/templates/series-detail.html @@ -46,6 +46,7 @@ {{else}}
    + {{if .SiteCompleted}}

    {{.SiteCompleted}}

    {{end}} "):], "
    ") + end := open + inner + len("") + outer + len("") + return body[start:end] +} + // The finish route is the owner's one writer: a finish press stamps // finished_at and answers with the swapped detail-meta fragment carrying the // "finished ago" mark, and an un-finish press writes zero and answers -- 2.52.0 From f85908d3b416f71b39108f88edda1874f0f9f76b Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 00:36:12 +0700 Subject: [PATCH 15/20] web: a failure names itself on the page (issue #167) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Series list row's fact line gains the failure word and its age in one mark before the Notes chips, spelled through outcomeWord — the one vocabulary shared with the Lanes chips. The detail page adds one sentence beside the correction control while a Reader's number stands behind a failure past the owner window, without naming the Reader. Each Lanes Site name is a plain link to that Site's failing Series; the outcome chips stay unlinked, and outcomeChips records why the promise is withdrawn. --- backend/internal/web/admin_lanes.go | 25 ++-- backend/internal/web/admin_series.go | 22 ++++ backend/internal/web/admin_series_detail.go | 13 ++ backend/internal/web/templates/lanes.html | 2 +- .../internal/web/templates/series-detail.html | 1 + .../internal/web/templates/series-list.html | 2 +- backend/web_test.go | 116 +++++++++++++++++- 7 files changed, 171 insertions(+), 10 deletions(-) diff --git a/backend/internal/web/admin_lanes.go b/backend/internal/web/admin_lanes.go index c70b972..d6fcb85 100644 --- a/backend/internal/web/admin_lanes.go +++ b/backend/internal/web/admin_lanes.go @@ -56,6 +56,10 @@ type laneRow struct { // offer Resume from the moment the owner presses Pause, with no pass // having run to record it (issue #147). Paused bool + // FailingHref is the one navigation the row offers: the Site's name + // links to that Site's failing Series. The chips beside it stay + // unlinked; the withdrawn promise lives on outcomeChips (issue #167). + FailingHref string } // chip is one named outcome count over the owner's window. @@ -223,6 +227,7 @@ func buildLaneRow(p store.LanePass, o store.SiteOutcomes, now time.Time) laneRow if p.GapMS > 0 { row.Gap = (time.Duration(p.GapMS) * time.Millisecond).Truncate(time.Second).String() } + row.FailingHref = seriesListHref(store.SeriesFilterFailing, p.Site, "", 0) row.Chips = outcomeChips(o) row.HasChips = len(row.Chips) > 0 row.StatePhrase, row.StateGood, row.Attention = laneState(p, now) @@ -232,18 +237,24 @@ func buildLaneRow(p store.LanePass, o store.SiteOutcomes, now time.Time) laneRow // outcomeChips lists a Site's nonzero window sums in the taxonomy's fixed // order, so the chips never reorder as the window changes. None observed is -// written by the template, not drawn as a confident zero count. +// written by the template, not drawn as a confident zero count. The names are +// spelled through outcomeWord, the one vocabulary the failure surface shares +// with the Series list's fact line (issue #167). The counts stay unlinked +// permanently — a withdrawn promise: two of the six words write no per-Series +// state, and the other four count attempts inside the owner window while the +// failing filter lists Series failing now, so neither set contains the other +// and no chip can be a door to its list. func outcomeChips(o store.SiteOutcomes) []chip { fixed := []struct { name string count int }{ - {"refused", o.Refused}, - {"unreachable", o.Unreachable}, - {"no chapter", o.NoChapter}, - {"unfetchable", o.Unfetchable}, - {"not found", o.NotFound}, - {"errors", o.Errors}, + {outcomeWord("refused"), o.Refused}, + {outcomeWord("unreachable"), o.Unreachable}, + {outcomeWord("no_chapter"), o.NoChapter}, + {outcomeWord("unfetchable"), o.Unfetchable}, + {outcomeWord("not_found"), o.NotFound}, + {outcomeWord("errors"), o.Errors}, } var out []chip for _, f := range fixed { diff --git a/backend/internal/web/admin_series.go b/backend/internal/web/admin_series.go index 32ab04d..2f2ae2e 100644 --- a/backend/internal/web/admin_series.go +++ b/backend/internal/web/admin_series.go @@ -123,6 +123,11 @@ type seriesRowView struct { // page, where a press that retires a Series from the Lane is on purpose // and confirm-gated (issue #158). Finished bool + // Failure names the standing failure and how long it has stood — "not + // found · 3d ago", "" while no failure row stands. Its own field, never a + // Notes chip: the chips cap at two plus a tail, so the one fact that + // names the failure would be the most likely to be truncated away. + Failure string } // adminSeries renders the filterable, bookmarkable Series list: filter, Site, @@ -683,6 +688,9 @@ func seriesRow(a store.AdminSeries, i int, now time.Time) seriesRowView { row.Ch = "—" } row.Age = checkedAge(now, a.LatestCheckedAt) + if a.FailureOutcome != "" { + row.Failure = outcomeWord(a.FailureOutcome) + " · " + checkedAge(now, a.FailingSince) + } notes := seriesNotes(a, now) if n := len(notes); n > 2 { row.Notes, row.More = notes[:2], n-2 @@ -693,6 +701,20 @@ func seriesRow(a store.AdminSeries, i int, now time.Time) seriesRowView { return row } +// outcomeWord spells the wire failure word as the Lanes chips spell it — a +// space, not the underscore: not_found reads "not found", no_chapter "no +// chapter". The remaining words are their own spelling, so an unknown word +// degrades to itself rather than vanishing from the page. +func outcomeWord(wire string) string { + switch wire { + case "not_found": + return "not found" + case "no_chapter": + return "no chapter" + } + return wire +} + // seriesNotes are a row's hygiene chips in the design's order: no URL, no // cover, orphan, stale, reader sighting. func seriesNotes(a store.AdminSeries, now time.Time) []string { diff --git a/backend/internal/web/admin_series_detail.go b/backend/internal/web/admin_series_detail.go index 9d0f3d1..08f6bde 100644 --- a/backend/internal/web/admin_series_detail.go +++ b/backend/internal/web/admin_series_detail.go @@ -64,6 +64,11 @@ type seriesDetailView struct { // stands. It rides the meta fragment both presses swap, so the answer // itself shows how long the Series has been finished. FinishedSince string + // Unverified is the sentence beside the Latest Chapter correction + // control while a Reader's number stands behind a failure past the + // owner window: the value is unconfirmed and the owner should not trust + // it. It never names the Reader. "" otherwise. + Unverified string } // adminSeriesDetail renders one Series' page, keyed by the composite @@ -151,6 +156,14 @@ func (h *Handler) seriesDetailView(a store.AdminSeries) seriesDetailView { v.Corrected = correctedAge(time.Now(), a.LatestCorrectedAt) v.Finished = a.FinishedAt != 0 v.FinishedSince = finishedAge(time.Now(), a.FinishedAt) + // Unverified: a Reader's number standing behind a failure that has outlived + // the owner window is a number nobody has re-checked since — say so next to + // the correction control, without naming the Reader. A machine read or a + // failure still inside the window carries no sentence; the failure itself + // has not yet outlived the twelve hours' worth of trust. + if a.RaisedByReader && a.FailureOutcome != "" && a.FailingSince < time.Now().Add(-ownerWindow).UnixMilli() { + v.Unverified = "Unverified Reader number: the page has been failing for over 12h, so this Reader-reported chapter is unconfirmed." + } // Provenance: the actor class behind the current number, evaluated in the // order the classes outrank one another — the owner's stamp, which a diff --git a/backend/internal/web/templates/lanes.html b/backend/internal/web/templates/lanes.html index c4a359e..0bf79ac 100644 --- a/backend/internal/web/templates/lanes.html +++ b/backend/internal/web/templates/lanes.html @@ -26,7 +26,7 @@ {{range .Rows}}
    - {{.Site}} + {{.Site}} {{.Due}} {{.Checked}} {{.Gap}} diff --git a/backend/internal/web/templates/series-detail.html b/backend/internal/web/templates/series-detail.html index c2204de..21b619f 100644 --- a/backend/internal/web/templates/series-detail.html +++ b/backend/internal/web/templates/series-detail.html @@ -19,6 +19,7 @@
    + {{if .Unverified}}

    {{.Unverified}}

    {{end}}

    Repair series URL

    diff --git a/backend/internal/web/templates/series-list.html b/backend/internal/web/templates/series-list.html index f0af815..f79a690 100644 --- a/backend/internal/web/templates/series-list.html +++ b/backend/internal/web/templates/series-list.html @@ -47,7 +47,7 @@ {{.Ch}} {{.Age}} {{.Readers}} - {{range .Notes}}{{.}}{{end}}{{if .More}}+{{.More}}{{end}}{{if .Finished}}finished{{end}} + {{if .Failure}}{{.Failure}}{{end}}{{range .Notes}}{{.}}{{end}}{{if .More}}+{{.More}}{{end}}{{if .Finished}}finished{{end}} {{if .CanPoll}}Check now{{end}}{{if .CanRemove}}{{end}} {{if .RemovalRefused}}a Reader has bookmarked this Series again{{end}} diff --git a/backend/web_test.go b/backend/web_test.go index 08d191a..dcd4405 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -918,7 +918,7 @@ func TestLanesRenderFromSeededRowsAfterRestart(t *testing.T) { seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.Add(-time.Minute).UnixMilli(), Due: 5, Checked: 5, GapMS: 20_000}) body := lanesBody(t, router, st) - if !strings.Contains(body, `class="c-site">asura`) { + if !strings.Contains(body, `asura`) { t.Fatalf("asura row missing from a restart-read database:\n%s", body) } // The fragment carries its own single timer and answers the swap it asked @@ -989,6 +989,31 @@ func TestNamedOutcomeChips(t *testing.T) { } } +// Each Site row on the Lanes page carries one plain navigation link to that +// Site's failing Series — a navigation, not a number: the outcome chips +// beside it stay unlinked permanently, so the row offers one click and one +// answer. The clean Site links too, into an empty list (issue #167). +func TestLaneSiteLinksToFailingSeriesAndChipsStayUnlinked(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + now := time.Now() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), NotFound: 6}) + seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli()}) + + body := lanesBody(t, router, st) + if !strings.Contains(body, `asura`) { + t.Errorf("the failing Site's name is not a link to its failing Series:\n%s", body) + } + if !strings.Contains(body, `demonic`) { + t.Errorf("the clean Site's name is not a link to its (empty) failing Series:\n%s", body) + } + if !strings.Contains(body, `not found 6`) { + t.Errorf("the outcome chip no longer renders beside the link:\n%s", body) + } + if got := strings.Count(body, ` · ", and a Series +// with no failure row renders no marker at all. The marker is its own field, +// never a Notes chip — the chips cap at two plus a tail, so the failure +// would be the fact most likely to be truncated away (issue #167). +func TestSeriesListFailureMarkerAges(t *testing.T) { + st, dsn := newTestStoreURL(t) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + now := time.Now() + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:hour", url: "u", cover: "c", checkedAt: now.UnixMilli(), latestNum: floatPtr(1), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:months", url: "u", cover: "c", checkedAt: now.UnixMilli(), latestNum: floatPtr(1), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:fine", url: "u", cover: "c", checkedAt: now.UnixMilli(), latestNum: floatPtr(1), bookmarks: 1}) + if err := st.RecordSeriesFailure("asura", "hour", "not_found", now.Add(-time.Hour).UnixMilli()); err != nil { + t.Fatalf("seed failure row on hour: %v", err) + } + if err := st.RecordSeriesFailure("asura", "months", "no_chapter", now.Add(-90*24*time.Hour).UnixMilli()); err != nil { + t.Fatalf("seed failure row on months: %v", err) + } + + body := adminSeriesPage(t, newRouter(st, testConfig()), st, "") + if !strings.Contains(body, `not found · 1h ago`) { + t.Errorf("the one-hour failure lacks its marker:\n%s", body) + } + if !strings.Contains(body, `no chapter · 90d ago`) { + t.Errorf("the three-month failure lacks its marker:\n%s", body) + } + // The clean row carries none, and neither word leaks anywhere else on + // the page: exactly one marker per failing row. + if strings.Count(body, "not found") != 1 || strings.Count(body, "no chapter") != 1 { + t.Errorf("failure words appear more than once, or on a clean row:\n%s", body) + } +} + // The per-Series page renders every Series-level fact the admin read model // holds for the key the list row already shows: title, the composite key with // Site and kind, the Latest Chapter, the check age and the anonymous Reader @@ -2768,6 +2831,57 @@ func TestAdminSeriesDetailRendersMarks(t *testing.T) { } } +// The detail page adds one sentence beside the Latest Chapter correction +// control only while the number came from a Reader and the failure row has +// stood past the owner window: a Reader-attributed failing Series carries it, +// a machine-read failing Series does not, and neither does a +// Reader-attributed Series whose failure is still inside the window. The +// sentence names no Reader (issue #167). +func TestAdminSeriesDetailUnverifiedSentence(t *testing.T) { + st, dsn := newTestStoreURL(t) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + now := time.Now() + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:reader-fail", url: "u", cover: "c", checkedAt: now.UnixMilli(), latestNum: floatPtr(2), bookmarks: 1, raisedBy: true}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:machine-fail", url: "u", cover: "c", checkedAt: now.UnixMilli(), latestNum: floatPtr(1), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:reader-recent", url: "u", cover: "c", checkedAt: now.UnixMilli(), latestNum: floatPtr(3), bookmarks: 1, raisedBy: true}) + if err := st.RecordSeriesFailure("asura", "reader-fail", "not_found", now.Add(-24*time.Hour).UnixMilli()); err != nil { + t.Fatalf("seed failure row on reader-fail: %v", err) + } + if err := st.RecordSeriesFailure("asura", "machine-fail", "not_found", now.Add(-24*time.Hour).UnixMilli()); err != nil { + t.Fatalf("seed failure row on machine-fail: %v", err) + } + if err := st.RecordSeriesFailure("asura", "reader-recent", "not_found", now.Add(-time.Hour).UnixMilli()); err != nil { + t.Fatalf("seed failure row on reader-recent: %v", err) + } + srv := newRouter(st, testConfig()) + + want := "Unverified Reader number: the page has been failing for over 12h, so this Reader-reported chapter is unconfirmed." + fetch := func(key string) string { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/admin/series/"+key, nil) + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + srv.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("GET /admin/series/%s status = %d, want 200", key, rr.Code) + } + return rr.Body.String() + } + if body := fetch("asura:reader-fail"); !strings.Contains(body, want) { + t.Errorf("a Reader-attributed failing Series lacks the unverified sentence:\n%s", body) + } + if body := fetch("asura:machine-fail"); strings.Contains(body, want) { + t.Errorf("a machine-read failing Series carries the unverified sentence:\n%s", body) + } + if body := fetch("asura:reader-recent"); strings.Contains(body, want) { + t.Errorf("a Reader-attributed Series failing inside the window carries the sentence:\n%s", body) + } +} + // The provenance line beside the chapter names the actor class behind the // value — "machine read" for a checked Series, "correction" for the owner's // stamp, "sighting" for a Reader-raised one — and appears nowhere in the -- 2.52.0 From 28fef689ecee93658a9da82a51a5dab492299d4c Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 02:44:45 +0700 Subject: [PATCH 16/20] #172: three more owner-notice conditions: no-browser-route, sidecar-down, adapter-broken FaultsFrom judges three more faults at the shared twelve-hour OwnerWindow: a no-browser-route Site's refusing run (browser-backed Sites excluded), a sidecar no Lane has reached (one site-wide fault), and a Site where more than half of Series hold an old no-chapter failure row. recordPass fills the inputs from three new store reads (RefusingSince, SidecarOK, NoChapterShare), each failing independently and never failing a pass, and the clear loop forgets the empty-Site row too so a lifted sidecar-down fires again on return. --- backend/internal/latest/faults.go | 117 ++++++- backend/internal/latest/poller.go | 71 +++- backend/internal/latest/poller_test.go | 431 +++++++++++++++++++++++++ backend/internal/store/store.go | 95 ++++++ backend/internal/store/store_test.go | 159 +++++++++ 5 files changed, 853 insertions(+), 20 deletions(-) diff --git a/backend/internal/latest/faults.go b/backend/internal/latest/faults.go index 88c810f..a7e6531 100644 --- a/backend/internal/latest/faults.go +++ b/backend/internal/latest/faults.go @@ -14,6 +14,25 @@ import ( // words (no-browser-route, sidecar-down, adapter-broken); this ticket // declares only the stall. const ConditionStall = "stall" +// ConditionNoBrowserRoute is the owner-notice machine word for a Site whose +// challenge refuses for longer than the owner window with no browser route +// to clear it — the 403-with-interstitial the reader maps to +// errChallengeHeld (read.go), which plain TLS cannot clear. The word is the +// message's footer and its suppression key; it is wire-stable. +const ConditionNoBrowserRoute = "no-browser-route" + +// ConditionSidecarDown is the owner-notice machine word for a browser +// sidecar no Lane has reached for longer than the owner window: every +// browser-backed Lane's latest pass is a sidecar skip. The word is the +// message's footer and its suppression key; it is wire-stable, and its +// suppression row holds the empty Site (AC4). +const ConditionSidecarDown = "sidecar-down" + +// ConditionAdapterBroken is the owner-notice machine word for a Site whose +// adapter stopped finding chapters: more than half of its Series hold an +// old no-chapter failure row. The word is the message's footer and its +// suppression key; it is wire-stable. +const ConditionAdapterBroken = "adapter-broken" // OwnerWindow is the class-level staleness boundary every owner-notice // condition measures against — the same twelve hours the Lanes page's @@ -33,6 +52,22 @@ type Fault struct { // conditions can add inputs without changing either caller. type FaultInput struct { Passes []store.LanePass + // RefusingSince is, per Site, the unix ms when that Site's current + // unbroken run of refusing passes began, or absent when its latest pass + // did not refuse. Its zero value is an empty map, which contributes no + // fault. + RefusingSince map[string]int64 + + // SidecarOK is, per browser-backed Site, the unix ms of that Site's most + // recent pass that actually reached the sidecar. Zero when the pass log + // holds none — an asleep Lane never reached it and never counts as + // evidence either way. Its zero value is an empty map. + SidecarOK map[string]int64 + + // NoChapterShare is, per Site, the share of that Site's Series holding a + // no-chapter failure row older than the owner window. Its zero value is + // an empty map, which contributes no fault. + NoChapterShare map[string]float64 } // FaultsFrom judges the owner-notice conditions from durable rows alone, so @@ -42,12 +77,27 @@ type FaultInput struct { // value and no refusal — exactly the row the mid-loop browser loss writes // (see the comment at the outcomeUnreachable return in runLanePass), so a // Lane that owed Polls, made none, and has nothing to say for it is a fault. -// The no-refusal clause is AC6's amendment: the twice-refused break happens -// inside the pass loop, not on an early return, so a newly-gated Site would -// otherwise send two messages. A pause is excluded by Skip == "" (SkipPaused): -// the owner's own act is not reported back (AC10). The episode began at the -// pass that produced the row; the stall test itself has no age clause — the -// class-level twelve hours belongs to #172's conditions. +// The three #172 conditions share the same OwnerWindow boundary and the same +// fail-open shape: an input's absence contributes no fault, never a false +// one. +// +// - no-browser-route: a Site whose refusing run began before the window +// and that has no browser route to clear the challenge (AC2). Both +// refusal shapes count — the gate's skip='refusing' rows and the loop's +// refused>0 rows (the twice-refused break writes skip="") — so the run +// stays unbroken across them, and one healthy pass ends it. +// - sidecar-down: every browser-backed Site's latest pass is a sidecar +// skip — SkipSidecarDown or SkipNoFetcher, the two early returns that +// record a skip value — and each Site's most recent sidecar-reaching +// pass is older than the window (AC4). The skip clause is what keeps +// SkipAsleep out: a Lane under both wake thresholds is the commonest +// healthy state, never reached the sidecar, and would otherwise age into +// a false alarm. Emitted once, with Site "" (AC4's one row per episode). +// - adapter-broken: more than half of one Site's Series hold a no-chapter +// failure row older than the window (AC6). Strictly above half: the +// filter is the tool, and one Site change makes hundreds of rows, so a +// single failing Series never fires (AC7). The episode's age is the +// window — the old rows prove the episode is at least that old. func FaultsFrom(in FaultInput, now time.Time) []Fault { var faults []Fault for _, p := range in.Passes { @@ -55,9 +105,50 @@ func FaultsFrom(in FaultInput, now time.Time) []Fault { faults = append(faults, Fault{Condition: ConditionStall, Site: p.Site, Since: p.RanAt}) } } + cutoff := now.Add(-OwnerWindow).UnixMilli() + for site, since := range in.RefusingSince { + if since < cutoff && !isBrowserSite(site) { + faults = append(faults, Fault{Condition: ConditionNoBrowserRoute, Site: site, Since: since}) + } + } + if since, down := sidecarDownSince(in.Passes, in.SidecarOK, cutoff); down { + faults = append(faults, Fault{Condition: ConditionSidecarDown, Site: "", Since: since}) + } + for site, share := range in.NoChapterShare { + if share > 0.5 { + faults = append(faults, Fault{Condition: ConditionAdapterBroken, Site: site, Since: now.Add(-OwnerWindow).UnixMilli()}) + } + } return faults } +// sidecarDownSince reports whether no browser Lane has reached the sidecar +// for longer than the owner window and, when it has, the last moment any +// Lane reached it. The skip clause — every browser-backed Site's latest pass +// must be SkipSidecarDown or SkipNoFetcher — keeps SkipAsleep out (see +// FaultsFrom). A Site with no pass row at all is not judged down either: a +// fresh database is not a dead sidecar. +func sidecarDownSince(passes []store.LanePass, ok map[string]int64, cutoff int64) (since int64, down bool) { + latest := make(map[string]store.LanePass, len(passes)) + for _, p := range passes { + latest[p.Site] = p + } + for _, site := range browserBackedSites() { + p, found := latest[site] + if !found || (p.Skip != SkipSidecarDown && p.Skip != SkipNoFetcher) { + return 0, false + } + reached := ok[site] + if reached > 0 && reached >= cutoff { + return 0, false + } + if reached > since { + since = reached + } + } + return since, true +} + // Notifier delivers one owner notice. The poller neither retries nor queues: // an error is logged and the suppression row left unwritten, so the next pass // tries again while the condition holds. @@ -69,7 +160,7 @@ type Notifier interface { // clear loop in recordPass: a condition absent from a pass's fault list // forgets its episode, so the next occurrence sends again. #172 extends the // list when it adds its conditions. -var ownerNoticeConditions = []string{ConditionStall} +var ownerNoticeConditions = []string{ConditionStall, ConditionNoBrowserRoute, ConditionSidecarDown, ConditionAdapterBroken} // noticeFor renders one fault's message: the description sentence — condition, // age, repair — and the deep link the embed's title points at. Each condition @@ -80,6 +171,18 @@ func noticeFor(f Fault, row store.LanePass, now time.Time) (sentence, href strin return fmt.Sprintf( "%s owed %d Polls and made none — %s; check the Lane's browser sidecar and the Site's challenge state", f.Site, row.Due, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes" + case ConditionNoBrowserRoute: + return fmt.Sprintf( + "%s has refused for %s with no browser route — the challenge does not clear on plain TLS; redeploy or add a browser route", + f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes" + case ConditionSidecarDown: + return fmt.Sprintf( + "no browser Lane has reached the sidecar for %s — the browser sidecar is down; start or repair the browser machine", + humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes" + case ConditionAdapterBroken: + return fmt.Sprintf( + "more than half of %s's Series have failed no-chapter reads for at least %s — the Site's layout changed and the adapter is broken", + f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes" } return "", "" } diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index f43d4b3..d6da47a 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -568,18 +568,50 @@ func (p *Poller) recordPass(ctx context.Context, rec passRecord, fig passFigures // ownerNotices judges the owner-notice conditions for the pass just recorded // and fires (issue #171). It sits in recordPass because that deferred call is -// the one place every return path passes through: two of the four conditions -// occur on early returns and the success path can never see them. Per fault: -// NoticeSent → send → MarkNoticeSent, so a fault lasting a month sends one -// message, not one per pass; a condition absent from this pass's fault list -// forgets its episode, so the next occurrence sends again. Everything here is -// best-effort: a failed send, a failed store read and a failed notice write -// are all logged and never change the pass's outcome counts or its return -// value. The clear runs even when Notify is nil, so a deployment that turns -// the webhook off does not leave stale rows that suppress the first real -// notice after it is turned back on. +// the one place every return path passes through: three of the four +// conditions occur on early returns and the success path can never see them. +// The judgement reads the whole pass log plus three derived reads — the +// other Lanes' latest passes, each Site's refusing-run start, its last +// sidecar-reaching pass, and its no-chapter share — so one pass judges every +// condition (issue #172). Per fault: NoticeSent → send → MarkNoticeSent, so +// a fault lasting a month sends one message, not one per pass; a condition +// absent from this pass's fault list forgets its episode, so the next +// occurrence sends again. Everything here is best-effort: a failed send, a +// failed store read and a failed notice write are all logged and never +// change the pass's outcome counts or its return value. The clear runs even +// when Notify is nil, so a deployment that turns the webhook off does not +// leave stale rows that suppress the first real notice after it is turned +// back on. func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) { - faults := FaultsFrom(FaultInput{Passes: []store.LanePass{row}}, p.Now()) + now := p.Now() + in := FaultInput{Passes: []store.LanePass{row}} + // Each read fails independently: a failure logs and contributes no fault, + // never a false one. + if passes, err := p.Store.LatestLanePasses(); err != nil { + log.Printf("latest poll %s: latest lane passes: %v", row.Site, err) + } else { + in.Passes = passes + } + if since, err := p.Store.RefusingSince(now.UnixMilli()); err != nil { + log.Printf("latest poll %s: refusing since: %v", row.Site, err) + } else { + in.RefusingSince = since + } + if ok, err := p.Store.SidecarOK(browserBackedSites()); err != nil { + log.Printf("latest poll %s: sidecar ok: %v", row.Site, err) + } else { + in.SidecarOK = ok + } + if share, err := p.Store.NoChapterShare(now.Add(-OwnerWindow).UnixMilli()); err != nil { + log.Printf("latest poll %s: no-chapter share: %v", row.Site, err) + } else { + in.NoChapterShare = share + } + faults := FaultsFrom(in, now) + bySite := make(map[string]store.LanePass, len(in.Passes)) + for _, pass := range in.Passes { + bySite[pass.Site] = pass + } for _, f := range faults { if p.Notify == nil { continue @@ -592,14 +624,20 @@ func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) { if sent { continue } - sentence, href := noticeFor(f, row, p.Now()) + // The fault's own Site's pass renders its sentence — a stall judged + // from another Lane's pass must not quote this pass's figures. + pass, ok := bySite[f.Site] + if !ok { + pass = row + } + sentence, href := noticeFor(f, pass, now) if err := p.Notify.Notify(ctx, f, sentence, href); err != nil { // The stamp stays unset: no queue, no backoff — the condition is // durable, so the next pass tries again while it holds. log.Printf("latest poll %s: owner notice %s: %v", row.Site, f.Condition, err) continue } - if err := p.Store.MarkNoticeSent(f.Condition, f.Site, p.Now().UnixMilli()); err != nil { + if err := p.Store.MarkNoticeSent(f.Condition, f.Site, now.UnixMilli()); err != nil { log.Printf("latest poll %s: mark notice sent: %v", row.Site, err) } } @@ -609,6 +647,13 @@ func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) { log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) } } + // The site-wide conditions suppress under the empty Site; clear that + // row too, so a lifted sidecar-down fires again when it returns. + if !hasFault(faults, cond, "") { + if err := p.Store.ClearNotice(cond, ""); err != nil { + log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) + } + } } } diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index f7aff16..db8bf86 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -3181,6 +3181,437 @@ func TestFaultsFromStall(t *testing.T) { } } +// The #172 conditions are judged from the durable inputs alone, like the +// stall. A refusal is only a fault when the Site has no browser route to +// clear it (AC2's "browser Site" exclusions), and only once it is older than +// the owner window. +func TestFaultsFromNoBrowserRoute(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + old := now.Add(-2 * OwnerWindow).UnixMilli() // a two-day refusal + fresh := now.Add(-OwnerWindow / 2).UnixMilli() // inside the window + tests := []struct { + name string + in FaultInput + want int + }{ + { + name: "plain-TLS Sites refusing for two days are faults", + in: FaultInput{RefusingSince: map[string]int64{"asura": old, "demonic": now.Add(-3 * OwnerWindow).UnixMilli()}}, + want: 2, + }, + { + name: "a browser-backed Site's refusal is a route it has, not a fault", + in: FaultInput{RefusingSince: map[string]int64{"comix": old, "kagane": old, "novelfull": old}}, + want: 0, + }, + { + name: "a fresh refusal is not old enough", + in: FaultInput{RefusingSince: map[string]int64{"asura": fresh}}, + want: 0, + }, + { + name: "no refusal is no fault", + in: FaultInput{}, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + faults := FaultsFrom(tt.in, now) + if got := len(faults); got != tt.want { + t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want) + } + for _, f := range faults { + if f.Condition != ConditionNoBrowserRoute || f.Site == "" || f.Since != tt.in.RefusingSince[f.Site] { + t.Fatalf("fault = %+v, want no-browser-route on the refusing Site since its run began", f) + } + } + }) + } +} + +// sidecar-down is one site-wide fault: every browser Lane's latest pass must +// be a sidecar skip (SkipSidecarDown or SkipNoFetcher — the skip clause keeps +// an asleep Lane out) and each Lane's most recent sidecar-reaching pass must +// be older than the window. The fault's Since is the last moment any Lane +// reached the sidecar. +func TestFaultsFromSidecarDown(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + oldReach := now.Add(-2 * OwnerWindow).UnixMilli() + skip := func(site string) store.LanePass { + return store.LanePass{Site: site, RanAt: oldReach, Skip: SkipSidecarDown} + } + allSkipped := []store.LanePass{skip("comix"), skip("kagane"), skip("novelfull")} + reached := now.Add(-OwnerWindow / 2).UnixMilli() // inside the window + tests := []struct { + name string + in FaultInput + want int + wantSince int64 + }{ + { + name: "every browser Lane sidecar-skipped past the window is one fault", + in: FaultInput{Passes: allSkipped, SidecarOK: map[string]int64{"comix": oldReach}}, + want: 1, + wantSince: oldReach, + }, + { + name: "a browser Lane asleep for two days sends nothing", + in: FaultInput{ + Passes: []store.LanePass{ + skip("comix"), + {Site: "kagane", RanAt: oldReach, Skip: SkipAsleep}, + skip("novelfull"), + }, + }, + want: 0, + }, + { + name: "a Lane that reached the sidecar inside the window is not down", + in: FaultInput{ + Passes: allSkipped, + SidecarOK: map[string]int64{"comix": reached, "kagane": reached, "novelfull": reached}, + }, + want: 0, + }, + { + name: "a browser Site with no pass row is not judged down", + in: FaultInput{ + Passes: []store.LanePass{skip("comix"), skip("kagane")}, + }, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + faults := FaultsFrom(tt.in, now) + if got := len(faults); got != tt.want { + t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want) + } + for _, f := range faults { + if f.Condition != ConditionSidecarDown || f.Site != "" || f.Since != tt.wantSince { + t.Fatalf("fault = %+v, want one site-wide sidecar-down since %d", f, tt.wantSince) + } + } + }) + } +} + +// adapter-broken fires strictly above half: a float share judges a +// four-Series Site and a 200-Series Site on the same scale, exactly half +// stays quiet, and a single failing Series never reaches it. +func TestFaultsFromAdapterBroken(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + tests := []struct { + name string + in FaultInput + want int + }{ + { + name: "three of four Series failing is a fault", + in: FaultInput{NoChapterShare: map[string]float64{"asura": 3.0 / 4.0}}, + want: 1, + }, + { + name: "exactly half is not a fault", + in: FaultInput{NoChapterShare: map[string]float64{"asura": 1.0 / 2.0}}, + want: 0, + }, + { + name: "one of two hundred is not a fault", + in: FaultInput{NoChapterShare: map[string]float64{"asura": 1.0 / 200.0}}, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + faults := FaultsFrom(tt.in, now) + if got := len(faults); got != tt.want { + t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want) + } + for _, f := range faults { + if f.Condition != ConditionAdapterBroken || f.Site != "asura" || f.Since != now.Add(-OwnerWindow).UnixMilli() { + t.Fatalf("fault = %+v, want adapter-broken on asura since the window", f) + } + } + }) + } +} +// seedRefusingRun writes a refusing pass and the gate row that follows it, +// so a Site's run of refusing passes begins at start. +func seedRefusingRun(t *testing.T, s *store.Store, site string, start time.Time) { + t.Helper() + for i, row := range []store.LanePass{ + {Site: site, RanAt: start.UnixMilli(), Skip: "", Refused: 2}, + {Site: site, RanAt: start.Add(time.Minute).UnixMilli(), Skip: SkipRefusing}, + } { + if err := s.RecordLanePass(row, -1); err != nil { + t.Fatalf("seed refusing run %d: %v", i, err) + } + } +} + +// seedSidecarSkips writes one sidecar-skipped pass for every browser-backed +// Lane, all at the same time, so the pass log shows a sidecar that has been +// unreachable since then. +func seedSidecarSkips(t *testing.T, s *store.Store, at time.Time) { + t.Helper() + for _, site := range browserBackedSites() { + if err := s.RecordLanePass(store.LanePass{Site: site, RanAt: at.UnixMilli(), Skip: SkipSidecarDown}, -1); err != nil { + t.Fatalf("seed sidecar skip %s: %v", site, err) + } + } +} + +// seedNoChapter writes an old no-chapter failure row for each Series id. +func seedNoChapter(t *testing.T, s *store.Store, site string, ids []string, failingSince int64) { + t.Helper() + for _, id := range ids { + if err := s.RecordSeriesFailure(site, id, "no_chapter", failingSince); err != nil { + t.Fatalf("seed no-chapter failure %s:%s: %v", site, id, err) + } + } +} + +// no-browser-route fires once while the refusal holds, and again after it +// lifts and returns: the suppression row is the whole state, the gate row of +// a second refusing pass is the same episode, a healthy pass in between +// breaks the run and clears the row, and a later run that has aged past the +// window sends again (AC1, AC9). +func TestOwnerNoticeNoBrowserRouteFiresOncePerEpisode(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + s, _ := newTestStore(t) + seedForCheck(t, s, "asura:r1", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + seedForCheck(t, s, "asura:r2", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + refusing := &fakeFetcher{status: 403} + notifier := &fakeNotifier{} + p := newTestPoller(t, s, refusing, now) + p.Now = func() time.Time { return now } + p.Notify = notifier + + // A run that began before the window: seed two-day-old refusing rows, + // then a fresh pass that refuses again — the run is unbroken and still + // old, so the owner is told once. + seedRefusingRun(t, s, "asura", now.Add(-2*OwnerWindow)) + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after first refusing pass = %d, want 1", got) + } + if f := notifier.fault(0); f.Condition != ConditionNoBrowserRoute || f.Site != "asura" || f.Since != now.Add(-2*OwnerWindow).UnixMilli() { + t.Fatalf("fault = %+v, want no-browser-route on asura since the run began", f) + } + if sent, err := s.NoticeSent(ConditionNoBrowserRoute, "asura"); err != nil || !sent { + t.Fatalf("NoticeSent after first refusing pass = %v, %v; want true, nil", sent, err) + } + + // A second refusing pass — the refusal gate now returns early — is the + // same episode: the gate row continues the run, no second message. + now = now.Add(time.Minute) + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after gated refusing pass = %d, want 1 (one per episode)", got) + } + + // A healthy pass in between breaks the run: the condition lifts and the + // suppression row is cleared. + now = now.Add(RefuseBackoff + time.Minute) + seedForCheck(t, s, "asura:r1", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + seedForCheck(t, s, "asura:r2", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + p.Fetch = &fakeFetcher{body: asuraSeriesFixture, status: 200} + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after healthy pass = %d, want 1 (a healthy pass sends nothing)", got) + } + if sent, err := s.NoticeSent(ConditionNoBrowserRoute, "asura"); err != nil || sent { + t.Fatalf("NoticeSent after healthy pass = %v, %v; want false, nil (row cleared)", sent, err) + } + + // A later run that has aged past the window again is a new episode. + runStart := now.Add(2 * OwnerWindow) + seedRefusingRun(t, s, "asura", runStart) + now = runStart.Add(2 * OwnerWindow) + seedForCheck(t, s, "asura:r1", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + seedForCheck(t, s, "asura:r2", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0) + p.Fetch = refusing + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 2 { + t.Fatalf("notices after the returned refusal = %d, want 2 (a new episode)", got) + } + if f := notifier.fault(1); f.Condition != ConditionNoBrowserRoute || f.Site != "asura" || f.Since != runStart.UnixMilli() { + t.Fatalf("fault = %+v, want no-browser-route on asura since the new run began", f) + } +} + +// A browser-backed Site's refusal sends nothing (AC2): it has a route, so a +// two-day refusal is a browser-side problem, not the no-browser-route fault. +func TestOwnerNoticeBrowserRefusalSendsNothing(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + s, _ := newTestStore(t) + for i := 1; i <= 6; i++ { + seedForCheck(t, s, fmt.Sprintf("comix:b%d", i), fmt.Sprintf("https://comix.to/title/b%d", i), 0) + } + notifier := &fakeNotifier{} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.BrowserFetch = &fakeFetcher{status: 403} + p.Notify = notifier + + seedRefusingRun(t, s, "comix", now.Add(-2*OwnerWindow)) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 0 { + t.Fatalf("notices = %d, want 0 (a browser Site's refusal is not a no-browser-route)", got) + } + if sent, err := s.NoticeSent(ConditionNoBrowserRoute, "comix"); err != nil || sent { + t.Fatalf("NoticeSent = %v, %v; want false, nil", sent, err) + } +} + +// sidecar-down fires once while no Lane reaches the sidecar, and again after +// it returns and dies again (AC4, AC9): the suppression row holds the empty +// Site, the first Lane to notice writes it and the others stay quiet, a +// healthy browser pass clears it, and a later outage is a new episode. +func TestOwnerNoticeSidecarDownFiresOncePerEpisode(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + s, _ := newTestStore(t) + notifier := &fakeNotifier{} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.Notify = notifier + + // Seed the pass log so every browser Lane's latest pass is a sidecar + // skip older than the window, then a fresh pass that skips too. + seedSidecarSkips(t, s, now.Add(-2*OwnerWindow)) + p.setBrowserDown(now) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after first sidecar-skipped pass = %d, want 1", got) + } + if f := notifier.fault(0); f.Condition != ConditionSidecarDown || f.Site != "" { + t.Fatalf("fault = %+v, want one site-wide sidecar-down", f) + } + if sent, err := s.NoticeSent(ConditionSidecarDown, ""); err != nil || !sent { + t.Fatalf("NoticeSent after first pass = %v, %v; want true, nil", sent, err) + } + + // Another Lane's sidecar-skipped pass is the same episode: still one + // message. + p.runLanePass(context.Background(), "kagane", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after another Lane's skipped pass = %d, want 1 (one per episode)", got) + } + + // A healthy browser pass reaches the sidecar: the condition lifts and the + // suppression row is cleared. Five due Series wake the browser, and the + // series ids carry the fixture's id prefix so the scoped reader finds its + // chapters. + now = now.Add(RefuseBackoff + time.Minute) + for i := 1; i <= 5; i++ { + seedForCheck(t, s, fmt.Sprintf("comix:h%d", i), "https://comix.to/title/n8we-dungeons-and-crayons", 0) + } + p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after healthy pass = %d, want 1 (a healthy pass sends nothing)", got) + } + if sent, err := s.NoticeSent(ConditionSidecarDown, ""); err != nil || sent { + t.Fatalf("NoticeSent after healthy pass = %v, %v; want false, nil (row cleared)", sent, err) + } + + // The sidecar dies again: a new episode, told again. + now = now.Add(2 * OwnerWindow) + p.setBrowserDown(now) + seedSidecarSkips(t, s, now.Add(-time.Minute)) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 2 { + t.Fatalf("notices after the returned outage = %d, want 2 (a new episode)", got) + } +} + +// A browser Lane asleep under both wake thresholds sends nothing: it never +// reached the sidecar, but its skip is not a sidecar skip, so it cannot age +// into a false alarm (AC9). +func TestOwnerNoticeSidecarDownAsleepSendsNothing(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + s, _ := newTestStore(t) + notifier := &fakeNotifier{} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.Notify = notifier + + seedSidecarSkips(t, s, now.Add(-2*OwnerWindow)) + if err := s.RecordLanePass(store.LanePass{Site: "kagane", RanAt: now.Add(-2*OwnerWindow + time.Minute).UnixMilli(), Skip: SkipAsleep}, -1); err != nil { + t.Fatalf("seed asleep pass: %v", err) + } + p.setBrowserDown(now) + p.runLanePass(context.Background(), "comix", false) + if got := notifier.callCount(); got != 0 { + t.Fatalf("notices = %d, want 0 (an asleep Lane is not a down sidecar)", got) + } + if sent, err := s.NoticeSent(ConditionSidecarDown, ""); err != nil || sent { + t.Fatalf("NoticeSent = %v, %v; want false, nil", sent, err) + } +} + +// adapter-broken fires once while more than half of a Site's Series hold an +// old no-chapter failure row, and again after the failures clear and return: +// the suppression row is the whole state, a pass that clears the failures +// forgets the episode, and a later return sends again. The share is judged +// from durable rows, so the pass itself may be healthy (AC6, AC9). +func TestOwnerNoticeAdapterBrokenFiresOncePerEpisode(t *testing.T) { + now := time.UnixMilli(5_000_000_000) + s, _ := newTestStore(t) + for i := 1; i <= 4; i++ { + seedForCheck(t, s, fmt.Sprintf("asura:a%d", i), fmt.Sprintf("https://asurascans.com/comics/a%d", i), now.UnixMilli()) + } + notifier := &fakeNotifier{} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.Now = func() time.Time { return now } + p.Notify = notifier + + // Three of four Series hold an old no-chapter row: the first pass fires + // one adapter-broken fault. + seedNoChapter(t, s, "asura", []string{"a1", "a2", "a3"}, now.Add(-2*OwnerWindow).UnixMilli()) + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after first pass = %d, want 1", got) + } + if f := notifier.fault(0); f.Condition != ConditionAdapterBroken || f.Site != "asura" || f.Since != now.Add(-OwnerWindow).UnixMilli() { + t.Fatalf("fault = %+v, want adapter-broken on asura since the window", f) + } + if sent, err := s.NoticeSent(ConditionAdapterBroken, "asura"); err != nil || !sent { + t.Fatalf("NoticeSent after first pass = %v, %v; want true, nil", sent, err) + } + + // A second pass is the same episode: still one message. + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after second pass = %d, want 1 (one per episode)", got) + } + + // The failures clear: the condition lifts and the suppression row is + // cleared. + for _, id := range []string{"a1", "a2", "a3"} { + if err := s.ClearSeriesFailure("asura", id); err != nil { + t.Fatalf("clear failure %s: %v", id, err) + } + } + now = now.Add(RefuseBackoff + time.Minute) + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 1 { + t.Fatalf("notices after cleared failures = %d, want 1 (a healthy share sends nothing)", got) + } + if sent, err := s.NoticeSent(ConditionAdapterBroken, "asura"); err != nil || sent { + t.Fatalf("NoticeSent after cleared failures = %v, %v; want false, nil (row cleared)", sent, err) + } + + // The failures return: a new episode, told again. + seedNoChapter(t, s, "asura", []string{"a1", "a2", "a3"}, now.Add(-2*OwnerWindow).UnixMilli()) + now = now.Add(time.Minute) + p.runLanePass(context.Background(), "asura", false) + if got := notifier.callCount(); got != 2 { + t.Fatalf("notices after the returned failures = %d, want 2 (a new episode)", got) + } +} // The stall fires exactly once while it holds, and again after it lifts and // returns: the suppression row is the whole state, so the second stall-shaped // pass is the same episode, a healthy pass in between clears the row, and the diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index 2f579a3..ed32a60 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1227,6 +1227,101 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) { return out, rows.Err() } +// RefusingSince reports, per Site, when that Site's current unbroken run of +// refusing passes began: a pass refuses when the Lane gate returned early +// (skip = 'refusing') or the pass loop counted refusals (refused > 0), so a +// Site parked in refusal backoff keeps its run unbroken. A Site whose +// latest pass did not refuse is absent. The run is bounded by the pass +// log's retention — a run older than the log answers with the oldest row +// present — and nothing after now counts: the cutoff is the caller's clock. +func (s *Store) RefusingSince(now int64) (map[string]int64, error) { + rows, err := s.db.Query(` + SELECT site, MIN(ran_at) + FROM poll_passes p + WHERE ran_at <= $1 + AND (refused > 0 OR skip = 'refusing') + AND ran_at > COALESCE(( + SELECT MAX(q.ran_at) FROM poll_passes q + WHERE q.site = p.site AND q.ran_at <= $1 + AND NOT (q.refused > 0 OR q.skip = 'refusing') + ), 0) + GROUP BY site`, now) + if err != nil { + return nil, fmt.Errorf("query refusing since: %w", err) + } + defer rows.Close() + + out := map[string]int64{} + for rows.Next() { + var site string + var since int64 + if err := rows.Scan(&site, &since); err != nil { + return nil, fmt.Errorf("scan refusing since: %w", err) + } + out[site] = since + } + return out, rows.Err() +} + +// SidecarOK reports, per Site in sites, the unix ms of that Site's most +// recent pass that actually reached the sidecar: the pass ran its loop +// (skip = '') and no Series read lost Chrome (unreachable = 0) — a +// challenge answer still reached the sidecar, a lost sidecar did not. A +// Site with no such pass is absent: an asleep Lane never reached it, so it +// is absent too and cannot age into a sidecar-down alarm by itself. +func (s *Store) SidecarOK(sites []string) (map[string]int64, error) { + rows, err := s.db.Query(` + SELECT DISTINCT ON (site) site, ran_at + FROM poll_passes + WHERE site = ANY($1) AND skip = '' AND unreachable = 0 + ORDER BY site, ran_at DESC`, sites) + if err != nil { + return nil, fmt.Errorf("query sidecar ok: %w", err) + } + defer rows.Close() + + out := map[string]int64{} + for rows.Next() { + var site string + var ranAt int64 + if err := rows.Scan(&site, &ranAt); err != nil { + return nil, fmt.Errorf("scan sidecar ok: %w", err) + } + out[site] = ranAt + } + return out, rows.Err() +} + +// NoChapterShare reports, per Site, the share of that Site's Series holding +// a no-chapter failure row older than cutoff: old rows over the Site's +// whole Series count, so a four-Series Site and a 200-Series Site are +// judged on the same scale. A Site with no Series has no share and is +// absent. +func (s *Store) NoChapterShare(cutoff int64) (map[string]float64, error) { + rows, err := s.db.Query(` + SELECT s.site, + (COUNT(*) FILTER (WHERE f.outcome = 'no_chapter' AND f.failing_since < $1))::float8 + / (COUNT(*)::float8) + FROM series s + LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id + GROUP BY s.site + ORDER BY s.site`, cutoff) + if err != nil { + return nil, fmt.Errorf("query no-chapter share: %w", err) + } + defer rows.Close() + + out := map[string]float64{} + for rows.Next() { + var site string + var share float64 + if err := rows.Scan(&site, &share); err != nil { + return nil, fmt.Errorf("scan no-chapter share: %w", err) + } + out[site] = share + } + return out, rows.Err() +} // SetLaneRefusal persists a Site's refusal backoff stamp without touching its // pause. until is supplied by the caller's clock. func (s *Store) SetLaneRefusal(site string, until int64) error { diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index b70093d..5b9a3b8 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "io/fs" + "fmt" "os" "path/filepath" "strconv" @@ -2738,3 +2739,161 @@ func TestOwnerNoticeRows(t *testing.T) { t.Fatalf("ClearNotice on a missing row: %v", err) } } +// RefusingSince reads one Site's current unbroken run of refusing passes +// (issue #172). A refusing pass is one the Lane gate returned early from +// (skip 'refusing') or one whose loop counted refusals (refused > 0) — the +// two shapes a persistently-challenged Site alternates between, so the run +// must not break when the gate row follows the refusal row. A Site whose +// latest pass did not refuse is absent, nothing after the caller's clock +// counts, and a run older than the log answers with the oldest row present. +func TestRefusingSince(t *testing.T) { + s := newTestStore(t) + seed := func(site string, ranAt int64, skip string, refused int) { + t.Helper() + if err := s.RecordLanePass(LanePass{Site: site, RanAt: ranAt, Skip: skip, Refused: refused}, -1); err != nil { + t.Fatalf("RecordLanePass(%s/%d): %v", site, ranAt, err) + } + } + // asura: a refusing run broken by a healthy pass, then resumed; the + // current run began at the pass after the break. + seed("asura", 100, "", 2) + seed("asura", 200, "", 0) + seed("asura", 300, "refusing", 0) + seed("asura", 400, "", 2) + // demonic: the latest pass is healthy — no run, absent. + seed("demonic", 100, "", 2) + seed("demonic", 200, "", 0) + // novelfull: a healthy pass after now must not break the run — the run + // is judged as of the caller's clock. + seed("novelfull", 100, "", 2) + seed("novelfull", 600, "", 0) + // comix: an unbroken run reaches back to the oldest row present. + seed("comix", 100, "", 2) + seed("comix", 200, "refusing", 0) + + got, err := s.RefusingSince(450) + if err != nil { + t.Fatalf("RefusingSince: %v", err) + } + want := map[string]int64{"asura": 300, "novelfull": 100, "comix": 100} + if len(got) != len(want) { + t.Fatalf("RefusingSince = %v, want %v", got, want) + } + for site, since := range want { + if got[site] != since { + t.Fatalf("RefusingSince[%s] = %d, want %d (got %v)", site, got[site], since, got) + } + } + if _, ok := got["demonic"]; ok { + t.Fatalf("RefusingSince names demonic, whose latest pass did not refuse: %v", got) + } +} + +// SidecarOK reads, per Site, the most recent pass that actually reached the +// sidecar (issue #172): the pass ran its loop (skip '') and no read lost +// Chrome (unreachable 0). A challenge answer still reached the sidecar, a +// lost sidecar and every skip value did not, and a Site with no qualifying +// pass — an asleep Lane included — is absent. +func TestSidecarOK(t *testing.T) { + s := newTestStore(t) + seed := func(site string, ranAt int64, skip string, unreachable int) { + t.Helper() + if err := s.RecordLanePass(LanePass{Site: site, RanAt: ranAt, Skip: skip, Unreachable: unreachable}, -1); err != nil { + t.Fatalf("RecordLanePass(%s/%d): %v", site, ranAt, err) + } + } + // comix: only the skip='' unreachable=0 pass reached the sidecar; the + // mid-loop browser-loss row (skip '', unreachable > 0) and the skip + // values never did. + seed("comix", 100, "", 1) + seed("comix", 200, "sidecar-down", 0) + seed("comix", 300, "", 0) + seed("comix", 400, "refusing", 0) + // kagane: two passes reached the sidecar; the newest wins. + seed("kagane", 150, "", 0) + seed("kagane", 250, "", 0) + // novelfull: an asleep Lane never reached it. + seed("novelfull", 120, "asleep", 0) + + got, err := s.SidecarOK([]string{"comix", "kagane", "novelfull", "lightnovelworld"}) + if err != nil { + t.Fatalf("SidecarOK: %v", err) + } + want := map[string]int64{"comix": 300, "kagane": 250} + if len(got) != len(want) { + t.Fatalf("SidecarOK = %v, want %v", got, want) + } + for site, ranAt := range want { + if got[site] != ranAt { + t.Fatalf("SidecarOK[%s] = %d, want %d (got %v)", site, got[site], ranAt, got) + } + } + for _, site := range []string{"novelfull", "lightnovelworld"} { + if _, ok := got[site]; ok { + t.Fatalf("SidecarOK names %s, which never reached the sidecar: %v", site, got) + } + } +} + +// NoChapterShare reads, per Site, the share of its Series holding a +// no-chapter failure row older than the cutoff (issue #172): old rows over +// the Site's whole Series count, so a four-Series Site and a 200-Series +// Site are judged on the same scale. A fresh no-chapter row and a row of +// any other outcome do not count. +func TestNoChapterShare(t *testing.T) { + s := newTestStore(t) + for i := 1; i <= 4; i++ { + seedForCheck(t, s, fmt.Sprintf("asura:a%d", i), fmt.Sprintf("https://asurascans.com/comics/a%d", i), 0) + } + seedForCheck(t, s, "comix:c1", "https://comix.to/title/c1", 0) + seedForCheck(t, s, "comix:c2", "https://comix.to/title/c2", 0) + seedForCheck(t, s, "demonic:d1", "https://demonicscans.org/series/d1", 0) + for i := 1; i <= 200; i++ { + seedForCheck(t, s, fmt.Sprintf("novelfull:n%d", i), fmt.Sprintf("https://novelfull.com/n%d.html", i), 0) + } + + const cutoff = 1000 + oldNoChapter := func(site, seriesID string) { + t.Helper() + if err := s.RecordSeriesFailure(site, seriesID, "no_chapter", 100); err != nil { + t.Fatalf("seed no-chapter failure %s:%s: %v", site, seriesID, err) + } + } + // asura: three of four hold an old no-chapter row; the fourth's + // no-chapter row is fresh — it is not old, so it does not count. + oldNoChapter("asura", "a1") + oldNoChapter("asura", "a2") + oldNoChapter("asura", "a3") + if err := s.RecordSeriesFailure("asura", "a4", "no_chapter", 2000); err != nil { + t.Fatalf("seed fresh no-chapter failure: %v", err) + } + // comix: an old no-chapter row and an old errors row — the errors row + // is not a no-chapter row, so the share is 1/2, the exact boundary. + oldNoChapter("comix", "c1") + if err := s.RecordSeriesFailure("comix", "c2", "errors", 100); err != nil { + t.Fatalf("seed errors failure: %v", err) + } + // demonic and novelfull: one old no-chapter row each, against sites of + // one and two hundred Series. + oldNoChapter("demonic", "d1") + oldNoChapter("novelfull", "n1") + + got, err := s.NoChapterShare(cutoff) + if err != nil { + t.Fatalf("NoChapterShare: %v", err) + } + checks := []struct { + site string + want float64 + }{ + {"asura", 3.0 / 4.0}, + {"comix", 1.0 / 2.0}, + {"demonic", 1.0 / 1.0}, + {"novelfull", 1.0 / 200.0}, + } + for _, c := range checks { + if got[c.site] != c.want { + t.Fatalf("NoChapterShare[%s] = %v, want %v", c.site, got[c.site], c.want) + } + } +} -- 2.52.0 From b58945894dc462c1448004dc8ecbf40076957a23 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 02:47:21 +0700 Subject: [PATCH 17/20] #172: floor sidecar-down's age at the window when no Lane ever reached the sidecar A fresh database whose browser Lanes skipped from day one has no sidecar-reaching pass, so sidecarDownSince returned Since 0 and the notice rendered the age since the epoch. Report the window itself: 'at least twelve hours' is the honest floor the condition guarantees. --- backend/internal/latest/faults.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/internal/latest/faults.go b/backend/internal/latest/faults.go index a7e6531..2cd0852 100644 --- a/backend/internal/latest/faults.go +++ b/backend/internal/latest/faults.go @@ -146,6 +146,12 @@ func sidecarDownSince(passes []store.LanePass, ok map[string]int64, cutoff int64 since = reached } } + // No Lane's retained pass log shows a sidecar reach: the honest age is + // the window itself — "at least twelve hours" — not the epoch, which + // humanAge would render as tens of thousands of days. + if since == 0 { + since = cutoff + } return since, true } -- 2.52.0 From c655586202b86e50fc0b92504a0ee7ec8f89ea20 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 02:48:14 +0700 Subject: [PATCH 18/20] #172: clear the empty-Site suppression row only for sidecar-down Only sidecar-down ever writes a Site=='' row, so the loop's empty-Site clear for the other three conditions was a no-op delete on every pass. Move it out of the per-condition loop, scoped to ConditionSidecarDown. --- backend/internal/latest/poller.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index d6da47a..088ae49 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -647,12 +647,13 @@ func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) { log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) } } - // The site-wide conditions suppress under the empty Site; clear that - // row too, so a lifted sidecar-down fires again when it returns. - if !hasFault(faults, cond, "") { - if err := p.Store.ClearNotice(cond, ""); err != nil { - log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) - } + } + // sidecar-down suppresses under the empty Site — one row across all + // browser Lanes (AC4) — so clear that row when it is absent from this + // pass's fault list, and a lifted sidecar fires again when it returns. + if !hasFault(faults, ConditionSidecarDown, "") { + if err := p.Store.ClearNotice(ConditionSidecarDown, ""); err != nil { + log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) } } } -- 2.52.0 From 820f57a7faec197059ee42351b872e6af1763055 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 11:24:09 +0700 Subject: [PATCH 19/20] #173: page verdict shares FaultsFrom judgement so push and landing cannot disagree --- backend/internal/latest/sites.go | 6 ++ backend/internal/web/admin_overview.go | 56 +++++++++++----- backend/web_test.go | 92 +++++++++++++++++++++++++- 3 files changed, 137 insertions(+), 17 deletions(-) diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index 5289346..7628a8d 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -623,6 +623,12 @@ func SiteNames() []string { return names } +// BrowserBackedSites is derived from the registry: the Sites whose pages are +// read through the browser sidecar. Sorted so callers that range it (the +// browser fetcher's dispatch) see a stable order instead of map-iteration +// noise. Exported so the web layer shares the same set the poller does. +func BrowserBackedSites() []string { return browserBackedSites() } + // browserBackedSites is derived from the registry: the Sites whose pages are // read through the browser sidecar. Sorted so callers that range it (the // browser fetcher's dispatch) see a stable order instead of map-iteration diff --git a/backend/internal/web/admin_overview.go b/backend/internal/web/admin_overview.go index aa9db27..9d1ca86 100644 --- a/backend/internal/web/admin_overview.go +++ b/backend/internal/web/admin_overview.go @@ -2,8 +2,10 @@ package web import ( "fmt" + "log" "time" + "bookmarkmanager/backend/internal/latest" "bookmarkmanager/backend/internal/store" ) @@ -92,7 +94,31 @@ func (h *Handler) overviewView() (overviewView, error) { view := overviewView{Waiting: waiting(passes)} view.Unchecked = totals[store.SeriesFilterStale] + totals[store.SeriesFilterNeverChecked] - view.Verdict, view.HasCounts = overviewVerdict(passes, now) + var refusingSince map[string]int64 + if m, err := h.store.RefusingSince(now.UnixMilli()); err != nil { + log.Printf("admin overview: refusing since: %v", err) + } else { + refusingSince = m + } + var sidecarOK map[string]int64 + if m, err := h.store.SidecarOK(latest.BrowserBackedSites()); err != nil { + log.Printf("admin overview: sidecar ok: %v", err) + } else { + sidecarOK = m + } + var noChapterShare map[string]float64 + if m, err := h.store.NoChapterShare(cutoff); err != nil { + log.Printf("admin overview: no-chapter share: %v", err) + } else { + noChapterShare = m + } + faults := latest.FaultsFrom(latest.FaultInput{ + Passes: passes, + RefusingSince: refusingSince, + SidecarOK: sidecarOK, + NoChapterShare: noChapterShare, + }, now) + view.Verdict, view.HasCounts = overviewVerdict(passes, faults) // The hygiene figures, in seriesFilterOrder's tail: the problem filters // in permanent-then-fixable order, then the informational tail — finished @@ -165,28 +191,26 @@ func door(label string, count int, href string) fig { return fig{Label: label, Href: href, Count: count} } -// overviewVerdict decides the landing page's one line from the latest pass -// per Site: no passes at all is "no Lane has reported yet" — never confident -// zeroes; otherwise the count of Lanes whose last pass needs the owner, or -// "all lanes healthy". The count comes from the same laneState judgement the -// Lanes page colours on, so the two pages cannot disagree on what a fault is. -func overviewVerdict(passes []store.LanePass, now time.Time) (phrase string, counts bool) { +// overviewVerdict decides the landing page's one line: no passes at all is +// "no Lane has reported yet" — never confident zeroes; otherwise the count of +// faults from the shared FaultsFrom judgement, so the page and the push +// cannot disagree. Zero faults is "all lanes healthy". A sidecar-down fault +// carries Site == "" and is still one fault. The Lanes page's per-row +// laneState is a different question (is this Lane's last pass healthy) from +// the notice class, and its known false positives live there deliberately, so +// the two now differ. +func overviewVerdict(passes []store.LanePass, faults []latest.Fault) (phrase string, counts bool) { if len(passes) == 0 { return "no Lane has reported yet", false } - attention := 0 - for _, p := range passes { - if _, _, attn := laneState(p, now); attn { - attention++ - } - } - if attention == 0 { + n := len(faults) + if n == 0 { return "all lanes healthy", true } - if attention == 1 { + if n == 1 { return "1 lane needs a look", true } - return fmt.Sprintf("%d lanes need a look", attention), true + return fmt.Sprintf("%d lanes need a look", n), true } // waiting sums Due over the latest pass per Site: how many Series the Lanes diff --git a/backend/web_test.go b/backend/web_test.go index 816204e..d5bc62c 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -3099,7 +3099,7 @@ func TestOverviewVerdictThreeStates(t *testing.T) { }) t.Run("lanes need a look", func(t *testing.T) { - seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Skip: latest.SkipRefusing}) + seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Due: 1, Checked: 0, Skip: "", Refused: 0}) body := overviewBody(t, router, st) if !strings.Contains(body, "1 lane needs a look") { t.Errorf("attention verdict missing:\n%s", body) @@ -3118,6 +3118,96 @@ func TestOverviewVerdictThreeStates(t *testing.T) { }) } +// TestOverviewVerdictFromFaults mirrors the four fault inputs the notifier judges +// (stall, no-browser-route, sidecar-down, adapter-broken); plus healthy and +// virgin. Each seeds the durable rows the page reads, with no poller running. +func TestOverviewVerdictFromFaults(t *testing.T) { + t.Run("stall is a fault", func(t *testing.T) { + router, st := newWebTestServer(t, testConfig()) + now := time.Now() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Due: 3, Checked: 0, Skip: "", Refused: 0}) + body := overviewBody(t, router, st) + if !strings.Contains(body, "1 lane needs a look") { + t.Errorf("stall fault verdict missing:\n%s", body) + } + }) + + t.Run("no-browser-route is a fault on plain TLS site", func(t *testing.T) { + router, st := newWebTestServer(t, testConfig()) + old := time.Now().Add(-13 * time.Hour).UnixMilli() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: old, Skip: latest.SkipRefusing}) + body := overviewBody(t, router, st) + if !strings.Contains(body, "1 lane needs a look") { + t.Errorf("no-browser-route fault verdict missing:\n%s", body) + } + }) + + t.Run("sidecar-down is one fault even though every browser lane is down", func(t *testing.T) { + router, st := newWebTestServer(t, testConfig()) + now := time.Now().UnixMilli() + for _, site := range []string{"comix", "kagane", "novelfull"} { + seedPass(t, st, store.LanePass{Site: site, RanAt: now, Skip: latest.SkipSidecarDown}) + } + body := overviewBody(t, router, st) + if !strings.Contains(body, "1 lane needs a look") { + t.Errorf("sidecar-down fault should count as one, want %q:\n%s", "1 lane needs a look", body) + } + if strings.Contains(body, "3 lanes need a look") { + t.Errorf("sidecar-down fault must not count lanes, got 3:\n%s", body) + } + }) + + t.Run("adapter-broken is a fault over half no_chapter", func(t *testing.T) { + st, dsn := newTestStoreURL(t) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + router := newRouter(st, testConfig()) + // Four series of the same site; three hold an old no_chapter row (>12h). + now := time.Now() + old := now.Add(-13 * time.Hour).UnixMilli() + for i := 1; i <= 4; i++ { + key := fmt.Sprintf("asura:a%d", i) + url := fmt.Sprintf("https://asurascans.com/comics/a%d", i) + seedSeriesRow(t, st, db, seriesRowSeed{key: key, url: url, cover: "aaa", checkedAt: now.UnixMilli(), latestNum: floatPtr(1), bookmarks: 1}) + } + for i := 1; i <= 3; i++ { + if err := st.RecordSeriesFailure("asura", fmt.Sprintf("a%d", i), "no_chapter", old); err != nil { + t.Fatalf("seed no_chapter failure: %v", err) + } + } + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Due: 1, Checked: 1}) + body := overviewBody(t, router, st) + if !strings.Contains(body, "1 lane needs a look") { + t.Errorf("adapter-broken fault verdict missing:\n%s", body) + } + }) + + t.Run("healthy when no fault holds", func(t *testing.T) { + router, st := newWebTestServer(t, testConfig()) + now := time.Now() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Due: 1, Checked: 1}) + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.UnixMilli(), Skip: latest.SkipAsleep, Due: 1}) + body := overviewBody(t, router, st) + if !strings.Contains(body, "all lanes healthy") { + t.Errorf("healthy verdict missing:\n%s", body) + } + }) + + t.Run("virgin still draws no figures", func(t *testing.T) { + virgin, fresh := newWebTestServer(t, testConfig()) + body := overviewBody(t, virgin, fresh) + if !strings.Contains(body, "no Lane has reported yet") { + t.Errorf("virgin verdict missing:\n%s", body) + } + if strings.Contains(body, "all lanes healthy") || strings.Contains(body, "series waiting") { + t.Errorf("virgin verdict draws confident zeroes:\n%s", body) + } + }) +} + // The Overview never refreshes itself: the Lane rest is an hour, so a timer // would re-run a cross-Series join to redraw identical rows. Only the Lanes // block carries a refresh attribute. -- 2.52.0 From 8e6f4d2053abfa4ac5dbab266b8e951e99e82d16 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 23 Aug 2026 11:26:30 +0700 Subject: [PATCH 20/20] review: fix overviewView docstring to document fail-open fault reads --- backend/internal/web/admin_overview.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/internal/web/admin_overview.go b/backend/internal/web/admin_overview.go index 9d1ca86..10b71a7 100644 --- a/backend/internal/web/admin_overview.go +++ b/backend/internal/web/admin_overview.go @@ -64,8 +64,11 @@ type siteRow struct { // overviewView assembles the landing page from the store's read model: one // SeriesShapes pass per filter summed in Go (the shipped surface offers ten // grouped passes, not a stats query — #140), the pass log's latest pass per -// Site, and the roster. A failure in any read is a 500 with a logged reason, -// never a page of silent zeroes. +// Site, and the roster. A failure in the SeriesShapes, pass or roster reads +// is a 500 with a logged reason, never a page of silent zeroes. The three +// fault-input reads (RefusingSince, SidecarOK, NoChapterShare) fail open: +// a failing read logs and contributes no fault, so the landing page still +// renders — the same fail-open the poller uses for owner notices. func (h *Handler) overviewView() (overviewView, error) { now := time.Now() cutoff := now.Add(-ownerWindow).UnixMilli() -- 2.52.0