Spec #137: per-Series poll failure state, completion hint, and outbound owner notification #174

Merged
sulthan merged 30 commits from spec-137 into main 2026-08-23 11:31:11 +07:00
5 changed files with 853 additions and 20 deletions
Showing only changes of commit 28fef689ec - Show all commits
+110 -7
View File
@@ -14,6 +14,25 @@ import (
// words (no-browser-route, sidecar-down, adapter-broken); this ticket // words (no-browser-route, sidecar-down, adapter-broken); this ticket
// declares only the stall. // declares only the stall.
const ConditionStall = "stall" const ConditionStall = "stall"
// ConditionNoBrowserRoute is the owner-notice machine word for a Site whose
// challenge refuses for longer than the owner window with no browser route
// to clear it — the 403-with-interstitial the reader maps to
// errChallengeHeld (read.go), which plain TLS cannot clear. The word is the
// message's footer and its suppression key; it is wire-stable.
const ConditionNoBrowserRoute = "no-browser-route"
// ConditionSidecarDown is the owner-notice machine word for a browser
// sidecar no Lane has reached for longer than the owner window: every
// browser-backed Lane's latest pass is a sidecar skip. The word is the
// message's footer and its suppression key; it is wire-stable, and its
// suppression row holds the empty Site (AC4).
const ConditionSidecarDown = "sidecar-down"
// ConditionAdapterBroken is the owner-notice machine word for a Site whose
// adapter stopped finding chapters: more than half of its Series hold an
// old no-chapter failure row. The word is the message's footer and its
// suppression key; it is wire-stable.
const ConditionAdapterBroken = "adapter-broken"
// OwnerWindow is the class-level staleness boundary every owner-notice // OwnerWindow is the class-level staleness boundary every owner-notice
// condition measures against — the same twelve hours the Lanes page's // condition measures against — the same twelve hours the Lanes page's
@@ -33,6 +52,22 @@ type Fault struct {
// conditions can add inputs without changing either caller. // conditions can add inputs without changing either caller.
type FaultInput struct { type FaultInput struct {
Passes []store.LanePass Passes []store.LanePass
// RefusingSince is, per Site, the unix ms when that Site's current
// unbroken run of refusing passes began, or absent when its latest pass
// did not refuse. Its zero value is an empty map, which contributes no
// fault.
RefusingSince map[string]int64
// SidecarOK is, per browser-backed Site, the unix ms of that Site's most
// recent pass that actually reached the sidecar. Zero when the pass log
// holds none — an asleep Lane never reached it and never counts as
// evidence either way. Its zero value is an empty map.
SidecarOK map[string]int64
// NoChapterShare is, per Site, the share of that Site's Series holding a
// no-chapter failure row older than the owner window. Its zero value is
// an empty map, which contributes no fault.
NoChapterShare map[string]float64
} }
// FaultsFrom judges the owner-notice conditions from durable rows alone, so // FaultsFrom judges the owner-notice conditions from durable rows alone, so
@@ -42,12 +77,27 @@ type FaultInput struct {
// value and no refusal — exactly the row the mid-loop browser loss writes // value and no refusal — exactly the row the mid-loop browser loss writes
// (see the comment at the outcomeUnreachable return in runLanePass), so a // (see the comment at the outcomeUnreachable return in runLanePass), so a
// Lane that owed Polls, made none, and has nothing to say for it is a fault. // Lane that owed Polls, made none, and has nothing to say for it is a fault.
// The no-refusal clause is AC6's amendment: the twice-refused break happens // The three #172 conditions share the same OwnerWindow boundary and the same
// inside the pass loop, not on an early return, so a newly-gated Site would // fail-open shape: an input's absence contributes no fault, never a false
// otherwise send two messages. A pause is excluded by Skip == "" (SkipPaused): // one.
// the owner's own act is not reported back (AC10). The episode began at the //
// pass that produced the row; the stall test itself has no age clause — the // - no-browser-route: a Site whose refusing run began before the window
// class-level twelve hours belongs to #172's conditions. // and that has no browser route to clear the challenge (AC2). Both
// refusal shapes count — the gate's skip='refusing' rows and the loop's
// refused>0 rows (the twice-refused break writes skip="") — so the run
// stays unbroken across them, and one healthy pass ends it.
// - sidecar-down: every browser-backed Site's latest pass is a sidecar
// skip — SkipSidecarDown or SkipNoFetcher, the two early returns that
// record a skip value — and each Site's most recent sidecar-reaching
// pass is older than the window (AC4). The skip clause is what keeps
// SkipAsleep out: a Lane under both wake thresholds is the commonest
// healthy state, never reached the sidecar, and would otherwise age into
// a false alarm. Emitted once, with Site "" (AC4's one row per episode).
// - adapter-broken: more than half of one Site's Series hold a no-chapter
// failure row older than the window (AC6). Strictly above half: the
// filter is the tool, and one Site change makes hundreds of rows, so a
// single failing Series never fires (AC7). The episode's age is the
// window — the old rows prove the episode is at least that old.
func FaultsFrom(in FaultInput, now time.Time) []Fault { func FaultsFrom(in FaultInput, now time.Time) []Fault {
var faults []Fault var faults []Fault
for _, p := range in.Passes { for _, p := range in.Passes {
@@ -55,9 +105,50 @@ func FaultsFrom(in FaultInput, now time.Time) []Fault {
faults = append(faults, Fault{Condition: ConditionStall, Site: p.Site, Since: p.RanAt}) faults = append(faults, Fault{Condition: ConditionStall, Site: p.Site, Since: p.RanAt})
} }
} }
cutoff := now.Add(-OwnerWindow).UnixMilli()
for site, since := range in.RefusingSince {
if since < cutoff && !isBrowserSite(site) {
faults = append(faults, Fault{Condition: ConditionNoBrowserRoute, Site: site, Since: since})
}
}
if since, down := sidecarDownSince(in.Passes, in.SidecarOK, cutoff); down {
faults = append(faults, Fault{Condition: ConditionSidecarDown, Site: "", Since: since})
}
for site, share := range in.NoChapterShare {
if share > 0.5 {
faults = append(faults, Fault{Condition: ConditionAdapterBroken, Site: site, Since: now.Add(-OwnerWindow).UnixMilli()})
}
}
return faults return faults
} }
// sidecarDownSince reports whether no browser Lane has reached the sidecar
// for longer than the owner window and, when it has, the last moment any
// Lane reached it. The skip clause — every browser-backed Site's latest pass
// must be SkipSidecarDown or SkipNoFetcher — keeps SkipAsleep out (see
// FaultsFrom). A Site with no pass row at all is not judged down either: a
// fresh database is not a dead sidecar.
func sidecarDownSince(passes []store.LanePass, ok map[string]int64, cutoff int64) (since int64, down bool) {
latest := make(map[string]store.LanePass, len(passes))
for _, p := range passes {
latest[p.Site] = p
}
for _, site := range browserBackedSites() {
p, found := latest[site]
if !found || (p.Skip != SkipSidecarDown && p.Skip != SkipNoFetcher) {
return 0, false
}
reached := ok[site]
if reached > 0 && reached >= cutoff {
return 0, false
}
if reached > since {
since = reached
}
}
return since, true
}
// Notifier delivers one owner notice. The poller neither retries nor queues: // Notifier delivers one owner notice. The poller neither retries nor queues:
// an error is logged and the suppression row left unwritten, so the next pass // an error is logged and the suppression row left unwritten, so the next pass
// tries again while the condition holds. // tries again while the condition holds.
@@ -69,7 +160,7 @@ type Notifier interface {
// clear loop in recordPass: a condition absent from a pass's fault list // clear loop in recordPass: a condition absent from a pass's fault list
// forgets its episode, so the next occurrence sends again. #172 extends the // forgets its episode, so the next occurrence sends again. #172 extends the
// list when it adds its conditions. // list when it adds its conditions.
var ownerNoticeConditions = []string{ConditionStall} var ownerNoticeConditions = []string{ConditionStall, ConditionNoBrowserRoute, ConditionSidecarDown, ConditionAdapterBroken}
// noticeFor renders one fault's message: the description sentence — condition, // noticeFor renders one fault's message: the description sentence — condition,
// age, repair — and the deep link the embed's title points at. Each condition // age, repair — and the deep link the embed's title points at. Each condition
@@ -80,6 +171,18 @@ func noticeFor(f Fault, row store.LanePass, now time.Time) (sentence, href strin
return fmt.Sprintf( return fmt.Sprintf(
"%s owed %d Polls and made none — %s; check the Lane's browser sidecar and the Site's challenge state", "%s owed %d Polls and made none — %s; check the Lane's browser sidecar and the Site's challenge state",
f.Site, row.Due, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes" f.Site, row.Due, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
case ConditionNoBrowserRoute:
return fmt.Sprintf(
"%s has refused for %s with no browser route — the challenge does not clear on plain TLS; redeploy or add a browser route",
f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
case ConditionSidecarDown:
return fmt.Sprintf(
"no browser Lane has reached the sidecar for %s — the browser sidecar is down; start or repair the browser machine",
humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
case ConditionAdapterBroken:
return fmt.Sprintf(
"more than half of %s's Series have failed no-chapter reads for at least %s — the Site's layout changed and the adapter is broken",
f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
} }
return "", "" return "", ""
} }
+58 -13
View File
@@ -568,18 +568,50 @@ func (p *Poller) recordPass(ctx context.Context, rec passRecord, fig passFigures
// ownerNotices judges the owner-notice conditions for the pass just recorded // ownerNotices judges the owner-notice conditions for the pass just recorded
// and fires (issue #171). It sits in recordPass because that deferred call is // and fires (issue #171). It sits in recordPass because that deferred call is
// the one place every return path passes through: two of the four conditions // the one place every return path passes through: three of the four
// occur on early returns and the success path can never see them. Per fault: // conditions occur on early returns and the success path can never see them.
// NoticeSent → send → MarkNoticeSent, so a fault lasting a month sends one // The judgement reads the whole pass log plus three derived reads — the
// message, not one per pass; a condition absent from this pass's fault list // other Lanes' latest passes, each Site's refusing-run start, its last
// forgets its episode, so the next occurrence sends again. Everything here is // sidecar-reaching pass, and its no-chapter share — so one pass judges every
// best-effort: a failed send, a failed store read and a failed notice write // condition (issue #172). Per fault: NoticeSent → send → MarkNoticeSent, so
// are all logged and never change the pass's outcome counts or its return // a fault lasting a month sends one message, not one per pass; a condition
// value. The clear runs even when Notify is nil, so a deployment that turns // absent from this pass's fault list forgets its episode, so the next
// the webhook off does not leave stale rows that suppress the first real // occurrence sends again. Everything here is best-effort: a failed send, a
// notice after it is turned back on. // failed store read and a failed notice write are all logged and never
// change the pass's outcome counts or its return value. The clear runs even
// when Notify is nil, so a deployment that turns the webhook off does not
// leave stale rows that suppress the first real notice after it is turned
// back on.
func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) { func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) {
faults := FaultsFrom(FaultInput{Passes: []store.LanePass{row}}, p.Now()) now := p.Now()
in := FaultInput{Passes: []store.LanePass{row}}
// Each read fails independently: a failure logs and contributes no fault,
// never a false one.
if passes, err := p.Store.LatestLanePasses(); err != nil {
log.Printf("latest poll %s: latest lane passes: %v", row.Site, err)
} else {
in.Passes = passes
}
if since, err := p.Store.RefusingSince(now.UnixMilli()); err != nil {
log.Printf("latest poll %s: refusing since: %v", row.Site, err)
} else {
in.RefusingSince = since
}
if ok, err := p.Store.SidecarOK(browserBackedSites()); err != nil {
log.Printf("latest poll %s: sidecar ok: %v", row.Site, err)
} else {
in.SidecarOK = ok
}
if share, err := p.Store.NoChapterShare(now.Add(-OwnerWindow).UnixMilli()); err != nil {
log.Printf("latest poll %s: no-chapter share: %v", row.Site, err)
} else {
in.NoChapterShare = share
}
faults := FaultsFrom(in, now)
bySite := make(map[string]store.LanePass, len(in.Passes))
for _, pass := range in.Passes {
bySite[pass.Site] = pass
}
for _, f := range faults { for _, f := range faults {
if p.Notify == nil { if p.Notify == nil {
continue continue
@@ -592,14 +624,20 @@ func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) {
if sent { if sent {
continue continue
} }
sentence, href := noticeFor(f, row, p.Now()) // The fault's own Site's pass renders its sentence — a stall judged
// from another Lane's pass must not quote this pass's figures.
pass, ok := bySite[f.Site]
if !ok {
pass = row
}
sentence, href := noticeFor(f, pass, now)
if err := p.Notify.Notify(ctx, f, sentence, href); err != nil { if err := p.Notify.Notify(ctx, f, sentence, href); err != nil {
// The stamp stays unset: no queue, no backoff — the condition is // The stamp stays unset: no queue, no backoff — the condition is
// durable, so the next pass tries again while it holds. // durable, so the next pass tries again while it holds.
log.Printf("latest poll %s: owner notice %s: %v", row.Site, f.Condition, err) log.Printf("latest poll %s: owner notice %s: %v", row.Site, f.Condition, err)
continue continue
} }
if err := p.Store.MarkNoticeSent(f.Condition, f.Site, p.Now().UnixMilli()); err != nil { if err := p.Store.MarkNoticeSent(f.Condition, f.Site, now.UnixMilli()); err != nil {
log.Printf("latest poll %s: mark notice sent: %v", row.Site, err) log.Printf("latest poll %s: mark notice sent: %v", row.Site, err)
} }
} }
@@ -609,6 +647,13 @@ func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) {
log.Printf("latest poll %s: clear owner notice: %v", row.Site, err) log.Printf("latest poll %s: clear owner notice: %v", row.Site, err)
} }
} }
// The site-wide conditions suppress under the empty Site; clear that
// row too, so a lifted sidecar-down fires again when it returns.
if !hasFault(faults, cond, "") {
if err := p.Store.ClearNotice(cond, ""); err != nil {
log.Printf("latest poll %s: clear owner notice: %v", row.Site, err)
}
}
} }
} }
+431
View File
@@ -3181,6 +3181,437 @@ func TestFaultsFromStall(t *testing.T) {
} }
} }
// The #172 conditions are judged from the durable inputs alone, like the
// stall. A refusal is only a fault when the Site has no browser route to
// clear it (AC2's "browser Site" exclusions), and only once it is older than
// the owner window.
func TestFaultsFromNoBrowserRoute(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
old := now.Add(-2 * OwnerWindow).UnixMilli() // a two-day refusal
fresh := now.Add(-OwnerWindow / 2).UnixMilli() // inside the window
tests := []struct {
name string
in FaultInput
want int
}{
{
name: "plain-TLS Sites refusing for two days are faults",
in: FaultInput{RefusingSince: map[string]int64{"asura": old, "demonic": now.Add(-3 * OwnerWindow).UnixMilli()}},
want: 2,
},
{
name: "a browser-backed Site's refusal is a route it has, not a fault",
in: FaultInput{RefusingSince: map[string]int64{"comix": old, "kagane": old, "novelfull": old}},
want: 0,
},
{
name: "a fresh refusal is not old enough",
in: FaultInput{RefusingSince: map[string]int64{"asura": fresh}},
want: 0,
},
{
name: "no refusal is no fault",
in: FaultInput{},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
faults := FaultsFrom(tt.in, now)
if got := len(faults); got != tt.want {
t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want)
}
for _, f := range faults {
if f.Condition != ConditionNoBrowserRoute || f.Site == "" || f.Since != tt.in.RefusingSince[f.Site] {
t.Fatalf("fault = %+v, want no-browser-route on the refusing Site since its run began", f)
}
}
})
}
}
// sidecar-down is one site-wide fault: every browser Lane's latest pass must
// be a sidecar skip (SkipSidecarDown or SkipNoFetcher — the skip clause keeps
// an asleep Lane out) and each Lane's most recent sidecar-reaching pass must
// be older than the window. The fault's Since is the last moment any Lane
// reached the sidecar.
func TestFaultsFromSidecarDown(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
oldReach := now.Add(-2 * OwnerWindow).UnixMilli()
skip := func(site string) store.LanePass {
return store.LanePass{Site: site, RanAt: oldReach, Skip: SkipSidecarDown}
}
allSkipped := []store.LanePass{skip("comix"), skip("kagane"), skip("novelfull")}
reached := now.Add(-OwnerWindow / 2).UnixMilli() // inside the window
tests := []struct {
name string
in FaultInput
want int
wantSince int64
}{
{
name: "every browser Lane sidecar-skipped past the window is one fault",
in: FaultInput{Passes: allSkipped, SidecarOK: map[string]int64{"comix": oldReach}},
want: 1,
wantSince: oldReach,
},
{
name: "a browser Lane asleep for two days sends nothing",
in: FaultInput{
Passes: []store.LanePass{
skip("comix"),
{Site: "kagane", RanAt: oldReach, Skip: SkipAsleep},
skip("novelfull"),
},
},
want: 0,
},
{
name: "a Lane that reached the sidecar inside the window is not down",
in: FaultInput{
Passes: allSkipped,
SidecarOK: map[string]int64{"comix": reached, "kagane": reached, "novelfull": reached},
},
want: 0,
},
{
name: "a browser Site with no pass row is not judged down",
in: FaultInput{
Passes: []store.LanePass{skip("comix"), skip("kagane")},
},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
faults := FaultsFrom(tt.in, now)
if got := len(faults); got != tt.want {
t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want)
}
for _, f := range faults {
if f.Condition != ConditionSidecarDown || f.Site != "" || f.Since != tt.wantSince {
t.Fatalf("fault = %+v, want one site-wide sidecar-down since %d", f, tt.wantSince)
}
}
})
}
}
// adapter-broken fires strictly above half: a float share judges a
// four-Series Site and a 200-Series Site on the same scale, exactly half
// stays quiet, and a single failing Series never reaches it.
func TestFaultsFromAdapterBroken(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
tests := []struct {
name string
in FaultInput
want int
}{
{
name: "three of four Series failing is a fault",
in: FaultInput{NoChapterShare: map[string]float64{"asura": 3.0 / 4.0}},
want: 1,
},
{
name: "exactly half is not a fault",
in: FaultInput{NoChapterShare: map[string]float64{"asura": 1.0 / 2.0}},
want: 0,
},
{
name: "one of two hundred is not a fault",
in: FaultInput{NoChapterShare: map[string]float64{"asura": 1.0 / 200.0}},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
faults := FaultsFrom(tt.in, now)
if got := len(faults); got != tt.want {
t.Fatalf("FaultsFrom = %+v, want %d fault(s)", faults, tt.want)
}
for _, f := range faults {
if f.Condition != ConditionAdapterBroken || f.Site != "asura" || f.Since != now.Add(-OwnerWindow).UnixMilli() {
t.Fatalf("fault = %+v, want adapter-broken on asura since the window", f)
}
}
})
}
}
// seedRefusingRun writes a refusing pass and the gate row that follows it,
// so a Site's run of refusing passes begins at start.
func seedRefusingRun(t *testing.T, s *store.Store, site string, start time.Time) {
t.Helper()
for i, row := range []store.LanePass{
{Site: site, RanAt: start.UnixMilli(), Skip: "", Refused: 2},
{Site: site, RanAt: start.Add(time.Minute).UnixMilli(), Skip: SkipRefusing},
} {
if err := s.RecordLanePass(row, -1); err != nil {
t.Fatalf("seed refusing run %d: %v", i, err)
}
}
}
// seedSidecarSkips writes one sidecar-skipped pass for every browser-backed
// Lane, all at the same time, so the pass log shows a sidecar that has been
// unreachable since then.
func seedSidecarSkips(t *testing.T, s *store.Store, at time.Time) {
t.Helper()
for _, site := range browserBackedSites() {
if err := s.RecordLanePass(store.LanePass{Site: site, RanAt: at.UnixMilli(), Skip: SkipSidecarDown}, -1); err != nil {
t.Fatalf("seed sidecar skip %s: %v", site, err)
}
}
}
// seedNoChapter writes an old no-chapter failure row for each Series id.
func seedNoChapter(t *testing.T, s *store.Store, site string, ids []string, failingSince int64) {
t.Helper()
for _, id := range ids {
if err := s.RecordSeriesFailure(site, id, "no_chapter", failingSince); err != nil {
t.Fatalf("seed no-chapter failure %s:%s: %v", site, id, err)
}
}
}
// no-browser-route fires once while the refusal holds, and again after it
// lifts and returns: the suppression row is the whole state, the gate row of
// a second refusing pass is the same episode, a healthy pass in between
// breaks the run and clears the row, and a later run that has aged past the
// window sends again (AC1, AC9).
func TestOwnerNoticeNoBrowserRouteFiresOncePerEpisode(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
s, _ := newTestStore(t)
seedForCheck(t, s, "asura:r1", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
seedForCheck(t, s, "asura:r2", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
refusing := &fakeFetcher{status: 403}
notifier := &fakeNotifier{}
p := newTestPoller(t, s, refusing, now)
p.Now = func() time.Time { return now }
p.Notify = notifier
// A run that began before the window: seed two-day-old refusing rows,
// then a fresh pass that refuses again — the run is unbroken and still
// old, so the owner is told once.
seedRefusingRun(t, s, "asura", now.Add(-2*OwnerWindow))
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after first refusing pass = %d, want 1", got)
}
if f := notifier.fault(0); f.Condition != ConditionNoBrowserRoute || f.Site != "asura" || f.Since != now.Add(-2*OwnerWindow).UnixMilli() {
t.Fatalf("fault = %+v, want no-browser-route on asura since the run began", f)
}
if sent, err := s.NoticeSent(ConditionNoBrowserRoute, "asura"); err != nil || !sent {
t.Fatalf("NoticeSent after first refusing pass = %v, %v; want true, nil", sent, err)
}
// A second refusing pass — the refusal gate now returns early — is the
// same episode: the gate row continues the run, no second message.
now = now.Add(time.Minute)
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after gated refusing pass = %d, want 1 (one per episode)", got)
}
// A healthy pass in between breaks the run: the condition lifts and the
// suppression row is cleared.
now = now.Add(RefuseBackoff + time.Minute)
seedForCheck(t, s, "asura:r1", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
seedForCheck(t, s, "asura:r2", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
p.Fetch = &fakeFetcher{body: asuraSeriesFixture, status: 200}
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after healthy pass = %d, want 1 (a healthy pass sends nothing)", got)
}
if sent, err := s.NoticeSent(ConditionNoBrowserRoute, "asura"); err != nil || sent {
t.Fatalf("NoticeSent after healthy pass = %v, %v; want false, nil (row cleared)", sent, err)
}
// A later run that has aged past the window again is a new episode.
runStart := now.Add(2 * OwnerWindow)
seedRefusingRun(t, s, "asura", runStart)
now = runStart.Add(2 * OwnerWindow)
seedForCheck(t, s, "asura:r1", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
seedForCheck(t, s, "asura:r2", "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", 0)
p.Fetch = refusing
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 2 {
t.Fatalf("notices after the returned refusal = %d, want 2 (a new episode)", got)
}
if f := notifier.fault(1); f.Condition != ConditionNoBrowserRoute || f.Site != "asura" || f.Since != runStart.UnixMilli() {
t.Fatalf("fault = %+v, want no-browser-route on asura since the new run began", f)
}
}
// A browser-backed Site's refusal sends nothing (AC2): it has a route, so a
// two-day refusal is a browser-side problem, not the no-browser-route fault.
func TestOwnerNoticeBrowserRefusalSendsNothing(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
s, _ := newTestStore(t)
for i := 1; i <= 6; i++ {
seedForCheck(t, s, fmt.Sprintf("comix:b%d", i), fmt.Sprintf("https://comix.to/title/b%d", i), 0)
}
notifier := &fakeNotifier{}
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
p.Now = func() time.Time { return now }
p.BrowserFetch = &fakeFetcher{status: 403}
p.Notify = notifier
seedRefusingRun(t, s, "comix", now.Add(-2*OwnerWindow))
p.runLanePass(context.Background(), "comix", false)
if got := notifier.callCount(); got != 0 {
t.Fatalf("notices = %d, want 0 (a browser Site's refusal is not a no-browser-route)", got)
}
if sent, err := s.NoticeSent(ConditionNoBrowserRoute, "comix"); err != nil || sent {
t.Fatalf("NoticeSent = %v, %v; want false, nil", sent, err)
}
}
// sidecar-down fires once while no Lane reaches the sidecar, and again after
// it returns and dies again (AC4, AC9): the suppression row holds the empty
// Site, the first Lane to notice writes it and the others stay quiet, a
// healthy browser pass clears it, and a later outage is a new episode.
func TestOwnerNoticeSidecarDownFiresOncePerEpisode(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
s, _ := newTestStore(t)
notifier := &fakeNotifier{}
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
p.Now = func() time.Time { return now }
p.Notify = notifier
// Seed the pass log so every browser Lane's latest pass is a sidecar
// skip older than the window, then a fresh pass that skips too.
seedSidecarSkips(t, s, now.Add(-2*OwnerWindow))
p.setBrowserDown(now)
p.runLanePass(context.Background(), "comix", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after first sidecar-skipped pass = %d, want 1", got)
}
if f := notifier.fault(0); f.Condition != ConditionSidecarDown || f.Site != "" {
t.Fatalf("fault = %+v, want one site-wide sidecar-down", f)
}
if sent, err := s.NoticeSent(ConditionSidecarDown, ""); err != nil || !sent {
t.Fatalf("NoticeSent after first pass = %v, %v; want true, nil", sent, err)
}
// Another Lane's sidecar-skipped pass is the same episode: still one
// message.
p.runLanePass(context.Background(), "kagane", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after another Lane's skipped pass = %d, want 1 (one per episode)", got)
}
// A healthy browser pass reaches the sidecar: the condition lifts and the
// suppression row is cleared. Five due Series wake the browser, and the
// series ids carry the fixture's id prefix so the scoped reader finds its
// chapters.
now = now.Add(RefuseBackoff + time.Minute)
for i := 1; i <= 5; i++ {
seedForCheck(t, s, fmt.Sprintf("comix:h%d", i), "https://comix.to/title/n8we-dungeons-and-crayons", 0)
}
p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200}
p.runLanePass(context.Background(), "comix", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after healthy pass = %d, want 1 (a healthy pass sends nothing)", got)
}
if sent, err := s.NoticeSent(ConditionSidecarDown, ""); err != nil || sent {
t.Fatalf("NoticeSent after healthy pass = %v, %v; want false, nil (row cleared)", sent, err)
}
// The sidecar dies again: a new episode, told again.
now = now.Add(2 * OwnerWindow)
p.setBrowserDown(now)
seedSidecarSkips(t, s, now.Add(-time.Minute))
p.runLanePass(context.Background(), "comix", false)
if got := notifier.callCount(); got != 2 {
t.Fatalf("notices after the returned outage = %d, want 2 (a new episode)", got)
}
}
// A browser Lane asleep under both wake thresholds sends nothing: it never
// reached the sidecar, but its skip is not a sidecar skip, so it cannot age
// into a false alarm (AC9).
func TestOwnerNoticeSidecarDownAsleepSendsNothing(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
s, _ := newTestStore(t)
notifier := &fakeNotifier{}
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
p.Now = func() time.Time { return now }
p.Notify = notifier
seedSidecarSkips(t, s, now.Add(-2*OwnerWindow))
if err := s.RecordLanePass(store.LanePass{Site: "kagane", RanAt: now.Add(-2*OwnerWindow + time.Minute).UnixMilli(), Skip: SkipAsleep}, -1); err != nil {
t.Fatalf("seed asleep pass: %v", err)
}
p.setBrowserDown(now)
p.runLanePass(context.Background(), "comix", false)
if got := notifier.callCount(); got != 0 {
t.Fatalf("notices = %d, want 0 (an asleep Lane is not a down sidecar)", got)
}
if sent, err := s.NoticeSent(ConditionSidecarDown, ""); err != nil || sent {
t.Fatalf("NoticeSent = %v, %v; want false, nil", sent, err)
}
}
// adapter-broken fires once while more than half of a Site's Series hold an
// old no-chapter failure row, and again after the failures clear and return:
// the suppression row is the whole state, a pass that clears the failures
// forgets the episode, and a later return sends again. The share is judged
// from durable rows, so the pass itself may be healthy (AC6, AC9).
func TestOwnerNoticeAdapterBrokenFiresOncePerEpisode(t *testing.T) {
now := time.UnixMilli(5_000_000_000)
s, _ := newTestStore(t)
for i := 1; i <= 4; i++ {
seedForCheck(t, s, fmt.Sprintf("asura:a%d", i), fmt.Sprintf("https://asurascans.com/comics/a%d", i), now.UnixMilli())
}
notifier := &fakeNotifier{}
p := newTestPoller(t, s, &fakeFetcher{status: 200}, now)
p.Now = func() time.Time { return now }
p.Notify = notifier
// Three of four Series hold an old no-chapter row: the first pass fires
// one adapter-broken fault.
seedNoChapter(t, s, "asura", []string{"a1", "a2", "a3"}, now.Add(-2*OwnerWindow).UnixMilli())
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after first pass = %d, want 1", got)
}
if f := notifier.fault(0); f.Condition != ConditionAdapterBroken || f.Site != "asura" || f.Since != now.Add(-OwnerWindow).UnixMilli() {
t.Fatalf("fault = %+v, want adapter-broken on asura since the window", f)
}
if sent, err := s.NoticeSent(ConditionAdapterBroken, "asura"); err != nil || !sent {
t.Fatalf("NoticeSent after first pass = %v, %v; want true, nil", sent, err)
}
// A second pass is the same episode: still one message.
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after second pass = %d, want 1 (one per episode)", got)
}
// The failures clear: the condition lifts and the suppression row is
// cleared.
for _, id := range []string{"a1", "a2", "a3"} {
if err := s.ClearSeriesFailure("asura", id); err != nil {
t.Fatalf("clear failure %s: %v", id, err)
}
}
now = now.Add(RefuseBackoff + time.Minute)
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 1 {
t.Fatalf("notices after cleared failures = %d, want 1 (a healthy share sends nothing)", got)
}
if sent, err := s.NoticeSent(ConditionAdapterBroken, "asura"); err != nil || sent {
t.Fatalf("NoticeSent after cleared failures = %v, %v; want false, nil (row cleared)", sent, err)
}
// The failures return: a new episode, told again.
seedNoChapter(t, s, "asura", []string{"a1", "a2", "a3"}, now.Add(-2*OwnerWindow).UnixMilli())
now = now.Add(time.Minute)
p.runLanePass(context.Background(), "asura", false)
if got := notifier.callCount(); got != 2 {
t.Fatalf("notices after the returned failures = %d, want 2 (a new episode)", got)
}
}
// The stall fires exactly once while it holds, and again after it lifts and // The stall fires exactly once while it holds, and again after it lifts and
// returns: the suppression row is the whole state, so the second stall-shaped // returns: the suppression row is the whole state, so the second stall-shaped
// pass is the same episode, a healthy pass in between clears the row, and the // pass is the same episode, a healthy pass in between clears the row, and the
+95
View File
@@ -1227,6 +1227,101 @@ func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) {
return out, rows.Err() return out, rows.Err()
} }
// RefusingSince reports, per Site, when that Site's current unbroken run of
// refusing passes began: a pass refuses when the Lane gate returned early
// (skip = 'refusing') or the pass loop counted refusals (refused > 0), so a
// Site parked in refusal backoff keeps its run unbroken. A Site whose
// latest pass did not refuse is absent. The run is bounded by the pass
// log's retention — a run older than the log answers with the oldest row
// present — and nothing after now counts: the cutoff is the caller's clock.
func (s *Store) RefusingSince(now int64) (map[string]int64, error) {
rows, err := s.db.Query(`
SELECT site, MIN(ran_at)
FROM poll_passes p
WHERE ran_at <= $1
AND (refused > 0 OR skip = 'refusing')
AND ran_at > COALESCE((
SELECT MAX(q.ran_at) FROM poll_passes q
WHERE q.site = p.site AND q.ran_at <= $1
AND NOT (q.refused > 0 OR q.skip = 'refusing')
), 0)
GROUP BY site`, now)
if err != nil {
return nil, fmt.Errorf("query refusing since: %w", err)
}
defer rows.Close()
out := map[string]int64{}
for rows.Next() {
var site string
var since int64
if err := rows.Scan(&site, &since); err != nil {
return nil, fmt.Errorf("scan refusing since: %w", err)
}
out[site] = since
}
return out, rows.Err()
}
// SidecarOK reports, per Site in sites, the unix ms of that Site's most
// recent pass that actually reached the sidecar: the pass ran its loop
// (skip = '') and no Series read lost Chrome (unreachable = 0) — a
// challenge answer still reached the sidecar, a lost sidecar did not. A
// Site with no such pass is absent: an asleep Lane never reached it, so it
// is absent too and cannot age into a sidecar-down alarm by itself.
func (s *Store) SidecarOK(sites []string) (map[string]int64, error) {
rows, err := s.db.Query(`
SELECT DISTINCT ON (site) site, ran_at
FROM poll_passes
WHERE site = ANY($1) AND skip = '' AND unreachable = 0
ORDER BY site, ran_at DESC`, sites)
if err != nil {
return nil, fmt.Errorf("query sidecar ok: %w", err)
}
defer rows.Close()
out := map[string]int64{}
for rows.Next() {
var site string
var ranAt int64
if err := rows.Scan(&site, &ranAt); err != nil {
return nil, fmt.Errorf("scan sidecar ok: %w", err)
}
out[site] = ranAt
}
return out, rows.Err()
}
// NoChapterShare reports, per Site, the share of that Site's Series holding
// a no-chapter failure row older than cutoff: old rows over the Site's
// whole Series count, so a four-Series Site and a 200-Series Site are
// judged on the same scale. A Site with no Series has no share and is
// absent.
func (s *Store) NoChapterShare(cutoff int64) (map[string]float64, error) {
rows, err := s.db.Query(`
SELECT s.site,
(COUNT(*) FILTER (WHERE f.outcome = 'no_chapter' AND f.failing_since < $1))::float8
/ (COUNT(*)::float8)
FROM series s
LEFT JOIN poll_failures f ON f.site = s.site AND f.series_id = s.series_id
GROUP BY s.site
ORDER BY s.site`, cutoff)
if err != nil {
return nil, fmt.Errorf("query no-chapter share: %w", err)
}
defer rows.Close()
out := map[string]float64{}
for rows.Next() {
var site string
var share float64
if err := rows.Scan(&site, &share); err != nil {
return nil, fmt.Errorf("scan no-chapter share: %w", err)
}
out[site] = share
}
return out, rows.Err()
}
// SetLaneRefusal persists a Site's refusal backoff stamp without touching its // SetLaneRefusal persists a Site's refusal backoff stamp without touching its
// pause. until is supplied by the caller's clock. // pause. until is supplied by the caller's clock.
func (s *Store) SetLaneRefusal(site string, until int64) error { func (s *Store) SetLaneRefusal(site string, until int64) error {
+159
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex" "encoding/hex"
"errors" "errors"
"io/fs" "io/fs"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
@@ -2738,3 +2739,161 @@ func TestOwnerNoticeRows(t *testing.T) {
t.Fatalf("ClearNotice on a missing row: %v", err) t.Fatalf("ClearNotice on a missing row: %v", err)
} }
} }
// RefusingSince reads one Site's current unbroken run of refusing passes
// (issue #172). A refusing pass is one the Lane gate returned early from
// (skip 'refusing') or one whose loop counted refusals (refused > 0) — the
// two shapes a persistently-challenged Site alternates between, so the run
// must not break when the gate row follows the refusal row. A Site whose
// latest pass did not refuse is absent, nothing after the caller's clock
// counts, and a run older than the log answers with the oldest row present.
func TestRefusingSince(t *testing.T) {
s := newTestStore(t)
seed := func(site string, ranAt int64, skip string, refused int) {
t.Helper()
if err := s.RecordLanePass(LanePass{Site: site, RanAt: ranAt, Skip: skip, Refused: refused}, -1); err != nil {
t.Fatalf("RecordLanePass(%s/%d): %v", site, ranAt, err)
}
}
// asura: a refusing run broken by a healthy pass, then resumed; the
// current run began at the pass after the break.
seed("asura", 100, "", 2)
seed("asura", 200, "", 0)
seed("asura", 300, "refusing", 0)
seed("asura", 400, "", 2)
// demonic: the latest pass is healthy — no run, absent.
seed("demonic", 100, "", 2)
seed("demonic", 200, "", 0)
// novelfull: a healthy pass after now must not break the run — the run
// is judged as of the caller's clock.
seed("novelfull", 100, "", 2)
seed("novelfull", 600, "", 0)
// comix: an unbroken run reaches back to the oldest row present.
seed("comix", 100, "", 2)
seed("comix", 200, "refusing", 0)
got, err := s.RefusingSince(450)
if err != nil {
t.Fatalf("RefusingSince: %v", err)
}
want := map[string]int64{"asura": 300, "novelfull": 100, "comix": 100}
if len(got) != len(want) {
t.Fatalf("RefusingSince = %v, want %v", got, want)
}
for site, since := range want {
if got[site] != since {
t.Fatalf("RefusingSince[%s] = %d, want %d (got %v)", site, got[site], since, got)
}
}
if _, ok := got["demonic"]; ok {
t.Fatalf("RefusingSince names demonic, whose latest pass did not refuse: %v", got)
}
}
// SidecarOK reads, per Site, the most recent pass that actually reached the
// sidecar (issue #172): the pass ran its loop (skip '') and no read lost
// Chrome (unreachable 0). A challenge answer still reached the sidecar, a
// lost sidecar and every skip value did not, and a Site with no qualifying
// pass — an asleep Lane included — is absent.
func TestSidecarOK(t *testing.T) {
s := newTestStore(t)
seed := func(site string, ranAt int64, skip string, unreachable int) {
t.Helper()
if err := s.RecordLanePass(LanePass{Site: site, RanAt: ranAt, Skip: skip, Unreachable: unreachable}, -1); err != nil {
t.Fatalf("RecordLanePass(%s/%d): %v", site, ranAt, err)
}
}
// comix: only the skip='' unreachable=0 pass reached the sidecar; the
// mid-loop browser-loss row (skip '', unreachable > 0) and the skip
// values never did.
seed("comix", 100, "", 1)
seed("comix", 200, "sidecar-down", 0)
seed("comix", 300, "", 0)
seed("comix", 400, "refusing", 0)
// kagane: two passes reached the sidecar; the newest wins.
seed("kagane", 150, "", 0)
seed("kagane", 250, "", 0)
// novelfull: an asleep Lane never reached it.
seed("novelfull", 120, "asleep", 0)
got, err := s.SidecarOK([]string{"comix", "kagane", "novelfull", "lightnovelworld"})
if err != nil {
t.Fatalf("SidecarOK: %v", err)
}
want := map[string]int64{"comix": 300, "kagane": 250}
if len(got) != len(want) {
t.Fatalf("SidecarOK = %v, want %v", got, want)
}
for site, ranAt := range want {
if got[site] != ranAt {
t.Fatalf("SidecarOK[%s] = %d, want %d (got %v)", site, got[site], ranAt, got)
}
}
for _, site := range []string{"novelfull", "lightnovelworld"} {
if _, ok := got[site]; ok {
t.Fatalf("SidecarOK names %s, which never reached the sidecar: %v", site, got)
}
}
}
// NoChapterShare reads, per Site, the share of its Series holding a
// no-chapter failure row older than the cutoff (issue #172): old rows over
// the Site's whole Series count, so a four-Series Site and a 200-Series
// Site are judged on the same scale. A fresh no-chapter row and a row of
// any other outcome do not count.
func TestNoChapterShare(t *testing.T) {
s := newTestStore(t)
for i := 1; i <= 4; i++ {
seedForCheck(t, s, fmt.Sprintf("asura:a%d", i), fmt.Sprintf("https://asurascans.com/comics/a%d", i), 0)
}
seedForCheck(t, s, "comix:c1", "https://comix.to/title/c1", 0)
seedForCheck(t, s, "comix:c2", "https://comix.to/title/c2", 0)
seedForCheck(t, s, "demonic:d1", "https://demonicscans.org/series/d1", 0)
for i := 1; i <= 200; i++ {
seedForCheck(t, s, fmt.Sprintf("novelfull:n%d", i), fmt.Sprintf("https://novelfull.com/n%d.html", i), 0)
}
const cutoff = 1000
oldNoChapter := func(site, seriesID string) {
t.Helper()
if err := s.RecordSeriesFailure(site, seriesID, "no_chapter", 100); err != nil {
t.Fatalf("seed no-chapter failure %s:%s: %v", site, seriesID, err)
}
}
// asura: three of four hold an old no-chapter row; the fourth's
// no-chapter row is fresh — it is not old, so it does not count.
oldNoChapter("asura", "a1")
oldNoChapter("asura", "a2")
oldNoChapter("asura", "a3")
if err := s.RecordSeriesFailure("asura", "a4", "no_chapter", 2000); err != nil {
t.Fatalf("seed fresh no-chapter failure: %v", err)
}
// comix: an old no-chapter row and an old errors row — the errors row
// is not a no-chapter row, so the share is 1/2, the exact boundary.
oldNoChapter("comix", "c1")
if err := s.RecordSeriesFailure("comix", "c2", "errors", 100); err != nil {
t.Fatalf("seed errors failure: %v", err)
}
// demonic and novelfull: one old no-chapter row each, against sites of
// one and two hundred Series.
oldNoChapter("demonic", "d1")
oldNoChapter("novelfull", "n1")
got, err := s.NoChapterShare(cutoff)
if err != nil {
t.Fatalf("NoChapterShare: %v", err)
}
checks := []struct {
site string
want float64
}{
{"asura", 3.0 / 4.0},
{"comix", 1.0 / 2.0},
{"demonic", 1.0 / 1.0},
{"novelfull", 1.0 / 200.0},
}
for _, c := range checks {
if got[c.site] != c.want {
t.Fatalf("NoChapterShare[%s] = %v, want %v", c.site, got[c.site], c.want)
}
}
}