28fef689ec
FaultsFrom judges three more faults at the shared twelve-hour OwnerWindow: a no-browser-route Site's refusing run (browser-backed Sites excluded), a sidecar no Lane has reached (one site-wide fault), and a Site where more than half of Series hold an old no-chapter failure row. recordPass fills the inputs from three new store reads (RefusingSince, SidecarOK, NoChapterShare), each failing independently and never failing a pass, and the clear loop forgets the empty-Site row too so a lifted sidecar-down fires again on return.
205 lines
9.2 KiB
Go
205 lines
9.2 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"bookmarkmanager/backend/internal/store"
|
|
)
|
|
|
|
// ConditionStall is the owner-notice machine word for a Lane that owed Polls,
|
|
// made none, and has nothing to say for it. The word is the message's footer
|
|
// and its suppression key; it is wire-stable. #172 declares the other three
|
|
// words (no-browser-route, sidecar-down, adapter-broken); this ticket
|
|
// declares only the stall.
|
|
const ConditionStall = "stall"
|
|
// ConditionNoBrowserRoute is the owner-notice machine word for a Site whose
|
|
// challenge refuses for longer than the owner window with no browser route
|
|
// to clear it — the 403-with-interstitial the reader maps to
|
|
// errChallengeHeld (read.go), which plain TLS cannot clear. The word is the
|
|
// message's footer and its suppression key; it is wire-stable.
|
|
const ConditionNoBrowserRoute = "no-browser-route"
|
|
|
|
// ConditionSidecarDown is the owner-notice machine word for a browser
|
|
// sidecar no Lane has reached for longer than the owner window: every
|
|
// browser-backed Lane's latest pass is a sidecar skip. The word is the
|
|
// message's footer and its suppression key; it is wire-stable, and its
|
|
// suppression row holds the empty Site (AC4).
|
|
const ConditionSidecarDown = "sidecar-down"
|
|
|
|
// ConditionAdapterBroken is the owner-notice machine word for a Site whose
|
|
// adapter stopped finding chapters: more than half of its Series hold an
|
|
// old no-chapter failure row. The word is the message's footer and its
|
|
// suppression key; it is wire-stable.
|
|
const ConditionAdapterBroken = "adapter-broken"
|
|
|
|
// OwnerWindow is the class-level staleness boundary every owner-notice
|
|
// condition measures against — the same twelve hours the Lanes page's
|
|
// "not checked in 12h" filter uses (internal/web/admin.go). Declared here
|
|
// once so #172's three conditions and the admin filters share one figure.
|
|
const OwnerWindow = 12 * time.Hour
|
|
|
|
// Fault is one condition the owner is told about, judged from durable rows
|
|
// alone. Site is "" for a fault that is not one Site's.
|
|
type Fault struct {
|
|
Condition string // one of the Condition* words
|
|
Site string
|
|
Since int64 // unix ms the episode began; the message's age
|
|
}
|
|
|
|
// FaultInput is everything the judgement reads. A struct so #172's three
|
|
// conditions can add inputs without changing either caller.
|
|
type FaultInput struct {
|
|
Passes []store.LanePass
|
|
// RefusingSince is, per Site, the unix ms when that Site's current
|
|
// unbroken run of refusing passes began, or absent when its latest pass
|
|
// did not refuse. Its zero value is an empty map, which contributes no
|
|
// fault.
|
|
RefusingSince map[string]int64
|
|
|
|
// SidecarOK is, per browser-backed Site, the unix ms of that Site's most
|
|
// recent pass that actually reached the sidecar. Zero when the pass log
|
|
// holds none — an asleep Lane never reached it and never counts as
|
|
// evidence either way. Its zero value is an empty map.
|
|
SidecarOK map[string]int64
|
|
|
|
// NoChapterShare is, per Site, the share of that Site's Series holding a
|
|
// no-chapter failure row older than the owner window. Its zero value is
|
|
// an empty map, which contributes no fault.
|
|
NoChapterShare map[string]float64
|
|
}
|
|
|
|
// FaultsFrom judges the owner-notice conditions from durable rows alone, so
|
|
// the poller and the landing page cannot disagree about what a fault is.
|
|
//
|
|
// A Lane stalls when its latest pass shows due > 0, none checked, no skip
|
|
// value and no refusal — exactly the row the mid-loop browser loss writes
|
|
// (see the comment at the outcomeUnreachable return in runLanePass), so a
|
|
// Lane that owed Polls, made none, and has nothing to say for it is a fault.
|
|
// The three #172 conditions share the same OwnerWindow boundary and the same
|
|
// fail-open shape: an input's absence contributes no fault, never a false
|
|
// one.
|
|
//
|
|
// - no-browser-route: a Site whose refusing run began before the window
|
|
// and that has no browser route to clear the challenge (AC2). Both
|
|
// refusal shapes count — the gate's skip='refusing' rows and the loop's
|
|
// refused>0 rows (the twice-refused break writes skip="") — so the run
|
|
// stays unbroken across them, and one healthy pass ends it.
|
|
// - sidecar-down: every browser-backed Site's latest pass is a sidecar
|
|
// skip — SkipSidecarDown or SkipNoFetcher, the two early returns that
|
|
// record a skip value — and each Site's most recent sidecar-reaching
|
|
// pass is older than the window (AC4). The skip clause is what keeps
|
|
// SkipAsleep out: a Lane under both wake thresholds is the commonest
|
|
// healthy state, never reached the sidecar, and would otherwise age into
|
|
// a false alarm. Emitted once, with Site "" (AC4's one row per episode).
|
|
// - adapter-broken: more than half of one Site's Series hold a no-chapter
|
|
// failure row older than the window (AC6). Strictly above half: the
|
|
// filter is the tool, and one Site change makes hundreds of rows, so a
|
|
// single failing Series never fires (AC7). The episode's age is the
|
|
// window — the old rows prove the episode is at least that old.
|
|
func FaultsFrom(in FaultInput, now time.Time) []Fault {
|
|
var faults []Fault
|
|
for _, p := range in.Passes {
|
|
if p.Due > 0 && p.Checked == 0 && p.Skip == "" && p.Refused == 0 {
|
|
faults = append(faults, Fault{Condition: ConditionStall, Site: p.Site, Since: p.RanAt})
|
|
}
|
|
}
|
|
cutoff := now.Add(-OwnerWindow).UnixMilli()
|
|
for site, since := range in.RefusingSince {
|
|
if since < cutoff && !isBrowserSite(site) {
|
|
faults = append(faults, Fault{Condition: ConditionNoBrowserRoute, Site: site, Since: since})
|
|
}
|
|
}
|
|
if since, down := sidecarDownSince(in.Passes, in.SidecarOK, cutoff); down {
|
|
faults = append(faults, Fault{Condition: ConditionSidecarDown, Site: "", Since: since})
|
|
}
|
|
for site, share := range in.NoChapterShare {
|
|
if share > 0.5 {
|
|
faults = append(faults, Fault{Condition: ConditionAdapterBroken, Site: site, Since: now.Add(-OwnerWindow).UnixMilli()})
|
|
}
|
|
}
|
|
return faults
|
|
}
|
|
|
|
// sidecarDownSince reports whether no browser Lane has reached the sidecar
|
|
// for longer than the owner window and, when it has, the last moment any
|
|
// Lane reached it. The skip clause — every browser-backed Site's latest pass
|
|
// must be SkipSidecarDown or SkipNoFetcher — keeps SkipAsleep out (see
|
|
// FaultsFrom). A Site with no pass row at all is not judged down either: a
|
|
// fresh database is not a dead sidecar.
|
|
func sidecarDownSince(passes []store.LanePass, ok map[string]int64, cutoff int64) (since int64, down bool) {
|
|
latest := make(map[string]store.LanePass, len(passes))
|
|
for _, p := range passes {
|
|
latest[p.Site] = p
|
|
}
|
|
for _, site := range browserBackedSites() {
|
|
p, found := latest[site]
|
|
if !found || (p.Skip != SkipSidecarDown && p.Skip != SkipNoFetcher) {
|
|
return 0, false
|
|
}
|
|
reached := ok[site]
|
|
if reached > 0 && reached >= cutoff {
|
|
return 0, false
|
|
}
|
|
if reached > since {
|
|
since = reached
|
|
}
|
|
}
|
|
return since, true
|
|
}
|
|
|
|
// Notifier delivers one owner notice. The poller neither retries nor queues:
|
|
// an error is logged and the suppression row left unwritten, so the next pass
|
|
// tries again while the condition holds.
|
|
type Notifier interface {
|
|
Notify(ctx context.Context, f Fault, sentence, href string) error
|
|
}
|
|
|
|
// ownerNoticeConditions is every condition this package judges, for the
|
|
// clear loop in recordPass: a condition absent from a pass's fault list
|
|
// forgets its episode, so the next occurrence sends again. #172 extends the
|
|
// list when it adds its conditions.
|
|
var ownerNoticeConditions = []string{ConditionStall, ConditionNoBrowserRoute, ConditionSidecarDown, ConditionAdapterBroken}
|
|
|
|
// noticeFor renders one fault's message: the description sentence — condition,
|
|
// age, repair — and the deep link the embed's title points at. Each condition
|
|
// provides its own wording; the stall is the only one today (issue #171).
|
|
func noticeFor(f Fault, row store.LanePass, now time.Time) (sentence, href string) {
|
|
switch f.Condition {
|
|
case ConditionStall:
|
|
return fmt.Sprintf(
|
|
"%s owed %d Polls and made none — %s; check the Lane's browser sidecar and the Site's challenge state",
|
|
f.Site, row.Due, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
|
case ConditionNoBrowserRoute:
|
|
return fmt.Sprintf(
|
|
"%s has refused for %s with no browser route — the challenge does not clear on plain TLS; redeploy or add a browser route",
|
|
f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
|
case ConditionSidecarDown:
|
|
return fmt.Sprintf(
|
|
"no browser Lane has reached the sidecar for %s — the browser sidecar is down; start or repair the browser machine",
|
|
humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
|
case ConditionAdapterBroken:
|
|
return fmt.Sprintf(
|
|
"more than half of %s's Series have failed no-chapter reads for at least %s — the Site's layout changed and the adapter is broken",
|
|
f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
|
}
|
|
return "", ""
|
|
}
|
|
|
|
// humanAge renders a duration the way an owner reads it in a message: minutes
|
|
// under an hour, then hours, then days; a stall that was just born reads
|
|
// "just now".
|
|
func humanAge(d time.Duration) string {
|
|
switch {
|
|
case d < time.Minute:
|
|
return "just now"
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%dm", int(d.Minutes()))
|
|
case d < 24*time.Hour:
|
|
return fmt.Sprintf("%dh", int(d.Hours()))
|
|
default:
|
|
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
|
}
|
|
}
|