Implements spec #137 (spec 4 of 4 from wayfinder map #114). Closes #137. Tickets: #164, #165, #166, #167, #168, #169, #170, #171, #172, #173 — all closed, landed on this branch. ## Summary - #164/#168: sixth outcome word `not_found`; per-Site completed marker predicate. - #165/#169: `poll_failures` row is the failure state; the pass remembers a Site-reported completion. - #166/#171: two new admin filters (`failing`, `unverified`); outbound owner notification + stall condition. - #167/#170: a failure names itself on the Series page; the completion hint reaches the owner and decides nothing. - #172: the other three fault conditions (no-browser-route, sidecar-down, adapter-broken) feeding the notifier. - #173: the landing verdict line shares the same `latest.FaultsFrom` judgement the notifier uses, so the page and the push cannot disagree. `cd backend && go test ./...` green on the merged branch (8 packages). Reviewed-on: #174 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #174.
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// ConditionStall is the owner-notice machine word for a Lane that owed Polls,
|
||||
// made none, and has nothing to say for it. The word is the message's footer
|
||||
// and its suppression key; it is wire-stable. #172 declares the other three
|
||||
// words (no-browser-route, sidecar-down, adapter-broken); this ticket
|
||||
// declares only the stall.
|
||||
const ConditionStall = "stall"
|
||||
// ConditionNoBrowserRoute is the owner-notice machine word for a Site whose
|
||||
// challenge refuses for longer than the owner window with no browser route
|
||||
// to clear it — the 403-with-interstitial the reader maps to
|
||||
// errChallengeHeld (read.go), which plain TLS cannot clear. The word is the
|
||||
// message's footer and its suppression key; it is wire-stable.
|
||||
const ConditionNoBrowserRoute = "no-browser-route"
|
||||
|
||||
// ConditionSidecarDown is the owner-notice machine word for a browser
|
||||
// sidecar no Lane has reached for longer than the owner window: every
|
||||
// browser-backed Lane's latest pass is a sidecar skip. The word is the
|
||||
// message's footer and its suppression key; it is wire-stable, and its
|
||||
// suppression row holds the empty Site (AC4).
|
||||
const ConditionSidecarDown = "sidecar-down"
|
||||
|
||||
// ConditionAdapterBroken is the owner-notice machine word for a Site whose
|
||||
// adapter stopped finding chapters: more than half of its Series hold an
|
||||
// old no-chapter failure row. The word is the message's footer and its
|
||||
// suppression key; it is wire-stable.
|
||||
const ConditionAdapterBroken = "adapter-broken"
|
||||
|
||||
// OwnerWindow is the class-level staleness boundary every owner-notice
|
||||
// condition measures against — the same twelve hours the Lanes page's
|
||||
// "not checked in 12h" filter uses (internal/web/admin.go). Declared here
|
||||
// once so #172's three conditions and the admin filters share one figure.
|
||||
const OwnerWindow = 12 * time.Hour
|
||||
|
||||
// Fault is one condition the owner is told about, judged from durable rows
|
||||
// alone. Site is "" for a fault that is not one Site's.
|
||||
type Fault struct {
|
||||
Condition string // one of the Condition* words
|
||||
Site string
|
||||
Since int64 // unix ms the episode began; the message's age
|
||||
}
|
||||
|
||||
// FaultInput is everything the judgement reads. A struct so #172's three
|
||||
// conditions can add inputs without changing either caller.
|
||||
type FaultInput struct {
|
||||
Passes []store.LanePass
|
||||
// RefusingSince is, per Site, the unix ms when that Site's current
|
||||
// unbroken run of refusing passes began, or absent when its latest pass
|
||||
// did not refuse. Its zero value is an empty map, which contributes no
|
||||
// fault.
|
||||
RefusingSince map[string]int64
|
||||
|
||||
// SidecarOK is, per browser-backed Site, the unix ms of that Site's most
|
||||
// recent pass that actually reached the sidecar. Zero when the pass log
|
||||
// holds none — an asleep Lane never reached it and never counts as
|
||||
// evidence either way. Its zero value is an empty map.
|
||||
SidecarOK map[string]int64
|
||||
|
||||
// NoChapterShare is, per Site, the share of that Site's Series holding a
|
||||
// no-chapter failure row older than the owner window. Its zero value is
|
||||
// an empty map, which contributes no fault.
|
||||
NoChapterShare map[string]float64
|
||||
}
|
||||
|
||||
// FaultsFrom judges the owner-notice conditions from durable rows alone, so
|
||||
// the poller and the landing page cannot disagree about what a fault is.
|
||||
//
|
||||
// A Lane stalls when its latest pass shows due > 0, none checked, no skip
|
||||
// value and no refusal — exactly the row the mid-loop browser loss writes
|
||||
// (see the comment at the outcomeUnreachable return in runLanePass), so a
|
||||
// Lane that owed Polls, made none, and has nothing to say for it is a fault.
|
||||
// The three #172 conditions share the same OwnerWindow boundary and the same
|
||||
// fail-open shape: an input's absence contributes no fault, never a false
|
||||
// one.
|
||||
//
|
||||
// - no-browser-route: a Site whose refusing run began before the window
|
||||
// and that has no browser route to clear the challenge (AC2). Both
|
||||
// refusal shapes count — the gate's skip='refusing' rows and the loop's
|
||||
// refused>0 rows (the twice-refused break writes skip="") — so the run
|
||||
// stays unbroken across them, and one healthy pass ends it.
|
||||
// - sidecar-down: every browser-backed Site's latest pass is a sidecar
|
||||
// skip — SkipSidecarDown or SkipNoFetcher, the two early returns that
|
||||
// record a skip value — and each Site's most recent sidecar-reaching
|
||||
// pass is older than the window (AC4). The skip clause is what keeps
|
||||
// SkipAsleep out: a Lane under both wake thresholds is the commonest
|
||||
// healthy state, never reached the sidecar, and would otherwise age into
|
||||
// a false alarm. Emitted once, with Site "" (AC4's one row per episode).
|
||||
// - adapter-broken: more than half of one Site's Series hold a no-chapter
|
||||
// failure row older than the window (AC6). Strictly above half: the
|
||||
// filter is the tool, and one Site change makes hundreds of rows, so a
|
||||
// single failing Series never fires (AC7). The episode's age is the
|
||||
// window — the old rows prove the episode is at least that old.
|
||||
func FaultsFrom(in FaultInput, now time.Time) []Fault {
|
||||
var faults []Fault
|
||||
for _, p := range in.Passes {
|
||||
if p.Due > 0 && p.Checked == 0 && p.Skip == "" && p.Refused == 0 {
|
||||
faults = append(faults, Fault{Condition: ConditionStall, Site: p.Site, Since: p.RanAt})
|
||||
}
|
||||
}
|
||||
cutoff := now.Add(-OwnerWindow).UnixMilli()
|
||||
for site, since := range in.RefusingSince {
|
||||
if since < cutoff && !isBrowserSite(site) {
|
||||
faults = append(faults, Fault{Condition: ConditionNoBrowserRoute, Site: site, Since: since})
|
||||
}
|
||||
}
|
||||
if since, down := sidecarDownSince(in.Passes, in.SidecarOK, cutoff); down {
|
||||
faults = append(faults, Fault{Condition: ConditionSidecarDown, Site: "", Since: since})
|
||||
}
|
||||
for site, share := range in.NoChapterShare {
|
||||
if share > 0.5 {
|
||||
faults = append(faults, Fault{Condition: ConditionAdapterBroken, Site: site, Since: now.Add(-OwnerWindow).UnixMilli()})
|
||||
}
|
||||
}
|
||||
return faults
|
||||
}
|
||||
|
||||
// sidecarDownSince reports whether no browser Lane has reached the sidecar
|
||||
// for longer than the owner window and, when it has, the last moment any
|
||||
// Lane reached it. The skip clause — every browser-backed Site's latest pass
|
||||
// must be SkipSidecarDown or SkipNoFetcher — keeps SkipAsleep out (see
|
||||
// FaultsFrom). A Site with no pass row at all is not judged down either: a
|
||||
// fresh database is not a dead sidecar.
|
||||
func sidecarDownSince(passes []store.LanePass, ok map[string]int64, cutoff int64) (since int64, down bool) {
|
||||
latest := make(map[string]store.LanePass, len(passes))
|
||||
for _, p := range passes {
|
||||
latest[p.Site] = p
|
||||
}
|
||||
for _, site := range browserBackedSites() {
|
||||
p, found := latest[site]
|
||||
if !found || (p.Skip != SkipSidecarDown && p.Skip != SkipNoFetcher) {
|
||||
return 0, false
|
||||
}
|
||||
reached := ok[site]
|
||||
if reached > 0 && reached >= cutoff {
|
||||
return 0, false
|
||||
}
|
||||
if reached > since {
|
||||
since = reached
|
||||
}
|
||||
}
|
||||
// No Lane's retained pass log shows a sidecar reach: the honest age is
|
||||
// the window itself — "at least twelve hours" — not the epoch, which
|
||||
// humanAge would render as tens of thousands of days.
|
||||
if since == 0 {
|
||||
since = cutoff
|
||||
}
|
||||
return since, true
|
||||
}
|
||||
|
||||
// Notifier delivers one owner notice. The poller neither retries nor queues:
|
||||
// an error is logged and the suppression row left unwritten, so the next pass
|
||||
// tries again while the condition holds.
|
||||
type Notifier interface {
|
||||
Notify(ctx context.Context, f Fault, sentence, href string) error
|
||||
}
|
||||
|
||||
// ownerNoticeConditions is every condition this package judges, for the
|
||||
// clear loop in recordPass: a condition absent from a pass's fault list
|
||||
// forgets its episode, so the next occurrence sends again. #172 extends the
|
||||
// list when it adds its conditions.
|
||||
var ownerNoticeConditions = []string{ConditionStall, ConditionNoBrowserRoute, ConditionSidecarDown, ConditionAdapterBroken}
|
||||
|
||||
// noticeFor renders one fault's message: the description sentence — condition,
|
||||
// age, repair — and the deep link the embed's title points at. Each condition
|
||||
// provides its own wording; the stall is the only one today (issue #171).
|
||||
func noticeFor(f Fault, row store.LanePass, now time.Time) (sentence, href string) {
|
||||
switch f.Condition {
|
||||
case ConditionStall:
|
||||
return fmt.Sprintf(
|
||||
"%s owed %d Polls and made none — %s; check the Lane's browser sidecar and the Site's challenge state",
|
||||
f.Site, row.Due, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
||||
case ConditionNoBrowserRoute:
|
||||
return fmt.Sprintf(
|
||||
"%s has refused for %s with no browser route — the challenge does not clear on plain TLS; redeploy or add a browser route",
|
||||
f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
||||
case ConditionSidecarDown:
|
||||
return fmt.Sprintf(
|
||||
"no browser Lane has reached the sidecar for %s — the browser sidecar is down; start or repair the browser machine",
|
||||
humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
||||
case ConditionAdapterBroken:
|
||||
return fmt.Sprintf(
|
||||
"more than half of %s's Series have failed no-chapter reads for at least %s — the Site's layout changed and the adapter is broken",
|
||||
f.Site, humanAge(now.Sub(time.UnixMilli(f.Since)))), "/admin/lanes"
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// humanAge renders a duration the way an owner reads it in a message: minutes
|
||||
// under an hour, then hours, then days; a stall that was just born reads
|
||||
// "just now".
|
||||
func humanAge(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Minute:
|
||||
return "just now"
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||
case d < 24*time.Hour:
|
||||
return fmt.Sprintf("%dh", int(d.Hours()))
|
||||
default:
|
||||
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,10 @@ type Poller struct {
|
||||
// CoverBytesFetch is optional; it handles plain-TLS sources through the
|
||||
// same failure-isolated prefetch path.
|
||||
CoverBytesFetch CoverBytesFetcher
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
// Notify delivers owner notices. Nil disables the whole path (issue #171):
|
||||
// the poller is not the place a missing webhook becomes an error.
|
||||
Notify Notifier
|
||||
Now func() time.Time // injected so tests can freeze it
|
||||
// eligibleCount reports how many of a Site's Series are eligible for
|
||||
// polling, defaulting to Store.EligibleSeriesCount. Injected so tests can
|
||||
// fail the count alone: the eligible query shares the due query's tables,
|
||||
@@ -273,7 +276,9 @@ const (
|
||||
// (issue #141). The classification the read already makes is counted, never a
|
||||
// second taxonomy: refused is the Site holding a challenge, unreachable the
|
||||
// browser interrupting, noChapter a 200 with real HTML but no chapter links,
|
||||
// unfetchable the host pin or a missing fetcher, and errors everything else.
|
||||
// unfetchable the host pin or a missing fetcher, notFound a 4xx other than
|
||||
// the 403 refusal — the Site answered with a client status — and errors
|
||||
// everything else.
|
||||
type readOutcome int
|
||||
|
||||
const (
|
||||
@@ -283,14 +288,33 @@ const (
|
||||
outcomeNoChapter
|
||||
outcomeUnfetchable
|
||||
outcomeError
|
||||
outcomeNotFound
|
||||
)
|
||||
// word returns the wire spelling this outcome stores in poll_failures — the
|
||||
// same strings the pass log's columns use (C1, issue #164). Success, refusal
|
||||
// and browser loss return "" so recordFailure's "no statement" case is one
|
||||
// return: a challenge or a lost sidecar is no evidence about any particular
|
||||
// Series (ADR-0016).
|
||||
func (o readOutcome) word() string {
|
||||
switch o {
|
||||
case outcomeNotFound:
|
||||
return "not_found"
|
||||
case outcomeNoChapter:
|
||||
return "no_chapter"
|
||||
case outcomeUnfetchable:
|
||||
return "unfetchable"
|
||||
case outcomeError:
|
||||
return "errors"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// outcomeCounts are the five named outcome counts of one pass. A success
|
||||
// count is derived, never stored: checked minus the four, with unreachable
|
||||
// excluded because the sidecar-loss path returns before the checked counter
|
||||
// increments (issue #141).
|
||||
// outcomeCounts are the six named outcome counts of one pass. A success
|
||||
// count is derived, never stored: checked minus the five named failures,
|
||||
// with unreachable excluded because the sidecar-loss path returns before the
|
||||
// checked counter increments (issue #141).
|
||||
type outcomeCounts struct {
|
||||
refused, unreachable, noChapter, unfetchable, errors int
|
||||
refused, unreachable, noChapter, unfetchable, errors, notFound int
|
||||
}
|
||||
|
||||
func (c *outcomeCounts) add(o readOutcome) {
|
||||
@@ -303,6 +327,8 @@ func (c *outcomeCounts) add(o readOutcome) {
|
||||
c.noChapter++
|
||||
case outcomeUnfetchable:
|
||||
c.unfetchable++
|
||||
case outcomeNotFound:
|
||||
c.notFound++
|
||||
case outcomeError:
|
||||
c.errors++
|
||||
}
|
||||
@@ -334,7 +360,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.
|
||||
// carries the previous pass's forward inside recordPass.
|
||||
fig := passFigures{}
|
||||
rec := passRecord{site: name, ranAt: now.UnixMilli()}
|
||||
defer func() { p.recordPass(rec, fig) }()
|
||||
defer func() { p.recordPass(ctx, rec, fig) }()
|
||||
|
||||
// One Lane row read at the top of a pass, serving two gates (issue #139).
|
||||
// Both stamps outlive our process, so the gates read the durable row
|
||||
@@ -451,6 +477,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.
|
||||
}
|
||||
}
|
||||
outcome := p.checkOne(ctx, sr)
|
||||
p.recordFailure(sr, outcome)
|
||||
if outcome == outcomeUnreachable {
|
||||
// The mid-loop browser loss writes an empty skip on purpose: the
|
||||
// pass returns before the checked counter increments, so its row
|
||||
@@ -503,8 +530,9 @@ type passFigures struct {
|
||||
// pass's due, gap, clamped and checked forward rather than stating zeroes it
|
||||
// did not measure; the skip column says why it declined, so the zeroes that
|
||||
// remain (due-query, no-fetcher) read as explanations rather than
|
||||
// measurements.
|
||||
func (p *Poller) recordPass(rec passRecord, fig passFigures) {
|
||||
// measurements. Once the row is durable, the owner-notice judgement runs
|
||||
// beside it (issue #171).
|
||||
func (p *Poller) recordPass(ctx context.Context, rec passRecord, fig passFigures) {
|
||||
row := store.LanePass{
|
||||
Site: rec.site,
|
||||
RanAt: rec.ranAt,
|
||||
@@ -517,6 +545,7 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) {
|
||||
Unreachable: rec.counts.unreachable,
|
||||
NoChapter: rec.counts.noChapter,
|
||||
Unfetchable: rec.counts.unfetchable,
|
||||
NotFound: rec.counts.notFound,
|
||||
Errors: rec.counts.errors,
|
||||
}
|
||||
if row.GapMS == 0 {
|
||||
@@ -532,7 +561,111 @@ func (p *Poller) recordPass(rec passRecord, fig passFigures) {
|
||||
}
|
||||
if err := p.Store.RecordLanePass(row, rec.ranAt-lanePassRetention.Milliseconds()); err != nil {
|
||||
log.Printf("latest poll %s: record lane pass: %v", rec.site, err)
|
||||
return
|
||||
}
|
||||
p.ownerNotices(ctx, row)
|
||||
}
|
||||
|
||||
// ownerNotices judges the owner-notice conditions for the pass just recorded
|
||||
// and fires (issue #171). It sits in recordPass because that deferred call is
|
||||
// the one place every return path passes through: three of the four
|
||||
// conditions occur on early returns and the success path can never see them.
|
||||
// The judgement reads the whole pass log plus three derived reads — the
|
||||
// other Lanes' latest passes, each Site's refusing-run start, its last
|
||||
// sidecar-reaching pass, and its no-chapter share — so one pass judges every
|
||||
// condition (issue #172). Per fault: NoticeSent → send → MarkNoticeSent, so
|
||||
// a fault lasting a month sends one message, not one per pass; a condition
|
||||
// absent from this pass's fault list forgets its episode, so the next
|
||||
// occurrence sends again. Everything here is best-effort: a failed send, a
|
||||
// failed store read and a failed notice write are all logged and never
|
||||
// change the pass's outcome counts or its return value. The clear runs even
|
||||
// when Notify is nil, so a deployment that turns the webhook off does not
|
||||
// leave stale rows that suppress the first real notice after it is turned
|
||||
// back on.
|
||||
func (p *Poller) ownerNotices(ctx context.Context, row store.LanePass) {
|
||||
now := p.Now()
|
||||
in := FaultInput{Passes: []store.LanePass{row}}
|
||||
// Each read fails independently: a failure logs and contributes no fault,
|
||||
// never a false one.
|
||||
if passes, err := p.Store.LatestLanePasses(); err != nil {
|
||||
log.Printf("latest poll %s: latest lane passes: %v", row.Site, err)
|
||||
} else {
|
||||
in.Passes = passes
|
||||
}
|
||||
if since, err := p.Store.RefusingSince(now.UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %s: refusing since: %v", row.Site, err)
|
||||
} else {
|
||||
in.RefusingSince = since
|
||||
}
|
||||
if ok, err := p.Store.SidecarOK(browserBackedSites()); err != nil {
|
||||
log.Printf("latest poll %s: sidecar ok: %v", row.Site, err)
|
||||
} else {
|
||||
in.SidecarOK = ok
|
||||
}
|
||||
if share, err := p.Store.NoChapterShare(now.Add(-OwnerWindow).UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %s: no-chapter share: %v", row.Site, err)
|
||||
} else {
|
||||
in.NoChapterShare = share
|
||||
}
|
||||
faults := FaultsFrom(in, now)
|
||||
bySite := make(map[string]store.LanePass, len(in.Passes))
|
||||
for _, pass := range in.Passes {
|
||||
bySite[pass.Site] = pass
|
||||
}
|
||||
for _, f := range faults {
|
||||
if p.Notify == nil {
|
||||
continue
|
||||
}
|
||||
sent, err := p.Store.NoticeSent(f.Condition, f.Site)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %s: notice sent: %v", row.Site, err)
|
||||
continue
|
||||
}
|
||||
if sent {
|
||||
continue
|
||||
}
|
||||
// The fault's own Site's pass renders its sentence — a stall judged
|
||||
// from another Lane's pass must not quote this pass's figures.
|
||||
pass, ok := bySite[f.Site]
|
||||
if !ok {
|
||||
pass = row
|
||||
}
|
||||
sentence, href := noticeFor(f, pass, now)
|
||||
if err := p.Notify.Notify(ctx, f, sentence, href); err != nil {
|
||||
// The stamp stays unset: no queue, no backoff — the condition is
|
||||
// durable, so the next pass tries again while it holds.
|
||||
log.Printf("latest poll %s: owner notice %s: %v", row.Site, f.Condition, err)
|
||||
continue
|
||||
}
|
||||
if err := p.Store.MarkNoticeSent(f.Condition, f.Site, now.UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %s: mark notice sent: %v", row.Site, err)
|
||||
}
|
||||
}
|
||||
for _, cond := range ownerNoticeConditions {
|
||||
if !hasFault(faults, cond, row.Site) {
|
||||
if err := p.Store.ClearNotice(cond, row.Site); err != nil {
|
||||
log.Printf("latest poll %s: clear owner notice: %v", row.Site, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// sidecar-down suppresses under the empty Site — one row across all
|
||||
// browser Lanes (AC4) — so clear that row when it is absent from this
|
||||
// pass's fault list, and a lifted sidecar fires again when it returns.
|
||||
if !hasFault(faults, ConditionSidecarDown, "") {
|
||||
if err := p.Store.ClearNotice(ConditionSidecarDown, ""); err != nil {
|
||||
log.Printf("latest poll %s: clear owner notice: %v", row.Site, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hasFault reports whether faults hold the given condition for the site.
|
||||
func hasFault(faults []Fault, condition, site string) bool {
|
||||
for _, f := range faults {
|
||||
if f.Condition == condition && f.Site == site {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// countEligible routes the eligible count through the test seam when one is
|
||||
@@ -607,13 +740,35 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D
|
||||
}
|
||||
return oldest
|
||||
}
|
||||
// recordFailure keeps one Series' failure row in step with its read
|
||||
// (ADR-0016): a failure word is upserted, a successful read deletes the row,
|
||||
// and refused or unreachable issue no statement at all. Called for every
|
||||
// outcome from the pass loop, so the four failure words and the success path
|
||||
// share one write point, and a forced Poll that reads the page clears through
|
||||
// the ordinary success path — no branch of its own. Log a store failure and
|
||||
// carry on: this is best-effort, and no single bad Series may stall a Lane.
|
||||
func (p *Poller) recordFailure(sr store.Series, outcome readOutcome) {
|
||||
if outcome == outcomeSuccess {
|
||||
if err := p.Store.ClearSeriesFailure(sr.Site, sr.SeriesID); err != nil {
|
||||
log.Printf("latest poll %q: clear failure: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
word := outcome.word()
|
||||
if word == "" {
|
||||
return
|
||||
}
|
||||
if err := p.Store.RecordSeriesFailure(sr.Site, sr.SeriesID, word, p.Now().UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %q: record failure: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// checkOne re-checks one series. Every failure path here is "log and move on":
|
||||
// the poller is a best-effort enhancement, and no single bad series may stall a
|
||||
// Lane or take down the process. The returned outcome classifies the read for
|
||||
// the pass row (issue #141), so the Lane can count a refusal, a lost browser,
|
||||
// a chapter-less page, an unfetchable address or a transport error without
|
||||
// re-deriving the taxonomy.
|
||||
// a chapter-less page, an unfetchable address, a missing page or a transport
|
||||
// error without re-deriving the taxonomy.
|
||||
func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
@@ -657,6 +812,9 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut
|
||||
if errors.Is(err, errBrowserInterrupted) {
|
||||
return outcomeUnreachable
|
||||
}
|
||||
if errors.Is(err, errNotFound) {
|
||||
return outcomeNotFound
|
||||
}
|
||||
return outcomeError
|
||||
}
|
||||
// A legacy cover source is healed independently of the page read.
|
||||
@@ -669,6 +827,25 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOut
|
||||
} else {
|
||||
p.fillBlankCover(ctx, sr, facts.Cover)
|
||||
}
|
||||
|
||||
// Learned from the successful read: the write sits after the error switch
|
||||
// (a refused, unreachable or errored read reaches nothing) and before the
|
||||
// returns below — a completed page whose chapter number did not change
|
||||
// still has to write. The transition is zero-versus-nonzero, not the
|
||||
// stamp's value: a Series still completed keeps its original stamp, so the
|
||||
// age #170 prints is "since the Site first said so"; one that stopped
|
||||
// being completed is zeroed.
|
||||
stamp := int64(0)
|
||||
if facts.SiteCompleted {
|
||||
stamp = p.Now().UnixMilli()
|
||||
}
|
||||
if (sr.SiteCompletedAt == 0) != (stamp == 0) {
|
||||
if err := p.Store.SetSiteCompletedAt(sr.Site, sr.SeriesID, stamp); err != nil {
|
||||
// Best-effort, like every poller write: never change the outcome
|
||||
// word the pass counts.
|
||||
log.Printf("latest poll %q: set site completed: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
if !facts.HasLatest {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a rest instead of hot-looping.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,15 +6,19 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// seriesRead carries the two facts the poll and the acquirer both extract
|
||||
// from a series page. Persistence, stamps and scheduling stay with the
|
||||
// callers, so the policies that keep the two flows distinct (stamp order,
|
||||
// rests) are not swallowed by the module.
|
||||
// seriesRead carries the facts the poll and the acquirer both extract from a
|
||||
// series page. Persistence, stamps and scheduling stay with the callers, so
|
||||
// the policies that keep the two flows distinct (stamp order, rests) are not
|
||||
// swallowed by the module.
|
||||
type seriesRead struct {
|
||||
Latest latestChapter
|
||||
HasLatest bool
|
||||
Cover string
|
||||
HasCover bool
|
||||
// SiteCompleted is whether the Site's own completed value was on the page.
|
||||
// A challenge body and a redesign both read false — an absent hint, never
|
||||
// a claim (issue #168).
|
||||
SiteCompleted bool
|
||||
// BodyLen is the fetched body's length, surfaced because the no-chapter
|
||||
// log uses it to tell a markup change from a body the size cap cut short.
|
||||
BodyLen int
|
||||
@@ -25,15 +29,18 @@ type seriesRead struct {
|
||||
// errChallengeHeld (browser.go) is the outcome of a Site that answered with
|
||||
// its interstitial — status 403 (cf-mitigated) or a challenge page body — and
|
||||
// is how a Lane tells a refusal from an ordinary failure (issue #100).
|
||||
// errNotFound marks any other 4xx: the page is gone — a fact the owner can act
|
||||
// on — as distinct from a Site or database that is merely unwell (issue #164).
|
||||
var (
|
||||
errNotFetchable = errors.New("series url not fetchable")
|
||||
errNoFetcher = errors.New("no fetcher for site")
|
||||
errNotFound = errors.New("series page not found")
|
||||
)
|
||||
|
||||
// readSeriesPage performs the series-page read the poll and the acquirer have
|
||||
// in common: gate the address, choose the route, fetch the page, extract the
|
||||
// Latest Chapter and the Cover address. It persists nothing and stamps
|
||||
// nothing.
|
||||
// Latest Chapter, the Cover address and the Site's completed value. It
|
||||
// persists nothing and stamps nothing.
|
||||
//
|
||||
// series_url arrives in a client-supplied PUT body (PUT /bookmarks/{key}
|
||||
// accepts any string), so the gate is not an optimisation against burning a
|
||||
@@ -59,6 +66,9 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errChallengeHeld, seriesURL, status)
|
||||
}
|
||||
if status != 200 {
|
||||
if status >= 400 && status < 500 {
|
||||
return seriesRead{}, fmt.Errorf("%w: fetch %s: status %d", errNotFound, seriesURL, status)
|
||||
}
|
||||
return seriesRead{}, fmt.Errorf("fetch %s: status %d", seriesURL, status)
|
||||
}
|
||||
if isInterstitial(body) {
|
||||
@@ -69,5 +79,12 @@ func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fe
|
||||
}
|
||||
latest, hasLatest := latestChapterFrom(site, seriesURL, body)
|
||||
cover, hasCover := coverFrom(site, seriesURL, body)
|
||||
return seriesRead{Latest: latest, HasLatest: hasLatest, Cover: cover, HasCover: hasCover, BodyLen: len(body)}, nil
|
||||
return seriesRead{
|
||||
Latest: latest,
|
||||
HasLatest: hasLatest,
|
||||
Cover: cover,
|
||||
HasCover: hasCover,
|
||||
SiteCompleted: siteCompletedFrom(site, seriesURL, body),
|
||||
BodyLen: len(body),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -22,10 +22,10 @@ type latestChapter struct {
|
||||
|
||||
// site answers the fixed questions every series-page read asks of its Site
|
||||
// (ADR-0009): the host its addresses must carry, how to find the Latest
|
||||
// Chapter and the Cover address in a body, and — for a Site behind a
|
||||
// JavaScript challenge — how to read its payload from a cleared tab. One
|
||||
// entry describes everything about one Site, and nowhere else gets to compare
|
||||
// the site string.
|
||||
// Chapter and the Cover address in a body, whether the Site calls the work
|
||||
// completed, and — for a Site behind a JavaScript challenge — how to read its
|
||||
// payload from a cleared tab. One entry describes everything about one Site,
|
||||
// and nowhere else gets to compare the site string.
|
||||
type site struct {
|
||||
// Host is the exact hostname a series_url for this Site must carry.
|
||||
Host string
|
||||
@@ -33,6 +33,10 @@ type site struct {
|
||||
LatestChapter func(seriesURL, body string) (latestChapter, bool)
|
||||
// Cover finds the Cover address in a fetched body.
|
||||
Cover func(seriesURL, body string) (string, bool)
|
||||
// Completed reports whether this body carries the Site's own completed
|
||||
// value. False for a body that carries any other value, and false for a
|
||||
// failed extraction — never an error and never a third state.
|
||||
Completed func(seriesURL, body string) bool
|
||||
// Rest is how long a Series of this Site rests between Polls.
|
||||
Rest time.Duration
|
||||
// Gap is the Lane's strictest pace: at least one second must pass between
|
||||
@@ -299,34 +303,42 @@ func kaganeCoverURL(body string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func comixCoverURL(seriesURL, body string) string {
|
||||
// comixDetailQuery returns the ["manga","detail","<id>"] query entry of
|
||||
// comix's initial-data JSON, or nil. The cover and completed reads share the
|
||||
// scoped lookup so a "recommended" strip entry can never contribute either
|
||||
// answer.
|
||||
func comixDetailQuery(seriesURL, body string) json.RawMessage {
|
||||
id, ok := comixSeriesID(seriesURL)
|
||||
if !ok {
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
data := comixInitialDataRe.FindStringSubmatch(body)
|
||||
if data == nil {
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
var state struct {
|
||||
Queries map[string]json.RawMessage `json:"queries"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(data[1]), &state); err != nil {
|
||||
return nil
|
||||
}
|
||||
return state.Queries[`["manga","detail","`+id+`"]`]
|
||||
}
|
||||
|
||||
func comixCoverURL(seriesURL, body string) string {
|
||||
detail := comixDetailQuery(seriesURL, body)
|
||||
if len(detail) == 0 {
|
||||
return ""
|
||||
}
|
||||
raw := state.Queries[`["manga","detail","`+id+`"]`]
|
||||
if len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
var detail struct {
|
||||
var entry struct {
|
||||
Poster struct {
|
||||
Medium string `json:"medium"`
|
||||
} `json:"poster"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &detail); err != nil {
|
||||
if err := json.Unmarshal(detail, &entry); err != nil {
|
||||
return ""
|
||||
}
|
||||
return publishedCoverURL(detail.Poster.Medium)
|
||||
return publishedCoverURL(entry.Poster.Medium)
|
||||
}
|
||||
|
||||
// ogImageCover reads the og:image metadata shared by asura, demonic and
|
||||
@@ -360,6 +372,87 @@ func coverFrom(site, seriesURL, body string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// asuraStatusRe matches the status value inside the escaped astro-island
|
||||
// props blob, in both the " form the served document carries and the "
|
||||
// form a decoded copy would. "completed" is the only true value: "dropped"
|
||||
// is scanlation editorial (the work itself continues elsewhere) and hiatus is
|
||||
// its own value.
|
||||
var asuraStatusRe = regexp.MustCompile(`(?:"|")status(?:"|"):\[0,(?:"|")completed(?:"|")\]`)
|
||||
|
||||
func asuraCompleted(_, body string) bool {
|
||||
return asuraStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// demonicStatusRe matches the info block's status pair: a Status label <li>
|
||||
// immediately followed by the value <li>. The site's whole status vocabulary
|
||||
// is {Ongoing, Completed} (its advanced-search status filter), so the literal
|
||||
// Completed value is the entire signal.
|
||||
var demonicStatusRe = regexp.MustCompile(`<li[^>]*>\s*Status\s*</li>\s*<li[^>]*>\s*Completed\s*</li>`)
|
||||
|
||||
func demonicCompleted(_, body string) bool {
|
||||
return demonicStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// comixCompleted reads "status" from the scoped detail entry only;
|
||||
// "finished" is the completed value, and on_hiatus and discontinued are
|
||||
// distinct values.
|
||||
func comixCompleted(seriesURL, body string) bool {
|
||||
detail := comixDetailQuery(seriesURL, body)
|
||||
if len(detail) == 0 {
|
||||
return false
|
||||
}
|
||||
var entry struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
if err := json.Unmarshal(detail, &entry); err != nil {
|
||||
return false
|
||||
}
|
||||
return entry.Status == "finished"
|
||||
}
|
||||
|
||||
// kaganeCompleted reads publication_status only: upload_status is the
|
||||
// release's state, and the two provably diverge ('Cause Calypso Can,
|
||||
// 2026-08-19: publication Ongoing, upload Hiatus), so a Completed upload
|
||||
// must never read as a Completed work.
|
||||
func kaganeCompleted(_, body string) bool {
|
||||
var series struct {
|
||||
PublicationStatus string `json:"publication_status"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &series); err != nil {
|
||||
return false
|
||||
}
|
||||
return series.PublicationStatus == "Completed"
|
||||
}
|
||||
|
||||
// novelfullStatusRe matches the info panel's status link. The page's whole
|
||||
// status vocabulary is {Ongoing, Completed} (the "OnGoing" spelling aliases
|
||||
// "Ongoing" on the taxonomy), so the Completed href is the signal.
|
||||
var novelfullStatusRe = regexp.MustCompile(`href="/status/Completed"`)
|
||||
|
||||
func novelfullCompleted(_, body string) bool {
|
||||
return novelfullStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// lnwCompleted matches creativeWorkStatus in the head's JSON-LD block. The
|
||||
// whole body is scanned and the comment marker is not required, unlike
|
||||
// lnwLatestChapter: the status block sits ahead of the visitor-writable
|
||||
// thread, and hiatus maps to a distinct PotentialActionStatus value.
|
||||
var lnwStatusRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`)
|
||||
|
||||
func lnwCompleted(_, body string) bool {
|
||||
return lnwStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// siteCompletedFrom reports whether the Site calls this work completed, via
|
||||
// the Site's registry entry. False for an unknown site, a challenge body and
|
||||
// a redesign alike: an absent hint, never a claim.
|
||||
func siteCompletedFrom(site, seriesURL, body string) bool {
|
||||
if fn := sites[site].Completed; fn != nil {
|
||||
return fn(seriesURL, body)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// metaContent returns the content of the first <meta> whose attrName is
|
||||
// attrValue. It keeps scanning after an empty match so a later published cover
|
||||
// is not hidden by an empty tag.
|
||||
@@ -449,6 +542,7 @@ var sites = map[string]site{
|
||||
Host: "asurascans.com",
|
||||
LatestChapter: asuraLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Completed: asuraCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
@@ -456,6 +550,7 @@ var sites = map[string]site{
|
||||
Host: "demonicscans.org",
|
||||
LatestChapter: demonicLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Completed: demonicCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
@@ -463,6 +558,7 @@ var sites = map[string]site{
|
||||
Host: "comix.to",
|
||||
LatestChapter: comixLatestChapter,
|
||||
Cover: comixCoverEntry,
|
||||
Completed: comixCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
@@ -479,6 +575,7 @@ var sites = map[string]site{
|
||||
Host: "kagane.to",
|
||||
LatestChapter: kaganeLatestChapter,
|
||||
Cover: kaganeCoverEntry,
|
||||
Completed: kaganeCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
@@ -493,6 +590,7 @@ var sites = map[string]site{
|
||||
Host: "novelfull.com",
|
||||
LatestChapter: novelfullLatestChapter,
|
||||
Cover: novelfullCoverEntry,
|
||||
Completed: novelfullCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
@@ -507,6 +605,7 @@ var sites = map[string]site{
|
||||
Host: "lightnovelworld.net",
|
||||
LatestChapter: lnwLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Completed: lnwCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
@@ -524,6 +623,12 @@ func SiteNames() []string {
|
||||
return names
|
||||
}
|
||||
|
||||
// BrowserBackedSites is derived from the registry: the Sites whose pages are
|
||||
// read through the browser sidecar. Sorted so callers that range it (the
|
||||
// browser fetcher's dispatch) see a stable order instead of map-iteration
|
||||
// noise. Exported so the web layer shares the same set the poller does.
|
||||
func BrowserBackedSites() []string { return browserBackedSites() }
|
||||
|
||||
// browserBackedSites is derived from the registry: the Sites whose pages are
|
||||
// read through the browser sidecar. Sorted so callers that range it (the
|
||||
// browser fetcher's dispatch) see a stable order instead of map-iteration
|
||||
|
||||
@@ -188,6 +188,104 @@ const lnwSeriesFixture = `
|
||||
</div>
|
||||
`
|
||||
|
||||
// Completed-marker fixtures, trimmed from the live pages fetched 2026-08-22
|
||||
// for the verification step below. Selector-removed rows delete the marker
|
||||
// from the consts and must answer false.
|
||||
|
||||
// Trimmed from https://asurascans.com/comics/solo-leveling (301 to
|
||||
// /comics/solo-leveling-b60d532c) fetched 2026-08-22 by a curl probe from
|
||||
// this machine. The astro-island props are HTML-escaped in the served
|
||||
// document, so the quotes arrive as ".
|
||||
const asuraCompletedFixture = `
|
||||
"bookmarkCount":[0,39128],"status":[0,"completed"],"type":[0,"manhwa"]
|
||||
`
|
||||
|
||||
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
|
||||
// fetched 2026-08-22 by a curl probe.
|
||||
const asuraOngoingFixture = `
|
||||
"bookmarkCount":[0,54027],"status":[0,"ongoing"],"type":[0,"manhwa"]
|
||||
`
|
||||
|
||||
// Trimmed from https://demonicscans.org/manga/Solo-Leveling fetched
|
||||
// 2026-08-22 by a curl probe. The info block is sloppy: bare <li>s inside a
|
||||
// <div>, label and value as a sibling pair.
|
||||
const demonicCompletedFixture = `
|
||||
<div class="flex flex-row">
|
||||
<li style="width:150px;color:#b2b2b2;">Status</li>
|
||||
<li>Completed</li>
|
||||
</div>
|
||||
`
|
||||
|
||||
// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer
|
||||
// fetched 2026-08-22 by a curl probe. Same block, ongoing value.
|
||||
const demonicOngoingFixture = `
|
||||
<div class="flex flex-row">
|
||||
<li style="width:150px;color:#b2b2b2;">Status</li>
|
||||
<li>Ongoing</li>
|
||||
</div>
|
||||
`
|
||||
|
||||
// Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 by a
|
||||
// cleared Chrome tab (the CDP sidecar: the Series URL is fetched in-tab, the
|
||||
// same server-rendered payload the poll reads). The served page escapes the
|
||||
// query map keys as \u0022; the fixture carries the decoded form, which
|
||||
// parses to the same key. The recommended strip on this very page carries a
|
||||
// finished entry; only the ["manga","detail","q77m"] entry counts.
|
||||
const comixCompletedFixture = `<script type="application/json" id="initial-data">{"queries":{"[\"manga\",\"recommended\",\"q77m\",1]":{"items":[{"hid":"pz65","title":"Wangan Midnight: C1 Runner","status":"finished"}]},"[\"manga\",\"detail\",\"q77m\"]":{"id":13211,"hid":"q77m","title":"Countach","status":"finished","originalLanguage":"ja"}}}</script>`
|
||||
|
||||
// Trimmed from https://comix.to/title/n8we-dungeons-and-crayons fetched
|
||||
// 2026-08-22 the same way (keys in their decoded form, as above). The
|
||||
// recommended strip includes a finished entry; the target detail is
|
||||
// releasing, and only the detail read counts.
|
||||
const comixOngoingFixture = `<script type="application/json" id="initial-data">{"queries":{"[\"manga\",\"recommended\",\"n8we\",1]":{"items":[{"hid":"g2rk","title":"On the Way to Meet Mom","status":"finished"}]},"[\"manga\",\"detail\",\"n8we\"]":{"id":1429,"hid":"n8we","title":"Dungeons and Crayons","status":"releasing","originalLanguage":"ko"}}}</script>`
|
||||
|
||||
// Trimmed from GET https://kagane.to/api/v2/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b
|
||||
// fetched 2026-08-22 in a cleared Chrome tab (plain TLS serves the Cloudflare
|
||||
// challenge).
|
||||
const kaganeOngoingFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Ongoing"}`
|
||||
|
||||
// Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6
|
||||
// fetched 2026-08-22 the same way.
|
||||
const kaganeCompletedFixture = `{"series_id":"019c29c3-5abd-70e6-9efc-1f1f22d839f6","title":"Real Account 1-17","publication_status":"Completed","upload_status":"Completed"}`
|
||||
|
||||
// Composed, not a single live trim: upload Completed alongside a
|
||||
// non-Completed publication status is the research note's inferred inverse
|
||||
// case and did not turn up in the ~1900-series live scan of 2026-08-22 (both
|
||||
// field vocabularies are live-verified; the note's live divergence is 'Cause
|
||||
// Calypso Can, publication Ongoing + upload Hiatus). The predicate must read
|
||||
// publication_status only.
|
||||
const kaganeDivergentFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Completed"}`
|
||||
|
||||
// Trimmed from https://novelfull.com/reverend-insanity.html fetched
|
||||
// 2026-08-22 in a cleared Chrome tab after the plain-TLS probe was served the
|
||||
// challenge (time-varying; plain curl answered 403 the same day).
|
||||
const novelfullCompletedFixture = `<div><h3>Status:</h3><a href="/status/Completed">Completed</a></div>`
|
||||
|
||||
// Trimmed from https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html
|
||||
// fetched 2026-08-22 the same way.
|
||||
const novelfullOngoingFixture = `<div><h3>Status:</h3><a href="/status/Ongoing">Ongoing</a></div>`
|
||||
|
||||
// Trimmed from the JSON-LD block in the head of
|
||||
// https://lightnovelworld.net/novel/a-will-eternal/ fetched 2026-08-22. The
|
||||
// block sits ahead of the wpdiscuz thread, so the predicate reads the whole
|
||||
// body and needs no comment marker, unlike lnwLatestChapter.
|
||||
const lnwCompletedFixture = `<script type="application/ld+json">{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "Book",
|
||||
"name": "A Will Eternal",
|
||||
"creativeWorkStatus": "https://schema.org/CompletedActionStatus"
|
||||
}</script>`
|
||||
|
||||
// Trimmed from
|
||||
// https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/
|
||||
// fetched 2026-08-22. Same block, ongoing value.
|
||||
const lnwOngoingFixture = `<script type="application/ld+json">{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "Book",
|
||||
"name": "All Jobs and Classes I Just Wanted One Skill Not Them All",
|
||||
"creativeWorkStatus": "https://schema.org/ActiveActionStatus"
|
||||
}</script>`
|
||||
|
||||
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
|
||||
// (redirected to ...-00dcbf97) on 2026-08-10.
|
||||
const asuraCoverFixture = `<meta property="og:image" content="https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp">`
|
||||
@@ -473,3 +571,178 @@ func TestLatestChapterFrom(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiteCompletedFrom(t *testing.T) {
|
||||
const asuraURL = "https://asurascans.com/comics/solo-leveling-b60d532c"
|
||||
const demonicURL = "https://demonicscans.org/manga/Solo-Leveling"
|
||||
const comixURL = "https://comix.to/title/q77m-countach"
|
||||
const kaganeURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
const novelfullURL = "https://novelfull.com/reverend-insanity.html"
|
||||
const lnwURL = "https://lightnovelworld.net/novel/a-will-eternal/"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
site string
|
||||
seriesURL string
|
||||
body string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "asura completed via escaped props status",
|
||||
site: "asura", seriesURL: asuraURL, body: asuraCompletedFixture,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "asura ongoing value is not completed",
|
||||
site: "asura",
|
||||
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
||||
body: asuraOngoingFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "asura with the status key removed",
|
||||
site: "asura", seriesURL: asuraURL,
|
||||
body: strings.ReplaceAll(asuraCompletedFixture, `"status":[0,"completed"]`, ""),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "asura challenge page",
|
||||
site: "asura", seriesURL: asuraURL, body: challengeFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "demonic completed info-block pair",
|
||||
site: "demonic", seriesURL: demonicURL, body: demonicCompletedFixture,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "demonic ongoing value is not completed",
|
||||
site: "demonic",
|
||||
seriesURL: "https://demonicscans.org/manga/Catastrophic-Necromancer",
|
||||
body: demonicOngoingFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "demonic with the value li removed",
|
||||
site: "demonic", seriesURL: demonicURL,
|
||||
body: strings.ReplaceAll(demonicCompletedFixture, "<li>Completed</li>", ""),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "comix recommended finished does not count",
|
||||
site: "comix",
|
||||
seriesURL: "https://comix.to/title/n8we-dungeons-and-crayons",
|
||||
body: comixOngoingFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "comix detail finished wins over a finished recommended strip",
|
||||
site: "comix",
|
||||
seriesURL: comixURL,
|
||||
body: comixCompletedFixture,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "comix with the detail status removed",
|
||||
site: "comix", seriesURL: comixURL,
|
||||
body: strings.ReplaceAll(comixCompletedFixture, `"status":"finished",`, ""),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "comix challenge page",
|
||||
site: "comix", seriesURL: comixURL, body: challengeFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "kagane publication Completed",
|
||||
site: "kagane",
|
||||
seriesURL: "https://kagane.to/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6",
|
||||
body: kaganeCompletedFixture,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "kagane upload Completed does not count",
|
||||
site: "kagane", seriesURL: kaganeURL, body: kaganeDivergentFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "kagane ongoing values are not completed",
|
||||
site: "kagane", seriesURL: kaganeURL, body: kaganeOngoingFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "kagane with publication_status removed",
|
||||
site: "kagane", seriesURL: kaganeURL,
|
||||
body: strings.ReplaceAll(kaganeCompletedFixture, `"publication_status":"Completed",`, ""),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "kagane challenge page",
|
||||
site: "kagane", seriesURL: kaganeURL, body: challengeFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "novelfull status link Completed",
|
||||
site: "novelfull",
|
||||
seriesURL: novelfullURL,
|
||||
body: novelfullCompletedFixture,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "novelfull ongoing link is not completed",
|
||||
site: "novelfull",
|
||||
seriesURL: "https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html",
|
||||
body: novelfullOngoingFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "novelfull with the status link removed",
|
||||
site: "novelfull", seriesURL: novelfullURL,
|
||||
body: strings.ReplaceAll(novelfullCompletedFixture, `<a href="/status/Completed">Completed</a>`, ""),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "novelfull challenge page",
|
||||
site: "novelfull", seriesURL: novelfullURL, body: challengeFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld JSON-LD CompletedActionStatus",
|
||||
site: "lightnovelworld",
|
||||
seriesURL: lnwURL,
|
||||
body: lnwCompletedFixture,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld ActiveActionStatus is not completed",
|
||||
site: "lightnovelworld",
|
||||
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||
body: lnwOngoingFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld with creativeWorkStatus removed",
|
||||
site: "lightnovelworld", seriesURL: lnwURL,
|
||||
body: strings.ReplaceAll(lnwCompletedFixture, `"creativeWorkStatus": "https://schema.org/CompletedActionStatus"`, ""),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld challenge page",
|
||||
site: "lightnovelworld", seriesURL: lnwURL, body: challengeFixture,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "unknown site",
|
||||
site: "mangadex", seriesURL: "https://mangadex.org/title/x", body: asuraCompletedFixture,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := siteCompletedFrom(tt.site, tt.seriesURL, tt.body); got != tt.want {
|
||||
t.Errorf("siteCompletedFrom(%q, %q, body) = %v, want %v", tt.site, tt.seriesURL, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user