From 78ce251b4dc641db67e0e6dadc01d551b26fbe34 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Tue, 11 Aug 2026 13:12:39 +0700 Subject: [PATCH 1/2] #88: live canary for the lnw comment-thread marker --- backend/internal/latest/smoke_lnw_test.go | 104 ++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 backend/internal/latest/smoke_lnw_test.go diff --git a/backend/internal/latest/smoke_lnw_test.go b/backend/internal/latest/smoke_lnw_test.go new file mode 100644 index 0000000..dae69cd --- /dev/null +++ b/backend/internal/latest/smoke_lnw_test.go @@ -0,0 +1,104 @@ +package latest + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + "testing" + "time" +) + +// lnwSeriesPageFloor is the smallest body that can still be a whole +// lightnovelworld Series page. Whole pages measured 685 KB..1.18 MB on +// 2026-08-11 and carry the marker at ~94% of the document, so a body under +// 100 KB is a challenge, a notice, or a truncated read — asserting on it +// would report the marker missing when it was never fetched. +const lnwSeriesPageFloor = 100 << 10 + +// TestSmokeLnwCommentBoundary is the live proof that the comment-thread marker +// the lightnovelworld chapter scan truncates at (lnwCommentMarker, +// "wpd-threads") still holds on the Site. The scan depends on it: when the +// marker vanishes every Series is skipped and logged — correct, but silent +// until a Reader notices their Latest Chapter has stopped moving. It runs only +// when SMOKE_LNW_SERIES_URL is set — the URL of the live Series page to check. +// The immortality-simulator page measured 2026-08-11 +// (docs/research/lightnovelworld-chapter-vs-series-slug.md) is the default to +// point it at: +// +// SMOKE_LNW_SERIES_URL=https://lightnovelworld.net/novel/immortality-simulator/ go test -v -run TestSmokeLnwCommentBoundary ./internal/latest +// +// A red run means the Site's markup has moved — the marker is gone, occurs +// more than once, or no longer follows the last chapter anchor — and the scan +// in sites.go is now skipping this Site. Revisit sites.go before anything +// else; the test is not flaky. A Cloudflare challenge or a non-200 is +// distinguished from a marker failure by the "not a marker failure" messages +// below, which carry the observed status and body length. +func TestSmokeLnwCommentBoundary(t *testing.T) { + seriesURL := os.Getenv("SMOKE_LNW_SERIES_URL") + if seriesURL == "" { + t.Skip("SMOKE_LNW_SERIES_URL unset") + } + if !fetchableSeriesURL("lightnovelworld", seriesURL) { + t.Fatalf("%q is not a fetchable lightnovelworld series URL", seriesURL) + } + + f, err := NewTLSFetcher() + if err != nil { + t.Fatalf("NewTLSFetcher: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + body, status, err := f.Get(ctx, seriesURL) + if err != nil { + t.Fatalf("Get: %v", err) + } + if status != http.StatusOK { + t.Fatalf("status = %d, body %d bytes — not a marker failure; the Site did not answer this IP with a Series page", status, len(body)) + } + if len(body) < lnwSeriesPageFloor { + t.Fatalf("body %d bytes — not a whole Series page (measured 685 KB..1.18 MB); not a marker failure, likely a Cloudflare challenge or a non-Series response", len(body)) + } + if !lnwChapterRe.MatchString(body) { + t.Fatalf("no chapter anchor in %d bytes — not a lightnovelworld Series page; not a marker failure, likely a Cloudflare challenge or a non-Series response", len(body)) + } + if failures := checkLnwCommentBoundary(body); len(failures) > 0 { + t.Fatalf("%s (body %d bytes)", strings.Join(failures, "; "), len(body)) + } + t.Logf("ok: %q once at byte %d, body %d bytes", lnwCommentMarker, strings.Index(body, lnwCommentMarker), len(body)) +} + +// checkLnwCommentBoundary verifies the three marker assertions against a +// fetched Series body: the marker occurs exactly once, every chapter anchor +// precedes it, and at least one anchor precedes it at all. It returns one +// human-readable failure per broken assertion — with observed offsets and body +// length — and empty when the page is healthy. +func checkLnwCommentBoundary(body string) []string { + n := strings.Count(body, lnwCommentMarker) + if n == 0 { + return []string{fmt.Sprintf("%q occurs 0 times in %d bytes, want exactly 1", lnwCommentMarker, len(body))} + } + if n != 1 { + return []string{fmt.Sprintf("%q occurs %d times in %d bytes (first at byte %d), want exactly 1", lnwCommentMarker, n, len(body), strings.Index(body, lnwCommentMarker))} + } + markerIdx := strings.Index(body, lnwCommentMarker) + lastAnchor, before := -1, 0 + for _, m := range lnwChapterRe.FindAllStringIndex(body, -1) { + if m[0] < markerIdx { + before++ + } + lastAnchor = m[0] + } + if lastAnchor == -1 { + return []string{fmt.Sprintf("no chapter anchor in the %d-byte body", len(body))} + } + var failures []string + if lastAnchor >= markerIdx { + failures = append(failures, fmt.Sprintf("last chapter anchor at byte %d does not precede the marker at byte %d", lastAnchor, markerIdx)) + } + if before == 0 { + failures = append(failures, fmt.Sprintf("no chapter anchor before the marker at byte %d — the truncated prefix the scan sees yields nothing", markerIdx)) + } + return failures +} From 3396c50aac8b7892fe703991d159e8e1836bf0ff Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Tue, 11 Aug 2026 13:32:14 +0700 Subject: [PATCH 2/2] =?UTF-8?q?#88:=20review=20=E2=80=94=20drop=20dead=20a?= =?UTF-8?q?nchor=20branch,=20single=20marker=20scan=20in=20canary=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/latest/smoke_lnw_test.go | 25 ++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/backend/internal/latest/smoke_lnw_test.go b/backend/internal/latest/smoke_lnw_test.go index dae69cd..cd3e589 100644 --- a/backend/internal/latest/smoke_lnw_test.go +++ b/backend/internal/latest/smoke_lnw_test.go @@ -63,10 +63,11 @@ func TestSmokeLnwCommentBoundary(t *testing.T) { if !lnwChapterRe.MatchString(body) { t.Fatalf("no chapter anchor in %d bytes — not a lightnovelworld Series page; not a marker failure, likely a Cloudflare challenge or a non-Series response", len(body)) } + markerIdx := strings.Index(body, lnwCommentMarker) if failures := checkLnwCommentBoundary(body); len(failures) > 0 { t.Fatalf("%s (body %d bytes)", strings.Join(failures, "; "), len(body)) } - t.Logf("ok: %q once at byte %d, body %d bytes", lnwCommentMarker, strings.Index(body, lnwCommentMarker), len(body)) + t.Logf("ok: %q once at byte %d, body %d bytes", lnwCommentMarker, markerIdx, len(body)) } // checkLnwCommentBoundary verifies the three marker assertions against a @@ -75,29 +76,25 @@ func TestSmokeLnwCommentBoundary(t *testing.T) { // human-readable failure per broken assertion — with observed offsets and body // length — and empty when the page is healthy. func checkLnwCommentBoundary(body string) []string { - n := strings.Count(body, lnwCommentMarker) - if n == 0 { - return []string{fmt.Sprintf("%q occurs 0 times in %d bytes, want exactly 1", lnwCommentMarker, len(body))} - } - if n != 1 { - return []string{fmt.Sprintf("%q occurs %d times in %d bytes (first at byte %d), want exactly 1", lnwCommentMarker, n, len(body), strings.Index(body, lnwCommentMarker))} - } markerIdx := strings.Index(body, lnwCommentMarker) - lastAnchor, before := -1, 0 + switch n := strings.Count(body, lnwCommentMarker); { + case n == 0: + return []string{fmt.Sprintf("%q occurs 0 times in %d bytes, want exactly 1", lnwCommentMarker, len(body))} + case n != 1: + return []string{fmt.Sprintf("%q occurs %d times in %d bytes (first at byte %d), want exactly 1", lnwCommentMarker, n, len(body), markerIdx)} + } + lastAnchor, anchorsBefore := -1, 0 for _, m := range lnwChapterRe.FindAllStringIndex(body, -1) { if m[0] < markerIdx { - before++ + anchorsBefore++ } lastAnchor = m[0] } - if lastAnchor == -1 { - return []string{fmt.Sprintf("no chapter anchor in the %d-byte body", len(body))} - } var failures []string if lastAnchor >= markerIdx { failures = append(failures, fmt.Sprintf("last chapter anchor at byte %d does not precede the marker at byte %d", lastAnchor, markerIdx)) } - if before == 0 { + if anchorsBefore == 0 { failures = append(failures, fmt.Sprintf("no chapter anchor before the marker at byte %d — the truncated prefix the scan sees yields nothing", markerIdx)) } return failures