From ecce453d1315d0faf0c7212556b16cc19aa2dc16 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Tue, 11 Aug 2026 12:59:21 +0700 Subject: [PATCH] Unscope lnw chapter scan, truncate at comment thread (#87) The Poll scoped the lightnovelworld scan to the slug derived from the stored address, so a Series publishing chapters under more than one Chapter Slug froze its Latest Chapter. The branch now matches any chapter-shaped address on the host (lnwChapterRe) and scans only the body before the wpdiscuz thread marker (wpd-threads); a body without the marker is skipped and logged with its length, never scanned whole. The size-cap comment now states the measured ~3.5x headroom. --- backend/internal/latest/fetch.go | 5 +-- backend/internal/latest/sites.go | 49 ++++++++++++++++++++------- backend/internal/latest/sites_test.go | 29 ++++++++++++---- 3 files changed, 63 insertions(+), 20 deletions(-) diff --git a/backend/internal/latest/fetch.go b/backend/internal/latest/fetch.go index 866bed4..72d965e 100644 --- a/backend/internal/latest/fetch.go +++ b/backend/internal/latest/fetch.go @@ -11,8 +11,9 @@ import ( ) // maxBodyBytes caps what a single series page can cost in memory. Real pages -// measured 100-400 KB on 2026-07-26, so this is roughly 10x headroom and mostly -// guards against a proxy handing back something enormous. +// measured 100-400 KB on 2026-07-26; lightnovelworld runs larger — 685 KB and +// 1.18 MB measured 2026-08-11 — so the headroom there is roughly 3.5x, and the +// cap mostly guards against a proxy handing back something enormous. const maxBodyBytes = 4 << 20 // chromeUA matches the client profile below. A Chrome fingerprint paired with a diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index a4b1225..7b2f2f5 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -3,6 +3,7 @@ package latest import ( "encoding/json" "html" + "log" "net/url" "regexp" "strconv" @@ -58,11 +59,26 @@ var kaganeChapterRe = regexp.MustCompile(`"chapter_no":"([0-9.]+)"`) // "//chapter-[-].html". Verified live 2026-08-05. var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`) -// lnwSlugRe does the same for lightnovelworld, whose series pages live under -// /novel// while its chapter URLs are flat at the site root: -// "/-chapter-/", absolute in the page's own anchors. Verified live -// 2026-08-05. -var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`) +// lnwChapterRe matches any chapter-shaped address on lightnovelworld. Unlike +// asura, novelfull and comix — which scope to their stored series slug so a +// foreign chapter link cannot contribute — this Site's chapter addresses carry +// the Chapter Slug, which is not the Series identity: one Series may publish +// under several Chapter Slugs (measured 2026-08-11: a sampled novel serves +// 1-99 under one slug and 100-423 under another), so no stored-slug pattern can +// cover a Series' whole list. An unscoped match is safe because +// latestChapterFrom truncates the body at the comment thread before scanning +// (lnwCommentMarker); without that, a visitor's comment could set the Latest +// Chapter on the shared Series row. +var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-([0-9.]+)/`) + +// lnwCommentMarker is the boundary of lightnovelworld's server-rendered +// wpdiscuz comment thread. It occurs exactly once per page and follows every +// chapter anchor (measured 2026-08-11, +// docs/research/lightnovelworld-chapter-vs-series-slug.md §6), so cutting the +// body at its first occurrence keeps the whole chapter list while excluding a +// region any visitor can write to. Absent means the page shape changed: the +// body is skipped, never scanned whole. +const lnwCommentMarker = "wpd-threads" // latestChapterFrom returns the highest chapter number body advertises for this // series. ok is false when the body yields nothing usable — an unknown site, an @@ -82,6 +98,11 @@ var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`) // such guarantee — demonicChapterRe matches any chaptered.php?manga= anchor // with no per-series scoping, because the stored series_id for demonic is a // slug, not the numeric id the URL carries, so it cannot easily be scoped. +// lightnovelworld is unscoped for a stronger reason: its chapter addresses +// carry the Chapter Slug, which is not the Series identity, so no stored-slug +// pattern can cover a Series' whole list (see lnwChapterRe). Instead of +// scoping, its branch scans only the body before the comment thread marker and +// skips the page when the marker is absent. func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) { var re *regexp.Regexp switch site { @@ -126,15 +147,19 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) { // and neither may contribute to the maximum. re = regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`) case "lightnovelworld": - u, err := url.Parse(seriesURL) - if err != nil { + // The comment thread below the chapter list is the one region of the + // page any visitor can write to, so the scan never reads past it (see + // lnwChapterRe). A body without the marker is skipped, never scanned + // whole — a redesign must degrade into staleness, not into a wrong + // shared value. The body length distinguishes a genuine markup change + // from a body the size cap cut short. + i := strings.Index(body, lnwCommentMarker) + if i < 0 { + log.Printf("latest poll %q: no %s marker in %d bytes; skipping", seriesURL, lnwCommentMarker, len(body)) return latestChapter{}, false } - m := lnwSlugRe.FindStringSubmatch(u.Path) - if m == nil { - return latestChapter{}, false - } - re = regexp.MustCompile(`lightnovelworld\.net/` + regexp.QuoteMeta(m[1]) + `-chapter-([0-9.]+)/`) + body = body[:i] + re = lnwChapterRe default: return latestChapter{}, false } diff --git a/backend/internal/latest/sites_test.go b/backend/internal/latest/sites_test.go index f1c4876..6afd0af 100644 --- a/backend/internal/latest/sites_test.go +++ b/backend/internal/latest/sites_test.go @@ -1,6 +1,9 @@ package latest -import "testing" +import ( + "strings" + "testing" +) // Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af // fetched 2026-07-26. The first anchor is the "First Chapter" shortcut: it is a @@ -408,19 +411,33 @@ func TestLatestChapterFrom(t *testing.T) { body: novelfullSeriesFixture, wantOK: false, }, + // Stored before the slug split, so the address carries the ...-not + // Chapter Slug; the 100-423 block under the other slug must still win. + // The body is the chapter-list portion of lnwSeriesFixture with the + // comment block omitted (that is the next case's job); the marker is + // kept, because a body without it is skipped, not scanned. { - name: "lightnovelworld takes the max on a split-slug novel", + name: "lightnovelworld max spans both chapter slugs", + site: "lightnovelworld", + seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not/", + body: strings.SplitN(lnwSeriesFixture, lnwCommentMarker, 2)[0] + lnwCommentMarker, + wantOK: true, wantNum: 423, wantLabel: "Chapter 423", + }, + { + name: "lightnovelworld comment anchor cannot set the latest chapter", site: "lightnovelworld", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", body: lnwSeriesFixture, wantOK: true, wantNum: 423, wantLabel: "Chapter 423", }, + // Marker removed from the fixture, comment block still present: a + // redesign must degrade into a skip, never into the comment's number. { - name: "lightnovelworld tolerates a series url with no trailing slash", + name: "lightnovelworld body without the comment marker is skipped", site: "lightnovelworld", - seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", - body: lnwSeriesFixture, - wantOK: true, wantNum: 423, wantLabel: "Chapter 423", + seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", + body: strings.ReplaceAll(lnwSeriesFixture, lnwCommentMarker, ""), + wantOK: false, }, { name: "lightnovelworld yields nothing on a challenge page",