diff --git a/backend/internal/latest/fetch.go b/backend/internal/latest/fetch.go index 866bed4..72d965e 100644 --- a/backend/internal/latest/fetch.go +++ b/backend/internal/latest/fetch.go @@ -11,8 +11,9 @@ import ( ) // maxBodyBytes caps what a single series page can cost in memory. Real pages -// measured 100-400 KB on 2026-07-26, so this is roughly 10x headroom and mostly -// guards against a proxy handing back something enormous. +// measured 100-400 KB on 2026-07-26; lightnovelworld runs larger — 685 KB and +// 1.18 MB measured 2026-08-11 — so the headroom there is roughly 3.5x, and the +// cap mostly guards against a proxy handing back something enormous. const maxBodyBytes = 4 << 20 // chromeUA matches the client profile below. A Chrome fingerprint paired with a diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index a4b1225..0c354ec 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -3,6 +3,7 @@ package latest import ( "encoding/json" "html" + "log" "net/url" "regexp" "strconv" @@ -58,11 +59,26 @@ var kaganeChapterRe = regexp.MustCompile(`"chapter_no":"([0-9.]+)"`) // "//chapter-[-].html". Verified live 2026-08-05. var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`) -// lnwSlugRe does the same for lightnovelworld, whose series pages live under -// /novel// while its chapter URLs are flat at the site root: -// "/-chapter-/", absolute in the page's own anchors. Verified live -// 2026-08-05. -var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`) +// lnwChapterRe matches any chapter-shaped address on lightnovelworld. Unlike +// asura, novelfull and comix — which scope to their stored series slug so a +// foreign chapter link cannot contribute — this Site's chapter addresses carry +// the Chapter Slug, which is not the Series identity: one Series may publish +// under several Chapter Slugs (measured 2026-08-11: a sampled novel serves +// 1-99 under one slug and 100-423 under another), so no stored-slug pattern can +// cover a Series' whole list. An unscoped match is safe because +// latestChapterFrom truncates the body at the comment thread before scanning +// (lnwCommentMarker); without that, a visitor's comment could set the Latest +// Chapter on the shared Series row. +var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-([0-9.]+)/`) + +// lnwCommentMarker is the boundary of lightnovelworld's server-rendered +// wpdiscuz comment thread. It occurs exactly once per page and follows every +// chapter anchor (measured 2026-08-11, +// docs/research/lightnovelworld-chapter-vs-series-slug.md §6), so cutting the +// body at its first occurrence keeps the whole chapter list while excluding a +// region any visitor can write to. Absent means the page shape changed: the +// body is skipped, never scanned whole. +const lnwCommentMarker = "wpd-threads" // latestChapterFrom returns the highest chapter number body advertises for this // series. ok is false when the body yields nothing usable — an unknown site, an @@ -82,6 +98,7 @@ var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`) // such guarantee — demonicChapterRe matches any chaptered.php?manga= anchor // with no per-series scoping, because the stored series_id for demonic is a // slug, not the numeric id the URL carries, so it cannot easily be scoped. +// lightnovelworld is unscoped and body-truncated instead — see lnwChapterRe. func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) { var re *regexp.Regexp switch site { @@ -126,15 +143,19 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) { // and neither may contribute to the maximum. re = regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`) case "lightnovelworld": - u, err := url.Parse(seriesURL) - if err != nil { + // The comment thread below the chapter list is the one region of the + // page any visitor can write to, so the scan never reads past it (see + // lnwChapterRe). A body without the marker is skipped, never scanned + // whole — a redesign must degrade into staleness, not into a wrong + // shared value; the logged body length tells a markup change from a + // body the size cap cut short. + i := strings.Index(body, lnwCommentMarker) + if i < 0 { + log.Printf("latest poll %q: no %s marker in %d bytes", seriesURL, lnwCommentMarker, len(body)) return latestChapter{}, false } - m := lnwSlugRe.FindStringSubmatch(u.Path) - if m == nil { - return latestChapter{}, false - } - re = regexp.MustCompile(`lightnovelworld\.net/` + regexp.QuoteMeta(m[1]) + `-chapter-([0-9.]+)/`) + body = body[:i] + re = lnwChapterRe default: return latestChapter{}, false } diff --git a/backend/internal/latest/sites_test.go b/backend/internal/latest/sites_test.go index f1c4876..d665b25 100644 --- a/backend/internal/latest/sites_test.go +++ b/backend/internal/latest/sites_test.go @@ -1,6 +1,9 @@ package latest -import "testing" +import ( + "strings" + "testing" +) // Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af // fetched 2026-07-26. The first anchor is the "First Chapter" shortcut: it is a @@ -408,19 +411,33 @@ func TestLatestChapterFrom(t *testing.T) { body: novelfullSeriesFixture, wantOK: false, }, + // Stored before the slug split, so the address carries the ...-not + // Chapter Slug; the 100-423 block under the other slug must still win. + // The body is the chapter-list portion of lnwSeriesFixture with the + // comment block omitted; the marker is kept, because a body without it + // is skipped, not scanned. { - name: "lightnovelworld takes the max on a split-slug novel", + name: "lightnovelworld max spans both chapter slugs", + site: "lightnovelworld", + seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not/", + body: strings.SplitN(lnwSeriesFixture, lnwCommentMarker, 2)[0] + lnwCommentMarker, + wantOK: true, wantNum: 423, wantLabel: "Chapter 423", + }, + { + name: "lightnovelworld comment anchor cannot set the latest chapter", site: "lightnovelworld", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", body: lnwSeriesFixture, wantOK: true, wantNum: 423, wantLabel: "Chapter 423", }, + // Marker removed from the fixture, comment block still present: a + // redesign must degrade into a skip, never into the comment's number. { - name: "lightnovelworld tolerates a series url with no trailing slash", + name: "lightnovelworld body without the comment marker is skipped", site: "lightnovelworld", - seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", - body: lnwSeriesFixture, - wantOK: true, wantNum: 423, wantLabel: "Chapter 423", + seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", + body: strings.ReplaceAll(lnwSeriesFixture, lnwCommentMarker, ""), + wantOK: false, }, { name: "lightnovelworld yields nothing on a challenge page",