sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Unscope lnw chapter scan, truncate at comment thread (#87)

Browse Source 
The Poll scoped the lightnovelworld scan to the slug derived from the
stored address, so a Series publishing chapters under more than one
Chapter Slug froze its Latest Chapter. The branch now matches any
chapter-shaped address on the host (lnwChapterRe) and scans only the
body before the wpdiscuz thread marker (wpd-threads); a body without
the marker is skipped and logged with its length, never scanned whole.
The size-cap comment now states the measured ~3.5x headroom.
This commit is contained in:
Sulthan Zaki
2026-08-11 12:59:21 +07:00
parent e864770eb8
commit ecce453d13
3 changed files with 63 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/latest/fetch.go
+3 -2
View File
@@ -11,8 +11,9 @@ import (
)
// maxBodyBytes caps what a single series page can cost in memory. Real pages
// measured 100-400 KB on 2026-07-26, so this is roughly 10x headroom and mostly
// guards against a proxy handing back something enormous.
// measured 100-400 KB on 2026-07-26; lightnovelworld runs larger — 685 KB and
// 1.18 MB measured 2026-08-11 — so the headroom there is roughly 3.5x, and the
// cap mostly guards against a proxy handing back something enormous.
const maxBodyBytes = 4 << 20
// chromeUA matches the client profile below. A Chrome fingerprint paired with a
backend/internal/latest/sites.go
+37 -12
View File
@@ -3,6 +3,7 @@ package latest
import (
"encoding/json"
"html"
"log"
"net/url"
"regexp"
"strconv"
@@ -58,11 +59,26 @@ var kaganeChapterRe = regexp.MustCompile(`"chapter_no":"([0-9.]+)"`)
// "/<slug>/chapter-<n>[-<title-slug>].html". Verified live 2026-08-05.
var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`)
// lnwSlugRe does the same for lightnovelworld, whose series pages live under
// /novel/<slug>/ while its chapter URLs are flat at the site root:
// "/<slug>-chapter-<n>/", absolute in the page's own anchors. Verified live
// 2026-08-05.
var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`)
// lnwChapterRe matches any chapter-shaped address on lightnovelworld. Unlike
// asura, novelfull and comix — which scope to their stored series slug so a
// foreign chapter link cannot contribute — this Site's chapter addresses carry
// the Chapter Slug, which is not the Series identity: one Series may publish
// under several Chapter Slugs (measured 2026-08-11: a sampled novel serves
// 1-99 under one slug and 100-423 under another), so no stored-slug pattern can
// cover a Series' whole list. An unscoped match is safe because
// latestChapterFrom truncates the body at the comment thread before scanning
// (lnwCommentMarker); without that, a visitor's comment could set the Latest
// Chapter on the shared Series row.
var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-([0-9.]+)/`)
// lnwCommentMarker is the boundary of lightnovelworld's server-rendered
// wpdiscuz comment thread. It occurs exactly once per page and follows every
// chapter anchor (measured 2026-08-11,
// docs/research/lightnovelworld-chapter-vs-series-slug.md §6), so cutting the
// body at its first occurrence keeps the whole chapter list while excluding a
// region any visitor can write to. Absent means the page shape changed: the
// body is skipped, never scanned whole.
const lnwCommentMarker = "wpd-threads"
// latestChapterFrom returns the highest chapter number body advertises for this
// series. ok is false when the body yields nothing usable — an unknown site, an
@@ -82,6 +98,11 @@ var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`)
// such guarantee — demonicChapterRe matches any chaptered.php?manga=<id> anchor
// with no per-series scoping, because the stored series_id for demonic is a
// slug, not the numeric id the URL carries, so it cannot easily be scoped.
// lightnovelworld is unscoped for a stronger reason: its chapter addresses
// carry the Chapter Slug, which is not the Series identity, so no stored-slug
// pattern can cover a Series' whole list (see lnwChapterRe). Instead of
// scoping, its branch scans only the body before the comment thread marker and
// skips the page when the marker is absent.
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
var re *regexp.Regexp
switch site {
@@ -126,15 +147,19 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
// and neither may contribute to the maximum.
re = regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`)
case "lightnovelworld":
u, err := url.Parse(seriesURL)
if err != nil {
// The comment thread below the chapter list is the one region of the
// page any visitor can write to, so the scan never reads past it (see
// lnwChapterRe). A body without the marker is skipped, never scanned
// whole — a redesign must degrade into staleness, not into a wrong
// shared value. The body length distinguishes a genuine markup change
// from a body the size cap cut short.
i := strings.Index(body, lnwCommentMarker)
if i < 0 {
log.Printf("latest poll %q: no %s marker in %d bytes; skipping", seriesURL, lnwCommentMarker, len(body))
return latestChapter{}, false
}
m := lnwSlugRe.FindStringSubmatch(u.Path)
if m == nil {
return latestChapter{}, false
}
re = regexp.MustCompile(`lightnovelworld\.net/` + regexp.QuoteMeta(m[1]) + `-chapter-([0-9.]+)/`)
body = body[:i]
re = lnwChapterRe
default:
return latestChapter{}, false
}
backend/internal/latest/sites_test.go
+23 -6
View File
@@ -1,6 +1,9 @@
package latest
import "testing"
import (
"strings"
"testing"
)
// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
// fetched 2026-07-26. The first anchor is the "First Chapter" shortcut: it is a
@@ -408,19 +411,33 @@ func TestLatestChapterFrom(t *testing.T) {
body: novelfullSeriesFixture,
wantOK: false,
},
// Stored before the slug split, so the address carries the ...-not
// Chapter Slug; the 100-423 block under the other slug must still win.
// The body is the chapter-list portion of lnwSeriesFixture with the
// comment block omitted (that is the next case's job); the marker is
// kept, because a body without it is skipped, not scanned.
{
name: "lightnovelworld takes the max on a split-slug novel",
name: "lightnovelworld max spans both chapter slugs",
site: "lightnovelworld",
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not/",
body: strings.SplitN(lnwSeriesFixture, lnwCommentMarker, 2)[0] + lnwCommentMarker,
wantOK: true, wantNum: 423, wantLabel: "Chapter 423",
},
{
name: "lightnovelworld comment anchor cannot set the latest chapter",
site: "lightnovelworld",
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
body: lnwSeriesFixture,
wantOK: true, wantNum: 423, wantLabel: "Chapter 423",
},
// Marker removed from the fixture, comment block still present: a
// redesign must degrade into a skip, never into the comment's number.
{
name: "lightnovelworld tolerates a series url with no trailing slash",
name: "lightnovelworld body without the comment marker is skipped",
site: "lightnovelworld",
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all",
body: lnwSeriesFixture,
wantOK: true, wantNum: 423, wantLabel: "Chapter 423",
seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
body: strings.ReplaceAll(lnwSeriesFixture, lnwCommentMarker, ""),
wantOK: false,
},
{
name: "lightnovelworld yields nothing on a challenge page",