Unscope lnw chapter scan, truncate at comment thread (#87)
The Poll scoped the lightnovelworld scan to the slug derived from the stored address, so a Series publishing chapters under more than one Chapter Slug froze its Latest Chapter. The branch now matches any chapter-shaped address on the host (lnwChapterRe) and scans only the body before the wpdiscuz thread marker (wpd-threads); a body without the marker is skipped and logged with its length, never scanned whole. The size-cap comment now states the measured ~3.5x headroom.
This commit is contained in:
@@ -3,6 +3,7 @@ package latest
|
||||
import (
|
||||
"encoding/json"
|
||||
"html"
|
||||
"log"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -58,11 +59,26 @@ var kaganeChapterRe = regexp.MustCompile(`"chapter_no":"([0-9.]+)"`)
|
||||
// "/<slug>/chapter-<n>[-<title-slug>].html". Verified live 2026-08-05.
|
||||
var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`)
|
||||
|
||||
// lnwSlugRe does the same for lightnovelworld, whose series pages live under
|
||||
// /novel/<slug>/ while its chapter URLs are flat at the site root:
|
||||
// "/<slug>-chapter-<n>/", absolute in the page's own anchors. Verified live
|
||||
// 2026-08-05.
|
||||
var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`)
|
||||
// lnwChapterRe matches any chapter-shaped address on lightnovelworld. Unlike
|
||||
// asura, novelfull and comix — which scope to their stored series slug so a
|
||||
// foreign chapter link cannot contribute — this Site's chapter addresses carry
|
||||
// the Chapter Slug, which is not the Series identity: one Series may publish
|
||||
// under several Chapter Slugs (measured 2026-08-11: a sampled novel serves
|
||||
// 1-99 under one slug and 100-423 under another), so no stored-slug pattern can
|
||||
// cover a Series' whole list. An unscoped match is safe because
|
||||
// latestChapterFrom truncates the body at the comment thread before scanning
|
||||
// (lnwCommentMarker); without that, a visitor's comment could set the Latest
|
||||
// Chapter on the shared Series row.
|
||||
var lnwChapterRe = regexp.MustCompile(`lightnovelworld\.net/[a-z0-9-]+-chapter-([0-9.]+)/`)
|
||||
|
||||
// lnwCommentMarker is the boundary of lightnovelworld's server-rendered
|
||||
// wpdiscuz comment thread. It occurs exactly once per page and follows every
|
||||
// chapter anchor (measured 2026-08-11,
|
||||
// docs/research/lightnovelworld-chapter-vs-series-slug.md §6), so cutting the
|
||||
// body at its first occurrence keeps the whole chapter list while excluding a
|
||||
// region any visitor can write to. Absent means the page shape changed: the
|
||||
// body is skipped, never scanned whole.
|
||||
const lnwCommentMarker = "wpd-threads"
|
||||
|
||||
// latestChapterFrom returns the highest chapter number body advertises for this
|
||||
// series. ok is false when the body yields nothing usable — an unknown site, an
|
||||
@@ -82,6 +98,11 @@ var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`)
|
||||
// such guarantee — demonicChapterRe matches any chaptered.php?manga=<id> anchor
|
||||
// with no per-series scoping, because the stored series_id for demonic is a
|
||||
// slug, not the numeric id the URL carries, so it cannot easily be scoped.
|
||||
// lightnovelworld is unscoped for a stronger reason: its chapter addresses
|
||||
// carry the Chapter Slug, which is not the Series identity, so no stored-slug
|
||||
// pattern can cover a Series' whole list (see lnwChapterRe). Instead of
|
||||
// scoping, its branch scans only the body before the comment thread marker and
|
||||
// skips the page when the marker is absent.
|
||||
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
|
||||
var re *regexp.Regexp
|
||||
switch site {
|
||||
@@ -126,15 +147,19 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
|
||||
// and neither may contribute to the maximum.
|
||||
re = regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`)
|
||||
case "lightnovelworld":
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil {
|
||||
// The comment thread below the chapter list is the one region of the
|
||||
// page any visitor can write to, so the scan never reads past it (see
|
||||
// lnwChapterRe). A body without the marker is skipped, never scanned
|
||||
// whole — a redesign must degrade into staleness, not into a wrong
|
||||
// shared value. The body length distinguishes a genuine markup change
|
||||
// from a body the size cap cut short.
|
||||
i := strings.Index(body, lnwCommentMarker)
|
||||
if i < 0 {
|
||||
log.Printf("latest poll %q: no %s marker in %d bytes; skipping", seriesURL, lnwCommentMarker, len(body))
|
||||
return latestChapter{}, false
|
||||
}
|
||||
m := lnwSlugRe.FindStringSubmatch(u.Path)
|
||||
if m == nil {
|
||||
return latestChapter{}, false
|
||||
}
|
||||
re = regexp.MustCompile(`lightnovelworld\.net/` + regexp.QuoteMeta(m[1]) + `-chapter-([0-9.]+)/`)
|
||||
body = body[:i]
|
||||
re = lnwChapterRe
|
||||
default:
|
||||
return latestChapter{}, false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user