rebrand: MangaBM → BookmarkManager, add novel library support (#15)

Two intertwined changes — the rebrand and the novel library were developed on the same branch because the novel UI plumbing is part of the new "Bookmark Manager" wordmark in the web shell. ## What it does - **Rebrand**: MangaBM → BookmarkManager across the Go module, compose stack, env vars, Traefik hostnames, container/image names, userscript storage prefixes (`mangabm:cache` → `bmgr:manga:cache`, `mangabm:queue` → `bmgr:manga:queue`), and docs. - **Novel library**: same backend, two libraries. New `kind` column splits bookmarks into `manga` / `novel`; PUT validates it. Two userscripts: - `manga-bookmark.user.js` — unchanged behaviour, just stamps its own `kind`. - `novel-bookmark.user.js` — separate Violentmonkey install with adapters for **novelfull.com** (polled via headless browser — Cloudflare JS challenge) and **lightnovelworld.net** (polled via plain TLS). - **Web UI**: library switch on the app shell. Login art, libswitch, and novel-site colours from the Cinder design snapshot. ## Plumbing - `addedColumns` ALTER for `kind` runs on first start after upgrade; every pre-existing row is backfilled to `'manga'`. No manual SQL, no down-time. - `ALLOWED_ORIGINS` gains the two novel sites. - New `NOVEL_USERSCRIPT_PATH` env (default `/userscript/novel-bookmark.user.js`), bindmounted alongside the manga script. - Traefik router names `mangabm*` → `bmapi*` / `bmweb*`. ## Test status - `go test ./...` — green - `node --test userscript/test/logic.test.js` — 34 pass - `node --test userscript/test/novel-logic.test.js` — 11 pass - `node --check` on both userscripts — clean ## Notes for the redeploy .env keys were renamed (`MANGA_API_HOST` → `BOOKMARK_API_HOST`, `MANGA_WEB_HOST` → `BOOKMARK_WEB_HOST`). Update DNS / Traefik labels on the prod override before pulling, otherwise the public hostnames go dark. See the redeploy instructions I'll post next to this PR. Reviewed-on: #15 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
2026-08-06 03:58:23 +07:00
parent c445762244
commit 4229c179b0
40 changed files with 2948 additions and 347 deletions
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"regexp"
+	"strings"
 	"sync"
 	"time"

@@ -24,18 +25,23 @@ var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
 // BrowserFetcher retrieves pages through a remote headless Chrome over the
 // DevTools Protocol.
 //
-// It exists for one reason: kagane.to sits behind a Cloudflare JavaScript
-// challenge. Verified 2026-08-03 from the deployment host, plain HTTP and
-// bogdanfinn/tls-client with a Chrome_133 profile both get 403 with
-// cf-mitigated: challenge on every path, including the API, robots.txt and
-// images. Clearing it requires executing the challenge script, which only a
-// real browser does.
+// It exists for one reason: kagane.to and novelfull.com sit behind a
+// Cloudflare JavaScript challenge. Verified 2026-08-03 (kagane) and 2026-08-05
+// (novelfull) from the deployment host, plain HTTP and bogdanfinn/tls-client
+// with a Chrome_133 profile both get 403 with cf-mitigated: challenge on every
+// path, including the API, robots.txt and images. Clearing it requires
+// executing the challenge script, which only a real browser does.
 //
 // The request is made *inside* the page rather than by extracting cf_clearance
 // and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent
 // and often the TLS fingerprint, so replaying it means keeping three things in
 // sync that break silently and separately. The browser's own cookie jar
 // persists across polls, so the challenge is solved once every few hours.
+//
+// The two sites differ in how the chapter list is read: kagane serves it from
+// a JSON API that must be called from inside the page (so the request carries
+// the clearance cookie), while novelfull renders it into the HTML so the
+// cleared DOM is the payload.
 type BrowserFetcher struct {
 	allocCtx context.Context
 	cancel   context.CancelFunc
@@ -73,14 +79,16 @@ func (f *BrowserFetcher) Close() {
 	f.cancel()
 }

-// Get navigates to seriesURL, lets any challenge resolve, then reads the site's
-// JSON API from inside the page so the request carries the clearance cookie.
-// The returned body is API JSON, which is what latestChapterFrom's kagane case
-// expects — it is not HTML.
+// Get navigates to seriesURL, lets any challenge resolve, then reads either the
+// site's JSON API (kagane) from inside the page so the request carries the
+// clearance cookie, or the served HTML itself (novelfull) — see
+// novelfullSeriesURL for the latter case. The returned body is whatever the
+// site's chapter list lives in, which is what latestChapterFrom's per-site
+// switch expects.
 func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
-	apiURL, ok := kaganeAPIURL(seriesURL)
-	if !ok {
-		return "", 0, fmt.Errorf("not a fetchable kagane series url: %q", seriesURL)
+	apiURL, isKagane := kaganeAPIURL(seriesURL)
+	if !isKagane && !novelfullSeriesURL(seriesURL) {
+		return "", 0, fmt.Errorf("not a fetchable browser series url: %q", seriesURL)
 	}

 	f.mu.Lock()
@@ -101,16 +109,26 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int
 	}()

 	var body string
+	// kagane's chapter list is only in its JSON API, which must be called from
+	// inside the page so the request carries the clearance cookie. novelfull
+	// renders its chapters into the HTML, so the cleared DOM is the answer.
+	// chromedp.OuterHTML returns a QueryAction and chromedp.Evaluate an
+	// EvaluateAction, so the variable has to be the interface both implement.
+	var read chromedp.Action = chromedp.OuterHTML("html", &body, chromedp.ByQuery)
+	if isKagane {
+		read = chromedp.Evaluate(
+			`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
+			&body,
+			awaitPromise,
+		)
+	}
+
 	err := chromedp.Run(tabCtx,
 		chromedp.Navigate(seriesURL),
 		// The challenge reloads the page itself when it passes; waiting for the
 		// site's own root element is what tells us we are through it.
 		chromedp.WaitReady("body", chromedp.ByQuery),
-		chromedp.Evaluate(
-			`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
-			&body,
-			awaitPromise,
-		),
+		read,
 	)
 	if err != nil {
 		return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
@@ -139,6 +157,18 @@ func kaganeAPIURL(seriesURL string) (string, bool) {
 	return "https://kagane.to/api/v2/series/" + m[1], true
 }

+// novelfullSeriesURL reports whether seriesURL is a novelfull series page this
+// fetcher will open. novelfull's chapter list is in the served HTML, so unlike
+// kagane there is no API to call from inside the page — the challenge-cleared
+// DOM is the payload. The host is pinned here for the same reason kagane's is:
+// series_url is client-supplied and a headless browser is a strong SSRF
+// primitive.
+func novelfullSeriesURL(seriesURL string) bool {
+	u, err := url.Parse(seriesURL)
+	return err == nil && u.Scheme == "https" && u.Hostname() == "novelfull.com" &&
+		strings.HasSuffix(u.Path, ".html")
+}
+
 // awaitPromise makes Evaluate resolve the promise rather than returning a
 // serialised Promise object.
 func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
@@ -36,3 +36,24 @@ func TestKaganeAPIURL(t *testing.T) {
 		})
 	}
 }
+
+func TestNovelfullSeriesURL(t *testing.T) {
+	cases := []struct {
+		name string
+		url  string
+		want bool
+	}{
+		{"series page", "https://novelfull.com/reverend-insanity.html", true},
+		{"foreign host", "https://evil.example/reverend-insanity.html", false},
+		{"not https", "http://novelfull.com/reverend-insanity.html", false},
+		{"not a series page", "https://novelfull.com/genre/Fantasy", false},
+		{"garbage", "://nope", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := novelfullSeriesURL(tc.url); got != tc.want {
+				t.Fatalf("novelfullSeriesURL(%q) = %v, want %v", tc.url, got, tc.want)
+			}
+		})
+	}
+}
@@ -6,7 +6,7 @@ import (
 	"net/url"
 	"time"

-	"mangabm/backend/internal/store"
+	"bookmarkmanager/backend/internal/store"
 )

 // Fetcher retrieves a series page. It exists as an interface so tests can inject
@@ -44,12 +44,13 @@ type Poller struct {
 }

 // fetcherFor returns the fetcher a site needs, or nil when the site cannot be
-// fetched at all right now. kagane sits behind a Cloudflare JavaScript
-// challenge that no TLS fingerprint clears — verified 2026-08-03 from the
-// deployment host with the same Chrome profile TLSFetcher uses — so it is
-// browser-only or nothing.
+// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
+// JavaScript challenge that no TLS fingerprint clears — kagane verified
+// 2026-08-03, novelfull verified 2026-08-05, both against the same Chrome_133
+// profile TLSFetcher uses — so they are browser-only or nothing.
 func (p *Poller) fetcherFor(site string) Fetcher {
-	if site == "kagane" {
+	switch site {
+	case "kagane", "novelfull":
 		return p.BrowserFetch
 	}
 	return p.Fetch
@@ -212,13 +213,18 @@ func (p *Poller) checkOne(ctx context.Context, b store.Bookmark) {
 // a defence against the poller being used to probe arbitrary hosts from the
 // server's own network position, not just a check against wasted requests.
 //
-// kagane is held to a stricter rule: it is fetched by a headless browser, which
-// executes JavaScript and carries cookies, and is therefore a far stronger SSRF
-// primitive than an HTTP GET. Its host must match exactly, not merely be
-// non-empty.
+// Three sites are held to a stricter rule, each for a different reason:
+//
+//   - kagane and novelfull are fetched by a headless browser, which executes
+//     JavaScript and carries cookies, and is therefore a far stronger SSRF
+//     primitive than an HTTP GET. Their hosts must match exactly, not merely
+//     be non-empty.
+//   - lightnovelworld's parser regex hardcodes its host, so a URL anywhere
+//     else could never yield a match — reject it here rather than burn the
+//     request.
 func fetchableSeriesURL(site, seriesURL string) bool {
 	switch site {
-	case "asura", "demonic", "comix", "kagane":
+	case "asura", "demonic", "comix", "kagane", "novelfull", "lightnovelworld":
 	default:
 		return false
 	}
@@ -229,8 +235,17 @@ func fetchableSeriesURL(site, seriesURL string) bool {
 	if u.Scheme != "https" || u.Host == "" {
 		return false
 	}
-	if site == "kagane" {
+	switch site {
+	case "kagane":
 		return u.Hostname() == "kagane.to"
+	case "novelfull":
+		// Fetched by a real browser, same as kagane, so the host is pinned
+		// rather than merely non-empty.
+		return u.Hostname() == "novelfull.com"
+	case "lightnovelworld":
+		// Its parser regex hardcodes this host, so a URL anywhere else could
+		// never yield a match — reject it here rather than burn the request.
+		return u.Hostname() == "lightnovelworld.net"
 	}
 	return true
 }
@@ -8,7 +8,7 @@ import (
 	"testing"
 	"time"

-	"mangabm/backend/internal/store"
+	"bookmarkmanager/backend/internal/store"
 )

 // newTestStore opens a fresh SQLite store in a temp dir.
@@ -453,3 +453,49 @@ func TestKaganeUsesBrowserFetcher(t *testing.T) {
 		t.Errorf("LatestChapterNum = %v, want 41", got.LatestChapterNum)
 	}
 }
+
+func TestFetcherForRoutesNovelSites(t *testing.T) {
+	tls := &fakeFetcher{}
+	browser := &fakeFetcher{}
+	p := &Poller{Fetch: tls, BrowserFetch: browser}
+
+	cases := []struct {
+		site string
+		want Fetcher
+	}{
+		{"asura", tls},
+		{"lightnovelworld", tls},
+		{"kagane", browser},
+		{"novelfull", browser},
+	}
+	for _, tc := range cases {
+		t.Run(tc.site, func(t *testing.T) {
+			if got := p.fetcherFor(tc.site); got != tc.want {
+				t.Fatalf("fetcherFor(%q) = %v, want %v", tc.site, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
+	cases := []struct {
+		name string
+		site string
+		url  string
+		want bool
+	}{
+		{"novelfull on its own host", "novelfull", "https://novelfull.com/reverend-insanity.html", true},
+		{"novelfull on a foreign host", "novelfull", "https://evil.example/x.html", false},
+		{"novelfull over http", "novelfull", "http://novelfull.com/x.html", false},
+		{"lightnovelworld on its own host", "lightnovelworld", "https://lightnovelworld.net/novel/a-will-eternal/", true},
+		{"lightnovelworld on a foreign host", "lightnovelworld", "https://evil.example/novel/x/", false},
+		{"unknown site", "webnovel", "https://webnovel.com/x", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
+				t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
+			}
+		})
+	}
+}
@@ -1,11 +1,12 @@
 package latest

 import (
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"

-	"mangabm/backend/internal/store"
+	"bookmarkmanager/backend/internal/store"
 )

 // latestChapter is the newest chapter a series page advertises.
@@ -34,6 +35,17 @@ var comixSlugRe = regexp.MustCompile(`/title/([^/?#]+)`)
 // there are no anchors to scan.
 var kaganeChapterRe = regexp.MustCompile(`"chapter_no":"([0-9.]+)"`)

+// novelfullSlugRe pulls the series slug out of a stored series_url. novelfull
+// series pages are "/<slug>.html"; their chapter anchors are
+// "/<slug>/chapter-<n>[-<title-slug>].html". Verified live 2026-08-05.
+var novelfullSlugRe = regexp.MustCompile(`^/([^/?#]+)\.html$`)
+
+// lnwSlugRe does the same for lightnovelworld, whose series pages live under
+// /novel/<slug>/ while its chapter URLs are flat at the site root:
+// "/<slug>-chapter-<n>/", absolute in the page's own anchors. Verified live
+// 2026-08-05.
+var lnwSlugRe = regexp.MustCompile(`^/novel/([^/?#]+)/?$`)
+
 // latestChapterFrom returns the highest chapter number body advertises for this
 // series. ok is false when the body yields nothing usable — an unknown site, an
 // empty body, a Cloudflare challenge page, and a site redesign all land here,
@@ -86,6 +98,29 @@ func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
 		re = regexp.MustCompile(`"latestChapterUrl":"/title/` + regexp.QuoteMeta(id) + `-[^"]*-chapter-([0-9.]+)"`)
 	case "kagane":
 		re = kaganeChapterRe
+	case "novelfull":
+		u, err := url.Parse(seriesURL)
+		if err != nil {
+			return latestChapter{}, false
+		}
+		m := novelfullSlugRe.FindStringSubmatch(u.Path)
+		if m == nil {
+			return latestChapter{}, false
+		}
+		// Scoped to this series' slug for the same reason asura is: page 1
+		// carries a "latest chapters" widget and a "you may also like" strip,
+		// and neither may contribute to the maximum.
+		re = regexp.MustCompile(`/` + regexp.QuoteMeta(m[1]) + `/chapter-([0-9.]+)`)
+	case "lightnovelworld":
+		u, err := url.Parse(seriesURL)
+		if err != nil {
+			return latestChapter{}, false
+		}
+		m := lnwSlugRe.FindStringSubmatch(u.Path)
+		if m == nil {
+			return latestChapter{}, false
+		}
+		re = regexp.MustCompile(`lightnovelworld\.net/` + regexp.QuoteMeta(m[1]) + `-chapter-([0-9.]+)/`)
 	default:
 		return latestChapter{}, false
 	}
@@ -52,6 +52,33 @@ const kaganeAPIFixture = `
 {"book_id":"c","title":"Episode 40.5","chapter_no":"40.5","sort_no":40}]}
 `

+// Trimmed from https://novelfull.com/reverend-insanity.html fetched 2026-08-05.
+// The page carries a newest-first "latest chapters" widget above an
+// oldest-first paginated list, so the newest anchor is deliberately NOT last —
+// only a maximum finds it. The final anchor belongs to another series and must
+// be excluded by slug scoping.
+const novelfullSeriesFixture = `
+<div class="l-chapters">
+  <a href="/reverend-insanity/chapter-2334-fang-yuan-and-giant-sun.html">Chapter 2334</a>
+  <a href="/reverend-insanity/chapter-2333-three-venerables.html">Chapter 2333</a>
+</div>
+<ul class="list-chapter">
+  <li><a href="/reverend-insanity/chapter-1.html">Chapter 1</a></li>
+  <li><a href="/reverend-insanity/chapter-2.html">Chapter 2</a></li>
+</ul>
+<a href="/release-that-witch/chapter-9999.html">Chapter 9999</a>
+`
+
+// Trimmed from https://lightnovelworld.net/novel/a-will-eternal/ fetched
+// 2026-08-05. Its chapter anchors are absolute and flat — /<slug>-chapter-<n>/
+// at the site root, not under /novel/. The last anchor is another series'.
+const lnwSeriesFixture = `
+<a href="https://lightnovelworld.net/a-will-eternal-chapter-1/">Chapter 1</a>
+<a href="https://lightnovelworld.net/a-will-eternal-chapter-1317/">Chapter 1317</a>
+<a href="https://lightnovelworld.net/a-will-eternal-chapter-1298/">Chapter 1298</a>
+<a href="https://lightnovelworld.net/overgeared-chapter-9999/">Chapter 9999</a>
+`
+
 func TestLatestChapterFrom(t *testing.T) {
 	const asuraURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af"
 	const demonicURL = "https://demonicscans.org/manga/Catastrophic-Necromancer"
@@ -146,6 +173,48 @@ func TestLatestChapterFrom(t *testing.T) {
 			body:      challengeFixture,
 			wantOK:    false,
 		},
+		{
+			name:      "novelfull takes the max and ignores another series",
+			site:      "novelfull",
+			seriesURL: "https://novelfull.com/reverend-insanity.html",
+			body:      novelfullSeriesFixture,
+			wantOK:    true, wantNum: 2334, wantLabel: "Chapter 2334",
+		},
+		{
+			name:      "novelfull yields nothing on a challenge page",
+			site:      "novelfull",
+			seriesURL: "https://novelfull.com/reverend-insanity.html",
+			body:      challengeFixture,
+			wantOK:    false,
+		},
+		{
+			name:      "novelfull with an unparseable series url",
+			site:      "novelfull",
+			seriesURL: "https://novelfull.com/genre/Fantasy",
+			body:      novelfullSeriesFixture,
+			wantOK:    false,
+		},
+		{
+			name:      "lightnovelworld takes the max and ignores another series",
+			site:      "lightnovelworld",
+			seriesURL: "https://lightnovelworld.net/novel/a-will-eternal/",
+			body:      lnwSeriesFixture,
+			wantOK:    true, wantNum: 1317, wantLabel: "Chapter 1317",
+		},
+		{
+			name:      "lightnovelworld tolerates a series url with no trailing slash",
+			site:      "lightnovelworld",
+			seriesURL: "https://lightnovelworld.net/novel/a-will-eternal",
+			body:      lnwSeriesFixture,
+			wantOK:    true, wantNum: 1317, wantLabel: "Chapter 1317",
+		},
+		{
+			name:      "lightnovelworld yields nothing on a challenge page",
+			site:      "lightnovelworld",
+			seriesURL: "https://lightnovelworld.net/novel/a-will-eternal/",
+			body:      challengeFixture,
+			wantOK:    false,
+		},
 	}

 	for _, tt := range tests {