Add completed markers for all six sites (#168)

site gains a Completed predicate answering whether a fetched body carries the Site's own completed value: asura escaped props status, demonic info-block Status pair, comix scoped detail entry (shared comixDetailQuery helper with the cover read), kagane publication_status only, novelfull status link, lightnovelworld JSON-LD creativeWorkStatus. siteCompletedFrom dispatches it; an absent hint stays false. Fixtures trimmed from live pages fetched 2026-08-22; TestSiteCompletedFrom covers completed, ongoing, selector-removed, and challenge bodies per site, the kagane upload/publication divergence, and an unknown site.
2026-08-22 18:23:42 +07:00
parent 8e4fa6448e
commit e69e513be4
2 changed files with 382 additions and 14 deletions
@@ -22,10 +22,10 @@ type latestChapter struct {

 // site answers the fixed questions every series-page read asks of its Site
 // (ADR-0009): the host its addresses must carry, how to find the Latest
-// Chapter and the Cover address in a body, and — for a Site behind a
-// JavaScript challenge — how to read its payload from a cleared tab. One
-// entry describes everything about one Site, and nowhere else gets to compare
-// the site string.
+// Chapter and the Cover address in a body, whether the Site calls the work
+// completed, and — for a Site behind a JavaScript challenge — how to read its
+// payload from a cleared tab. One entry describes everything about one Site,
+// and nowhere else gets to compare the site string.
 type site struct {
 	// Host is the exact hostname a series_url for this Site must carry.
 	Host string
@@ -33,6 +33,10 @@ type site struct {
 	LatestChapter func(seriesURL, body string) (latestChapter, bool)
 	// Cover finds the Cover address in a fetched body.
 	Cover func(seriesURL, body string) (string, bool)
+	// Completed reports whether this body carries the Site's own completed
+	// value. False for a body that carries any other value, and false for a
+	// failed extraction — never an error and never a third state.
+	Completed func(seriesURL, body string) bool
 	// Rest is how long a Series of this Site rests between Polls.
 	Rest time.Duration
 	// Gap is the Lane's strictest pace: at least one second must pass between
@@ -299,34 +303,42 @@ func kaganeCoverURL(body string) string {
 	return ""
 }

-func comixCoverURL(seriesURL, body string) string {
+// comixDetailQuery returns the ["manga","detail","<id>"] query entry of
+// comix's initial-data JSON, or nil. The cover and completed reads share the
+// scoped lookup so a "recommended" strip entry can never contribute either
+// answer.
+func comixDetailQuery(seriesURL, body string) json.RawMessage {
 	id, ok := comixSeriesID(seriesURL)
 	if !ok {
-		return ""
+		return nil
 	}
 	data := comixInitialDataRe.FindStringSubmatch(body)
 	if data == nil {
-		return ""
+		return nil
 	}
 	var state struct {
 		Queries map[string]json.RawMessage `json:"queries"`
 	}
 	if err := json.Unmarshal([]byte(data[1]), &state); err != nil {
+		return nil
+	}
+	return state.Queries[`["manga","detail","`+id+`"]`]
+}
+
+func comixCoverURL(seriesURL, body string) string {
+	detail := comixDetailQuery(seriesURL, body)
+	if len(detail) == 0 {
 		return ""
 	}
-	raw := state.Queries[`["manga","detail","`+id+`"]`]
-	if len(raw) == 0 {
-		return ""
-	}
-	var detail struct {
+	var entry struct {
 		Poster struct {
 			Medium string `json:"medium"`
 		} `json:"poster"`
 	}
-	if err := json.Unmarshal(raw, &detail); err != nil {
+	if err := json.Unmarshal(detail, &entry); err != nil {
 		return ""
 	}
-	return publishedCoverURL(detail.Poster.Medium)
+	return publishedCoverURL(entry.Poster.Medium)
 }

 // ogImageCover reads the og:image metadata shared by asura, demonic and
@@ -360,6 +372,87 @@ func coverFrom(site, seriesURL, body string) (string, bool) {
 	return "", false
 }

+// asuraStatusRe matches the status value inside the escaped astro-island
+// props blob, in both the &quot; form the served document carries and the "
+// form a decoded copy would. "completed" is the only true value: "dropped"
+// is scanlation editorial (the work itself continues elsewhere) and hiatus is
+// its own value.
+var asuraStatusRe = regexp.MustCompile(`(?:&quot;|")status(?:&quot;|"):\[0,(?:&quot;|")completed(?:&quot;|")\]`)
+
+func asuraCompleted(_, body string) bool {
+	return asuraStatusRe.MatchString(body)
+}
+
+// demonicStatusRe matches the info block's status pair: a Status label <li>
+// immediately followed by the value <li>. The site's whole status vocabulary
+// is {Ongoing, Completed} (its advanced-search status filter), so the literal
+// Completed value is the entire signal.
+var demonicStatusRe = regexp.MustCompile(`<li[^>]*>\s*Status\s*</li>\s*<li[^>]*>\s*Completed\s*</li>`)
+
+func demonicCompleted(_, body string) bool {
+	return demonicStatusRe.MatchString(body)
+}
+
+// comixCompleted reads "status" from the scoped detail entry only;
+// "finished" is the completed value, and on_hiatus and discontinued are
+// distinct values.
+func comixCompleted(seriesURL, body string) bool {
+	detail := comixDetailQuery(seriesURL, body)
+	if len(detail) == 0 {
+		return false
+	}
+	var entry struct {
+		Status string `json:"status"`
+	}
+	if err := json.Unmarshal(detail, &entry); err != nil {
+		return false
+	}
+	return entry.Status == "finished"
+}
+
+// kaganeCompleted reads publication_status only: upload_status is the
+// release's state, and the two provably diverge ('Cause Calypso Can,
+// 2026-08-19: publication Ongoing, upload Hiatus), so a Completed upload
+// must never read as a Completed work.
+func kaganeCompleted(_, body string) bool {
+	var series struct {
+		PublicationStatus string `json:"publication_status"`
+	}
+	if err := json.Unmarshal([]byte(body), &series); err != nil {
+		return false
+	}
+	return series.PublicationStatus == "Completed"
+}
+
+// novelfullStatusRe matches the info panel's status link. The page's whole
+// status vocabulary is {Ongoing, Completed} (the "OnGoing" spelling aliases
+// "Ongoing" on the taxonomy), so the Completed href is the signal.
+var novelfullStatusRe = regexp.MustCompile(`href="/status/Completed"`)
+
+func novelfullCompleted(_, body string) bool {
+	return novelfullStatusRe.MatchString(body)
+}
+
+// lnwCompleted matches creativeWorkStatus in the head's JSON-LD block. The
+// whole body is scanned and the comment marker is not required, unlike
+// lnwLatestChapter: the status block sits ahead of the visitor-writable
+// thread, and hiatus maps to a distinct PotentialActionStatus value.
+var lnwCompletedRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`)
+
+func lnwCompleted(_, body string) bool {
+	return lnwCompletedRe.MatchString(body)
+}
+
+// siteCompletedFrom reports whether the Site calls this work completed, via
+// the Site's registry entry. False for an unknown site, a challenge body and
+// a redesign alike: an absent hint, never a claim.
+func siteCompletedFrom(site, seriesURL, body string) bool {
+	if fn := sites[site].Completed; fn != nil {
+		return fn(seriesURL, body)
+	}
+	return false
+}
+
 // metaContent returns the content of the first <meta> whose attrName is
 // attrValue. It keeps scanning after an empty match so a later published cover
 // is not hidden by an empty tag.
@@ -449,6 +542,7 @@ var sites = map[string]site{
 		Host:          "asurascans.com",
 		LatestChapter: asuraLatestChapter,
 		Cover:         ogImageCover,
+		Completed:     asuraCompleted,
 		Rest:          defaultRest,
 		Gap:           defaultGap,
 	},
@@ -456,6 +550,7 @@ var sites = map[string]site{
 		Host:          "demonicscans.org",
 		LatestChapter: demonicLatestChapter,
 		Cover:         ogImageCover,
+		Completed:     demonicCompleted,
 		Rest:          defaultRest,
 		Gap:           defaultGap,
 	},
@@ -463,6 +558,7 @@ var sites = map[string]site{
 		Host:          "comix.to",
 		LatestChapter: comixLatestChapter,
 		Cover:         comixCoverEntry,
+		Completed:     comixCompleted,
 		Rest:          defaultRest,
 		Gap:           defaultGap,
 		Browser: &browserRead{
@@ -479,6 +575,7 @@ var sites = map[string]site{
 		Host:          "kagane.to",
 		LatestChapter: kaganeLatestChapter,
 		Cover:         kaganeCoverEntry,
+		Completed:     kaganeCompleted,
 		Rest:          defaultRest,
 		Gap:           defaultGap,
 		Browser: &browserRead{
@@ -493,6 +590,7 @@ var sites = map[string]site{
 		Host:          "novelfull.com",
 		LatestChapter: novelfullLatestChapter,
 		Cover:         novelfullCoverEntry,
+		Completed:     novelfullCompleted,
 		Rest:          defaultRest,
 		Gap:           defaultGap,
 		Browser: &browserRead{
@@ -507,6 +605,7 @@ var sites = map[string]site{
 		Host:          "lightnovelworld.net",
 		LatestChapter: lnwLatestChapter,
 		Cover:         ogImageCover,
+		Completed:     lnwCompleted,
 		Rest:          defaultRest,
 		Gap:           defaultGap,
 	},
@@ -188,6 +188,100 @@ const lnwSeriesFixture = `
 </div>
 `

+// Completed-marker fixtures, trimmed from the live pages fetched 2026-08-22
+// for the verification step below. Selector-removed rows delete the marker
+// from the consts and must answer false.
+
+// Trimmed from https://asurascans.com/comics/solo-leveling (301 to
+// /comics/solo-leveling-b60d532c) fetched 2026-08-22. The astro-island props
+// are HTML-escaped in the served document, so the quotes arrive as &quot;.
+const asuraCompletedFixture = `
+&quot;bookmarkCount&quot;:[0,39128],&quot;status&quot;:[0,&quot;completed&quot;],&quot;type&quot;:[0,&quot;manhwa&quot;]
+`
+
+// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
+// fetched 2026-08-22.
+const asuraOngoingFixture = `
+&quot;bookmarkCount&quot;:[0,54027],&quot;status&quot;:[0,&quot;ongoing&quot;],&quot;type&quot;:[0,&quot;manhwa&quot;]
+`
+
+// Trimmed from https://demonicscans.org/manga/Solo-Leveling fetched
+// 2026-08-22. The info block is sloppy: bare <li>s inside a <div>, label and
+// value as a sibling pair.
+const demonicCompletedFixture = `
+<div class="flex flex-row">
+    <li style="width:150px;color:#b2b2b2;">Status</li>
+    <li>Completed</li>
+</div>
+`
+
+// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer
+// fetched 2026-08-22. Same block, ongoing value.
+const demonicOngoingFixture = `
+<div class="flex flex-row">
+    <li style="width:150px;color:#b2b2b2;">Status</li>
+    <li>Ongoing</li>
+</div>
+`
+
+// Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 over a
+// cleared Chrome tab (the in-tab fetch of the Series URL, the same
+// server-rendered payload the poll reads). The recommended strip on this very
+// page carries a finished entry; only the ["manga","detail","q77m"] entry
+// counts.
+const comixCompletedFixture = `<script type="application/json" id="initial-data">{"queries":{"[\"manga\",\"recommended\",\"q77m\",1]":{"items":[{"hid":"pz65","title":"Wangan Midnight: C1 Runner","status":"finished"}]},"[\"manga\",\"detail\",\"q77m\"]":{"id":13211,"hid":"q77m","title":"Countach","status":"finished","originalLanguage":"ja"}}}</script>`
+
+// Trimmed from https://comix.to/title/n8we-dungeons-and-crayons fetched
+// 2026-08-22 the same way. The recommended strip includes a finished entry;
+// the target detail is releasing, and only the detail read counts.
+const comixOngoingFixture = `<script type="application/json" id="initial-data">{"queries":{"[\"manga\",\"recommended\",\"n8we\",1]":{"items":[{"hid":"g2rk","title":"On the Way to Meet Mom","status":"finished"}]},"[\"manga\",\"detail\",\"n8we\"]":{"id":1429,"hid":"n8we","title":"Dungeons and Crayons","status":"releasing","originalLanguage":"ko"}}}</script>`
+
+// Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6
+// on 2026-08-22.
+const kaganeCompletedFixture = `{"series_id":"019c29c3-5abd-70e6-9efc-1f1f22d839f6","title":"Real Account 1-17","publication_status":"Completed","upload_status":"Completed"}`
+
+// Trimmed from GET https://kagane.to/api/v2/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b
+// on 2026-08-22.
+const kaganeOngoingFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Ongoing"}`
+
+// Composed, not a single live trim: upload Completed alongside a
+// non-Completed publication status is the research note's inferred inverse
+// case and did not turn up in the ~1900-series live scan of 2026-08-22 (both
+// field vocabularies are live-verified; the note's live divergence is 'Cause
+// Calypso Can, publication Ongoing + upload Hiatus). The predicate must read
+// publication_status only.
+const kaganeDivergentFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Completed"}`
+
+// Trimmed from https://novelfull.com/reverend-insanity.html fetched
+// 2026-08-22 (the challenge served to plain TLS cleared in a real Chrome the
+// same day).
+const novelfullCompletedFixture = `<div><h3>Status:</h3><a href="/status/Completed">Completed</a></div>`
+
+// Trimmed from https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html
+// fetched 2026-08-22.
+const novelfullOngoingFixture = `<div><h3>Status:</h3><a href="/status/Ongoing">Ongoing</a></div>`
+
+// Trimmed from the JSON-LD block in the head of
+// https://lightnovelworld.net/novel/a-will-eternal/ fetched 2026-08-22. The
+// block sits ahead of the wpdiscuz thread, so the predicate reads the whole
+// body and needs no comment marker, unlike lnwLatestChapter.
+const lnwCompletedFixture = `<script type="application/ld+json">{
+    "@context": "https://schema.org",
+    "@type": "Book",
+    "name": "A Will Eternal",
+    "creativeWorkStatus": "https://schema.org/CompletedActionStatus"
+}</script>`
+
+// Trimmed from
+// https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/
+// fetched 2026-08-22. Same block, ongoing value.
+const lnwOngoingFixture = `<script type="application/ld+json">{
+    "@context": "https://schema.org",
+    "@type": "Book",
+    "name": "All Jobs and Classes I Just Wanted One Skill Not Them All",
+    "creativeWorkStatus": "https://schema.org/ActiveActionStatus"
+}</script>`
+
 // Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af
 // (redirected to ...-00dcbf97) on 2026-08-10.
 const asuraCoverFixture = `<meta property="og:image" content="https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp">`
@@ -473,3 +567,178 @@ func TestLatestChapterFrom(t *testing.T) {
 		})
 	}
 }
+
+func TestSiteCompletedFrom(t *testing.T) {
+	const asuraURL = "https://asurascans.com/comics/solo-leveling-b60d532c"
+	const demonicURL = "https://demonicscans.org/manga/Solo-Leveling"
+	const comixURL = "https://comix.to/title/q77m-countach"
+	const kaganeURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
+	const novelfullURL = "https://novelfull.com/reverend-insanity.html"
+	const lnwURL = "https://lightnovelworld.net/novel/a-will-eternal/"
+
+	tests := []struct {
+		name      string
+		site      string
+		seriesURL string
+		body      string
+		want      bool
+	}{
+		{
+			name: "asura completed via escaped props status",
+			site: "asura", seriesURL: asuraURL, body: asuraCompletedFixture,
+			want: true,
+		},
+		{
+			name:      "asura ongoing value is not completed",
+			site:      "asura",
+			seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
+			body:      asuraOngoingFixture,
+			want:      false,
+		},
+		{
+			name: "asura with the status key removed",
+			site: "asura", seriesURL: asuraURL,
+			body: strings.ReplaceAll(asuraCompletedFixture, `&quot;status&quot;:[0,&quot;completed&quot;]`, ""),
+			want: false,
+		},
+		{
+			name: "asura challenge page",
+			site: "asura", seriesURL: asuraURL, body: challengeFixture,
+			want: false,
+		},
+		{
+			name: "demonic completed info-block pair",
+			site: "demonic", seriesURL: demonicURL, body: demonicCompletedFixture,
+			want: true,
+		},
+		{
+			name:      "demonic ongoing value is not completed",
+			site:      "demonic",
+			seriesURL: "https://demonicscans.org/manga/Catastrophic-Necromancer",
+			body:      demonicOngoingFixture,
+			want:      false,
+		},
+		{
+			name: "demonic with the value li removed",
+			site: "demonic", seriesURL: demonicURL,
+			body: strings.ReplaceAll(demonicCompletedFixture, "<li>Completed</li>", ""),
+			want: false,
+		},
+		{
+			name:      "comix recommended finished does not count",
+			site:      "comix",
+			seriesURL: "https://comix.to/title/n8we-dungeons-and-crayons",
+			body:      comixOngoingFixture,
+			want:      false,
+		},
+		{
+			name:      "comix detail finished wins over a finished recommended strip",
+			site:      "comix",
+			seriesURL: comixURL,
+			body:      comixCompletedFixture,
+			want:      true,
+		},
+		{
+			name: "comix with the detail status removed",
+			site: "comix", seriesURL: comixURL,
+			body: strings.ReplaceAll(comixCompletedFixture, `"status":"finished",`, ""),
+			want: false,
+		},
+		{
+			name: "comix challenge page",
+			site: "comix", seriesURL: comixURL, body: challengeFixture,
+			want: false,
+		},
+		{
+			name:      "kagane publication Completed",
+			site:      "kagane",
+			seriesURL: "https://kagane.to/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6",
+			body:      kaganeCompletedFixture,
+			want:      true,
+		},
+		{
+			name: "kagane upload Completed does not count",
+			site: "kagane", seriesURL: kaganeURL, body: kaganeDivergentFixture,
+			want: false,
+		},
+		{
+			name: "kagane ongoing values are not completed",
+			site: "kagane", seriesURL: kaganeURL, body: kaganeOngoingFixture,
+			want: false,
+		},
+		{
+			name: "kagane with publication_status removed",
+			site: "kagane", seriesURL: kaganeURL,
+			body: strings.ReplaceAll(kaganeCompletedFixture, `"publication_status":"Completed",`, ""),
+			want: false,
+		},
+		{
+			name: "kagane challenge page",
+			site: "kagane", seriesURL: kaganeURL, body: challengeFixture,
+			want: false,
+		},
+		{
+			name:      "novelfull status link Completed",
+			site:      "novelfull",
+			seriesURL: novelfullURL,
+			body:      novelfullCompletedFixture,
+			want:      true,
+		},
+		{
+			name:      "novelfull ongoing link is not completed",
+			site:      "novelfull",
+			seriesURL: "https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html",
+			body:      novelfullOngoingFixture,
+			want:      false,
+		},
+		{
+			name: "novelfull with the status link removed",
+			site: "novelfull", seriesURL: novelfullURL,
+			body: strings.ReplaceAll(novelfullCompletedFixture, `<a href="/status/Completed">Completed</a>`, ""),
+			want: false,
+		},
+		{
+			name: "novelfull challenge page",
+			site: "novelfull", seriesURL: novelfullURL, body: challengeFixture,
+			want: false,
+		},
+		{
+			name:      "lightnovelworld JSON-LD CompletedActionStatus",
+			site:      "lightnovelworld",
+			seriesURL: lnwURL,
+			body:      lnwCompletedFixture,
+			want:      true,
+		},
+		{
+			name:      "lightnovelworld ActiveActionStatus is not completed",
+			site:      "lightnovelworld",
+			seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
+			body:      lnwOngoingFixture,
+			want:      false,
+		},
+		{
+			name: "lightnovelworld with creativeWorkStatus removed",
+			site: "lightnovelworld", seriesURL: lnwURL,
+			body: strings.ReplaceAll(lnwCompletedFixture, `"creativeWorkStatus": "https://schema.org/CompletedActionStatus"`, ""),
+			want: false,
+		},
+		{
+			name: "lightnovelworld challenge page",
+			site: "lightnovelworld", seriesURL: lnwURL, body: challengeFixture,
+			want: false,
+		},
+		{
+			name: "unknown site",
+			site: "mangadex", seriesURL: "https://mangadex.org/title/x", body: asuraCompletedFixture,
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := siteCompletedFrom(tt.site, tt.seriesURL, tt.body); got != tt.want {
+				t.Errorf("siteCompletedFrom(%q, %q, body) = %v, want %v", tt.site, tt.seriesURL, got, tt.want)
+			}
+		})
+	}
+}