Add completed markers for all six sites (#168)
site gains a Completed predicate answering whether a fetched body carries the Site's own completed value: asura escaped props status, demonic info-block Status pair, comix scoped detail entry (shared comixDetailQuery helper with the cover read), kagane publication_status only, novelfull status link, lightnovelworld JSON-LD creativeWorkStatus. siteCompletedFrom dispatches it; an absent hint stays false. Fixtures trimmed from live pages fetched 2026-08-22; TestSiteCompletedFrom covers completed, ongoing, selector-removed, and challenge bodies per site, the kagane upload/publication divergence, and an unknown site.
This commit is contained in:
@@ -22,10 +22,10 @@ type latestChapter struct {
|
||||
|
||||
// site answers the fixed questions every series-page read asks of its Site
|
||||
// (ADR-0009): the host its addresses must carry, how to find the Latest
|
||||
// Chapter and the Cover address in a body, and — for a Site behind a
|
||||
// JavaScript challenge — how to read its payload from a cleared tab. One
|
||||
// entry describes everything about one Site, and nowhere else gets to compare
|
||||
// the site string.
|
||||
// Chapter and the Cover address in a body, whether the Site calls the work
|
||||
// completed, and — for a Site behind a JavaScript challenge — how to read its
|
||||
// payload from a cleared tab. One entry describes everything about one Site,
|
||||
// and nowhere else gets to compare the site string.
|
||||
type site struct {
|
||||
// Host is the exact hostname a series_url for this Site must carry.
|
||||
Host string
|
||||
@@ -33,6 +33,10 @@ type site struct {
|
||||
LatestChapter func(seriesURL, body string) (latestChapter, bool)
|
||||
// Cover finds the Cover address in a fetched body.
|
||||
Cover func(seriesURL, body string) (string, bool)
|
||||
// Completed reports whether this body carries the Site's own completed
|
||||
// value. False for a body that carries any other value, and false for a
|
||||
// failed extraction — never an error and never a third state.
|
||||
Completed func(seriesURL, body string) bool
|
||||
// Rest is how long a Series of this Site rests between Polls.
|
||||
Rest time.Duration
|
||||
// Gap is the Lane's strictest pace: at least one second must pass between
|
||||
@@ -299,34 +303,42 @@ func kaganeCoverURL(body string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func comixCoverURL(seriesURL, body string) string {
|
||||
// comixDetailQuery returns the ["manga","detail","<id>"] query entry of
|
||||
// comix's initial-data JSON, or nil. The cover and completed reads share the
|
||||
// scoped lookup so a "recommended" strip entry can never contribute either
|
||||
// answer.
|
||||
func comixDetailQuery(seriesURL, body string) json.RawMessage {
|
||||
id, ok := comixSeriesID(seriesURL)
|
||||
if !ok {
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
data := comixInitialDataRe.FindStringSubmatch(body)
|
||||
if data == nil {
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
var state struct {
|
||||
Queries map[string]json.RawMessage `json:"queries"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(data[1]), &state); err != nil {
|
||||
return nil
|
||||
}
|
||||
return state.Queries[`["manga","detail","`+id+`"]`]
|
||||
}
|
||||
|
||||
func comixCoverURL(seriesURL, body string) string {
|
||||
detail := comixDetailQuery(seriesURL, body)
|
||||
if len(detail) == 0 {
|
||||
return ""
|
||||
}
|
||||
raw := state.Queries[`["manga","detail","`+id+`"]`]
|
||||
if len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
var detail struct {
|
||||
var entry struct {
|
||||
Poster struct {
|
||||
Medium string `json:"medium"`
|
||||
} `json:"poster"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &detail); err != nil {
|
||||
if err := json.Unmarshal(detail, &entry); err != nil {
|
||||
return ""
|
||||
}
|
||||
return publishedCoverURL(detail.Poster.Medium)
|
||||
return publishedCoverURL(entry.Poster.Medium)
|
||||
}
|
||||
|
||||
// ogImageCover reads the og:image metadata shared by asura, demonic and
|
||||
@@ -360,6 +372,87 @@ func coverFrom(site, seriesURL, body string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// asuraStatusRe matches the status value inside the escaped astro-island
|
||||
// props blob, in both the " form the served document carries and the "
|
||||
// form a decoded copy would. "completed" is the only true value: "dropped"
|
||||
// is scanlation editorial (the work itself continues elsewhere) and hiatus is
|
||||
// its own value.
|
||||
var asuraStatusRe = regexp.MustCompile(`(?:"|")status(?:"|"):\[0,(?:"|")completed(?:"|")\]`)
|
||||
|
||||
func asuraCompleted(_, body string) bool {
|
||||
return asuraStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// demonicStatusRe matches the info block's status pair: a Status label <li>
|
||||
// immediately followed by the value <li>. The site's whole status vocabulary
|
||||
// is {Ongoing, Completed} (its advanced-search status filter), so the literal
|
||||
// Completed value is the entire signal.
|
||||
var demonicStatusRe = regexp.MustCompile(`<li[^>]*>\s*Status\s*</li>\s*<li[^>]*>\s*Completed\s*</li>`)
|
||||
|
||||
func demonicCompleted(_, body string) bool {
|
||||
return demonicStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// comixCompleted reads "status" from the scoped detail entry only;
|
||||
// "finished" is the completed value, and on_hiatus and discontinued are
|
||||
// distinct values.
|
||||
func comixCompleted(seriesURL, body string) bool {
|
||||
detail := comixDetailQuery(seriesURL, body)
|
||||
if len(detail) == 0 {
|
||||
return false
|
||||
}
|
||||
var entry struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
if err := json.Unmarshal(detail, &entry); err != nil {
|
||||
return false
|
||||
}
|
||||
return entry.Status == "finished"
|
||||
}
|
||||
|
||||
// kaganeCompleted reads publication_status only: upload_status is the
|
||||
// release's state, and the two provably diverge ('Cause Calypso Can,
|
||||
// 2026-08-19: publication Ongoing, upload Hiatus), so a Completed upload
|
||||
// must never read as a Completed work.
|
||||
func kaganeCompleted(_, body string) bool {
|
||||
var series struct {
|
||||
PublicationStatus string `json:"publication_status"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &series); err != nil {
|
||||
return false
|
||||
}
|
||||
return series.PublicationStatus == "Completed"
|
||||
}
|
||||
|
||||
// novelfullStatusRe matches the info panel's status link. The page's whole
|
||||
// status vocabulary is {Ongoing, Completed} (the "OnGoing" spelling aliases
|
||||
// "Ongoing" on the taxonomy), so the Completed href is the signal.
|
||||
var novelfullStatusRe = regexp.MustCompile(`href="/status/Completed"`)
|
||||
|
||||
func novelfullCompleted(_, body string) bool {
|
||||
return novelfullStatusRe.MatchString(body)
|
||||
}
|
||||
|
||||
// lnwCompleted matches creativeWorkStatus in the head's JSON-LD block. The
|
||||
// whole body is scanned and the comment marker is not required, unlike
|
||||
// lnwLatestChapter: the status block sits ahead of the visitor-writable
|
||||
// thread, and hiatus maps to a distinct PotentialActionStatus value.
|
||||
var lnwCompletedRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`)
|
||||
|
||||
func lnwCompleted(_, body string) bool {
|
||||
return lnwCompletedRe.MatchString(body)
|
||||
}
|
||||
|
||||
// siteCompletedFrom reports whether the Site calls this work completed, via
|
||||
// the Site's registry entry. False for an unknown site, a challenge body and
|
||||
// a redesign alike: an absent hint, never a claim.
|
||||
func siteCompletedFrom(site, seriesURL, body string) bool {
|
||||
if fn := sites[site].Completed; fn != nil {
|
||||
return fn(seriesURL, body)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// metaContent returns the content of the first <meta> whose attrName is
|
||||
// attrValue. It keeps scanning after an empty match so a later published cover
|
||||
// is not hidden by an empty tag.
|
||||
@@ -449,6 +542,7 @@ var sites = map[string]site{
|
||||
Host: "asurascans.com",
|
||||
LatestChapter: asuraLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Completed: asuraCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
@@ -456,6 +550,7 @@ var sites = map[string]site{
|
||||
Host: "demonicscans.org",
|
||||
LatestChapter: demonicLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Completed: demonicCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
@@ -463,6 +558,7 @@ var sites = map[string]site{
|
||||
Host: "comix.to",
|
||||
LatestChapter: comixLatestChapter,
|
||||
Cover: comixCoverEntry,
|
||||
Completed: comixCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
@@ -479,6 +575,7 @@ var sites = map[string]site{
|
||||
Host: "kagane.to",
|
||||
LatestChapter: kaganeLatestChapter,
|
||||
Cover: kaganeCoverEntry,
|
||||
Completed: kaganeCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
@@ -493,6 +590,7 @@ var sites = map[string]site{
|
||||
Host: "novelfull.com",
|
||||
LatestChapter: novelfullLatestChapter,
|
||||
Cover: novelfullCoverEntry,
|
||||
Completed: novelfullCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
Browser: &browserRead{
|
||||
@@ -507,6 +605,7 @@ var sites = map[string]site{
|
||||
Host: "lightnovelworld.net",
|
||||
LatestChapter: lnwLatestChapter,
|
||||
Cover: ogImageCover,
|
||||
Completed: lnwCompleted,
|
||||
Rest: defaultRest,
|
||||
Gap: defaultGap,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user