diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index 52f3824..b23d1b1 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -22,10 +22,10 @@ type latestChapter struct { // site answers the fixed questions every series-page read asks of its Site // (ADR-0009): the host its addresses must carry, how to find the Latest -// Chapter and the Cover address in a body, and — for a Site behind a -// JavaScript challenge — how to read its payload from a cleared tab. One -// entry describes everything about one Site, and nowhere else gets to compare -// the site string. +// Chapter and the Cover address in a body, whether the Site calls the work +// completed, and — for a Site behind a JavaScript challenge — how to read its +// payload from a cleared tab. One entry describes everything about one Site, +// and nowhere else gets to compare the site string. type site struct { // Host is the exact hostname a series_url for this Site must carry. Host string @@ -33,6 +33,10 @@ type site struct { LatestChapter func(seriesURL, body string) (latestChapter, bool) // Cover finds the Cover address in a fetched body. Cover func(seriesURL, body string) (string, bool) + // Completed reports whether this body carries the Site's own completed + // value. False for a body that carries any other value, and false for a + // failed extraction — never an error and never a third state. + Completed func(seriesURL, body string) bool // Rest is how long a Series of this Site rests between Polls. Rest time.Duration // Gap is the Lane's strictest pace: at least one second must pass between @@ -299,34 +303,42 @@ func kaganeCoverURL(body string) string { return "" } -func comixCoverURL(seriesURL, body string) string { +// comixDetailQuery returns the ["manga","detail",""] query entry of +// comix's initial-data JSON, or nil. The cover and completed reads share the +// scoped lookup so a "recommended" strip entry can never contribute either +// answer. +func comixDetailQuery(seriesURL, body string) json.RawMessage { id, ok := comixSeriesID(seriesURL) if !ok { - return "" + return nil } data := comixInitialDataRe.FindStringSubmatch(body) if data == nil { - return "" + return nil } var state struct { Queries map[string]json.RawMessage `json:"queries"` } if err := json.Unmarshal([]byte(data[1]), &state); err != nil { + return nil + } + return state.Queries[`["manga","detail","`+id+`"]`] +} + +func comixCoverURL(seriesURL, body string) string { + detail := comixDetailQuery(seriesURL, body) + if len(detail) == 0 { return "" } - raw := state.Queries[`["manga","detail","`+id+`"]`] - if len(raw) == 0 { - return "" - } - var detail struct { + var entry struct { Poster struct { Medium string `json:"medium"` } `json:"poster"` } - if err := json.Unmarshal(raw, &detail); err != nil { + if err := json.Unmarshal(detail, &entry); err != nil { return "" } - return publishedCoverURL(detail.Poster.Medium) + return publishedCoverURL(entry.Poster.Medium) } // ogImageCover reads the og:image metadata shared by asura, demonic and @@ -360,6 +372,87 @@ func coverFrom(site, seriesURL, body string) (string, bool) { return "", false } +// asuraStatusRe matches the status value inside the escaped astro-island +// props blob, in both the " form the served document carries and the " +// form a decoded copy would. "completed" is the only true value: "dropped" +// is scanlation editorial (the work itself continues elsewhere) and hiatus is +// its own value. +var asuraStatusRe = regexp.MustCompile(`(?:"|")status(?:"|"):\[0,(?:"|")completed(?:"|")\]`) + +func asuraCompleted(_, body string) bool { + return asuraStatusRe.MatchString(body) +} + +// demonicStatusRe matches the info block's status pair: a Status label
  • +// immediately followed by the value
  • . The site's whole status vocabulary +// is {Ongoing, Completed} (its advanced-search status filter), so the literal +// Completed value is the entire signal. +var demonicStatusRe = regexp.MustCompile(`]*>\s*Status\s*
    • \s*]*>\s*Completed\s*`) + +func demonicCompleted(_, body string) bool { + return demonicStatusRe.MatchString(body) +} + +// comixCompleted reads "status" from the scoped detail entry only; +// "finished" is the completed value, and on_hiatus and discontinued are +// distinct values. +func comixCompleted(seriesURL, body string) bool { + detail := comixDetailQuery(seriesURL, body) + if len(detail) == 0 { + return false + } + var entry struct { + Status string `json:"status"` + } + if err := json.Unmarshal(detail, &entry); err != nil { + return false + } + return entry.Status == "finished" +} + +// kaganeCompleted reads publication_status only: upload_status is the +// release's state, and the two provably diverge ('Cause Calypso Can, +// 2026-08-19: publication Ongoing, upload Hiatus), so a Completed upload +// must never read as a Completed work. +func kaganeCompleted(_, body string) bool { + var series struct { + PublicationStatus string `json:"publication_status"` + } + if err := json.Unmarshal([]byte(body), &series); err != nil { + return false + } + return series.PublicationStatus == "Completed" +} + +// novelfullStatusRe matches the info panel's status link. The page's whole +// status vocabulary is {Ongoing, Completed} (the "OnGoing" spelling aliases +// "Ongoing" on the taxonomy), so the Completed href is the signal. +var novelfullStatusRe = regexp.MustCompile(`href="/status/Completed"`) + +func novelfullCompleted(_, body string) bool { + return novelfullStatusRe.MatchString(body) +} + +// lnwCompleted matches creativeWorkStatus in the head's JSON-LD block. The +// whole body is scanned and the comment marker is not required, unlike +// lnwLatestChapter: the status block sits ahead of the visitor-writable +// thread, and hiatus maps to a distinct PotentialActionStatus value. +var lnwCompletedRe = regexp.MustCompile(`"creativeWorkStatus"\s*:\s*"https://schema\.org/CompletedActionStatus"`) + +func lnwCompleted(_, body string) bool { + return lnwCompletedRe.MatchString(body) +} + +// siteCompletedFrom reports whether the Site calls this work completed, via +// the Site's registry entry. False for an unknown site, a challenge body and +// a redesign alike: an absent hint, never a claim. +func siteCompletedFrom(site, seriesURL, body string) bool { + if fn := sites[site].Completed; fn != nil { + return fn(seriesURL, body) + } + return false +} + // metaContent returns the content of the first whose attrName is // attrValue. It keeps scanning after an empty match so a later published cover // is not hidden by an empty tag. @@ -449,6 +542,7 @@ var sites = map[string]site{ Host: "asurascans.com", LatestChapter: asuraLatestChapter, Cover: ogImageCover, + Completed: asuraCompleted, Rest: defaultRest, Gap: defaultGap, }, @@ -456,6 +550,7 @@ var sites = map[string]site{ Host: "demonicscans.org", LatestChapter: demonicLatestChapter, Cover: ogImageCover, + Completed: demonicCompleted, Rest: defaultRest, Gap: defaultGap, }, @@ -463,6 +558,7 @@ var sites = map[string]site{ Host: "comix.to", LatestChapter: comixLatestChapter, Cover: comixCoverEntry, + Completed: comixCompleted, Rest: defaultRest, Gap: defaultGap, Browser: &browserRead{ @@ -479,6 +575,7 @@ var sites = map[string]site{ Host: "kagane.to", LatestChapter: kaganeLatestChapter, Cover: kaganeCoverEntry, + Completed: kaganeCompleted, Rest: defaultRest, Gap: defaultGap, Browser: &browserRead{ @@ -493,6 +590,7 @@ var sites = map[string]site{ Host: "novelfull.com", LatestChapter: novelfullLatestChapter, Cover: novelfullCoverEntry, + Completed: novelfullCompleted, Rest: defaultRest, Gap: defaultGap, Browser: &browserRead{ @@ -507,6 +605,7 @@ var sites = map[string]site{ Host: "lightnovelworld.net", LatestChapter: lnwLatestChapter, Cover: ogImageCover, + Completed: lnwCompleted, Rest: defaultRest, Gap: defaultGap, }, diff --git a/backend/internal/latest/sites_test.go b/backend/internal/latest/sites_test.go index 989aeb0..95368c9 100644 --- a/backend/internal/latest/sites_test.go +++ b/backend/internal/latest/sites_test.go @@ -188,6 +188,100 @@ const lnwSeriesFixture = ` ` +// Completed-marker fixtures, trimmed from the live pages fetched 2026-08-22 +// for the verification step below. Selector-removed rows delete the marker +// from the consts and must answer false. + +// Trimmed from https://asurascans.com/comics/solo-leveling (301 to +// /comics/solo-leveling-b60d532c) fetched 2026-08-22. The astro-island props +// are HTML-escaped in the served document, so the quotes arrive as ". +const asuraCompletedFixture = ` +"bookmarkCount":[0,39128],"status":[0,"completed"],"type":[0,"manhwa"] +` + +// Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af +// fetched 2026-08-22. +const asuraOngoingFixture = ` +"bookmarkCount":[0,54027],"status":[0,"ongoing"],"type":[0,"manhwa"] +` + +// Trimmed from https://demonicscans.org/manga/Solo-Leveling fetched +// 2026-08-22. The info block is sloppy: bare
  • s inside a
    , label and +// value as a sibling pair. +const demonicCompletedFixture = ` +
    +
  • Status
    • +
  • Completed
    • + +` + +// Trimmed from https://demonicscans.org/manga/Catastrophic-Necromancer +// fetched 2026-08-22. Same block, ongoing value. +const demonicOngoingFixture = ` +
    +
  • Status
    • +
  • Ongoing
    • +
    +` + +// Trimmed from https://comix.to/title/q77m-countach fetched 2026-08-22 over a +// cleared Chrome tab (the in-tab fetch of the Series URL, the same +// server-rendered payload the poll reads). The recommended strip on this very +// page carries a finished entry; only the ["manga","detail","q77m"] entry +// counts. +const comixCompletedFixture = `` + +// Trimmed from https://comix.to/title/n8we-dungeons-and-crayons fetched +// 2026-08-22 the same way. The recommended strip includes a finished entry; +// the target detail is releasing, and only the detail read counts. +const comixOngoingFixture = `` + +// Trimmed from GET https://kagane.to/api/v2/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6 +// on 2026-08-22. +const kaganeCompletedFixture = `{"series_id":"019c29c3-5abd-70e6-9efc-1f1f22d839f6","title":"Real Account 1-17","publication_status":"Completed","upload_status":"Completed"}` + +// Trimmed from GET https://kagane.to/api/v2/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b +// on 2026-08-22. +const kaganeOngoingFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Ongoing"}` + +// Composed, not a single live trim: upload Completed alongside a +// non-Completed publication status is the research note's inferred inverse +// case and did not turn up in the ~1900-series live scan of 2026-08-22 (both +// field vocabularies are live-verified; the note's live divergence is 'Cause +// Calypso Can, publication Ongoing + upload Hiatus). The predicate must read +// publication_status only. +const kaganeDivergentFixture = `{"series_id":"019f84bc-9ba0-7ed9-86f5-8b905ec7c28b","title":"Infinite Decryption: The Strongest Level 0","publication_status":"Ongoing","upload_status":"Completed"}` + +// Trimmed from https://novelfull.com/reverend-insanity.html fetched +// 2026-08-22 (the challenge served to plain TLS cleared in a real Chrome the +// same day). +const novelfullCompletedFixture = `

    Status:

    Completed
    ` + +// Trimmed from https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html +// fetched 2026-08-22. +const novelfullOngoingFixture = `

    Status:

    Ongoing
    ` + +// Trimmed from the JSON-LD block in the head of +// https://lightnovelworld.net/novel/a-will-eternal/ fetched 2026-08-22. The +// block sits ahead of the wpdiscuz thread, so the predicate reads the whole +// body and needs no comment marker, unlike lnwLatestChapter. +const lnwCompletedFixture = `` + +// Trimmed from +// https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/ +// fetched 2026-08-22. Same block, ongoing value. +const lnwOngoingFixture = `` + // Trimmed from https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af // (redirected to ...-00dcbf97) on 2026-08-10. const asuraCoverFixture = `` @@ -473,3 +567,178 @@ func TestLatestChapterFrom(t *testing.T) { }) } } + +func TestSiteCompletedFrom(t *testing.T) { + const asuraURL = "https://asurascans.com/comics/solo-leveling-b60d532c" + const demonicURL = "https://demonicscans.org/manga/Solo-Leveling" + const comixURL = "https://comix.to/title/q77m-countach" + const kaganeURL = "https://kagane.to/series/019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + const novelfullURL = "https://novelfull.com/reverend-insanity.html" + const lnwURL = "https://lightnovelworld.net/novel/a-will-eternal/" + + tests := []struct { + name string + site string + seriesURL string + body string + want bool + }{ + { + name: "asura completed via escaped props status", + site: "asura", seriesURL: asuraURL, body: asuraCompletedFixture, + want: true, + }, + { + name: "asura ongoing value is not completed", + site: "asura", + seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", + body: asuraOngoingFixture, + want: false, + }, + { + name: "asura with the status key removed", + site: "asura", seriesURL: asuraURL, + body: strings.ReplaceAll(asuraCompletedFixture, `"status":[0,"completed"]`, ""), + want: false, + }, + { + name: "asura challenge page", + site: "asura", seriesURL: asuraURL, body: challengeFixture, + want: false, + }, + { + name: "demonic completed info-block pair", + site: "demonic", seriesURL: demonicURL, body: demonicCompletedFixture, + want: true, + }, + { + name: "demonic ongoing value is not completed", + site: "demonic", + seriesURL: "https://demonicscans.org/manga/Catastrophic-Necromancer", + body: demonicOngoingFixture, + want: false, + }, + { + name: "demonic with the value li removed", + site: "demonic", seriesURL: demonicURL, + body: strings.ReplaceAll(demonicCompletedFixture, "
  • Completed
    • ", ""), + want: false, + }, + { + name: "comix recommended finished does not count", + site: "comix", + seriesURL: "https://comix.to/title/n8we-dungeons-and-crayons", + body: comixOngoingFixture, + want: false, + }, + { + name: "comix detail finished wins over a finished recommended strip", + site: "comix", + seriesURL: comixURL, + body: comixCompletedFixture, + want: true, + }, + { + name: "comix with the detail status removed", + site: "comix", seriesURL: comixURL, + body: strings.ReplaceAll(comixCompletedFixture, `"status":"finished",`, ""), + want: false, + }, + { + name: "comix challenge page", + site: "comix", seriesURL: comixURL, body: challengeFixture, + want: false, + }, + { + name: "kagane publication Completed", + site: "kagane", + seriesURL: "https://kagane.to/series/019c29c3-5abd-70e6-9efc-1f1f22d839f6", + body: kaganeCompletedFixture, + want: true, + }, + { + name: "kagane upload Completed does not count", + site: "kagane", seriesURL: kaganeURL, body: kaganeDivergentFixture, + want: false, + }, + { + name: "kagane ongoing values are not completed", + site: "kagane", seriesURL: kaganeURL, body: kaganeOngoingFixture, + want: false, + }, + { + name: "kagane with publication_status removed", + site: "kagane", seriesURL: kaganeURL, + body: strings.ReplaceAll(kaganeCompletedFixture, `"publication_status":"Completed",`, ""), + want: false, + }, + { + name: "kagane challenge page", + site: "kagane", seriesURL: kaganeURL, body: challengeFixture, + want: false, + }, + { + name: "novelfull status link Completed", + site: "novelfull", + seriesURL: novelfullURL, + body: novelfullCompletedFixture, + want: true, + }, + { + name: "novelfull ongoing link is not completed", + site: "novelfull", + seriesURL: "https://novelfull.com/a-cunning-pervert-in-the-cultivation-world.html", + body: novelfullOngoingFixture, + want: false, + }, + { + name: "novelfull with the status link removed", + site: "novelfull", seriesURL: novelfullURL, + body: strings.ReplaceAll(novelfullCompletedFixture, `Completed`, ""), + want: false, + }, + { + name: "novelfull challenge page", + site: "novelfull", seriesURL: novelfullURL, body: challengeFixture, + want: false, + }, + { + name: "lightnovelworld JSON-LD CompletedActionStatus", + site: "lightnovelworld", + seriesURL: lnwURL, + body: lnwCompletedFixture, + want: true, + }, + { + name: "lightnovelworld ActiveActionStatus is not completed", + site: "lightnovelworld", + seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", + body: lnwOngoingFixture, + want: false, + }, + { + name: "lightnovelworld with creativeWorkStatus removed", + site: "lightnovelworld", seriesURL: lnwURL, + body: strings.ReplaceAll(lnwCompletedFixture, `"creativeWorkStatus": "https://schema.org/CompletedActionStatus"`, ""), + want: false, + }, + { + name: "lightnovelworld challenge page", + site: "lightnovelworld", seriesURL: lnwURL, body: challengeFixture, + want: false, + }, + { + name: "unknown site", + site: "mangadex", seriesURL: "https://mangadex.org/title/x", body: asuraCompletedFixture, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := siteCompletedFrom(tt.site, tt.seriesURL, tt.body); got != tt.want { + t.Errorf("siteCompletedFrom(%q, %q, body) = %v, want %v", tt.site, tt.seriesURL, got, tt.want) + } + }) + } +}