fix: address final-review findings on the latest-chapter poller

Wires the five poll env vars into docker-compose (the documented kill
switch was inert), skips fetching unknown sites and non-https URLs
before spending a request, and makes runOnce's summary log fire on
empty and cancelled ticks. Records the accepted non-atomic Get+Upsert
window and the one-interval startup delay in the docs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-26 18:18:11 +07:00
parent 28f2d32e45
commit e966182e6f
6 changed files with 100 additions and 7 deletions
+36 -4
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"log"
"net/url"
"time"
)
@@ -66,9 +67,6 @@ func (p *latestPoller) runOnce(ctx context.Context) {
log.Printf("latest poll: due query: %v", err)
return
}
if len(due) == 0 {
return
}
checked := 0
for i, b := range due {
@@ -79,13 +77,17 @@ func (p *latestPoller) runOnce(ctx context.Context) {
// from one server IP is the traffic shape most likely to move that IP's
// bot score. This is the server-side analogue of the userscript's "one
// series per navigation ... indistinguishable from browsing" (L455-456).
stopped := false
if i > 0 && p.stagger > 0 {
select {
case <-ctx.Done():
return
stopped = true
case <-time.After(p.stagger):
}
}
if stopped {
break
}
p.checkOne(ctx, b)
checked++
}
@@ -114,6 +116,18 @@ func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) {
return
}
// series_url is client-supplied (PUT /bookmarks/{key} accepts any string),
// so this is not just an optimisation against burning a request on an
// unknown site: without it, the server would issue a GET from its own
// network position to whatever URL a token-holder writes, including
// link-local/internal addresses or non-https schemes. The cooldown above
// is already consumed, so a row that never passes this check is retried at
// cooldown pace rather than hot-looping.
if !fetchableSeriesURL(b.Site, b.SeriesURL) {
log.Printf("latest poll %q: not fetchable: site=%q url=%q", b.Key, b.Site, b.SeriesURL)
return
}
body, status, err := p.fetch.Get(ctx, b.SeriesURL)
if err != nil {
log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
@@ -160,3 +174,21 @@ func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) {
}
log.Printf("latest poll %q: latest is now %s", b.Key, latest.Label)
}
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
// to parse and seriesURL is safe to hand to the fetcher: an https URL with a
// non-empty host. series_url comes from client-supplied PUT bodies, so this is
// a defence against the poller being used to probe arbitrary hosts from the
// server's own network position, not just a check against wasted requests.
func fetchableSeriesURL(site, seriesURL string) bool {
switch site {
case "asura", "demonic":
default:
return false
}
u, err := url.Parse(seriesURL)
if err != nil {
return false
}
return u.Scheme == "https" && u.Host != ""
}
+6 -3
View File
@@ -33,9 +33,12 @@ var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?ch
// The userscript's asura rule additionally requires the anchor text to match
// /Chapter\s+[\d.]+/i. That check exists only to skip the "First Chapter"
// shortcut, which points at chapter/1 and therefore can never win a maximum, so
// it is redundant here. Scoping the pattern to this series' own slug replaces it
// with a stronger guarantee: a chapter link belonging to some other series
// cannot contribute even if the page starts carrying them.
// it is redundant here. For asura, scoping the pattern to this series' own slug
// replaces it with a stronger guarantee: a chapter link belonging to some other
// series cannot contribute even if the page starts carrying them. demonic has no
// such guarantee — demonicChapterRe matches any chaptered.php?manga=<id> anchor
// with no per-series scoping, because the stored series_id for demonic is a
// slug, not the numeric id the URL carries, so it cannot easily be scoped.
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
var re *regexp.Regexp
switch site {
+42
View File
@@ -259,6 +259,48 @@ func TestRunOnceCorrectsDownward(t *testing.T) {
}
}
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
// reject anything that is not a known site with an https URL before spending a
// request on it — the cooldown still gets consumed either way.
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
tests := []struct {
name string
site string
seriesURL string
wantCalls int
}{
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
{"unparseable url", "asura", "http://[::1", 0},
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newTestStore(t)
key := tt.site + ":x"
if _, err := s.Upsert(Bookmark{
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
now := time.UnixMilli(4_000_000)
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
newTestPoller(t, s, f, now).checkOne(context.Background(), Bookmark{
Key: key, Site: tt.site, SeriesURL: tt.seriesURL,
})
if got := f.callCount(); got != tt.wantCalls {
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
}
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
}
})
}
}
// A cancelled context must abandon the batch rather than run it to completion.
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
s := newTestStore(t)