fix: address final-review findings on the latest-chapter poller
Wires the five poll env vars into docker-compose (the documented kill switch was inert), skips fetching unknown sites and non-https URLs before spending a request, and makes runOnce's summary log fire on empty and cancelled ticks. Records the accepted non-atomic Get+Upsert window and the one-interval startup delay in the docs. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+36
-4
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -66,9 +67,6 @@ func (p *latestPoller) runOnce(ctx context.Context) {
|
||||
log.Printf("latest poll: due query: %v", err)
|
||||
return
|
||||
}
|
||||
if len(due) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
checked := 0
|
||||
for i, b := range due {
|
||||
@@ -79,13 +77,17 @@ func (p *latestPoller) runOnce(ctx context.Context) {
|
||||
// from one server IP is the traffic shape most likely to move that IP's
|
||||
// bot score. This is the server-side analogue of the userscript's "one
|
||||
// series per navigation ... indistinguishable from browsing" (L455-456).
|
||||
stopped := false
|
||||
if i > 0 && p.stagger > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
stopped = true
|
||||
case <-time.After(p.stagger):
|
||||
}
|
||||
}
|
||||
if stopped {
|
||||
break
|
||||
}
|
||||
p.checkOne(ctx, b)
|
||||
checked++
|
||||
}
|
||||
@@ -114,6 +116,18 @@ func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) {
|
||||
return
|
||||
}
|
||||
|
||||
// series_url is client-supplied (PUT /bookmarks/{key} accepts any string),
|
||||
// so this is not just an optimisation against burning a request on an
|
||||
// unknown site: without it, the server would issue a GET from its own
|
||||
// network position to whatever URL a token-holder writes, including
|
||||
// link-local/internal addresses or non-https schemes. The cooldown above
|
||||
// is already consumed, so a row that never passes this check is retried at
|
||||
// cooldown pace rather than hot-looping.
|
||||
if !fetchableSeriesURL(b.Site, b.SeriesURL) {
|
||||
log.Printf("latest poll %q: not fetchable: site=%q url=%q", b.Key, b.Site, b.SeriesURL)
|
||||
return
|
||||
}
|
||||
|
||||
body, status, err := p.fetch.Get(ctx, b.SeriesURL)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
|
||||
@@ -160,3 +174,21 @@ func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) {
|
||||
}
|
||||
log.Printf("latest poll %q: latest is now %s", b.Key, latest.Label)
|
||||
}
|
||||
|
||||
// fetchableSeriesURL reports whether site is a site latestChapterFrom knows how
|
||||
// to parse and seriesURL is safe to hand to the fetcher: an https URL with a
|
||||
// non-empty host. series_url comes from client-supplied PUT bodies, so this is
|
||||
// a defence against the poller being used to probe arbitrary hosts from the
|
||||
// server's own network position, not just a check against wasted requests.
|
||||
func fetchableSeriesURL(site, seriesURL string) bool {
|
||||
switch site {
|
||||
case "asura", "demonic":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return u.Scheme == "https" && u.Host != ""
|
||||
}
|
||||
|
||||
@@ -33,9 +33,12 @@ var demonicChapterRe = regexp.MustCompile(`chaptered\.php\?manga=\d+&(?:amp;)?ch
|
||||
// The userscript's asura rule additionally requires the anchor text to match
|
||||
// /Chapter\s+[\d.]+/i. That check exists only to skip the "First Chapter"
|
||||
// shortcut, which points at chapter/1 and therefore can never win a maximum, so
|
||||
// it is redundant here. Scoping the pattern to this series' own slug replaces it
|
||||
// with a stronger guarantee: a chapter link belonging to some other series
|
||||
// cannot contribute even if the page starts carrying them.
|
||||
// it is redundant here. For asura, scoping the pattern to this series' own slug
|
||||
// replaces it with a stronger guarantee: a chapter link belonging to some other
|
||||
// series cannot contribute even if the page starts carrying them. demonic has no
|
||||
// such guarantee — demonicChapterRe matches any chaptered.php?manga=<id> anchor
|
||||
// with no per-series scoping, because the stored series_id for demonic is a
|
||||
// slug, not the numeric id the URL carries, so it cannot easily be scoped.
|
||||
func latestChapterFrom(site, seriesURL, body string) (latestChapter, bool) {
|
||||
var re *regexp.Regexp
|
||||
switch site {
|
||||
|
||||
@@ -259,6 +259,48 @@ func TestRunOnceCorrectsDownward(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// series_url is client-supplied via PUT /bookmarks/{key}, so checkOne must
|
||||
// reject anything that is not a known site with an https URL before spending a
|
||||
// request on it — the cooldown still gets consumed either way.
|
||||
func TestCheckOneValidatesSeriesURLBeforeFetching(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
site string
|
||||
seriesURL string
|
||||
wantCalls int
|
||||
}{
|
||||
{"unknown site", "mangadex", "https://mangadex.org/title/x", 0},
|
||||
{"http scheme", "asura", "http://asurascans.com/comics/x", 0},
|
||||
{"unparseable url", "asura", "http://[::1", 0},
|
||||
{"valid https asura", "asura", "https://asurascans.com/comics/x", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
key := tt.site + ":x"
|
||||
if _, err := s.Upsert(Bookmark{
|
||||
Key: key, Site: tt.site, SeriesID: "x", SeriesURL: tt.seriesURL,
|
||||
UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
now := time.UnixMilli(4_000_000)
|
||||
f := &fakeFetcher{body: asuraSeriesFixture, status: 200}
|
||||
newTestPoller(t, s, f, now).checkOne(context.Background(), Bookmark{
|
||||
Key: key, Site: tt.site, SeriesURL: tt.seriesURL,
|
||||
})
|
||||
|
||||
if got := f.callCount(); got != tt.wantCalls {
|
||||
t.Fatalf("fetch calls = %d, want %d", got, tt.wantCalls)
|
||||
}
|
||||
if got := readLatestCheckedAt(t, s, key); got != now.UnixMilli() {
|
||||
t.Fatalf("latest_checked_at = %d, want %d (cooldown must be consumed regardless)", got, now.UnixMilli())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A cancelled context must abandon the batch rather than run it to completion.
|
||||
func TestRunOnceStopsOnCancelledContext(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
Reference in New Issue
Block a user