feat: background poller for latest published chapter
Ticker goroutine reads bookmarks past their per-bookmark cooldown, fetches the series page, and writes latest_chapter through Get+Upsert so updated_at never moves and the list never reorders. The row is stamped before the fetch so a broken series waits out a cooldown instead of retrying every tick.
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fetcher retrieves a series page. It exists as an interface so tests can inject
|
||||
// a fake: nothing in the test suite may touch the network or the TLS client.
|
||||
type fetcher interface {
|
||||
Get(ctx context.Context, url string) (body string, status int, err error)
|
||||
}
|
||||
|
||||
// latestPoller re-checks each bookmarked series' newest published chapter on a
|
||||
// schedule, independent of the userscript's own in-browser checks. The two run
|
||||
// in parallel and report the same observable fact, so whichever writes last wins
|
||||
// and neither needs to know about the other.
|
||||
//
|
||||
// Two clocks, deliberately independent:
|
||||
//
|
||||
// - interval is how often this goroutine wakes up and looks.
|
||||
// - cooldown is how long one bookmark rests since its own last check.
|
||||
//
|
||||
// Only the cooldown is per bookmark, and it is enforced by the WHERE clause in
|
||||
// DueForLatestCheck rather than by any timer. Shortening interval therefore
|
||||
// cannot shorten anyone's cooldown; it only makes the poller wake up and find
|
||||
// nothing due more often.
|
||||
type latestPoller struct {
|
||||
store *Store
|
||||
fetch fetcher
|
||||
now func() time.Time // injected so tests can freeze it
|
||||
cooldown time.Duration
|
||||
interval time.Duration
|
||||
stagger time.Duration
|
||||
batch int
|
||||
}
|
||||
|
||||
// Run polls until ctx is cancelled.
|
||||
//
|
||||
// runOnce is called synchronously, so a batch that overruns the tick delays the
|
||||
// next one instead of stacking a second batch on top of it. That is the intended
|
||||
// failure mode for a misconfigured batch x stagger: a slower cadence, never
|
||||
// concurrent fetch storms.
|
||||
func (p *latestPoller) Run(ctx context.Context) {
|
||||
log.Printf("latest-chapter poller: interval=%s cooldown=%s batch=%d stagger=%s",
|
||||
p.interval, p.cooldown, p.batch, p.stagger)
|
||||
t := time.NewTicker(p.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("latest-chapter poller: stopped")
|
||||
return
|
||||
case <-t.C:
|
||||
p.runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runOnce processes one batch of due bookmarks.
|
||||
func (p *latestPoller) runOnce(ctx context.Context) {
|
||||
cutoff := p.now().Add(-p.cooldown).UnixMilli()
|
||||
due, err := p.store.DueForLatestCheck(cutoff, p.batch)
|
||||
if err != nil {
|
||||
log.Printf("latest poll: due query: %v", err)
|
||||
return
|
||||
}
|
||||
if len(due) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
checked := 0
|
||||
for i, b := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
// Staggered rather than fired together: a burst of simultaneous requests
|
||||
// from one server IP is the traffic shape most likely to move that IP's
|
||||
// bot score. This is the server-side analogue of the userscript's "one
|
||||
// series per navigation ... indistinguishable from browsing" (L455-456).
|
||||
if i > 0 && p.stagger > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(p.stagger):
|
||||
}
|
||||
}
|
||||
p.checkOne(ctx, b)
|
||||
checked++
|
||||
}
|
||||
// due vs checked is how you tell which constraint is binding: ticks that
|
||||
// report due=0 mean the cooldown is the limit, ticks that report due==batch
|
||||
// every time mean throughput is.
|
||||
log.Printf("latest poll: due=%d checked=%d", len(due), checked)
|
||||
}
|
||||
|
||||
// checkOne re-checks one series. Every failure path here is "log and move on":
|
||||
// the poller is a best-effort enhancement, and no single bad series may stall a
|
||||
// batch or take down the process.
|
||||
func (p *latestPoller) checkOne(ctx context.Context, b Bookmark) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("latest poll %q: recovered from panic: %v", b.Key, r)
|
||||
}
|
||||
}()
|
||||
|
||||
// Stamped before the fetch, not after, so an error, a timeout, or a shutdown
|
||||
// mid-request still consumes the cooldown. Otherwise a renamed or deleted
|
||||
// series would be retried on every single tick forever. The userscript
|
||||
// stamps in the same order and for the same reason (L471-473).
|
||||
if err := p.store.MarkLatestChecked(b.Key, p.now().UnixMilli()); err != nil {
|
||||
log.Printf("latest poll %q: mark checked: %v", b.Key, err)
|
||||
return
|
||||
}
|
||||
|
||||
body, status, err := p.fetch.Get(ctx, b.SeriesURL)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch %s: %v", b.Key, b.SeriesURL, err)
|
||||
return
|
||||
}
|
||||
if status != 200 {
|
||||
log.Printf("latest poll %q: fetch %s: status %d", b.Key, b.SeriesURL, status)
|
||||
return
|
||||
}
|
||||
|
||||
latest, ok := latestChapterFrom(b.Site, b.SeriesURL, body)
|
||||
if !ok {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||
log.Printf("latest poll %q: no chapter links in %d bytes", b.Key, len(body))
|
||||
return
|
||||
}
|
||||
|
||||
// Re-read: the row may have been updated or deleted while the fetch was in
|
||||
// flight, and writing b back wholesale would undo that.
|
||||
cur, found, err := p.store.Get(b.Key)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: reread: %v", b.Key, err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
// Equality, not >, mirroring the userscript (L427): a site that retracts a
|
||||
// chapter should correct the stored number downward.
|
||||
if cur.LatestChapterNum != nil && *cur.LatestChapterNum == latest.Num {
|
||||
return
|
||||
}
|
||||
|
||||
num := latest.Num
|
||||
cur.LatestChapter = latest.Label
|
||||
cur.LatestChapterNum = &num
|
||||
// A candidate only. last_chapter_num is untouched, so the CASE in Upsert
|
||||
// keeps the stored updated_at and the bookmark list does not reorder.
|
||||
cur.UpdatedAt = p.now().UnixMilli()
|
||||
if _, err := p.store.Upsert(cur); err != nil {
|
||||
log.Printf("latest poll %q: upsert: %v", b.Key, err)
|
||||
return
|
||||
}
|
||||
log.Printf("latest poll %q: latest is now %s", b.Key, latest.Label)
|
||||
}
|
||||
Reference in New Issue
Block a user