The Poll fills blank Covers for every Site and both Libraries (#61)

Permanently-blank Series rows heal on the next due cycle: the chapter-page
body already in hand is scanned for a Cover, bytes are stored, and the wire
address is set. A non-blank Cover is never replaced. Failures log against the
Series, leave the chapter poll alone, and retry on the next due cycle with no
separate queue. Kagane source-URL prefetch now lands on the same
content-addressed SetSeriesCover path as every other Site.
This commit is contained in:
2026-08-10 09:25:42 +07:00
parent e2c054e7ce
commit 6ca1b6fdf4
2 changed files with 288 additions and 40 deletions
+69 -40
View File
@@ -2,6 +2,7 @@ package latest
import (
"context"
"errors"
"log"
"net/url"
"slices"
@@ -57,60 +58,85 @@ type Poller struct {
var browserBackedSites = []string{"kagane", "novelfull"}
// fillBlankCover gives a Series its Cover when it has none. The blank state is
// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows
// created before acquisition existed, and rows whose creation-time fetch
// failed, both heal here. A non-blank CoverAddress is left alone — refetching
// would add a request per Series per cycle and change artwork under the Reader
// for no visible reason. A row that already carries a source URL is owned by
// prefetchCover instead; this path only extracts from the series page.
//
// Failures are logged against the Series and never returned: the chapter poll
// must not notice. A failed fill is retried the next time this Series is due;
// there is no separate retry queue.
func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, body string) {
if sr.CoverAddress != "" || sr.Cover != "" {
return
}
cover, ok := coverFrom(sr.Site, sr.SeriesURL, body)
if !ok {
return
}
p.storeCover(ctx, sr, cover)
}
// prefetchCover heals Series that already carry a third-party source URL but
// no stored address — the state left by client-supplied covers before
// acquisition moved server-side. Every Site takes the same path; only the
// byte fetcher differs (kagane needs the browser). New blanks have no source
// URL and go through fillBlankCover from the series page instead.
func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) {
if sr.Cover == "" {
if sr.Cover == "" || sr.CoverAddress != "" {
return
}
if sr.Site == "kagane" {
if p.CoverFetch == nil {
return
}
imageID, ok := store.KaganeImageID(sr.Cover)
if !ok {
return
}
_, _, found, err := p.Store.GetKaganeCover(imageID)
if err != nil {
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
return
}
if found {
return
}
body, contentType, err := p.CoverFetch.Image(ctx, imageID)
if err != nil {
log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err)
return
}
if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil {
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
}
return
}
if p.CoverBytesFetch == nil || sr.CoverAddress != "" {
return
}
// Bytes may already be stored from an earlier poll that ran before the
// Series carried an address; storing them again is free (they are
// content-addressed and immutable), and the point of the second call is
// the address, which is what makes the Cover visible on the wire.
body, contentType, found, err := p.Store.GetCover(sr.Cover)
if err != nil {
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
return
}
if !found {
body, contentType, err = p.CoverBytesFetch.Fetch(ctx, sr.Cover)
if err != nil {
log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err)
return
if found {
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil {
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
}
return
}
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil {
p.storeCover(ctx, sr, sr.Cover)
}
// storeCover fetches bytes for sourceURL and points the Series at them. Every
// failure is logged against the Series and swallowed so the chapter poll
// cannot see it.
func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL string) {
bytes, contentType, err := p.fetchCoverBytes(ctx, sr, sourceURL)
if err != nil {
log.Printf("latest poll %q: fetch cover %s: %v", sr.Key(), sourceURL, err)
return
}
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sourceURL, bytes, contentType); err != nil {
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
}
}
// fetchCoverBytes routes by Site: only kagane needs the browser for image
// bytes; every other Site's CDN answers plain TLS. Missing fetchers degrade to
// a blank Cover rather than falling back onto a path that cannot succeed.
func (p *Poller) fetchCoverBytes(ctx context.Context, sr store.Series, cover string) ([]byte, string, error) {
if sr.Site == "kagane" {
if p.CoverFetch == nil {
return nil, "", errors.New("no cover fetcher")
}
imageID, ok := store.KaganeImageID(cover)
if !ok {
return nil, "", errors.New("invalid kagane cover URL")
}
return p.CoverFetch.Image(ctx, imageID)
}
if p.CoverBytesFetch == nil {
return nil, "", errors.New("no cover fetcher")
}
return p.CoverBytesFetch.Fetch(ctx, cover)
}
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
// JavaScript challenge that no TLS fingerprint clears — kagane verified
@@ -234,6 +260,9 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
}
latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body)
// Cover fill is independent of the chapter signal: a page that lost its
// chapter list may keep its og:image, and a blank Series heals either way.
p.fillBlankCover(ctx, sr, body)
if !ok {
// Most likely a challenge page or a layout change. Either way the row is
// already stamped, so this waits out a cooldown instead of hot-looping.