The Poll fills blank Covers for every Site and both Libraries (#61)
Permanently-blank Series rows heal on the next due cycle: the chapter-page body already in hand is scanned for a Cover, bytes are stored, and the wire address is set. A non-blank Cover is never replaced. Failures log against the Series, leave the chapter poll alone, and retry on the next due cycle with no separate queue. Kagane source-URL prefetch now lands on the same content-addressed SetSeriesCover path as every other Site.
This commit is contained in:
@@ -2,6 +2,7 @@ package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/url"
|
||||
"slices"
|
||||
@@ -57,60 +58,85 @@ type Poller struct {
|
||||
|
||||
var browserBackedSites = []string{"kagane", "novelfull"}
|
||||
|
||||
// fillBlankCover gives a Series its Cover when it has none. The blank state is
|
||||
// what "no Cover yet" means on the wire (ADR-0007): permanently-blank rows
|
||||
// created before acquisition existed, and rows whose creation-time fetch
|
||||
// failed, both heal here. A non-blank CoverAddress is left alone — refetching
|
||||
// would add a request per Series per cycle and change artwork under the Reader
|
||||
// for no visible reason. A row that already carries a source URL is owned by
|
||||
// prefetchCover instead; this path only extracts from the series page.
|
||||
//
|
||||
// Failures are logged against the Series and never returned: the chapter poll
|
||||
// must not notice. A failed fill is retried the next time this Series is due;
|
||||
// there is no separate retry queue.
|
||||
func (p *Poller) fillBlankCover(ctx context.Context, sr store.Series, body string) {
|
||||
if sr.CoverAddress != "" || sr.Cover != "" {
|
||||
return
|
||||
}
|
||||
cover, ok := coverFrom(sr.Site, sr.SeriesURL, body)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p.storeCover(ctx, sr, cover)
|
||||
}
|
||||
|
||||
// prefetchCover heals Series that already carry a third-party source URL but
|
||||
// no stored address — the state left by client-supplied covers before
|
||||
// acquisition moved server-side. Every Site takes the same path; only the
|
||||
// byte fetcher differs (kagane needs the browser). New blanks have no source
|
||||
// URL and go through fillBlankCover from the series page instead.
|
||||
func (p *Poller) prefetchCover(ctx context.Context, sr store.Series) {
|
||||
if sr.Cover == "" {
|
||||
if sr.Cover == "" || sr.CoverAddress != "" {
|
||||
return
|
||||
}
|
||||
if sr.Site == "kagane" {
|
||||
if p.CoverFetch == nil {
|
||||
return
|
||||
}
|
||||
imageID, ok := store.KaganeImageID(sr.Cover)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
_, _, found, err := p.Store.GetKaganeCover(imageID)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
return
|
||||
}
|
||||
body, contentType, err := p.CoverFetch.Image(ctx, imageID)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
if err := p.Store.PutKaganeCover(imageID, body, contentType); err != nil {
|
||||
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.CoverBytesFetch == nil || sr.CoverAddress != "" {
|
||||
return
|
||||
}
|
||||
// Bytes may already be stored from an earlier poll that ran before the
|
||||
// Series carried an address; storing them again is free (they are
|
||||
// content-addressed and immutable), and the point of the second call is
|
||||
// the address, which is what makes the Cover visible on the wire.
|
||||
body, contentType, found, err := p.Store.GetCover(sr.Cover)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: read cover: %v", sr.Key(), err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
body, contentType, err = p.CoverBytesFetch.Fetch(ctx, sr.Cover)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch cover: %v", sr.Key(), err)
|
||||
return
|
||||
if found {
|
||||
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil {
|
||||
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sr.Cover, body, contentType); err != nil {
|
||||
p.storeCover(ctx, sr, sr.Cover)
|
||||
}
|
||||
|
||||
// storeCover fetches bytes for sourceURL and points the Series at them. Every
|
||||
// failure is logged against the Series and swallowed so the chapter poll
|
||||
// cannot see it.
|
||||
func (p *Poller) storeCover(ctx context.Context, sr store.Series, sourceURL string) {
|
||||
bytes, contentType, err := p.fetchCoverBytes(ctx, sr, sourceURL)
|
||||
if err != nil {
|
||||
log.Printf("latest poll %q: fetch cover %s: %v", sr.Key(), sourceURL, err)
|
||||
return
|
||||
}
|
||||
if err := p.Store.SetSeriesCover(sr.Site, sr.SeriesID, sourceURL, bytes, contentType); err != nil {
|
||||
log.Printf("latest poll %q: persist cover: %v", sr.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchCoverBytes routes by Site: only kagane needs the browser for image
|
||||
// bytes; every other Site's CDN answers plain TLS. Missing fetchers degrade to
|
||||
// a blank Cover rather than falling back onto a path that cannot succeed.
|
||||
func (p *Poller) fetchCoverBytes(ctx context.Context, sr store.Series, cover string) ([]byte, string, error) {
|
||||
if sr.Site == "kagane" {
|
||||
if p.CoverFetch == nil {
|
||||
return nil, "", errors.New("no cover fetcher")
|
||||
}
|
||||
imageID, ok := store.KaganeImageID(cover)
|
||||
if !ok {
|
||||
return nil, "", errors.New("invalid kagane cover URL")
|
||||
}
|
||||
return p.CoverFetch.Image(ctx, imageID)
|
||||
}
|
||||
if p.CoverBytesFetch == nil {
|
||||
return nil, "", errors.New("no cover fetcher")
|
||||
}
|
||||
return p.CoverBytesFetch.Fetch(ctx, cover)
|
||||
}
|
||||
|
||||
// fetcherFor returns the fetcher a site needs, or nil when the site cannot be
|
||||
// fetched at all right now. kagane and novelfull both sit behind a Cloudflare
|
||||
// JavaScript challenge that no TLS fingerprint clears — kagane verified
|
||||
@@ -234,6 +260,9 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) {
|
||||
}
|
||||
|
||||
latest, ok := latestChapterFrom(sr.Site, sr.SeriesURL, body)
|
||||
// Cover fill is independent of the chapter signal: a page that lost its
|
||||
// chapter list may keep its og:image, and a blank Series heals either way.
|
||||
p.fillBlankCover(ctx, sr, body)
|
||||
if !ok {
|
||||
// Most likely a challenge page or a layout change. Either way the row is
|
||||
// already stamped, so this waits out a cooldown instead of hot-looping.
|
||||
|
||||
Reference in New Issue
Block a user