Spec #135: owner data-correction actions — Latest Chapter, series_url, Cover, orphan removal #156

Merged
sulthan merged 15 commits from spec-135 into main 2026-08-22 12:10:41 +07:00
8 changed files with 337 additions and 77 deletions
Showing only changes of commit 448631c78e - Show all commits
+4 -4
View File
@@ -32,7 +32,7 @@ func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
// The wire URL is what a client actually requests, so the path under test // The wire URL is what a client actually requests, so the path under test
// is taken from it rather than rebuilt by hand. // is taken from it rather than rebuilt by hand.
wire := st.CoverWireURL(store.CoverAddress(sourceURL)) wire := st.CoverWireURL(store.CoverAddressForBytes([]byte("\x00webp-bytes")))
path, ok := strings.CutPrefix(wire, testCoverBaseURL) path, ok := strings.CutPrefix(wire, testCoverBaseURL)
if !ok { if !ok {
t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL) t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL)
@@ -57,7 +57,7 @@ func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
func TestPublicCoverRejectsUnknownAddress(t *testing.T) { func TestPublicCoverRejectsUnknownAddress(t *testing.T) {
srv, _ := newWebTestServer(t, testConfig()) srv, _ := newWebTestServer(t, testConfig())
cases := map[string]string{ cases := map[string]string{
"unknown": "/covers/" + store.CoverAddress("https://cdn.example/never-stored.jpg"), "unknown": "/covers/" + store.CoverAddressForBytes([]byte("never-stored")),
"malformed": "/covers/not-an-address", "malformed": "/covers/not-an-address",
"traversal": "/covers/../../etc/passwd", "traversal": "/covers/../../etc/passwd",
"empty": "/covers/", "empty": "/covers/",
@@ -86,7 +86,7 @@ func TestPublicCoverNeverEchoesNonImage(t *testing.T) {
} }
// A legitimate row, then the content type flipped behind the store's back: // A legitimate row, then the content type flipped behind the store's back:
// the bytes exist at the address, so only the type is hostile. // the bytes exist at the address, so only the type is hostile.
address := store.CoverAddress(sourceURL) address := store.CoverAddressForBytes([]byte("<script>"))
if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil { if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil {
t.Fatalf("seed row: %v", err) t.Fatalf("seed row: %v", err)
} }
@@ -127,7 +127,7 @@ func TestListRendersAcquiredCover(t *testing.T) {
if rr.Code != http.StatusOK { if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rr.Code) t.Fatalf("status = %d, want 200", rr.Code)
} }
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddress(sourceURL) + `"` want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("\xff\xd8jpeg")) + `"`
if !strings.Contains(rr.Body.String(), want) { if !strings.Contains(rr.Body.String(), want) {
t.Fatalf("rendered list does not contain %s", want) t.Fatalf("rendered list does not contain %s", want)
} }
+7 -7
View File
@@ -123,10 +123,10 @@ func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 { if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum) t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
} }
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want) t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want)
} }
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(acquireCoverURL)) body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
if err != nil || !ok { if err != nil || !ok {
t.Fatalf("CoverByAddress = %v, %v", ok, err) t.Fatalf("CoverByAddress = %v, %v", ok, err)
} }
@@ -182,7 +182,7 @@ func TestAcquireSkipsAnExistingSeries(t *testing.T) {
t.Fatalf("cover fetches = %d, want 1", got) t.Fatalf("cover fetches = %d, want 1", got)
} }
got := readBookmark(t, s, acquireKey) got := readBookmark(t, s, acquireKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want) t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want)
} }
} }
@@ -343,10 +343,10 @@ func TestAcquireKaganeCoverThroughBrowser(t *testing.T) {
t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc) t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc)
} }
got := readBookmark(t, s, kaganeKey) got := readBookmark(t, s, kaganeKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(kaganeCoverSrc); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want) t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
} }
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(kaganeCoverSrc)) body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
if err != nil || !ok { if err != nil || !ok {
t.Fatalf("CoverByAddress = %v, %v", ok, err) t.Fatalf("CoverByAddress = %v, %v", ok, err)
} }
@@ -381,7 +381,7 @@ func TestAcquireNovelfullCoverOverPlainTLS(t *testing.T) {
t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL) t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
} }
got := readBookmark(t, s, novelfullKey) got := readBookmark(t, s, novelfullKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want) t.Fatalf("Cover = %q, want %q", got.Cover, want)
} }
} }
@@ -451,7 +451,7 @@ func TestAcquireNovelfullCoverWithoutBrowser(t *testing.T) {
t.Fatalf("cover fetches = %d, want 1", got) t.Fatalf("cover fetches = %d, want 1", got)
} }
got := readBookmark(t, s, novelfullKey) got := readBookmark(t, s, novelfullKey)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want) t.Fatalf("Cover = %q, want %q", got.Cover, want)
} }
} }
+10 -14
View File
@@ -214,9 +214,9 @@ func TestRunOncePrefetchesPublicCover(t *testing.T) {
if got := covers.callCount(); got != 1 { if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got) t.Fatalf("cover fetch calls = %d, want 1", got)
} }
body, contentType, found, err := s.GetCover(coverURL) body, contentType, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
if err != nil || !found { if err != nil || !found {
t.Fatalf("GetCover: %v found=%v", err, found) t.Fatalf("CoverByAddress: %v found=%v", err, found)
} }
if string(body) != "cover-bytes" || contentType != "image/jpeg" { if string(body) != "cover-bytes" || contentType != "image/jpeg" {
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType) t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
@@ -663,7 +663,7 @@ func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
if err != nil || !found { if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found) t.Fatalf("Get: %v found=%v", err, found)
} }
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want) t.Fatalf("Cover = %q, want %q", got.Cover, want)
} }
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 { if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
@@ -811,7 +811,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
p.runOnce(context.Background()) p.runOnce(context.Background())
p.waitCovers() p.waitCovers()
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL)) body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
if err != nil || !ok { if err != nil || !ok {
t.Fatalf("CoverByAddress: %v found=%v", err, ok) t.Fatalf("CoverByAddress: %v found=%v", err, ok)
} }
@@ -821,7 +821,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
if got := covers.callCount(); got != 1 { if got := covers.callCount(); got != 1 {
t.Fatalf("cover fetch calls = %d, want 1", got) t.Fatalf("cover fetch calls = %d, want 1", got)
} }
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) { if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddressForBytes([]byte("cover-bytes")) {
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover) t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
} }
} }
@@ -914,7 +914,7 @@ func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
p.runOnce(context.Background()) p.runOnce(context.Background())
p.waitCovers() p.waitCovers()
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found { if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("not an image"))); err != nil || found {
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err) t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
} }
} }
@@ -940,7 +940,7 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
p.runOnce(context.Background()) p.runOnce(context.Background())
p.waitCovers() p.waitCovers()
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found { if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || found {
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err) t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
} }
} }
@@ -1038,7 +1038,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
seriesURL string seriesURL string
kind string kind string
body string body string
wantCover string
browser bool browser bool
}{ }{
{ {
@@ -1047,14 +1046,12 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
seriesID: "chronicles-of-the-demon-faction-f886a8af", seriesID: "chronicles-of-the-demon-faction-f886a8af",
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af", seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture, kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
}, },
{ {
name: "lightnovelworld novel", name: "lightnovelworld novel",
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld", key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/", seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture, kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
}, },
{ {
name: "kagane manga", name: "kagane manga",
@@ -1062,7 +1059,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787", seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787", seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true, kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
}, },
} }
for _, tc := range cases { for _, tc := range cases {
@@ -1086,7 +1082,7 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
p.waitCovers() p.waitCovers()
got := readBookmark(t, s, tc.key) got := readBookmark(t, s, tc.key)
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover) wantWire := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes"))
if got.Cover != wantWire { if got.Cover != wantWire {
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire) t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
} }
@@ -1143,7 +1139,7 @@ func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
t.Fatalf("cover fetch calls = %d, want 0", got) t.Fatalf("cover fetch calls = %d, want 0", got)
} }
got := readBookmark(t, s, key) got := readBookmark(t, s, key)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want) t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
} }
} }
@@ -1190,7 +1186,7 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
t.Fatalf("cover fetch calls after retry = %d, want 2", got) t.Fatalf("cover fetch calls after retry = %d, want 2", got)
} }
got := readBookmark(t, s, key) got := readBookmark(t, s, key)
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want { if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want) t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
} }
} }
+12 -8
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"net/http" "net/http"
"os" "os"
"strings"
"testing" "testing"
"time" "time"
@@ -108,10 +109,7 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
if ws == "" { if ws == "" {
t.Skip("SMOKE_BROWSER_WS_URL unset") t.Skip("SMOKE_BROWSER_WS_URL unset")
} }
const ( const seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
coverURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed"
)
s, _ := newTestStore(t) s, _ := newTestStore(t)
bf, err := NewBrowserFetcher(ws) bf, err := NewBrowserFetcher(ws)
if err != nil { if err != nil {
@@ -140,12 +138,13 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
if err != nil || !found { if err != nil || !found {
t.Fatalf("Get: %v found=%v", err, found) t.Fatalf("Get: %v found=%v", err, found)
} }
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want { addr, ok := strings.CutPrefix(got.Cover, testCoverBaseURL+"/covers/")
t.Fatalf("Cover = %q, want %q — the acquire path did not store the browser-fetched bytes", got.Cover, want) if !ok {
t.Fatalf("Cover = %q, want an address on %q — the acquire path did not store the browser-fetched bytes", got.Cover, testCoverBaseURL+"/covers/")
} }
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL)) body, contentType, ok, err := s.CoverByAddress(addr)
if err != nil || !ok { if err != nil || !ok {
t.Fatalf("CoverByAddress: %v found=%v", err, ok) t.Fatalf("CoverByAddress(%q): %v found=%v", addr, err, ok)
} }
if len(body) < 1000 { if len(body) < 1000 {
t.Fatalf("stored cover is %d bytes, want a real image", len(body)) t.Fatalf("stored cover is %d bytes, want a real image", len(body))
@@ -153,5 +152,10 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
if contentType != "image/webp" { if contentType != "image/webp" {
t.Fatalf("content type = %q, want image/webp", contentType) t.Fatalf("content type = %q, want image/webp", contentType)
} }
// The address the row carries is the bytes' own SHA-256: a re-art behind
// the same URL would be a different address, which is the whole point.
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes(body); got.Cover != want {
t.Fatalf("Cover = %q, want %q", got.Cover, want)
}
t.Logf("stored %d bytes of %s", len(body), contentType) t.Logf("stored %d bytes of %s", len(body), contentType)
} }
@@ -1,8 +1,15 @@
-- The Cover splits into two facts. `cover` keeps the third-party address the -- The Cover splits into two facts. `cover` keeps the third-party address the
-- bytes come from, which is what the acquisition path refetches and dedupes -- bytes come from, which is what the refetch path dedupes on; `cover_address`
-- on; `cover_address` is the content address of the bytes once they are -- is the content address of the bytes once they are actually stored, and is
-- actually stored, and is what the wire's absolute URL is built from. -- what the wire's absolute URL is built from.
-- --
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover -- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
-- that 404s", which is the distinction the API and the UI both depend on. -- that 404s", which is the distinction the API and the UI both depend on.
--
-- The content address was originally the hex SHA-256 of the source URL
-- (ADR-0007). Since ADR-0014 it is the hex SHA-256 of the bytes themselves,
-- so a re-art behind the same URL is a new address. Rows written before
-- ADR-0014 keep their URL-derived addresses; they are never rehashed and heal
-- into byte addressing on their first forced replacement. Both derivations
-- share the 64-hex-digit shape, so the serving guard is unchanged.
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT ''; ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
+81 -29
View File
@@ -704,67 +704,76 @@ func (s *Store) getCoverByAddress(address string) ([]byte, string, bool, error)
return body, contentType, true, nil return body, contentType, true, nil
} }
func (s *Store) putCover(sourceURL string, body []byte, contentType string) error { func (s *Store) putCover(sourceURL string, body []byte, contentType string) (string, error) {
stored, ok := CoverContentType(contentType) stored, ok := CoverContentType(contentType)
if !ok { if !ok {
return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType) return "", fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
} }
contentType = stored contentType = stored
address := coverSourceAddress(sourceURL) address := CoverAddressForBytes(body)
relativePath := coverRelativePath(address) relativePath := coverRelativePath(address)
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath)) coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
return fmt.Errorf("create cover shard: %w", err) return "", fmt.Errorf("create cover shard: %w", err)
} }
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*") tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
if err != nil { if err != nil {
return fmt.Errorf("create cover temp file: %w", err) return "", fmt.Errorf("create cover temp file: %w", err)
} }
tmpName := tmp.Name() tmpName := tmp.Name()
defer os.Remove(tmpName) defer os.Remove(tmpName)
if _, err := tmp.Write(body); err != nil { if _, err := tmp.Write(body); err != nil {
tmp.Close() tmp.Close()
return fmt.Errorf("write cover temp file: %w", err) return "", fmt.Errorf("write cover temp file: %w", err)
} }
if err := tmp.Sync(); err != nil { if err := tmp.Sync(); err != nil {
tmp.Close() tmp.Close()
return fmt.Errorf("sync cover temp file: %w", err) return "", fmt.Errorf("sync cover temp file: %w", err)
} }
if err := tmp.Close(); err != nil { if err := tmp.Close(); err != nil {
return fmt.Errorf("close cover temp file: %w", err) return "", fmt.Errorf("close cover temp file: %w", err)
} }
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) { if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("install cover file: %w", err) return "", fmt.Errorf("install cover file: %w", err)
} }
if _, err := s.db.Exec(` if _, err := s.db.Exec(`
INSERT INTO covers (address, path, content_type) INSERT INTO covers (address, path, content_type)
VALUES ($1, $2, $3) VALUES ($1, $2, $3)
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil { ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
return fmt.Errorf("record cover %q: %w", address, err) return "", fmt.Errorf("record cover %q: %w", address, err)
} }
return nil return address, nil
} }
// GetCover returns the immutable object addressed by its source URL. Missing // GetCover returns the immutable object a source URL's own hash names. Rows
// written before byte addressing (ADR-0014) are the only ones that ever reach
// it; it hashes the URL, so a byte-addressed Cover is invisible to it. Missing
// files are reported with ok=false so callers can retry acquisition later. // files are reported with ok=false so callers can retry acquisition later.
func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) { func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) {
return s.getCover(sourceURL) return s.getCover(sourceURL)
} }
// PutCover persists bytes under the source URL's content address. A later // PutCover persists bytes under their own content address (ADR-0014). A later
// write for the same URL cannot replace the immutable object. // write of the same bytes cannot replace the immutable object.
func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error { func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error {
return s.putCover(sourceURL, body, contentType) _, err := s.putCover(sourceURL, body, contentType)
return err
} }
// CoverAddress is the content address bytes fetched from sourceURL are stored // CoverAddressForBytes is the content address body is stored under: the hex
// under. It is a pure function of the URL, so the acquisition path can name a // SHA-256 of the bytes, so identical artwork is one address and a re-art a
// Cover before it has the bytes. // new one. Legacy rows were addressed from their source URL instead and are
func CoverAddress(sourceURL string) string { return coverSourceAddress(sourceURL) } // never rehashed — both derivations coexist (ADR-0014).
func CoverAddressForBytes(body []byte) string {
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:])
}
// coverAddressRe is the shape of a stored address: the hex SHA-256 of a source // coverAddressRe is the shape of a stored address: 64 lowercase hex digits —
// URL. Request paths reach CoverByAddress, so the shape is checked before the // the hex SHA-256 of the cover bytes, or of the source URL for legacy rows
// value is ever turned into a filesystem path. // (ADR-0014). Request paths reach CoverByAddress, so the shape is checked
// before the value is ever turned into a filesystem path; byte-derived
// addresses keep the same shape, so the guard is unchanged.
var coverAddressRe = regexp.MustCompile(`^[0-9a-f]{64}$`) var coverAddressRe = regexp.MustCompile(`^[0-9a-f]{64}$`)
// CoverByAddress returns the immutable object at one content address. An // CoverByAddress returns the immutable object at one content address. An
@@ -788,24 +797,67 @@ func (s *Store) CoverWireURL(address string) string {
return s.coverBaseURL + "/covers/" + address return s.coverBaseURL + "/covers/" + address
} }
// SetSeriesCover stores the bytes and points the Series at them, but only // SetSeriesCover stores the bytes and points the Series at their address, but
// while the Series has no Cover: acquisition at creation and the poll both // only while the Series has no Cover: acquisition at creation and the poll
// call this, and whichever arrives second must not overwrite the first. The // both call this, and whichever arrives second must not overwrite the first.
// bytes themselves are content-addressed and immutable, so storing them twice // The bytes themselves are content-addressed and immutable, so storing them
// is free. // twice is free. See ReplaceSeriesCover for the write that may move a Cover
// once one exists (ADR-0014).
func (s *Store) SetSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) error { func (s *Store) SetSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) error {
if err := s.putCover(sourceURL, body, contentType); err != nil { address, err := s.putCover(sourceURL, body, contentType)
if err != nil {
return err return err
} }
if _, err := s.db.Exec(` if _, err := s.db.Exec(`
UPDATE series SET cover = $3, cover_address = $4 UPDATE series SET cover = $3, cover_address = $4
WHERE site = $1 AND series_id = $2 AND cover_address = ''`, WHERE site = $1 AND series_id = $2 AND cover_address = ''`,
site, seriesID, sourceURL, coverSourceAddress(sourceURL)); err != nil { site, seriesID, sourceURL, address); err != nil {
return fmt.Errorf("set cover for %q: %w", site+":"+seriesID, err) return fmt.Errorf("set cover for %q: %w", site+":"+seriesID, err)
} }
return nil return nil
} }
// ReplaceSeriesCover stores the bytes and points the Series at their address
// whether or not one already exists, writing the current source URL alongside
// — the Forced Poll's installer and the only write that may move a Cover once
// one exists (ADR-0014). previous is the address the row held before the write
// ("" if it had none) and current the address of the bytes just stored; both
// are read and written in one transaction, so a concurrent replacement reports
// the exact displacement. previous == current means the Site served identical
// artwork, an honest no-op; otherwise previous is stranded — its bytes stay
// served under the covers table (ADR-0014), the row just no longer points at
// them.
func (s *Store) ReplaceSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) (previous, current string, err error) {
current, err = s.putCover(sourceURL, body, contentType)
if err != nil {
return "", "", err
}
tx, err := s.db.Begin()
if err != nil {
return "", "", fmt.Errorf("begin replace cover for %q: %w", site+":"+seriesID, err)
}
defer tx.Rollback()
err = tx.QueryRow(`
SELECT cover_address FROM series
WHERE site = $1 AND series_id = $2 FOR UPDATE`,
site, seriesID).Scan(&previous)
if errors.Is(err, sql.ErrNoRows) {
previous = ""
} else if err != nil {
return "", "", fmt.Errorf("read cover for %q: %w", site+":"+seriesID, err)
}
if _, err := tx.Exec(`
UPDATE series SET cover = $3, cover_address = $4
WHERE site = $1 AND series_id = $2`,
site, seriesID, sourceURL, current); err != nil {
return "", "", fmt.Errorf("replace cover for %q: %w", site+":"+seriesID, err)
}
if err := tx.Commit(); err != nil {
return "", "", fmt.Errorf("commit cover replace for %q: %w", site+":"+seriesID, err)
}
return previous, current, nil
}
// List returns every bookmark of one reader, newest activity first. // List returns every bookmark of one reader, newest activity first.
// Series-owned fields are joined in, so each Bookmark reads back whole and // Series-owned fields are joined in, so each Bookmark reads back whole and
// flat (ADR-0004). // flat (ADR-0004).
+127 -12
View File
@@ -850,7 +850,7 @@ func TestSetSeriesCoverDoesNotOverwrite(t *testing.T) {
if err != nil || !ok { if err != nil || !ok {
t.Fatalf("Get = %v, %v", ok, err) t.Fatalf("Get = %v, %v", ok, err)
} }
if want := "https://bookmarks.test/covers/" + CoverAddress(first); got.Cover != want { if want := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("first")); got.Cover != want {
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want) t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
} }
} }
@@ -869,7 +869,7 @@ func TestCoverByAddress(t *testing.T) {
t.Fatalf("SetSeriesCover: %v", err) t.Fatalf("SetSeriesCover: %v", err)
} }
body, contentType, ok, err := store.CoverByAddress(CoverAddress(source)) body, contentType, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("bytes")))
if err != nil || !ok { if err != nil || !ok {
t.Fatalf("CoverByAddress = %v, %v", ok, err) t.Fatalf("CoverByAddress = %v, %v", ok, err)
} }
@@ -877,8 +877,8 @@ func TestCoverByAddress(t *testing.T) {
t.Fatalf("CoverByAddress = %q, %q, want the stored bytes", body, contentType) t.Fatalf("CoverByAddress = %q, %q, want the stored bytes", body, contentType)
} }
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddress(source)[2:], for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddressForBytes([]byte("bytes"))[2:],
CoverAddress("never stored")} { CoverAddressForBytes([]byte("never stored"))} {
_, _, ok, err := store.CoverByAddress(address) _, _, ok, err := store.CoverByAddress(address)
if err != nil || ok { if err != nil || ok {
t.Fatalf("CoverByAddress(%q) = %v, %v, want a clean miss", address, ok, err) t.Fatalf("CoverByAddress(%q) = %v, %v, want a clean miss", address, ok, err)
@@ -913,7 +913,7 @@ func TestUpsertExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Upsert: %v", err) t.Fatalf("Upsert: %v", err)
} }
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired) wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
if got.Title != "Solo Leveling" || got.SeriesURL != "https://asurascans.com/comics/solo" || if got.Title != "Solo Leveling" || got.SeriesURL != "https://asurascans.com/comics/solo" ||
got.Cover != wantCover { got.Cover != wantCover {
t.Fatalf("stored = %+v, want original title/url/cover kept", got) t.Fatalf("stored = %+v, want original title/url/cover kept", got)
@@ -981,7 +981,7 @@ func TestDeleteKeepsSeriesRow(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("re-upsert: %v", err) t.Fatalf("re-upsert: %v", err)
} }
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired) wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
if stored.Title != "Solo Leveling" || stored.Cover != wantCover { if stored.Title != "Solo Leveling" || stored.Cover != wantCover {
t.Fatalf("re-bookmark = %+v, want title/cover from the surviving series row", stored) t.Fatalf("re-bookmark = %+v, want title/cover from the surviving series row", stored)
} }
@@ -1498,9 +1498,9 @@ func TestCoverPersistsAcrossReopen(t *testing.T) {
t.Fatalf("reopen: %v", err) t.Fatalf("reopen: %v", err)
} }
defer second.Close() defer second.Close()
got, contentType, ok, err := second.GetCover(sourceURL) got, contentType, ok, err := second.CoverByAddress(CoverAddressForBytes(body))
if err != nil { if err != nil {
t.Fatalf("GetCover: %v", err) t.Fatalf("CoverByAddress: %v", err)
} }
if !ok || !bytes.Equal(got, body) || contentType != "image/webp" { if !ok || !bytes.Equal(got, body) || contentType != "image/webp" {
t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body) t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body)
@@ -1539,7 +1539,7 @@ func TestCoverIsContentAddressedOnFilesystem(t *testing.T) {
} }
defer first.Close() defer first.Close()
addressBytes := sha256.Sum256([]byte(sourceURL)) addressBytes := sha256.Sum256(body)
address := hex.EncodeToString(addressBytes[:]) address := hex.EncodeToString(addressBytes[:])
wantPath := filepath.Join(address[:2], address[2:4], address) wantPath := filepath.Join(address[:2], address[2:4], address)
@@ -1570,9 +1570,9 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil { if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil {
t.Fatalf("PutCover: %v", err) t.Fatalf("PutCover: %v", err)
} }
got, contentType, ok, err := s.GetCover(sourceURL) got, contentType, ok, err := s.CoverByAddress(CoverAddressForBytes(want))
if err != nil { if err != nil {
t.Fatalf("GetCover: %v", err) t.Fatalf("CoverByAddress: %v", err)
} }
if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" { if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" {
t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want) t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want)
@@ -1580,7 +1580,7 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil { if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil {
t.Fatal("PutCover accepted a non-image") t.Fatal("PutCover accepted a non-image")
} }
if _, _, ok, err := s.GetCover("https://cdn.example/not-image"); err != nil || ok { if _, _, ok, err := s.CoverByAddress(CoverAddressForBytes([]byte("html"))); err != nil || ok {
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err) t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
} }
} }
@@ -2029,3 +2029,118 @@ func (s *Store) latestCorrectedAt(t *testing.T, site, seriesID string) int64 {
// num2 boxes a chapter number for the Bookmark fields that take a pointer. // num2 boxes a chapter number for the Bookmark fields that take a pointer.
func num2(f float64) *float64 { return &f } func num2(f float64) *float64 { return &f }
// --- Cover addressing (ADR-0014): the address is the bytes' SHA-256 ---
// The address is what makes a re-art visible at all, so the same bytes must
// always name the same address and different bytes different ones — and the
// address must keep the 64-hex-digit shape CoverByAddress's guard still checks
// before any request-supplied value becomes a filesystem path.
func TestCoverAddressForBytesIsDeterministicAndDistinct(t *testing.T) {
first := CoverAddressForBytes([]byte("art"))
again := CoverAddressForBytes([]byte("art"))
other := CoverAddressForBytes([]byte("artwork"))
if first != again {
t.Fatalf("same bytes gave %q then %q, want one address", first, again)
}
if first == other {
t.Fatalf("different bytes gave the same address %q", first)
}
if !coverAddressRe.MatchString(first) {
t.Fatalf("address %q is not the 64-hex-digit shape the serving guard checks", first)
}
}
// ReplaceSeriesCover is the forced-replacement installer: it moves a Cover
// whether or not one exists, writes the source URL alongside it, and reports
// the three outcomes the Forced Poll has to tell apart.
func TestReplaceSeriesCover(t *testing.T) {
store := newTestStore(t)
if _, err := store.Upsert(store.OwnerID(), Bookmark{
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
// A blank Cover: previous is "", and the row points at the new bytes.
previous, current, err := store.ReplaceSeriesCover("asura", "solo",
"https://cdn.asurascans.com/covers/solo.webp", []byte("first-art"), "image/webp")
if err != nil {
t.Fatalf("ReplaceSeriesCover on a blank: %v", err)
}
if previous != "" {
t.Fatalf("previous on a blank = %q, want empty", previous)
}
if want := CoverAddressForBytes([]byte("first-art")); current != want {
t.Fatalf("current = %q, want %q", current, want)
}
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
sr.Cover != "https://cdn.asurascans.com/covers/solo.webp" {
t.Fatalf("series after blank fill = %+v, want the new address and source URL", sr)
}
// A re-art: previous is the stranded address, current the new one.
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
if err != nil {
t.Fatalf("ReplaceSeriesCover over a filled Cover: %v", err)
}
if want := CoverAddressForBytes([]byte("first-art")); previous != want {
t.Fatalf("previous = %q, want the replaced address %q", previous, want)
}
if want := CoverAddressForBytes([]byte("second-art")); current != want {
t.Fatalf("current = %q, want %q", current, want)
}
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
sr.Cover != "https://cdn.asurascans.com/covers/solo-rebrand.webp" {
t.Fatalf("series after replacement = %+v, want the new address and source URL", sr)
}
// The replaced bytes stay served under their old address; nothing reclaims
// them in this ticket (the forced-poll wave does).
if _, _, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("first-art"))); err != nil || !ok {
t.Fatalf("superseded bytes = found %v, err %v, want still served", ok, err)
}
// The Site is serving the same artwork again: previous == current is the
// honest no-op the caller reports as "unchanged".
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
if err != nil {
t.Fatalf("ReplaceSeriesCover over identical bytes: %v", err)
}
if previous != current {
t.Fatalf("identical bytes: previous = %q, current = %q, want one address", previous, current)
}
if want := CoverAddressForBytes([]byte("second-art")); current != want {
t.Fatalf("current = %q, want %q", current, want)
}
}
// Rows written before byte addressing hold the hash of their source URL and
// are never rehashed: GetCover — the poller's heal path — keeps resolving
// them through coverSourceAddress.
func TestGetCoverResolvesLegacyURLDerivedAddress(t *testing.T) {
store := newTestStore(t)
source := "https://cdn.example/legacy.jpg"
legacy := coverSourceAddress(source)
relativePath := coverRelativePath(legacy)
coverPath := filepath.Join(store.coverDir, filepath.FromSlash(relativePath))
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
t.Fatalf("create shard dir: %v", err)
}
if err := os.WriteFile(coverPath, []byte("legacy-bytes"), 0o644); err != nil {
t.Fatalf("write legacy file: %v", err)
}
if _, err := store.db.Exec(
`INSERT INTO covers (address, path, content_type) VALUES ($1, $2, $3)`,
legacy, relativePath, "image/jpeg"); err != nil {
t.Fatalf("plant legacy row: %v", err)
}
body, contentType, ok, err := store.GetCover(source)
if err != nil || !ok {
t.Fatalf("GetCover on a legacy row = %v, %v, want found", ok, err)
}
if string(body) != "legacy-bytes" || contentType != "image/jpeg" {
t.Fatalf("legacy cover = (%q, %q), want the planted bytes", body, contentType)
}
}
@@ -0,0 +1,86 @@
# ADR-0014: Cover addresses derived from the bytes, not the source URL
Date: 2026-08-22
Status: accepted
## Decision
A Cover's content address is the hex SHA-256 of its **bytes**, not of the
source URL it was fetched from. `CoverAddressForBytes(body)` names the address
`putCover` stores under, `SetSeriesCover` and `ReplaceSeriesCover` point the
Series row at it, and the wire URL is built from it exactly as before — same
route, same 64-hex-digit shape, same immutability, only the input to the hash
changes. Rows written before this ADR keep their URL-derived addresses
forever: they are never rehashed on read, and they heal into byte addressing
only when a Forced Poll replaces them.
`ReplaceSeriesCover(site, seriesID, sourceURL, body, contentType)`
`(previous, current, error)` is the one write that may move a Cover once one
exists. It stores the bytes, then in one transaction locks the Series row,
reads the old `cover_address`, writes the new one and the source URL, and
reports both addresses: `previous == ""` means there was no Cover,
`previous == current` means the Site served identical artwork, and any other
pair names the stranded address.
## Why a future reader will find this surprising
The address is what makes a re-art visible at all. URL addressing collapses
every image behind a stable URL into one address, so a Series whose Cover
changes (a big-budget CPI blitz on a light novel is the standing example)
keeps serving its original cover bytes: the poll refetches the same URL,
hashes it, and the store records the same address, everyone happy except the
Reader. Nothing in the system can detect the change, because the address is a
pure function of the fetch target, and identical bytes written 1,000 times
are one blob on disk. Storing bytes we already know how to store is only a
few lines of work. **Rejecting that work is the surprising part, and the
answer is the Forced Poll wave**: for a corrupt/blank cover the poll's
fill-if-blank installer already worked, but for a *wrong but non-blank* cover
there was no write that would move it at all — only a manual truth in
`series.cover_address`, which is exactly the thing that must never be set by
hand. Byte addressing gives the replacement write a **new address to write**,
and with it a legitimate, transaction-safe mover.
## Considered options
**Keep URL addressing and add a generic "clear the cover" write.**
Rejected: clearing is a two-phase action (blank it, wait for the poll to
re-fill, hope the bytes changed in between) that cannot report what the
write did, and it makes the Series render cover-less in between. The
replacement write is atomic, reports its displacement, and has one effect:
the Series now points at bytes that actually came from its source URL.
**Address by URL, but salt it so a re-art is a new address.**
Rejected: the salt would have to live somewhere addressable (a stored per-
Series nonce), turning the address from a content fact into a mutable fact —
two rows could then hold identical bytes under different addresses and the
invariant "same bytes object" is gone.
## Consequences
- `store.CoverAddress` (URL-hash) is deleted; `CoverAddressForBytes` is
public so tests and the forced-poll wave can predict addresses from the
bytes fakes serve.
- Legacy URL-addressed rows are read-only facts: `GetCover(sourceURL)` keeps
resolving them (the poll heal path), and they are re-addressed only by a
forced replacement. Until one happens, they are invisible to byte-derived
lookups — the reverse direction was always true, so this side has no
migration and no lookup fan-out.
- A replaced Cover's old bytes stay on disk under their address (the `covers`
row is untouched — only the Series row moves). Nothing reclaims them
today; a later sweep is a small query over `covers` addresses not
referenced by any `series` row.
- `SetSeriesCover` keeps its `cover_address = ''` guard untouched: the
acquisition-at-creation and poll fill paths still may not overwrite a
non-blank Cover. The two installers are now deliberately different
functions instead of one function with a conditional.
- The address is still a filesystem path (≤64 hex chars, no separators), so
`coverAddressRe` and the sharding stay exactly as they are.
## Cost of reversing
The URL-hash side of the current rows is uncomputable from the rows alone: a
rollback would need every stored blob's source URL, a join to a table that
does not store it, or a refetch of every Series. Keeping both derivations
resolvable is cheaper than either, so the two derivations are documented in
the 0009 migration comment: no component may assume which derivation a
stored address came from, because the 64-hex shape hides it.