Spec #135: owner data-correction actions — Latest Chapter, series_url, Cover, orphan removal #156
@@ -32,7 +32,7 @@ func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
|
|||||||
|
|
||||||
// The wire URL is what a client actually requests, so the path under test
|
// The wire URL is what a client actually requests, so the path under test
|
||||||
// is taken from it rather than rebuilt by hand.
|
// is taken from it rather than rebuilt by hand.
|
||||||
wire := st.CoverWireURL(store.CoverAddress(sourceURL))
|
wire := st.CoverWireURL(store.CoverAddressForBytes([]byte("\x00webp-bytes")))
|
||||||
path, ok := strings.CutPrefix(wire, testCoverBaseURL)
|
path, ok := strings.CutPrefix(wire, testCoverBaseURL)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL)
|
t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL)
|
||||||
@@ -57,7 +57,7 @@ func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
|
|||||||
func TestPublicCoverRejectsUnknownAddress(t *testing.T) {
|
func TestPublicCoverRejectsUnknownAddress(t *testing.T) {
|
||||||
srv, _ := newWebTestServer(t, testConfig())
|
srv, _ := newWebTestServer(t, testConfig())
|
||||||
cases := map[string]string{
|
cases := map[string]string{
|
||||||
"unknown": "/covers/" + store.CoverAddress("https://cdn.example/never-stored.jpg"),
|
"unknown": "/covers/" + store.CoverAddressForBytes([]byte("never-stored")),
|
||||||
"malformed": "/covers/not-an-address",
|
"malformed": "/covers/not-an-address",
|
||||||
"traversal": "/covers/../../etc/passwd",
|
"traversal": "/covers/../../etc/passwd",
|
||||||
"empty": "/covers/",
|
"empty": "/covers/",
|
||||||
@@ -86,7 +86,7 @@ func TestPublicCoverNeverEchoesNonImage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// A legitimate row, then the content type flipped behind the store's back:
|
// A legitimate row, then the content type flipped behind the store's back:
|
||||||
// the bytes exist at the address, so only the type is hostile.
|
// the bytes exist at the address, so only the type is hostile.
|
||||||
address := store.CoverAddress(sourceURL)
|
address := store.CoverAddressForBytes([]byte("<script>"))
|
||||||
if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil {
|
if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil {
|
||||||
t.Fatalf("seed row: %v", err)
|
t.Fatalf("seed row: %v", err)
|
||||||
}
|
}
|
||||||
@@ -127,7 +127,7 @@ func TestListRendersAcquiredCover(t *testing.T) {
|
|||||||
if rr.Code != http.StatusOK {
|
if rr.Code != http.StatusOK {
|
||||||
t.Fatalf("status = %d, want 200", rr.Code)
|
t.Fatalf("status = %d, want 200", rr.Code)
|
||||||
}
|
}
|
||||||
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddress(sourceURL) + `"`
|
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("\xff\xd8jpeg")) + `"`
|
||||||
if !strings.Contains(rr.Body.String(), want) {
|
if !strings.Contains(rr.Body.String(), want) {
|
||||||
t.Fatalf("rendered list does not contain %s", want)
|
t.Fatalf("rendered list does not contain %s", want)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -123,10 +123,10 @@ func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
|
|||||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
||||||
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
||||||
}
|
}
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(acquireCoverURL))
|
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||||
}
|
}
|
||||||
@@ -182,7 +182,7 @@ func TestAcquireSkipsAnExistingSeries(t *testing.T) {
|
|||||||
t.Fatalf("cover fetches = %d, want 1", got)
|
t.Fatalf("cover fetches = %d, want 1", got)
|
||||||
}
|
}
|
||||||
got := readBookmark(t, s, acquireKey)
|
got := readBookmark(t, s, acquireKey)
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -343,10 +343,10 @@ func TestAcquireKaganeCoverThroughBrowser(t *testing.T) {
|
|||||||
t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc)
|
t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc)
|
||||||
}
|
}
|
||||||
got := readBookmark(t, s, kaganeKey)
|
got := readBookmark(t, s, kaganeKey)
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(kaganeCoverSrc); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(kaganeCoverSrc))
|
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||||
}
|
}
|
||||||
@@ -381,7 +381,7 @@ func TestAcquireNovelfullCoverOverPlainTLS(t *testing.T) {
|
|||||||
t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
|
t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
|
||||||
}
|
}
|
||||||
got := readBookmark(t, s, novelfullKey)
|
got := readBookmark(t, s, novelfullKey)
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -451,7 +451,7 @@ func TestAcquireNovelfullCoverWithoutBrowser(t *testing.T) {
|
|||||||
t.Fatalf("cover fetches = %d, want 1", got)
|
t.Fatalf("cover fetches = %d, want 1", got)
|
||||||
}
|
}
|
||||||
got := readBookmark(t, s, novelfullKey)
|
got := readBookmark(t, s, novelfullKey)
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -214,9 +214,9 @@ func TestRunOncePrefetchesPublicCover(t *testing.T) {
|
|||||||
if got := covers.callCount(); got != 1 {
|
if got := covers.callCount(); got != 1 {
|
||||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||||
}
|
}
|
||||||
body, contentType, found, err := s.GetCover(coverURL)
|
body, contentType, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
t.Fatalf("GetCover: %v found=%v", err, found)
|
t.Fatalf("CoverByAddress: %v found=%v", err, found)
|
||||||
}
|
}
|
||||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||||
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
||||||
@@ -663,7 +663,7 @@ func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
|
|||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
t.Fatalf("Get: %v found=%v", err, found)
|
t.Fatalf("Get: %v found=%v", err, found)
|
||||||
}
|
}
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
||||||
@@ -811,7 +811,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
|||||||
p.runOnce(context.Background())
|
p.runOnce(context.Background())
|
||||||
p.waitCovers()
|
p.waitCovers()
|
||||||
|
|
||||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
||||||
}
|
}
|
||||||
@@ -821,7 +821,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
|||||||
if got := covers.callCount(); got != 1 {
|
if got := covers.callCount(); got != 1 {
|
||||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||||
}
|
}
|
||||||
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) {
|
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddressForBytes([]byte("cover-bytes")) {
|
||||||
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -914,7 +914,7 @@ func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
|||||||
p.runOnce(context.Background())
|
p.runOnce(context.Background())
|
||||||
p.waitCovers()
|
p.waitCovers()
|
||||||
|
|
||||||
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("not an image"))); err != nil || found {
|
||||||
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -940,7 +940,7 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
|||||||
p.runOnce(context.Background())
|
p.runOnce(context.Background())
|
||||||
p.waitCovers()
|
p.waitCovers()
|
||||||
|
|
||||||
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || found {
|
||||||
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1038,7 +1038,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|||||||
seriesURL string
|
seriesURL string
|
||||||
kind string
|
kind string
|
||||||
body string
|
body string
|
||||||
wantCover string
|
|
||||||
browser bool
|
browser bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -1047,14 +1046,12 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|||||||
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
||||||
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
||||||
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
||||||
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "lightnovelworld novel",
|
name: "lightnovelworld novel",
|
||||||
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
|
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
|
||||||
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||||
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
||||||
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "kagane manga",
|
name: "kagane manga",
|
||||||
@@ -1062,7 +1059,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|||||||
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
||||||
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
||||||
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
||||||
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -1086,7 +1082,7 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
|||||||
p.waitCovers()
|
p.waitCovers()
|
||||||
|
|
||||||
got := readBookmark(t, s, tc.key)
|
got := readBookmark(t, s, tc.key)
|
||||||
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
|
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes"))
|
||||||
if got.Cover != wantWire {
|
if got.Cover != wantWire {
|
||||||
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
||||||
}
|
}
|
||||||
@@ -1143,7 +1139,7 @@ func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
|
|||||||
t.Fatalf("cover fetch calls = %d, want 0", got)
|
t.Fatalf("cover fetch calls = %d, want 0", got)
|
||||||
}
|
}
|
||||||
got := readBookmark(t, s, key)
|
got := readBookmark(t, s, key)
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1190,7 +1186,7 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
|
|||||||
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
||||||
}
|
}
|
||||||
got := readBookmark(t, s, key)
|
got := readBookmark(t, s, key)
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||||
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -108,10 +109,7 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
|
|||||||
if ws == "" {
|
if ws == "" {
|
||||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||||
}
|
}
|
||||||
const (
|
const seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
|
||||||
seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
|
|
||||||
coverURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed"
|
|
||||||
)
|
|
||||||
s, _ := newTestStore(t)
|
s, _ := newTestStore(t)
|
||||||
bf, err := NewBrowserFetcher(ws)
|
bf, err := NewBrowserFetcher(ws)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -140,12 +138,13 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
|
|||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
t.Fatalf("Get: %v found=%v", err, found)
|
t.Fatalf("Get: %v found=%v", err, found)
|
||||||
}
|
}
|
||||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
addr, ok := strings.CutPrefix(got.Cover, testCoverBaseURL+"/covers/")
|
||||||
t.Fatalf("Cover = %q, want %q — the acquire path did not store the browser-fetched bytes", got.Cover, want)
|
if !ok {
|
||||||
|
t.Fatalf("Cover = %q, want an address on %q — the acquire path did not store the browser-fetched bytes", got.Cover, testCoverBaseURL+"/covers/")
|
||||||
}
|
}
|
||||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
body, contentType, ok, err := s.CoverByAddress(addr)
|
||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
t.Fatalf("CoverByAddress(%q): %v found=%v", addr, err, ok)
|
||||||
}
|
}
|
||||||
if len(body) < 1000 {
|
if len(body) < 1000 {
|
||||||
t.Fatalf("stored cover is %d bytes, want a real image", len(body))
|
t.Fatalf("stored cover is %d bytes, want a real image", len(body))
|
||||||
@@ -153,5 +152,10 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
|
|||||||
if contentType != "image/webp" {
|
if contentType != "image/webp" {
|
||||||
t.Fatalf("content type = %q, want image/webp", contentType)
|
t.Fatalf("content type = %q, want image/webp", contentType)
|
||||||
}
|
}
|
||||||
|
// The address the row carries is the bytes' own SHA-256: a re-art behind
|
||||||
|
// the same URL would be a different address, which is the whole point.
|
||||||
|
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes(body); got.Cover != want {
|
||||||
|
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||||
|
}
|
||||||
t.Logf("stored %d bytes of %s", len(body), contentType)
|
t.Logf("stored %d bytes of %s", len(body), contentType)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,15 @@
|
|||||||
-- The Cover splits into two facts. `cover` keeps the third-party address the
|
-- The Cover splits into two facts. `cover` keeps the third-party address the
|
||||||
-- bytes come from, which is what the acquisition path refetches and dedupes
|
-- bytes come from, which is what the refetch path dedupes on; `cover_address`
|
||||||
-- on; `cover_address` is the content address of the bytes once they are
|
-- is the content address of the bytes once they are actually stored, and is
|
||||||
-- actually stored, and is what the wire's absolute URL is built from.
|
-- what the wire's absolute URL is built from.
|
||||||
--
|
--
|
||||||
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
|
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
|
||||||
-- that 404s", which is the distinction the API and the UI both depend on.
|
-- that 404s", which is the distinction the API and the UI both depend on.
|
||||||
|
--
|
||||||
|
-- The content address was originally the hex SHA-256 of the source URL
|
||||||
|
-- (ADR-0007). Since ADR-0014 it is the hex SHA-256 of the bytes themselves,
|
||||||
|
-- so a re-art behind the same URL is a new address. Rows written before
|
||||||
|
-- ADR-0014 keep their URL-derived addresses; they are never rehashed and heal
|
||||||
|
-- into byte addressing on their first forced replacement. Both derivations
|
||||||
|
-- share the 64-hex-digit shape, so the serving guard is unchanged.
|
||||||
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
|
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
|
||||||
|
|||||||
@@ -704,67 +704,76 @@ func (s *Store) getCoverByAddress(address string) ([]byte, string, bool, error)
|
|||||||
return body, contentType, true, nil
|
return body, contentType, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) putCover(sourceURL string, body []byte, contentType string) error {
|
func (s *Store) putCover(sourceURL string, body []byte, contentType string) (string, error) {
|
||||||
stored, ok := CoverContentType(contentType)
|
stored, ok := CoverContentType(contentType)
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
|
return "", fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
|
||||||
}
|
}
|
||||||
contentType = stored
|
contentType = stored
|
||||||
address := coverSourceAddress(sourceURL)
|
address := CoverAddressForBytes(body)
|
||||||
relativePath := coverRelativePath(address)
|
relativePath := coverRelativePath(address)
|
||||||
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
|
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
|
||||||
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
|
||||||
return fmt.Errorf("create cover shard: %w", err)
|
return "", fmt.Errorf("create cover shard: %w", err)
|
||||||
}
|
}
|
||||||
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
|
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create cover temp file: %w", err)
|
return "", fmt.Errorf("create cover temp file: %w", err)
|
||||||
}
|
}
|
||||||
tmpName := tmp.Name()
|
tmpName := tmp.Name()
|
||||||
defer os.Remove(tmpName)
|
defer os.Remove(tmpName)
|
||||||
if _, err := tmp.Write(body); err != nil {
|
if _, err := tmp.Write(body); err != nil {
|
||||||
tmp.Close()
|
tmp.Close()
|
||||||
return fmt.Errorf("write cover temp file: %w", err)
|
return "", fmt.Errorf("write cover temp file: %w", err)
|
||||||
}
|
}
|
||||||
if err := tmp.Sync(); err != nil {
|
if err := tmp.Sync(); err != nil {
|
||||||
tmp.Close()
|
tmp.Close()
|
||||||
return fmt.Errorf("sync cover temp file: %w", err)
|
return "", fmt.Errorf("sync cover temp file: %w", err)
|
||||||
}
|
}
|
||||||
if err := tmp.Close(); err != nil {
|
if err := tmp.Close(); err != nil {
|
||||||
return fmt.Errorf("close cover temp file: %w", err)
|
return "", fmt.Errorf("close cover temp file: %w", err)
|
||||||
}
|
}
|
||||||
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
|
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||||
return fmt.Errorf("install cover file: %w", err)
|
return "", fmt.Errorf("install cover file: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := s.db.Exec(`
|
if _, err := s.db.Exec(`
|
||||||
INSERT INTO covers (address, path, content_type)
|
INSERT INTO covers (address, path, content_type)
|
||||||
VALUES ($1, $2, $3)
|
VALUES ($1, $2, $3)
|
||||||
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
|
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
|
||||||
return fmt.Errorf("record cover %q: %w", address, err)
|
return "", fmt.Errorf("record cover %q: %w", address, err)
|
||||||
}
|
}
|
||||||
return nil
|
return address, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCover returns the immutable object addressed by its source URL. Missing
|
// GetCover returns the immutable object a source URL's own hash names. Rows
|
||||||
|
// written before byte addressing (ADR-0014) are the only ones that ever reach
|
||||||
|
// it; it hashes the URL, so a byte-addressed Cover is invisible to it. Missing
|
||||||
// files are reported with ok=false so callers can retry acquisition later.
|
// files are reported with ok=false so callers can retry acquisition later.
|
||||||
func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) {
|
func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) {
|
||||||
return s.getCover(sourceURL)
|
return s.getCover(sourceURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutCover persists bytes under the source URL's content address. A later
|
// PutCover persists bytes under their own content address (ADR-0014). A later
|
||||||
// write for the same URL cannot replace the immutable object.
|
// write of the same bytes cannot replace the immutable object.
|
||||||
func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error {
|
func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error {
|
||||||
return s.putCover(sourceURL, body, contentType)
|
_, err := s.putCover(sourceURL, body, contentType)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// CoverAddress is the content address bytes fetched from sourceURL are stored
|
// CoverAddressForBytes is the content address body is stored under: the hex
|
||||||
// under. It is a pure function of the URL, so the acquisition path can name a
|
// SHA-256 of the bytes, so identical artwork is one address and a re-art a
|
||||||
// Cover before it has the bytes.
|
// new one. Legacy rows were addressed from their source URL instead and are
|
||||||
func CoverAddress(sourceURL string) string { return coverSourceAddress(sourceURL) }
|
// never rehashed — both derivations coexist (ADR-0014).
|
||||||
|
func CoverAddressForBytes(body []byte) string {
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
// coverAddressRe is the shape of a stored address: the hex SHA-256 of a source
|
// coverAddressRe is the shape of a stored address: 64 lowercase hex digits —
|
||||||
// URL. Request paths reach CoverByAddress, so the shape is checked before the
|
// the hex SHA-256 of the cover bytes, or of the source URL for legacy rows
|
||||||
// value is ever turned into a filesystem path.
|
// (ADR-0014). Request paths reach CoverByAddress, so the shape is checked
|
||||||
|
// before the value is ever turned into a filesystem path; byte-derived
|
||||||
|
// addresses keep the same shape, so the guard is unchanged.
|
||||||
var coverAddressRe = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
var coverAddressRe = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||||
|
|
||||||
// CoverByAddress returns the immutable object at one content address. An
|
// CoverByAddress returns the immutable object at one content address. An
|
||||||
@@ -788,24 +797,67 @@ func (s *Store) CoverWireURL(address string) string {
|
|||||||
return s.coverBaseURL + "/covers/" + address
|
return s.coverBaseURL + "/covers/" + address
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetSeriesCover stores the bytes and points the Series at them, but only
|
// SetSeriesCover stores the bytes and points the Series at their address, but
|
||||||
// while the Series has no Cover: acquisition at creation and the poll both
|
// only while the Series has no Cover: acquisition at creation and the poll
|
||||||
// call this, and whichever arrives second must not overwrite the first. The
|
// both call this, and whichever arrives second must not overwrite the first.
|
||||||
// bytes themselves are content-addressed and immutable, so storing them twice
|
// The bytes themselves are content-addressed and immutable, so storing them
|
||||||
// is free.
|
// twice is free. See ReplaceSeriesCover for the write that may move a Cover
|
||||||
|
// once one exists (ADR-0014).
|
||||||
func (s *Store) SetSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) error {
|
func (s *Store) SetSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) error {
|
||||||
if err := s.putCover(sourceURL, body, contentType); err != nil {
|
address, err := s.putCover(sourceURL, body, contentType)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := s.db.Exec(`
|
if _, err := s.db.Exec(`
|
||||||
UPDATE series SET cover = $3, cover_address = $4
|
UPDATE series SET cover = $3, cover_address = $4
|
||||||
WHERE site = $1 AND series_id = $2 AND cover_address = ''`,
|
WHERE site = $1 AND series_id = $2 AND cover_address = ''`,
|
||||||
site, seriesID, sourceURL, coverSourceAddress(sourceURL)); err != nil {
|
site, seriesID, sourceURL, address); err != nil {
|
||||||
return fmt.Errorf("set cover for %q: %w", site+":"+seriesID, err)
|
return fmt.Errorf("set cover for %q: %w", site+":"+seriesID, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReplaceSeriesCover stores the bytes and points the Series at their address
|
||||||
|
// whether or not one already exists, writing the current source URL alongside
|
||||||
|
// — the Forced Poll's installer and the only write that may move a Cover once
|
||||||
|
// one exists (ADR-0014). previous is the address the row held before the write
|
||||||
|
// ("" if it had none) and current the address of the bytes just stored; both
|
||||||
|
// are read and written in one transaction, so a concurrent replacement reports
|
||||||
|
// the exact displacement. previous == current means the Site served identical
|
||||||
|
// artwork, an honest no-op; otherwise previous is stranded — its bytes stay
|
||||||
|
// served under the covers table (ADR-0014), the row just no longer points at
|
||||||
|
// them.
|
||||||
|
func (s *Store) ReplaceSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) (previous, current string, err error) {
|
||||||
|
current, err = s.putCover(sourceURL, body, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
tx, err := s.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("begin replace cover for %q: %w", site+":"+seriesID, err)
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
err = tx.QueryRow(`
|
||||||
|
SELECT cover_address FROM series
|
||||||
|
WHERE site = $1 AND series_id = $2 FOR UPDATE`,
|
||||||
|
site, seriesID).Scan(&previous)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
previous = ""
|
||||||
|
} else if err != nil {
|
||||||
|
return "", "", fmt.Errorf("read cover for %q: %w", site+":"+seriesID, err)
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(`
|
||||||
|
UPDATE series SET cover = $3, cover_address = $4
|
||||||
|
WHERE site = $1 AND series_id = $2`,
|
||||||
|
site, seriesID, sourceURL, current); err != nil {
|
||||||
|
return "", "", fmt.Errorf("replace cover for %q: %w", site+":"+seriesID, err)
|
||||||
|
}
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return "", "", fmt.Errorf("commit cover replace for %q: %w", site+":"+seriesID, err)
|
||||||
|
}
|
||||||
|
return previous, current, nil
|
||||||
|
}
|
||||||
|
|
||||||
// List returns every bookmark of one reader, newest activity first.
|
// List returns every bookmark of one reader, newest activity first.
|
||||||
// Series-owned fields are joined in, so each Bookmark reads back whole and
|
// Series-owned fields are joined in, so each Bookmark reads back whole and
|
||||||
// flat (ADR-0004).
|
// flat (ADR-0004).
|
||||||
|
|||||||
@@ -850,7 +850,7 @@ func TestSetSeriesCoverDoesNotOverwrite(t *testing.T) {
|
|||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("Get = %v, %v", ok, err)
|
t.Fatalf("Get = %v, %v", ok, err)
|
||||||
}
|
}
|
||||||
if want := "https://bookmarks.test/covers/" + CoverAddress(first); got.Cover != want {
|
if want := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("first")); got.Cover != want {
|
||||||
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -869,7 +869,7 @@ func TestCoverByAddress(t *testing.T) {
|
|||||||
t.Fatalf("SetSeriesCover: %v", err)
|
t.Fatalf("SetSeriesCover: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
body, contentType, ok, err := store.CoverByAddress(CoverAddress(source))
|
body, contentType, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("bytes")))
|
||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||||
}
|
}
|
||||||
@@ -877,8 +877,8 @@ func TestCoverByAddress(t *testing.T) {
|
|||||||
t.Fatalf("CoverByAddress = %q, %q, want the stored bytes", body, contentType)
|
t.Fatalf("CoverByAddress = %q, %q, want the stored bytes", body, contentType)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddress(source)[2:],
|
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddressForBytes([]byte("bytes"))[2:],
|
||||||
CoverAddress("never stored")} {
|
CoverAddressForBytes([]byte("never stored"))} {
|
||||||
_, _, ok, err := store.CoverByAddress(address)
|
_, _, ok, err := store.CoverByAddress(address)
|
||||||
if err != nil || ok {
|
if err != nil || ok {
|
||||||
t.Fatalf("CoverByAddress(%q) = %v, %v, want a clean miss", address, ok, err)
|
t.Fatalf("CoverByAddress(%q) = %v, %v, want a clean miss", address, ok, err)
|
||||||
@@ -913,7 +913,7 @@ func TestUpsertExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Upsert: %v", err)
|
t.Fatalf("Upsert: %v", err)
|
||||||
}
|
}
|
||||||
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired)
|
wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
|
||||||
if got.Title != "Solo Leveling" || got.SeriesURL != "https://asurascans.com/comics/solo" ||
|
if got.Title != "Solo Leveling" || got.SeriesURL != "https://asurascans.com/comics/solo" ||
|
||||||
got.Cover != wantCover {
|
got.Cover != wantCover {
|
||||||
t.Fatalf("stored = %+v, want original title/url/cover kept", got)
|
t.Fatalf("stored = %+v, want original title/url/cover kept", got)
|
||||||
@@ -981,7 +981,7 @@ func TestDeleteKeepsSeriesRow(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("re-upsert: %v", err)
|
t.Fatalf("re-upsert: %v", err)
|
||||||
}
|
}
|
||||||
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired)
|
wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
|
||||||
if stored.Title != "Solo Leveling" || stored.Cover != wantCover {
|
if stored.Title != "Solo Leveling" || stored.Cover != wantCover {
|
||||||
t.Fatalf("re-bookmark = %+v, want title/cover from the surviving series row", stored)
|
t.Fatalf("re-bookmark = %+v, want title/cover from the surviving series row", stored)
|
||||||
}
|
}
|
||||||
@@ -1498,9 +1498,9 @@ func TestCoverPersistsAcrossReopen(t *testing.T) {
|
|||||||
t.Fatalf("reopen: %v", err)
|
t.Fatalf("reopen: %v", err)
|
||||||
}
|
}
|
||||||
defer second.Close()
|
defer second.Close()
|
||||||
got, contentType, ok, err := second.GetCover(sourceURL)
|
got, contentType, ok, err := second.CoverByAddress(CoverAddressForBytes(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GetCover: %v", err)
|
t.Fatalf("CoverByAddress: %v", err)
|
||||||
}
|
}
|
||||||
if !ok || !bytes.Equal(got, body) || contentType != "image/webp" {
|
if !ok || !bytes.Equal(got, body) || contentType != "image/webp" {
|
||||||
t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body)
|
t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body)
|
||||||
@@ -1539,7 +1539,7 @@ func TestCoverIsContentAddressedOnFilesystem(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer first.Close()
|
defer first.Close()
|
||||||
|
|
||||||
addressBytes := sha256.Sum256([]byte(sourceURL))
|
addressBytes := sha256.Sum256(body)
|
||||||
address := hex.EncodeToString(addressBytes[:])
|
address := hex.EncodeToString(addressBytes[:])
|
||||||
wantPath := filepath.Join(address[:2], address[2:4], address)
|
wantPath := filepath.Join(address[:2], address[2:4], address)
|
||||||
|
|
||||||
@@ -1570,9 +1570,9 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
|
|||||||
if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil {
|
if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil {
|
||||||
t.Fatalf("PutCover: %v", err)
|
t.Fatalf("PutCover: %v", err)
|
||||||
}
|
}
|
||||||
got, contentType, ok, err := s.GetCover(sourceURL)
|
got, contentType, ok, err := s.CoverByAddress(CoverAddressForBytes(want))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GetCover: %v", err)
|
t.Fatalf("CoverByAddress: %v", err)
|
||||||
}
|
}
|
||||||
if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" {
|
if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" {
|
||||||
t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want)
|
t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want)
|
||||||
@@ -1580,7 +1580,7 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
|
|||||||
if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil {
|
if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil {
|
||||||
t.Fatal("PutCover accepted a non-image")
|
t.Fatal("PutCover accepted a non-image")
|
||||||
}
|
}
|
||||||
if _, _, ok, err := s.GetCover("https://cdn.example/not-image"); err != nil || ok {
|
if _, _, ok, err := s.CoverByAddress(CoverAddressForBytes([]byte("html"))); err != nil || ok {
|
||||||
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
|
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2029,3 +2029,118 @@ func (s *Store) latestCorrectedAt(t *testing.T, site, seriesID string) int64 {
|
|||||||
|
|
||||||
// num2 boxes a chapter number for the Bookmark fields that take a pointer.
|
// num2 boxes a chapter number for the Bookmark fields that take a pointer.
|
||||||
func num2(f float64) *float64 { return &f }
|
func num2(f float64) *float64 { return &f }
|
||||||
|
// --- Cover addressing (ADR-0014): the address is the bytes' SHA-256 ---
|
||||||
|
|
||||||
|
// The address is what makes a re-art visible at all, so the same bytes must
|
||||||
|
// always name the same address and different bytes different ones — and the
|
||||||
|
// address must keep the 64-hex-digit shape CoverByAddress's guard still checks
|
||||||
|
// before any request-supplied value becomes a filesystem path.
|
||||||
|
func TestCoverAddressForBytesIsDeterministicAndDistinct(t *testing.T) {
|
||||||
|
first := CoverAddressForBytes([]byte("art"))
|
||||||
|
again := CoverAddressForBytes([]byte("art"))
|
||||||
|
other := CoverAddressForBytes([]byte("artwork"))
|
||||||
|
if first != again {
|
||||||
|
t.Fatalf("same bytes gave %q then %q, want one address", first, again)
|
||||||
|
}
|
||||||
|
if first == other {
|
||||||
|
t.Fatalf("different bytes gave the same address %q", first)
|
||||||
|
}
|
||||||
|
if !coverAddressRe.MatchString(first) {
|
||||||
|
t.Fatalf("address %q is not the 64-hex-digit shape the serving guard checks", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReplaceSeriesCover is the forced-replacement installer: it moves a Cover
|
||||||
|
// whether or not one exists, writes the source URL alongside it, and reports
|
||||||
|
// the three outcomes the Forced Poll has to tell apart.
|
||||||
|
func TestReplaceSeriesCover(t *testing.T) {
|
||||||
|
store := newTestStore(t)
|
||||||
|
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||||
|
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("seed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A blank Cover: previous is "", and the row points at the new bytes.
|
||||||
|
previous, current, err := store.ReplaceSeriesCover("asura", "solo",
|
||||||
|
"https://cdn.asurascans.com/covers/solo.webp", []byte("first-art"), "image/webp")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReplaceSeriesCover on a blank: %v", err)
|
||||||
|
}
|
||||||
|
if previous != "" {
|
||||||
|
t.Fatalf("previous on a blank = %q, want empty", previous)
|
||||||
|
}
|
||||||
|
if want := CoverAddressForBytes([]byte("first-art")); current != want {
|
||||||
|
t.Fatalf("current = %q, want %q", current, want)
|
||||||
|
}
|
||||||
|
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
|
||||||
|
sr.Cover != "https://cdn.asurascans.com/covers/solo.webp" {
|
||||||
|
t.Fatalf("series after blank fill = %+v, want the new address and source URL", sr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A re-art: previous is the stranded address, current the new one.
|
||||||
|
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
|
||||||
|
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReplaceSeriesCover over a filled Cover: %v", err)
|
||||||
|
}
|
||||||
|
if want := CoverAddressForBytes([]byte("first-art")); previous != want {
|
||||||
|
t.Fatalf("previous = %q, want the replaced address %q", previous, want)
|
||||||
|
}
|
||||||
|
if want := CoverAddressForBytes([]byte("second-art")); current != want {
|
||||||
|
t.Fatalf("current = %q, want %q", current, want)
|
||||||
|
}
|
||||||
|
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
|
||||||
|
sr.Cover != "https://cdn.asurascans.com/covers/solo-rebrand.webp" {
|
||||||
|
t.Fatalf("series after replacement = %+v, want the new address and source URL", sr)
|
||||||
|
}
|
||||||
|
// The replaced bytes stay served under their old address; nothing reclaims
|
||||||
|
// them in this ticket (the forced-poll wave does).
|
||||||
|
if _, _, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("first-art"))); err != nil || !ok {
|
||||||
|
t.Fatalf("superseded bytes = found %v, err %v, want still served", ok, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The Site is serving the same artwork again: previous == current is the
|
||||||
|
// honest no-op the caller reports as "unchanged".
|
||||||
|
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
|
||||||
|
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReplaceSeriesCover over identical bytes: %v", err)
|
||||||
|
}
|
||||||
|
if previous != current {
|
||||||
|
t.Fatalf("identical bytes: previous = %q, current = %q, want one address", previous, current)
|
||||||
|
}
|
||||||
|
if want := CoverAddressForBytes([]byte("second-art")); current != want {
|
||||||
|
t.Fatalf("current = %q, want %q", current, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rows written before byte addressing hold the hash of their source URL and
|
||||||
|
// are never rehashed: GetCover — the poller's heal path — keeps resolving
|
||||||
|
// them through coverSourceAddress.
|
||||||
|
func TestGetCoverResolvesLegacyURLDerivedAddress(t *testing.T) {
|
||||||
|
store := newTestStore(t)
|
||||||
|
source := "https://cdn.example/legacy.jpg"
|
||||||
|
legacy := coverSourceAddress(source)
|
||||||
|
relativePath := coverRelativePath(legacy)
|
||||||
|
coverPath := filepath.Join(store.coverDir, filepath.FromSlash(relativePath))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create shard dir: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(coverPath, []byte("legacy-bytes"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write legacy file: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.db.Exec(
|
||||||
|
`INSERT INTO covers (address, path, content_type) VALUES ($1, $2, $3)`,
|
||||||
|
legacy, relativePath, "image/jpeg"); err != nil {
|
||||||
|
t.Fatalf("plant legacy row: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, contentType, ok, err := store.GetCover(source)
|
||||||
|
if err != nil || !ok {
|
||||||
|
t.Fatalf("GetCover on a legacy row = %v, %v, want found", ok, err)
|
||||||
|
}
|
||||||
|
if string(body) != "legacy-bytes" || contentType != "image/jpeg" {
|
||||||
|
t.Fatalf("legacy cover = (%q, %q), want the planted bytes", body, contentType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
# ADR-0014: Cover addresses derived from the bytes, not the source URL
|
||||||
|
|
||||||
|
Date: 2026-08-22
|
||||||
|
Status: accepted
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
A Cover's content address is the hex SHA-256 of its **bytes**, not of the
|
||||||
|
source URL it was fetched from. `CoverAddressForBytes(body)` names the address
|
||||||
|
`putCover` stores under, `SetSeriesCover` and `ReplaceSeriesCover` point the
|
||||||
|
Series row at it, and the wire URL is built from it exactly as before — same
|
||||||
|
route, same 64-hex-digit shape, same immutability, only the input to the hash
|
||||||
|
changes. Rows written before this ADR keep their URL-derived addresses
|
||||||
|
forever: they are never rehashed on read, and they heal into byte addressing
|
||||||
|
only when a Forced Poll replaces them.
|
||||||
|
|
||||||
|
`ReplaceSeriesCover(site, seriesID, sourceURL, body, contentType)`
|
||||||
|
`(previous, current, error)` is the one write that may move a Cover once one
|
||||||
|
exists. It stores the bytes, then in one transaction locks the Series row,
|
||||||
|
reads the old `cover_address`, writes the new one and the source URL, and
|
||||||
|
reports both addresses: `previous == ""` means there was no Cover,
|
||||||
|
`previous == current` means the Site served identical artwork, and any other
|
||||||
|
pair names the stranded address.
|
||||||
|
|
||||||
|
## Why a future reader will find this surprising
|
||||||
|
|
||||||
|
The address is what makes a re-art visible at all. URL addressing collapses
|
||||||
|
every image behind a stable URL into one address, so a Series whose Cover
|
||||||
|
changes (a big-budget CPI blitz on a light novel is the standing example)
|
||||||
|
keeps serving its original cover bytes: the poll refetches the same URL,
|
||||||
|
hashes it, and the store records the same address, everyone happy except the
|
||||||
|
Reader. Nothing in the system can detect the change, because the address is a
|
||||||
|
pure function of the fetch target, and identical bytes written 1,000 times
|
||||||
|
are one blob on disk. Storing bytes we already know how to store is only a
|
||||||
|
few lines of work. **Rejecting that work is the surprising part, and the
|
||||||
|
answer is the Forced Poll wave**: for a corrupt/blank cover the poll's
|
||||||
|
fill-if-blank installer already worked, but for a *wrong but non-blank* cover
|
||||||
|
there was no write that would move it at all — only a manual truth in
|
||||||
|
`series.cover_address`, which is exactly the thing that must never be set by
|
||||||
|
hand. Byte addressing gives the replacement write a **new address to write**,
|
||||||
|
and with it a legitimate, transaction-safe mover.
|
||||||
|
|
||||||
|
## Considered options
|
||||||
|
|
||||||
|
**Keep URL addressing and add a generic "clear the cover" write.**
|
||||||
|
Rejected: clearing is a two-phase action (blank it, wait for the poll to
|
||||||
|
re-fill, hope the bytes changed in between) that cannot report what the
|
||||||
|
write did, and it makes the Series render cover-less in between. The
|
||||||
|
replacement write is atomic, reports its displacement, and has one effect:
|
||||||
|
the Series now points at bytes that actually came from its source URL.
|
||||||
|
|
||||||
|
**Address by URL, but salt it so a re-art is a new address.**
|
||||||
|
Rejected: the salt would have to live somewhere addressable (a stored per-
|
||||||
|
Series nonce), turning the address from a content fact into a mutable fact —
|
||||||
|
two rows could then hold identical bytes under different addresses and the
|
||||||
|
invariant "same bytes object" is gone.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- `store.CoverAddress` (URL-hash) is deleted; `CoverAddressForBytes` is
|
||||||
|
public so tests and the forced-poll wave can predict addresses from the
|
||||||
|
bytes fakes serve.
|
||||||
|
- Legacy URL-addressed rows are read-only facts: `GetCover(sourceURL)` keeps
|
||||||
|
resolving them (the poll heal path), and they are re-addressed only by a
|
||||||
|
forced replacement. Until one happens, they are invisible to byte-derived
|
||||||
|
lookups — the reverse direction was always true, so this side has no
|
||||||
|
migration and no lookup fan-out.
|
||||||
|
- A replaced Cover's old bytes stay on disk under their address (the `covers`
|
||||||
|
row is untouched — only the Series row moves). Nothing reclaims them
|
||||||
|
today; a later sweep is a small query over `covers` addresses not
|
||||||
|
referenced by any `series` row.
|
||||||
|
- `SetSeriesCover` keeps its `cover_address = ''` guard untouched: the
|
||||||
|
acquisition-at-creation and poll fill paths still may not overwrite a
|
||||||
|
non-blank Cover. The two installers are now deliberately different
|
||||||
|
functions instead of one function with a conditional.
|
||||||
|
- The address is still a filesystem path (≤64 hex chars, no separators), so
|
||||||
|
`coverAddressRe` and the sharding stay exactly as they are.
|
||||||
|
|
||||||
|
## Cost of reversing
|
||||||
|
|
||||||
|
The URL-hash side of the current rows is uncomputable from the rows alone: a
|
||||||
|
rollback would need every stored blob's source URL, a join to a table that
|
||||||
|
does not store it, or a refetch of every Series. Keeping both derivations
|
||||||
|
resolvable is cheaper than either, so the two derivations are documented in
|
||||||
|
the 0009 migration comment: no component may assume which derivation a
|
||||||
|
stored address came from, because the 64-hex shape hides it.
|
||||||
Reference in New Issue
Block a user