Cover addresses derived from the bytes; ReplaceSeriesCover (#150)
# Conflicts: # backend/internal/store/store_test.go
This commit is contained in:
@@ -32,7 +32,7 @@ func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
|
||||
|
||||
// The wire URL is what a client actually requests, so the path under test
|
||||
// is taken from it rather than rebuilt by hand.
|
||||
wire := st.CoverWireURL(store.CoverAddress(sourceURL))
|
||||
wire := st.CoverWireURL(store.CoverAddressForBytes([]byte("\x00webp-bytes")))
|
||||
path, ok := strings.CutPrefix(wire, testCoverBaseURL)
|
||||
if !ok {
|
||||
t.Fatalf("wire URL %q is not on the public origin %q", wire, testCoverBaseURL)
|
||||
@@ -57,7 +57,7 @@ func TestPublicCoverServesStoredBytesUnauthenticated(t *testing.T) {
|
||||
func TestPublicCoverRejectsUnknownAddress(t *testing.T) {
|
||||
srv, _ := newWebTestServer(t, testConfig())
|
||||
cases := map[string]string{
|
||||
"unknown": "/covers/" + store.CoverAddress("https://cdn.example/never-stored.jpg"),
|
||||
"unknown": "/covers/" + store.CoverAddressForBytes([]byte("never-stored")),
|
||||
"malformed": "/covers/not-an-address",
|
||||
"traversal": "/covers/../../etc/passwd",
|
||||
"empty": "/covers/",
|
||||
@@ -86,7 +86,7 @@ func TestPublicCoverNeverEchoesNonImage(t *testing.T) {
|
||||
}
|
||||
// A legitimate row, then the content type flipped behind the store's back:
|
||||
// the bytes exist at the address, so only the type is hostile.
|
||||
address := store.CoverAddress(sourceURL)
|
||||
address := store.CoverAddressForBytes([]byte("<script>"))
|
||||
if err := st.SetSeriesCover("asura", "solo", sourceURL, []byte("<script>"), "image/png"); err != nil {
|
||||
t.Fatalf("seed row: %v", err)
|
||||
}
|
||||
@@ -127,7 +127,7 @@ func TestListRendersAcquiredCover(t *testing.T) {
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddress(sourceURL) + `"`
|
||||
want := `src="` + testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("\xff\xd8jpeg")) + `"`
|
||||
if !strings.Contains(rr.Body.String(), want) {
|
||||
t.Fatalf("rendered list does not contain %s", want)
|
||||
}
|
||||
|
||||
@@ -123,10 +123,10 @@ func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) {
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 {
|
||||
t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the absolute address %q", got.Cover, want)
|
||||
}
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(acquireCoverURL))
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||
}
|
||||
@@ -182,7 +182,7 @@ func TestAcquireSkipsAnExistingSeries(t *testing.T) {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, acquireKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(acquireCoverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the acquired one %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -343,10 +343,10 @@ func TestAcquireKaganeCoverThroughBrowser(t *testing.T) {
|
||||
t.Fatalf("browser cover fetched URL %q, want %q", got, kaganeCoverSrc)
|
||||
}
|
||||
got := readBookmark(t, s, kaganeKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(kaganeCoverSrc); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the content-addressed URL %q", got.Cover, want)
|
||||
}
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(kaganeCoverSrc))
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||
}
|
||||
@@ -381,7 +381,7 @@ func TestAcquireNovelfullCoverOverPlainTLS(t *testing.T) {
|
||||
t.Fatalf("cover fetched from %q, want %q", got, novelfullCoverURL)
|
||||
}
|
||||
got := readBookmark(t, s, novelfullKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -451,7 +451,7 @@ func TestAcquireNovelfullCoverWithoutBrowser(t *testing.T) {
|
||||
t.Fatalf("cover fetches = %d, want 1", got)
|
||||
}
|
||||
got := readBookmark(t, s, novelfullKey)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,9 +214,9 @@ func TestRunOncePrefetchesPublicCover(t *testing.T) {
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||
}
|
||||
body, contentType, found, err := s.GetCover(coverURL)
|
||||
body, contentType, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetCover: %v found=%v", err, found)
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, found)
|
||||
}
|
||||
if string(body) != "cover-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("stored cover = (%q, %q), want (cover-bytes, image/jpeg)", body, contentType)
|
||||
@@ -663,7 +663,7 @@ func TestNovelfullUsesTLSWhenNoBrowserFetcher(t *testing.T) {
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: %v found=%v", err, found)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(novelfullCoverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
if got.LatestChapterNum == nil || *got.LatestChapterNum != 2334 {
|
||||
@@ -811,7 +811,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes")))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
||||
}
|
||||
@@ -821,7 +821,7 @@ func TestRunOncePrefetchesKaganeCover(t *testing.T) {
|
||||
if got := covers.callCount(); got != 1 {
|
||||
t.Fatalf("cover fetch calls = %d, want 1", got)
|
||||
}
|
||||
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddress(coverURL) {
|
||||
if got := readBookmark(t, s, key); got.Cover != testCoverBaseURL+"/covers/"+store.CoverAddressForBytes([]byte("cover-bytes")) {
|
||||
t.Fatalf("wire Cover = %q, want content-addressed URL", got.Cover)
|
||||
}
|
||||
}
|
||||
@@ -914,7 +914,7 @@ func TestRunOnceRejectsInvalidKaganeCover(t *testing.T) {
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("not an image"))); err != nil || found {
|
||||
t.Fatalf("invalid cover persisted = %v, err %v; want missing", found, err)
|
||||
}
|
||||
}
|
||||
@@ -940,7 +940,7 @@ func TestRunOnceWithoutCoverFetcherStillPollsKagane(t *testing.T) {
|
||||
p.runOnce(context.Background())
|
||||
p.waitCovers()
|
||||
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddress(coverURL)); err != nil || found {
|
||||
if _, _, found, err := s.CoverByAddress(store.CoverAddressForBytes([]byte("cover-bytes"))); err != nil || found {
|
||||
t.Fatalf("cover after nil CoverFetch = found %v, err %v; want missing", found, err)
|
||||
}
|
||||
}
|
||||
@@ -1038,7 +1038,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
seriesURL string
|
||||
kind string
|
||||
body string
|
||||
wantCover string
|
||||
browser bool
|
||||
}{
|
||||
{
|
||||
@@ -1047,14 +1046,12 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
seriesID: "chronicles-of-the-demon-faction-f886a8af",
|
||||
seriesURL: "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af",
|
||||
kind: store.KindManga, body: asuraSeriesFixture + asuraCoverFixture,
|
||||
wantCover: "https://cdn.asurascans.com/asura-images/covers/chronicles-of-the-demon-faction.d4dcb8.webp",
|
||||
},
|
||||
{
|
||||
name: "lightnovelworld novel",
|
||||
key: "lightnovelworld:all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", site: "lightnovelworld",
|
||||
seriesID: "all-jobs-and-classes-i-just-wanted-one-skill-not-them-all", seriesURL: "https://lightnovelworld.net/novel/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all/",
|
||||
kind: store.KindNovel, body: lnwSeriesFixture + lnwCoverFixture,
|
||||
wantCover: "https://i1.wp.com/lightnovelworld.net/wp-content/uploads/2025/10/all-jobs-and-classes-i-just-wanted-one-skill-not-them-all.jpg",
|
||||
},
|
||||
{
|
||||
name: "kagane manga",
|
||||
@@ -1062,7 +1059,6 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
seriesID: "019fe11a-8670-7cf3-8343-0b02057d3787",
|
||||
seriesURL: "https://kagane.to/series/019fe11a-8670-7cf3-8343-0b02057d3787",
|
||||
kind: store.KindManga, body: kaganeAPIFixtureWithCover, browser: true,
|
||||
wantCover: "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
@@ -1086,7 +1082,7 @@ func TestRunOnceFillsBlankCoverFromSeriesPage(t *testing.T) {
|
||||
p.waitCovers()
|
||||
|
||||
got := readBookmark(t, s, tc.key)
|
||||
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddress(tc.wantCover)
|
||||
wantWire := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes"))
|
||||
if got.Cover != wantWire {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, wantWire)
|
||||
}
|
||||
@@ -1143,7 +1139,7 @@ func TestRunOnceDoesNotReplaceExistingCover(t *testing.T) {
|
||||
t.Fatalf("cover fetch calls = %d, want 0", got)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(first); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("first")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -1190,7 +1186,7 @@ func TestRunOnceRetriesFailedBlankCoverOnNextPoll(t *testing.T) {
|
||||
t.Fatalf("cover fetch calls after retry = %d, want 2", got)
|
||||
}
|
||||
got := readBookmark(t, s, key)
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes([]byte("cover-bytes")); got.Cover != want {
|
||||
t.Fatalf("Cover after retry = %q, want %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -108,10 +109,7 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
|
||||
if ws == "" {
|
||||
t.Skip("SMOKE_BROWSER_WS_URL unset")
|
||||
}
|
||||
const (
|
||||
seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
|
||||
coverURL = "https://kagane.to/api/v2/image/019fe11a-84c3-7fc3-a84b-88787374b617/compressed"
|
||||
)
|
||||
const seriesID = "019fe11a-8670-7cf3-8343-0b02057d3787"
|
||||
s, _ := newTestStore(t)
|
||||
bf, err := NewBrowserFetcher(ws)
|
||||
if err != nil {
|
||||
@@ -140,12 +138,13 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
|
||||
if err != nil || !found {
|
||||
t.Fatalf("Get: %v found=%v", err, found)
|
||||
}
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddress(coverURL); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q — the acquire path did not store the browser-fetched bytes", got.Cover, want)
|
||||
addr, ok := strings.CutPrefix(got.Cover, testCoverBaseURL+"/covers/")
|
||||
if !ok {
|
||||
t.Fatalf("Cover = %q, want an address on %q — the acquire path did not store the browser-fetched bytes", got.Cover, testCoverBaseURL+"/covers/")
|
||||
}
|
||||
body, contentType, ok, err := s.CoverByAddress(store.CoverAddress(coverURL))
|
||||
body, contentType, ok, err := s.CoverByAddress(addr)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress: %v found=%v", err, ok)
|
||||
t.Fatalf("CoverByAddress(%q): %v found=%v", addr, err, ok)
|
||||
}
|
||||
if len(body) < 1000 {
|
||||
t.Fatalf("stored cover is %d bytes, want a real image", len(body))
|
||||
@@ -153,5 +152,10 @@ func TestSmokeAcquireKaganeCover(t *testing.T) {
|
||||
if contentType != "image/webp" {
|
||||
t.Fatalf("content type = %q, want image/webp", contentType)
|
||||
}
|
||||
// The address the row carries is the bytes' own SHA-256: a re-art behind
|
||||
// the same URL would be a different address, which is the whole point.
|
||||
if want := testCoverBaseURL + "/covers/" + store.CoverAddressForBytes(body); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want %q", got.Cover, want)
|
||||
}
|
||||
t.Logf("stored %d bytes of %s", len(body), contentType)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
-- The Cover splits into two facts. `cover` keeps the third-party address the
|
||||
-- bytes come from, which is what the acquisition path refetches and dedupes
|
||||
-- on; `cover_address` is the content address of the bytes once they are
|
||||
-- actually stored, and is what the wire's absolute URL is built from.
|
||||
-- bytes come from, which is what the refetch path dedupes on; `cover_address`
|
||||
-- is the content address of the bytes once they are actually stored, and is
|
||||
-- what the wire's absolute URL is built from.
|
||||
--
|
||||
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
|
||||
-- that 404s", which is the distinction the API and the UI both depend on.
|
||||
--
|
||||
-- The content address was originally the hex SHA-256 of the source URL
|
||||
-- (ADR-0007). Since ADR-0014 it is the hex SHA-256 of the bytes themselves,
|
||||
-- so a re-art behind the same URL is a new address. Rows written before
|
||||
-- ADR-0014 keep their URL-derived addresses; they are never rehashed and heal
|
||||
-- into byte addressing on their first forced replacement. Both derivations
|
||||
-- share the 64-hex-digit shape, so the serving guard is unchanged.
|
||||
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
|
||||
|
||||
@@ -704,67 +704,76 @@ func (s *Store) getCoverByAddress(address string) ([]byte, string, bool, error)
|
||||
return body, contentType, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) putCover(sourceURL string, body []byte, contentType string) error {
|
||||
func (s *Store) putCover(sourceURL string, body []byte, contentType string) (string, error) {
|
||||
stored, ok := CoverContentType(contentType)
|
||||
if !ok {
|
||||
return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
|
||||
return "", fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
|
||||
}
|
||||
contentType = stored
|
||||
address := coverSourceAddress(sourceURL)
|
||||
address := CoverAddressForBytes(body)
|
||||
relativePath := coverRelativePath(address)
|
||||
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
|
||||
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create cover shard: %w", err)
|
||||
return "", fmt.Errorf("create cover shard: %w", err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create cover temp file: %w", err)
|
||||
return "", fmt.Errorf("create cover temp file: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if _, err := tmp.Write(body); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write cover temp file: %w", err)
|
||||
return "", fmt.Errorf("write cover temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("sync cover temp file: %w", err)
|
||||
return "", fmt.Errorf("sync cover temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close cover temp file: %w", err)
|
||||
return "", fmt.Errorf("close cover temp file: %w", err)
|
||||
}
|
||||
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("install cover file: %w", err)
|
||||
return "", fmt.Errorf("install cover file: %w", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO covers (address, path, content_type)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
|
||||
return fmt.Errorf("record cover %q: %w", address, err)
|
||||
return "", fmt.Errorf("record cover %q: %w", address, err)
|
||||
}
|
||||
return nil
|
||||
return address, nil
|
||||
}
|
||||
|
||||
// GetCover returns the immutable object addressed by its source URL. Missing
|
||||
// GetCover returns the immutable object a source URL's own hash names. Rows
|
||||
// written before byte addressing (ADR-0014) are the only ones that ever reach
|
||||
// it; it hashes the URL, so a byte-addressed Cover is invisible to it. Missing
|
||||
// files are reported with ok=false so callers can retry acquisition later.
|
||||
func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) {
|
||||
return s.getCover(sourceURL)
|
||||
}
|
||||
|
||||
// PutCover persists bytes under the source URL's content address. A later
|
||||
// write for the same URL cannot replace the immutable object.
|
||||
// PutCover persists bytes under their own content address (ADR-0014). A later
|
||||
// write of the same bytes cannot replace the immutable object.
|
||||
func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error {
|
||||
return s.putCover(sourceURL, body, contentType)
|
||||
_, err := s.putCover(sourceURL, body, contentType)
|
||||
return err
|
||||
}
|
||||
|
||||
// CoverAddress is the content address bytes fetched from sourceURL are stored
|
||||
// under. It is a pure function of the URL, so the acquisition path can name a
|
||||
// Cover before it has the bytes.
|
||||
func CoverAddress(sourceURL string) string { return coverSourceAddress(sourceURL) }
|
||||
// CoverAddressForBytes is the content address body is stored under: the hex
|
||||
// SHA-256 of the bytes, so identical artwork is one address and a re-art a
|
||||
// new one. Legacy rows were addressed from their source URL instead and are
|
||||
// never rehashed — both derivations coexist (ADR-0014).
|
||||
func CoverAddressForBytes(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// coverAddressRe is the shape of a stored address: the hex SHA-256 of a source
|
||||
// URL. Request paths reach CoverByAddress, so the shape is checked before the
|
||||
// value is ever turned into a filesystem path.
|
||||
// coverAddressRe is the shape of a stored address: 64 lowercase hex digits —
|
||||
// the hex SHA-256 of the cover bytes, or of the source URL for legacy rows
|
||||
// (ADR-0014). Request paths reach CoverByAddress, so the shape is checked
|
||||
// before the value is ever turned into a filesystem path; byte-derived
|
||||
// addresses keep the same shape, so the guard is unchanged.
|
||||
var coverAddressRe = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// CoverByAddress returns the immutable object at one content address. An
|
||||
@@ -788,24 +797,67 @@ func (s *Store) CoverWireURL(address string) string {
|
||||
return s.coverBaseURL + "/covers/" + address
|
||||
}
|
||||
|
||||
// SetSeriesCover stores the bytes and points the Series at them, but only
|
||||
// while the Series has no Cover: acquisition at creation and the poll both
|
||||
// call this, and whichever arrives second must not overwrite the first. The
|
||||
// bytes themselves are content-addressed and immutable, so storing them twice
|
||||
// is free.
|
||||
// SetSeriesCover stores the bytes and points the Series at their address, but
|
||||
// only while the Series has no Cover: acquisition at creation and the poll
|
||||
// both call this, and whichever arrives second must not overwrite the first.
|
||||
// The bytes themselves are content-addressed and immutable, so storing them
|
||||
// twice is free. See ReplaceSeriesCover for the write that may move a Cover
|
||||
// once one exists (ADR-0014).
|
||||
func (s *Store) SetSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) error {
|
||||
if err := s.putCover(sourceURL, body, contentType); err != nil {
|
||||
address, err := s.putCover(sourceURL, body, contentType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE series SET cover = $3, cover_address = $4
|
||||
WHERE site = $1 AND series_id = $2 AND cover_address = ''`,
|
||||
site, seriesID, sourceURL, coverSourceAddress(sourceURL)); err != nil {
|
||||
site, seriesID, sourceURL, address); err != nil {
|
||||
return fmt.Errorf("set cover for %q: %w", site+":"+seriesID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReplaceSeriesCover stores the bytes and points the Series at their address
|
||||
// whether or not one already exists, writing the current source URL alongside
|
||||
// — the Forced Poll's installer and the only write that may move a Cover once
|
||||
// one exists (ADR-0014). previous is the address the row held before the write
|
||||
// ("" if it had none) and current the address of the bytes just stored; both
|
||||
// are read and written in one transaction, so a concurrent replacement reports
|
||||
// the exact displacement. previous == current means the Site served identical
|
||||
// artwork, an honest no-op; otherwise previous is stranded — its bytes stay
|
||||
// served under the covers table (ADR-0014), the row just no longer points at
|
||||
// them.
|
||||
func (s *Store) ReplaceSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) (previous, current string, err error) {
|
||||
current, err = s.putCover(sourceURL, body, contentType)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("begin replace cover for %q: %w", site+":"+seriesID, err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
err = tx.QueryRow(`
|
||||
SELECT cover_address FROM series
|
||||
WHERE site = $1 AND series_id = $2 FOR UPDATE`,
|
||||
site, seriesID).Scan(&previous)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
previous = ""
|
||||
} else if err != nil {
|
||||
return "", "", fmt.Errorf("read cover for %q: %w", site+":"+seriesID, err)
|
||||
}
|
||||
if _, err := tx.Exec(`
|
||||
UPDATE series SET cover = $3, cover_address = $4
|
||||
WHERE site = $1 AND series_id = $2`,
|
||||
site, seriesID, sourceURL, current); err != nil {
|
||||
return "", "", fmt.Errorf("replace cover for %q: %w", site+":"+seriesID, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", "", fmt.Errorf("commit cover replace for %q: %w", site+":"+seriesID, err)
|
||||
}
|
||||
return previous, current, nil
|
||||
}
|
||||
|
||||
// List returns every bookmark of one reader, newest activity first.
|
||||
// Series-owned fields are joined in, so each Bookmark reads back whole and
|
||||
// flat (ADR-0004).
|
||||
|
||||
@@ -850,7 +850,7 @@ func TestSetSeriesCoverDoesNotOverwrite(t *testing.T) {
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("Get = %v, %v", ok, err)
|
||||
}
|
||||
if want := "https://bookmarks.test/covers/" + CoverAddress(first); got.Cover != want {
|
||||
if want := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("first")); got.Cover != want {
|
||||
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
|
||||
}
|
||||
}
|
||||
@@ -869,7 +869,7 @@ func TestCoverByAddress(t *testing.T) {
|
||||
t.Fatalf("SetSeriesCover: %v", err)
|
||||
}
|
||||
|
||||
body, contentType, ok, err := store.CoverByAddress(CoverAddress(source))
|
||||
body, contentType, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("bytes")))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("CoverByAddress = %v, %v", ok, err)
|
||||
}
|
||||
@@ -877,8 +877,8 @@ func TestCoverByAddress(t *testing.T) {
|
||||
t.Fatalf("CoverByAddress = %q, %q, want the stored bytes", body, contentType)
|
||||
}
|
||||
|
||||
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddress(source)[2:],
|
||||
CoverAddress("never stored")} {
|
||||
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddressForBytes([]byte("bytes"))[2:],
|
||||
CoverAddressForBytes([]byte("never stored"))} {
|
||||
_, _, ok, err := store.CoverByAddress(address)
|
||||
if err != nil || ok {
|
||||
t.Fatalf("CoverByAddress(%q) = %v, %v, want a clean miss", address, ok, err)
|
||||
@@ -913,7 +913,7 @@ func TestUpsertExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired)
|
||||
wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
|
||||
if got.Title != "Solo Leveling" || got.SeriesURL != "https://asurascans.com/comics/solo" ||
|
||||
got.Cover != wantCover {
|
||||
t.Fatalf("stored = %+v, want original title/url/cover kept", got)
|
||||
@@ -981,7 +981,7 @@ func TestDeleteKeepsSeriesRow(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("re-upsert: %v", err)
|
||||
}
|
||||
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired)
|
||||
wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
|
||||
if stored.Title != "Solo Leveling" || stored.Cover != wantCover {
|
||||
t.Fatalf("re-bookmark = %+v, want title/cover from the surviving series row", stored)
|
||||
}
|
||||
@@ -1498,9 +1498,9 @@ func TestCoverPersistsAcrossReopen(t *testing.T) {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer second.Close()
|
||||
got, contentType, ok, err := second.GetCover(sourceURL)
|
||||
got, contentType, ok, err := second.CoverByAddress(CoverAddressForBytes(body))
|
||||
if err != nil {
|
||||
t.Fatalf("GetCover: %v", err)
|
||||
t.Fatalf("CoverByAddress: %v", err)
|
||||
}
|
||||
if !ok || !bytes.Equal(got, body) || contentType != "image/webp" {
|
||||
t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body)
|
||||
@@ -1539,7 +1539,7 @@ func TestCoverIsContentAddressedOnFilesystem(t *testing.T) {
|
||||
}
|
||||
defer first.Close()
|
||||
|
||||
addressBytes := sha256.Sum256([]byte(sourceURL))
|
||||
addressBytes := sha256.Sum256(body)
|
||||
address := hex.EncodeToString(addressBytes[:])
|
||||
wantPath := filepath.Join(address[:2], address[2:4], address)
|
||||
|
||||
@@ -1570,9 +1570,9 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
|
||||
if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil {
|
||||
t.Fatalf("PutCover: %v", err)
|
||||
}
|
||||
got, contentType, ok, err := s.GetCover(sourceURL)
|
||||
got, contentType, ok, err := s.CoverByAddress(CoverAddressForBytes(want))
|
||||
if err != nil {
|
||||
t.Fatalf("GetCover: %v", err)
|
||||
t.Fatalf("CoverByAddress: %v", err)
|
||||
}
|
||||
if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" {
|
||||
t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want)
|
||||
@@ -1580,7 +1580,7 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
|
||||
if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil {
|
||||
t.Fatal("PutCover accepted a non-image")
|
||||
}
|
||||
if _, _, ok, err := s.GetCover("https://cdn.example/not-image"); err != nil || ok {
|
||||
if _, _, ok, err := s.CoverByAddress(CoverAddressForBytes([]byte("html"))); err != nil || ok {
|
||||
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
|
||||
}
|
||||
}
|
||||
@@ -2029,3 +2029,118 @@ func (s *Store) latestCorrectedAt(t *testing.T, site, seriesID string) int64 {
|
||||
|
||||
// num2 boxes a chapter number for the Bookmark fields that take a pointer.
|
||||
func num2(f float64) *float64 { return &f }
|
||||
// --- Cover addressing (ADR-0014): the address is the bytes' SHA-256 ---
|
||||
|
||||
// The address is what makes a re-art visible at all, so the same bytes must
|
||||
// always name the same address and different bytes different ones — and the
|
||||
// address must keep the 64-hex-digit shape CoverByAddress's guard still checks
|
||||
// before any request-supplied value becomes a filesystem path.
|
||||
func TestCoverAddressForBytesIsDeterministicAndDistinct(t *testing.T) {
|
||||
first := CoverAddressForBytes([]byte("art"))
|
||||
again := CoverAddressForBytes([]byte("art"))
|
||||
other := CoverAddressForBytes([]byte("artwork"))
|
||||
if first != again {
|
||||
t.Fatalf("same bytes gave %q then %q, want one address", first, again)
|
||||
}
|
||||
if first == other {
|
||||
t.Fatalf("different bytes gave the same address %q", first)
|
||||
}
|
||||
if !coverAddressRe.MatchString(first) {
|
||||
t.Fatalf("address %q is not the 64-hex-digit shape the serving guard checks", first)
|
||||
}
|
||||
}
|
||||
|
||||
// ReplaceSeriesCover is the forced-replacement installer: it moves a Cover
|
||||
// whether or not one exists, writes the source URL alongside it, and reports
|
||||
// the three outcomes the Forced Poll has to tell apart.
|
||||
func TestReplaceSeriesCover(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
if _, err := store.Upsert(store.OwnerID(), Bookmark{
|
||||
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
// A blank Cover: previous is "", and the row points at the new bytes.
|
||||
previous, current, err := store.ReplaceSeriesCover("asura", "solo",
|
||||
"https://cdn.asurascans.com/covers/solo.webp", []byte("first-art"), "image/webp")
|
||||
if err != nil {
|
||||
t.Fatalf("ReplaceSeriesCover on a blank: %v", err)
|
||||
}
|
||||
if previous != "" {
|
||||
t.Fatalf("previous on a blank = %q, want empty", previous)
|
||||
}
|
||||
if want := CoverAddressForBytes([]byte("first-art")); current != want {
|
||||
t.Fatalf("current = %q, want %q", current, want)
|
||||
}
|
||||
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
|
||||
sr.Cover != "https://cdn.asurascans.com/covers/solo.webp" {
|
||||
t.Fatalf("series after blank fill = %+v, want the new address and source URL", sr)
|
||||
}
|
||||
|
||||
// A re-art: previous is the stranded address, current the new one.
|
||||
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
|
||||
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
|
||||
if err != nil {
|
||||
t.Fatalf("ReplaceSeriesCover over a filled Cover: %v", err)
|
||||
}
|
||||
if want := CoverAddressForBytes([]byte("first-art")); previous != want {
|
||||
t.Fatalf("previous = %q, want the replaced address %q", previous, want)
|
||||
}
|
||||
if want := CoverAddressForBytes([]byte("second-art")); current != want {
|
||||
t.Fatalf("current = %q, want %q", current, want)
|
||||
}
|
||||
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
|
||||
sr.Cover != "https://cdn.asurascans.com/covers/solo-rebrand.webp" {
|
||||
t.Fatalf("series after replacement = %+v, want the new address and source URL", sr)
|
||||
}
|
||||
// The replaced bytes stay served under their old address; nothing reclaims
|
||||
// them in this ticket (the forced-poll wave does).
|
||||
if _, _, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("first-art"))); err != nil || !ok {
|
||||
t.Fatalf("superseded bytes = found %v, err %v, want still served", ok, err)
|
||||
}
|
||||
|
||||
// The Site is serving the same artwork again: previous == current is the
|
||||
// honest no-op the caller reports as "unchanged".
|
||||
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
|
||||
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
|
||||
if err != nil {
|
||||
t.Fatalf("ReplaceSeriesCover over identical bytes: %v", err)
|
||||
}
|
||||
if previous != current {
|
||||
t.Fatalf("identical bytes: previous = %q, current = %q, want one address", previous, current)
|
||||
}
|
||||
if want := CoverAddressForBytes([]byte("second-art")); current != want {
|
||||
t.Fatalf("current = %q, want %q", current, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Rows written before byte addressing hold the hash of their source URL and
|
||||
// are never rehashed: GetCover — the poller's heal path — keeps resolving
|
||||
// them through coverSourceAddress.
|
||||
func TestGetCoverResolvesLegacyURLDerivedAddress(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := "https://cdn.example/legacy.jpg"
|
||||
legacy := coverSourceAddress(source)
|
||||
relativePath := coverRelativePath(legacy)
|
||||
coverPath := filepath.Join(store.coverDir, filepath.FromSlash(relativePath))
|
||||
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
|
||||
t.Fatalf("create shard dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(coverPath, []byte("legacy-bytes"), 0o644); err != nil {
|
||||
t.Fatalf("write legacy file: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO covers (address, path, content_type) VALUES ($1, $2, $3)`,
|
||||
legacy, relativePath, "image/jpeg"); err != nil {
|
||||
t.Fatalf("plant legacy row: %v", err)
|
||||
}
|
||||
|
||||
body, contentType, ok, err := store.GetCover(source)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetCover on a legacy row = %v, %v, want found", ok, err)
|
||||
}
|
||||
if string(body) != "legacy-bytes" || contentType != "image/jpeg" {
|
||||
t.Fatalf("legacy cover = (%q, %q), want the planted bytes", body, contentType)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user