sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

#150: address covers by bytes; ReplaceSeriesCover

Browse Source
This commit is contained in:
Sulthan Zaki
2026-08-22 09:05:53 +07:00
parent 889f0f3f38
commit b9fc83217d
8 changed files with 338 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/store/migrations/0009_series_cover_address.sql
+10 -3
View File
@@ -1,8 +1,15 @@
-- The Cover splits into two facts. `cover` keeps the third-party address the
-- bytes come from, which is what the acquisition path refetches and dedupes
-- on; `cover_address` is the content address of the bytes once they are
-- actually stored, and is what the wire's absolute URL is built from.
-- bytes come from, which is what the refetch path dedupes on; `cover_address`
-- is the content address of the bytes once they are actually stored, and is
-- what the wire's absolute URL is built from.
--
-- Empty `cover_address` therefore means "no Cover yet" rather than "a Cover
-- that 404s", which is the distinction the API and the UI both depend on.
--
-- The content address was originally the hex SHA-256 of the source URL
-- (ADR-0007). Since ADR-0014 it is the hex SHA-256 of the bytes themselves,
-- so a re-art behind the same URL is a new address. Rows written before
-- ADR-0014 keep their URL-derived addresses; they are never rehashed and heal
-- into byte addressing on their first forced replacement. Both derivations
-- share the 64-hex-digit shape, so the serving guard is unchanged.
ALTER TABLE series ADD COLUMN cover_address text NOT NULL DEFAULT '';
backend/internal/store/store.go
+81 -29
View File
@@ -704,67 +704,76 @@ func (s *Store) getCoverByAddress(address string) ([]byte, string, bool, error)
return body, contentType, true, nil
}
func (s *Store) putCover(sourceURL string, body []byte, contentType string) error {
func (s *Store) putCover(sourceURL string, body []byte, contentType string) (string, error) {
stored, ok := CoverContentType(contentType)
if !ok {
return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
return "", fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
}
contentType = stored
address := coverSourceAddress(sourceURL)
address := CoverAddressForBytes(body)
relativePath := coverRelativePath(address)
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
return fmt.Errorf("create cover shard: %w", err)
return "", fmt.Errorf("create cover shard: %w", err)
}
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
if err != nil {
return fmt.Errorf("create cover temp file: %w", err)
return "", fmt.Errorf("create cover temp file: %w", err)
}
tmpName := tmp.Name()
defer os.Remove(tmpName)
if _, err := tmp.Write(body); err != nil {
tmp.Close()
return fmt.Errorf("write cover temp file: %w", err)
return "", fmt.Errorf("write cover temp file: %w", err)
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return fmt.Errorf("sync cover temp file: %w", err)
return "", fmt.Errorf("sync cover temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close cover temp file: %w", err)
return "", fmt.Errorf("close cover temp file: %w", err)
}
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("install cover file: %w", err)
return "", fmt.Errorf("install cover file: %w", err)
}
if _, err := s.db.Exec(`
INSERT INTO covers (address, path, content_type)
VALUES ($1, $2, $3)
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
return fmt.Errorf("record cover %q: %w", address, err)
return "", fmt.Errorf("record cover %q: %w", address, err)
}
return nil
return address, nil
}
// GetCover returns the immutable object addressed by its source URL. Missing
// GetCover returns the immutable object a source URL's own hash names. Rows
// written before byte addressing (ADR-0014) are the only ones that ever reach
// it; it hashes the URL, so a byte-addressed Cover is invisible to it. Missing
// files are reported with ok=false so callers can retry acquisition later.
func (s *Store) GetCover(sourceURL string) ([]byte, string, bool, error) {
return s.getCover(sourceURL)
}
// PutCover persists bytes under the source URL's content address. A later
// write for the same URL cannot replace the immutable object.
// PutCover persists bytes under their own content address (ADR-0014). A later
// write of the same bytes cannot replace the immutable object.
func (s *Store) PutCover(sourceURL string, body []byte, contentType string) error {
return s.putCover(sourceURL, body, contentType)
_, err := s.putCover(sourceURL, body, contentType)
return err
}
// CoverAddress is the content address bytes fetched from sourceURL are stored
// under. It is a pure function of the URL, so the acquisition path can name a
// Cover before it has the bytes.
func CoverAddress(sourceURL string) string { return coverSourceAddress(sourceURL) }
// CoverAddressForBytes is the content address body is stored under: the hex
// SHA-256 of the bytes, so identical artwork is one address and a re-art a
// new one. Legacy rows were addressed from their source URL instead and are
// never rehashed — both derivations coexist (ADR-0014).
func CoverAddressForBytes(body []byte) string {
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:])
}
// coverAddressRe is the shape of a stored address: the hex SHA-256 of a source
// URL. Request paths reach CoverByAddress, so the shape is checked before the
// value is ever turned into a filesystem path.
// coverAddressRe is the shape of a stored address: 64 lowercase hex digits —
// the hex SHA-256 of the cover bytes, or of the source URL for legacy rows
// (ADR-0014). Request paths reach CoverByAddress, so the shape is checked
// before the value is ever turned into a filesystem path; byte-derived
// addresses keep the same shape, so the guard is unchanged.
var coverAddressRe = regexp.MustCompile(`^[0-9a-f]{64}$`)
// CoverByAddress returns the immutable object at one content address. An
@@ -788,24 +797,67 @@ func (s *Store) CoverWireURL(address string) string {
return s.coverBaseURL + "/covers/" + address
}
// SetSeriesCover stores the bytes and points the Series at them, but only
// while the Series has no Cover: acquisition at creation and the poll both
// call this, and whichever arrives second must not overwrite the first. The
// bytes themselves are content-addressed and immutable, so storing them twice
// is free.
// SetSeriesCover stores the bytes and points the Series at their address, but
// only while the Series has no Cover: acquisition at creation and the poll
// both call this, and whichever arrives second must not overwrite the first.
// The bytes themselves are content-addressed and immutable, so storing them
// twice is free. See ReplaceSeriesCover for the write that may move a Cover
// once one exists (ADR-0014).
func (s *Store) SetSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) error {
if err := s.putCover(sourceURL, body, contentType); err != nil {
address, err := s.putCover(sourceURL, body, contentType)
if err != nil {
return err
}
if _, err := s.db.Exec(`
UPDATE series SET cover = $3, cover_address = $4
WHERE site = $1 AND series_id = $2 AND cover_address = ''`,
site, seriesID, sourceURL, coverSourceAddress(sourceURL)); err != nil {
site, seriesID, sourceURL, address); err != nil {
return fmt.Errorf("set cover for %q: %w", site+":"+seriesID, err)
}
return nil
}
// ReplaceSeriesCover stores the bytes and points the Series at their address
// whether or not one already exists, writing the current source URL alongside
// — the Forced Poll's installer and the only write that may move a Cover once
// one exists (ADR-0014). previous is the address the row held before the write
// ("" if it had none) and current the address of the bytes just stored; both
// are read and written in one transaction, so a concurrent replacement reports
// the exact displacement. previous == current means the Site served identical
// artwork, an honest no-op; otherwise previous is stranded — its bytes stay
// served under the covers table (ADR-0014), the row just no longer points at
// them.
func (s *Store) ReplaceSeriesCover(site, seriesID, sourceURL string, body []byte, contentType string) (previous, current string, err error) {
current, err = s.putCover(sourceURL, body, contentType)
if err != nil {
return "", "", err
}
tx, err := s.db.Begin()
if err != nil {
return "", "", fmt.Errorf("begin replace cover for %q: %w", site+":"+seriesID, err)
}
defer tx.Rollback()
err = tx.QueryRow(`
SELECT cover_address FROM series
WHERE site = $1 AND series_id = $2 FOR UPDATE`,
site, seriesID).Scan(&previous)
if errors.Is(err, sql.ErrNoRows) {
previous = ""
} else if err != nil {
return "", "", fmt.Errorf("read cover for %q: %w", site+":"+seriesID, err)
}
if _, err := tx.Exec(`
UPDATE series SET cover = $3, cover_address = $4
WHERE site = $1 AND series_id = $2`,
site, seriesID, sourceURL, current); err != nil {
return "", "", fmt.Errorf("replace cover for %q: %w", site+":"+seriesID, err)
}
if err := tx.Commit(); err != nil {
return "", "", fmt.Errorf("commit cover replace for %q: %w", site+":"+seriesID, err)
}
return previous, current, nil
}
// List returns every bookmark of one reader, newest activity first.
// Series-owned fields are joined in, so each Bookmark reads back whole and
// flat (ADR-0004).
backend/internal/store/store_test.go
+128 -12
View File
@@ -850,7 +850,7 @@ func TestSetSeriesCoverDoesNotOverwrite(t *testing.T) {
if err != nil || !ok {
t.Fatalf("Get = %v, %v", ok, err)
}
if want := "https://bookmarks.test/covers/" + CoverAddress(first); got.Cover != want {
if want := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("first")); got.Cover != want {
t.Fatalf("Cover = %q, want the first one %q", got.Cover, want)
}
}
@@ -869,7 +869,7 @@ func TestCoverByAddress(t *testing.T) {
t.Fatalf("SetSeriesCover: %v", err)
}
body, contentType, ok, err := store.CoverByAddress(CoverAddress(source))
body, contentType, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("bytes")))
if err != nil || !ok {
t.Fatalf("CoverByAddress = %v, %v", ok, err)
}
@@ -877,8 +877,8 @@ func TestCoverByAddress(t *testing.T) {
t.Fatalf("CoverByAddress = %q, %q, want the stored bytes", body, contentType)
}
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddress(source)[2:],
CoverAddress("never stored")} {
for _, address := range []string{"", "../../etc/passwd", "ZZ" + CoverAddressForBytes([]byte("bytes"))[2:],
CoverAddressForBytes([]byte("never stored"))} {
_, _, ok, err := store.CoverByAddress(address)
if err != nil || ok {
t.Fatalf("CoverByAddress(%q) = %v, %v, want a clean miss", address, ok, err)
@@ -913,7 +913,7 @@ func TestUpsertExistingSeriesIgnoresClientTitleCoverURL(t *testing.T) {
if err != nil {
t.Fatalf("Upsert: %v", err)
}
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired)
wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
if got.Title != "Solo Leveling" || got.SeriesURL != "https://asurascans.com/comics/solo" ||
got.Cover != wantCover {
t.Fatalf("stored = %+v, want original title/url/cover kept", got)
@@ -981,7 +981,7 @@ func TestDeleteKeepsSeriesRow(t *testing.T) {
if err != nil {
t.Fatalf("re-upsert: %v", err)
}
wantCover := "https://bookmarks.test/covers/" + CoverAddress(acquired)
wantCover := "https://bookmarks.test/covers/" + CoverAddressForBytes([]byte("bytes"))
if stored.Title != "Solo Leveling" || stored.Cover != wantCover {
t.Fatalf("re-bookmark = %+v, want title/cover from the surviving series row", stored)
}
@@ -1498,9 +1498,9 @@ func TestCoverPersistsAcrossReopen(t *testing.T) {
t.Fatalf("reopen: %v", err)
}
defer second.Close()
got, contentType, ok, err := second.GetCover(sourceURL)
got, contentType, ok, err := second.CoverByAddress(CoverAddressForBytes(body))
if err != nil {
t.Fatalf("GetCover: %v", err)
t.Fatalf("CoverByAddress: %v", err)
}
if !ok || !bytes.Equal(got, body) || contentType != "image/webp" {
t.Fatalf("stored cover = (%q, %q, %v), want (%q, image/webp, true)", got, contentType, ok, body)
@@ -1539,7 +1539,7 @@ func TestCoverIsContentAddressedOnFilesystem(t *testing.T) {
}
defer first.Close()
addressBytes := sha256.Sum256([]byte(sourceURL))
addressBytes := sha256.Sum256(body)
address := hex.EncodeToString(addressBytes[:])
wantPath := filepath.Join(address[:2], address[2:4], address)
@@ -1570,9 +1570,9 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
if err := s.PutCover(sourceURL, want, "image/jpeg"); err != nil {
t.Fatalf("PutCover: %v", err)
}
got, contentType, ok, err := s.GetCover(sourceURL)
got, contentType, ok, err := s.CoverByAddress(CoverAddressForBytes(want))
if err != nil {
t.Fatalf("GetCover: %v", err)
t.Fatalf("CoverByAddress: %v", err)
}
if !ok || !bytes.Equal(got, want) || contentType != "image/jpeg" {
t.Fatalf("GetCover = (%q, %q, %v), want (%q, image/jpeg, true)", got, contentType, ok, want)
@@ -1580,7 +1580,7 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) {
if err := s.PutCover("https://cdn.example/not-image", []byte("html"), "text/html"); err == nil {
t.Fatal("PutCover accepted a non-image")
}
if _, _, ok, err := s.GetCover("https://cdn.example/not-image"); err != nil || ok {
if _, _, ok, err := s.CoverByAddress(CoverAddressForBytes([]byte("html"))); err != nil || ok {
t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err)
}
}
@@ -1899,3 +1899,119 @@ func TestDueForLatestCheckForcedDoesNotOverrideURLOrJoin(t *testing.T) {
t.Fatalf("due = %v, want neither the URL-less nor the orphan series", due)
}
}
// --- Cover addressing (ADR-0014): the address is the bytes' SHA-256 ---
// The address is what makes a re-art visible at all, so the same bytes must
// always name the same address and different bytes different ones — and the
// address must keep the 64-hex-digit shape CoverByAddress's guard still checks
// before any request-supplied value becomes a filesystem path.
func TestCoverAddressForBytesIsDeterministicAndDistinct(t *testing.T) {
first := CoverAddressForBytes([]byte("art"))
again := CoverAddressForBytes([]byte("art"))
other := CoverAddressForBytes([]byte("artwork"))
if first != again {
t.Fatalf("same bytes gave %q then %q, want one address", first, again)
}
if first == other {
t.Fatalf("different bytes gave the same address %q", first)
}
if !coverAddressRe.MatchString(first) {
t.Fatalf("address %q is not the 64-hex-digit shape the serving guard checks", first)
}
}
// ReplaceSeriesCover is the forced-replacement installer: it moves a Cover
// whether or not one exists, writes the source URL alongside it, and reports
// the three outcomes the Forced Poll has to tell apart.
func TestReplaceSeriesCover(t *testing.T) {
store := newTestStore(t)
if _, err := store.Upsert(store.OwnerID(), Bookmark{
Key: "asura:solo", Site: "asura", SeriesID: "solo", UpdatedAt: 1000,
}); err != nil {
t.Fatalf("seed: %v", err)
}
// A blank Cover: previous is "", and the row points at the new bytes.
previous, current, err := store.ReplaceSeriesCover("asura", "solo",
"https://cdn.asurascans.com/covers/solo.webp", []byte("first-art"), "image/webp")
if err != nil {
t.Fatalf("ReplaceSeriesCover on a blank: %v", err)
}
if previous != "" {
t.Fatalf("previous on a blank = %q, want empty", previous)
}
if want := CoverAddressForBytes([]byte("first-art")); current != want {
t.Fatalf("current = %q, want %q", current, want)
}
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
sr.Cover != "https://cdn.asurascans.com/covers/solo.webp" {
t.Fatalf("series after blank fill = %+v, want the new address and source URL", sr)
}
// A re-art: previous is the stranded address, current the new one.
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
if err != nil {
t.Fatalf("ReplaceSeriesCover over a filled Cover: %v", err)
}
if want := CoverAddressForBytes([]byte("first-art")); previous != want {
t.Fatalf("previous = %q, want the replaced address %q", previous, want)
}
if want := CoverAddressForBytes([]byte("second-art")); current != want {
t.Fatalf("current = %q, want %q", current, want)
}
if sr := readSeries(t, store, "asura", "solo"); sr.CoverAddress != current ||
sr.Cover != "https://cdn.asurascans.com/covers/solo-rebrand.webp" {
t.Fatalf("series after replacement = %+v, want the new address and source URL", sr)
}
// The replaced bytes stay served under their old address; nothing reclaims
// them in this ticket (the forced-poll wave does).
if _, _, ok, err := store.CoverByAddress(CoverAddressForBytes([]byte("first-art"))); err != nil || !ok {
t.Fatalf("superseded bytes = found %v, err %v, want still served", ok, err)
}
// The Site is serving the same artwork again: previous == current is the
// honest no-op the caller reports as "unchanged".
previous, current, err = store.ReplaceSeriesCover("asura", "solo",
"https://cdn.asurascans.com/covers/solo-rebrand.webp", []byte("second-art"), "image/jpeg")
if err != nil {
t.Fatalf("ReplaceSeriesCover over identical bytes: %v", err)
}
if previous != current {
t.Fatalf("identical bytes: previous = %q, current = %q, want one address", previous, current)
}
if want := CoverAddressForBytes([]byte("second-art")); current != want {
t.Fatalf("current = %q, want %q", current, want)
}
}
// Rows written before byte addressing hold the hash of their source URL and
// are never rehashed: GetCover — the poller's heal path — keeps resolving
// them through coverSourceAddress.
func TestGetCoverResolvesLegacyURLDerivedAddress(t *testing.T) {
store := newTestStore(t)
source := "https://cdn.example/legacy.jpg"
legacy := coverSourceAddress(source)
relativePath := coverRelativePath(legacy)
coverPath := filepath.Join(store.coverDir, filepath.FromSlash(relativePath))
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
t.Fatalf("create shard dir: %v", err)
}
if err := os.WriteFile(coverPath, []byte("legacy-bytes"), 0o644); err != nil {
t.Fatalf("write legacy file: %v", err)
}
if _, err := store.db.Exec(
`INSERT INTO covers (address, path, content_type) VALUES ($1, $2, $3)`,
legacy, relativePath, "image/jpeg"); err != nil {
t.Fatalf("plant legacy row: %v", err)
}
body, contentType, ok, err := store.GetCover(source)
if err != nil || !ok {
t.Fatalf("GetCover on a legacy row = %v, %v, want found", ok, err)
}
if string(body) != "legacy-bytes" || contentType != "image/jpeg" {
t.Fatalf("legacy cover = (%q, %q), want the planted bytes", body, contentType)
}
}