Move cover bytes to content-addressed filesystem storage (#65)

Refs #56

## Summary

Moves Kagane cover bytes out of Postgres bytea storage into an immutable, content-addressed filesystem store. Reader-visible behavior remains unchanged: the existing session-gated route serves stored bytes, missing bytes use the existing browser fetch path, and no browser still returns a missing cover.

## Changes

- Added migration 0008, which drops the legacy `covers` table and recreates it with only `address`, `path`, and `content_type`. Existing byte rows are intentionally dropped.
- Added SHA-256 source-URL addressing with two-level sharding (`ab/cd/<sha256>`). Writes use a temp file plus atomic link; reads validate the stored relative path before opening it.
- Made `COVER_DIR` required in runtime config and Compose. Compose passes it as a Docker build argument and volume target, so custom durable paths keep image ownership, runtime config, and the named `cover-data` volume aligned.
- Updated every `store.Open` caller and documented configuration, deployment, backup, and troubleshooting behavior.
- Added filesystem, restart, migration-drop, no-browser, and content-addressing coverage.

## Verification

- `go test ./...`
- `CGO_ENABLED=0 go build ./...`
- `docker build --build-arg COVER_DIR=/data/covers -t manga-bookmark-cover-check-custom ./backend`
- `docker compose config --format json` confirms custom `COVER_DIR` is the volume target
- `git diff --check origin/main`
- LSP diagnostics clean for touched Go files

Parents #47 and #55 remain open as required by #56.

Reviewed-on: #65
Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com>
Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #65.
This commit is contained in:
2026-08-10 00:11:20 +07:00
committed by sulthan
parent 30c57bd39c
commit e8d1cba6c5
15 changed files with 264 additions and 42 deletions
+96 -21
View File
@@ -1,12 +1,16 @@
package store
import (
"crypto/sha256"
"database/sql"
"embed"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"regexp"
"slices"
"strconv"
@@ -225,7 +229,8 @@ type Store struct {
// ownerID is the seeded owner Reader (issue #22) — the only Reader with
// administrative reach (revoking another Reader's sessions). Every store
// method takes a reader id explicitly, so ownership is never implicit.
ownerID int64
ownerID int64
coverDir string
}
// OwnerID returns the seeded owner Reader's id: the administrator, and the
@@ -360,8 +365,22 @@ const allMigrations = 0
// Open connects to Postgres at url — a libpq connection URL such as
// "postgres://user:pass@host:5432/bookmarks?sslmode=disable" — brings its
// schema up to date, and seeds the owner Reader.
func Open(url string, owner Owner) (*Store, error) {
// schema up to date, seeds the owner Reader, and prepares cover storage.
func Open(url string, owner Owner, coverDir string) (*Store, error) {
if strings.TrimSpace(coverDir) == "" {
return nil, errors.New("cover directory is required")
}
if err := os.MkdirAll(coverDir, 0o755); err != nil {
return nil, fmt.Errorf("create cover directory: %w", err)
}
info, err := os.Stat(coverDir)
if err != nil {
return nil, fmt.Errorf("stat cover directory: %w", err)
}
if !info.IsDir() {
return nil, fmt.Errorf("cover directory %q is not a directory", coverDir)
}
db, err := sql.Open("pgx", url)
if err != nil {
return nil, fmt.Errorf("open postgres: %w", err)
@@ -396,7 +415,7 @@ func Open(url string, owner Owner) (*Store, error) {
db.Close()
return nil, fmt.Errorf("resolve owner: %w", err)
}
return &Store{db: db, ownerID: ownerID}, nil
return &Store{db: db, ownerID: ownerID, coverDir: coverDir}, nil
}
// seedOwner makes sure the configured owner exists as exactly one readers row.
@@ -550,40 +569,96 @@ func scanSeries(scan func(...any) error) (Series, error) {
// Close releases the underlying database handle.
func (s *Store) Close() error { return s.db.Close() }
// GetKaganeCover returns one persisted cover. Missing covers are reported with
// ok=false rather than as an error so the web handler can fetch them once.
func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) {
var (
body []byte
contentType string
)
func coverSourceAddress(sourceURL string) string {
sum := sha256.Sum256([]byte(sourceURL))
return hex.EncodeToString(sum[:])
}
func coverRelativePath(address string) string {
return address[:2] + "/" + address[2:4] + "/" + address
}
func kaganeCoverSourceURL(imageID string) string {
return "https://kagane.to/api/v2/image/" + imageID + "/compressed"
}
func (s *Store) getCover(sourceURL string) ([]byte, string, bool, error) {
address := coverSourceAddress(sourceURL)
var relativePath, contentType string
err := s.db.QueryRow(
`SELECT body, content_type FROM covers WHERE image_id = $1`, imageID,
).Scan(&body, &contentType)
`SELECT path, content_type FROM covers WHERE address = $1`, address,
).Scan(&relativePath, &contentType)
if errors.Is(err, sql.ErrNoRows) {
return nil, "", false, nil
}
if err != nil {
return nil, "", false, fmt.Errorf("get kagane cover %q: %w", imageID, err)
return nil, "", false, fmt.Errorf("get cover %q: %w", address, err)
}
expectedPath := coverRelativePath(address)
if relativePath != expectedPath {
return nil, "", false, fmt.Errorf("cover %q has unexpected path %q", address, relativePath)
}
body, err := os.ReadFile(filepath.Join(s.coverDir, filepath.FromSlash(relativePath)))
if errors.Is(err, fs.ErrNotExist) {
return nil, "", false, nil
}
if err != nil {
return nil, "", false, fmt.Errorf("read cover %q: %w", address, err)
}
return body, contentType, true, nil
}
// PutKaganeCover persists one fetched cover. Image ids are immutable, so a
// later fetch cannot replace the bytes already made durable.
func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error {
func (s *Store) putCover(sourceURL string, body []byte, contentType string) error {
if !IsKaganeCoverContentType(contentType) {
return fmt.Errorf("put kagane cover %q: unsupported content type %q", imageID, contentType)
return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
}
address := coverSourceAddress(sourceURL)
relativePath := coverRelativePath(address)
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
return fmt.Errorf("create cover shard: %w", err)
}
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
if err != nil {
return fmt.Errorf("create cover temp file: %w", err)
}
tmpName := tmp.Name()
defer os.Remove(tmpName)
if _, err := tmp.Write(body); err != nil {
tmp.Close()
return fmt.Errorf("write cover temp file: %w", err)
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return fmt.Errorf("sync cover temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close cover temp file: %w", err)
}
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("install cover file: %w", err)
}
if _, err := s.db.Exec(`
INSERT INTO covers (image_id, body, content_type)
INSERT INTO covers (address, path, content_type)
VALUES ($1, $2, $3)
ON CONFLICT (image_id) DO NOTHING`, imageID, body, contentType); err != nil {
return fmt.Errorf("put kagane cover %q: %w", imageID, err)
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
return fmt.Errorf("record cover %q: %w", address, err)
}
return nil
}
// GetKaganeCover returns one persisted cover. Missing covers are reported with
// ok=false rather than as an error so the web handler can fetch them once.
func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) {
return s.getCover(kaganeCoverSourceURL(imageID))
}
// PutKaganeCover persists one fetched cover. The source URL's content address
// makes each stored object immutable, so later writes for that URL are ignored.
func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error {
return s.putCover(kaganeCoverSourceURL(imageID), body, contentType)
}
// List returns every bookmark of one reader, newest activity first.
// Series-owned fields are joined in, so each Bookmark reads back whole and
// flat (ADR-0004).