Move cover bytes to content-addressed filesystem storage (#65)
Refs #56 ## Summary Moves Kagane cover bytes out of Postgres bytea storage into an immutable, content-addressed filesystem store. Reader-visible behavior remains unchanged: the existing session-gated route serves stored bytes, missing bytes use the existing browser fetch path, and no browser still returns a missing cover. ## Changes - Added migration 0008, which drops the legacy `covers` table and recreates it with only `address`, `path`, and `content_type`. Existing byte rows are intentionally dropped. - Added SHA-256 source-URL addressing with two-level sharding (`ab/cd/<sha256>`). Writes use a temp file plus atomic link; reads validate the stored relative path before opening it. - Made `COVER_DIR` required in runtime config and Compose. Compose passes it as a Docker build argument and volume target, so custom durable paths keep image ownership, runtime config, and the named `cover-data` volume aligned. - Updated every `store.Open` caller and documented configuration, deployment, backup, and troubleshooting behavior. - Added filesystem, restart, migration-drop, no-browser, and content-addressing coverage. ## Verification - `go test ./...` - `CGO_ENABLED=0 go build ./...` - `docker build --build-arg COVER_DIR=/data/covers -t manga-bookmark-cover-check-custom ./backend` - `docker compose config --format json` confirms custom `COVER_DIR` is the volume target - `git diff --check origin/main` - LSP diagnostics clean for touched Go files Parents #47 and #55 remain open as required by #56. Reviewed-on: #65 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #65.
This commit is contained in:
@@ -1,12 +1,16 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -225,7 +229,8 @@ type Store struct {
|
||||
// ownerID is the seeded owner Reader (issue #22) — the only Reader with
|
||||
// administrative reach (revoking another Reader's sessions). Every store
|
||||
// method takes a reader id explicitly, so ownership is never implicit.
|
||||
ownerID int64
|
||||
ownerID int64
|
||||
coverDir string
|
||||
}
|
||||
|
||||
// OwnerID returns the seeded owner Reader's id: the administrator, and the
|
||||
@@ -360,8 +365,22 @@ const allMigrations = 0
|
||||
|
||||
// Open connects to Postgres at url — a libpq connection URL such as
|
||||
// "postgres://user:pass@host:5432/bookmarks?sslmode=disable" — brings its
|
||||
// schema up to date, and seeds the owner Reader.
|
||||
func Open(url string, owner Owner) (*Store, error) {
|
||||
// schema up to date, seeds the owner Reader, and prepares cover storage.
|
||||
func Open(url string, owner Owner, coverDir string) (*Store, error) {
|
||||
if strings.TrimSpace(coverDir) == "" {
|
||||
return nil, errors.New("cover directory is required")
|
||||
}
|
||||
if err := os.MkdirAll(coverDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create cover directory: %w", err)
|
||||
}
|
||||
info, err := os.Stat(coverDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat cover directory: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("cover directory %q is not a directory", coverDir)
|
||||
}
|
||||
|
||||
db, err := sql.Open("pgx", url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open postgres: %w", err)
|
||||
@@ -396,7 +415,7 @@ func Open(url string, owner Owner) (*Store, error) {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("resolve owner: %w", err)
|
||||
}
|
||||
return &Store{db: db, ownerID: ownerID}, nil
|
||||
return &Store{db: db, ownerID: ownerID, coverDir: coverDir}, nil
|
||||
}
|
||||
|
||||
// seedOwner makes sure the configured owner exists as exactly one readers row.
|
||||
@@ -550,40 +569,96 @@ func scanSeries(scan func(...any) error) (Series, error) {
|
||||
// Close releases the underlying database handle.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// GetKaganeCover returns one persisted cover. Missing covers are reported with
|
||||
// ok=false rather than as an error so the web handler can fetch them once.
|
||||
func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) {
|
||||
var (
|
||||
body []byte
|
||||
contentType string
|
||||
)
|
||||
func coverSourceAddress(sourceURL string) string {
|
||||
sum := sha256.Sum256([]byte(sourceURL))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func coverRelativePath(address string) string {
|
||||
return address[:2] + "/" + address[2:4] + "/" + address
|
||||
}
|
||||
|
||||
func kaganeCoverSourceURL(imageID string) string {
|
||||
return "https://kagane.to/api/v2/image/" + imageID + "/compressed"
|
||||
}
|
||||
|
||||
func (s *Store) getCover(sourceURL string) ([]byte, string, bool, error) {
|
||||
address := coverSourceAddress(sourceURL)
|
||||
var relativePath, contentType string
|
||||
err := s.db.QueryRow(
|
||||
`SELECT body, content_type FROM covers WHERE image_id = $1`, imageID,
|
||||
).Scan(&body, &contentType)
|
||||
`SELECT path, content_type FROM covers WHERE address = $1`, address,
|
||||
).Scan(&relativePath, &contentType)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("get kagane cover %q: %w", imageID, err)
|
||||
return nil, "", false, fmt.Errorf("get cover %q: %w", address, err)
|
||||
}
|
||||
expectedPath := coverRelativePath(address)
|
||||
if relativePath != expectedPath {
|
||||
return nil, "", false, fmt.Errorf("cover %q has unexpected path %q", address, relativePath)
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(s.coverDir, filepath.FromSlash(relativePath)))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("read cover %q: %w", address, err)
|
||||
}
|
||||
return body, contentType, true, nil
|
||||
}
|
||||
|
||||
// PutKaganeCover persists one fetched cover. Image ids are immutable, so a
|
||||
// later fetch cannot replace the bytes already made durable.
|
||||
func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error {
|
||||
func (s *Store) putCover(sourceURL string, body []byte, contentType string) error {
|
||||
if !IsKaganeCoverContentType(contentType) {
|
||||
return fmt.Errorf("put kagane cover %q: unsupported content type %q", imageID, contentType)
|
||||
return fmt.Errorf("put cover %q: unsupported content type %q", sourceURL, contentType)
|
||||
}
|
||||
address := coverSourceAddress(sourceURL)
|
||||
relativePath := coverRelativePath(address)
|
||||
coverPath := filepath.Join(s.coverDir, filepath.FromSlash(relativePath))
|
||||
if err := os.MkdirAll(filepath.Dir(coverPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create cover shard: %w", err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(coverPath), ".cover-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create cover temp file: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if _, err := tmp.Write(body); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write cover temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("sync cover temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close cover temp file: %w", err)
|
||||
}
|
||||
if err := os.Link(tmpName, coverPath); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("install cover file: %w", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO covers (image_id, body, content_type)
|
||||
INSERT INTO covers (address, path, content_type)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (image_id) DO NOTHING`, imageID, body, contentType); err != nil {
|
||||
return fmt.Errorf("put kagane cover %q: %w", imageID, err)
|
||||
ON CONFLICT (address) DO NOTHING`, address, relativePath, contentType); err != nil {
|
||||
return fmt.Errorf("record cover %q: %w", address, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetKaganeCover returns one persisted cover. Missing covers are reported with
|
||||
// ok=false rather than as an error so the web handler can fetch them once.
|
||||
func (s *Store) GetKaganeCover(imageID string) ([]byte, string, bool, error) {
|
||||
return s.getCover(kaganeCoverSourceURL(imageID))
|
||||
}
|
||||
|
||||
// PutKaganeCover persists one fetched cover. The source URL's content address
|
||||
// makes each stored object immutable, so later writes for that URL are ignored.
|
||||
func (s *Store) PutKaganeCover(imageID string, body []byte, contentType string) error {
|
||||
return s.putCover(kaganeCoverSourceURL(imageID), body, contentType)
|
||||
}
|
||||
|
||||
// List returns every bookmark of one reader, newest activity first.
|
||||
// Series-owned fields are joined in, so each Bookmark reads back whole and
|
||||
// flat (ADR-0004).
|
||||
|
||||
Reference in New Issue
Block a user