sulthan/mangaBookmark
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backend)!: split Series from Bookmark, keeping the wire format flat

Browse Source 
Series becomes a shared row keyed (site, series_id) owning title, cover,
canonical URL, kind, latest chapter and last-checked time (ADR-0003).
A bookmark keeps only progress, favourite, lifecycle bucket, updated_at.

Store.Upsert decomposes one flat body across both tables in one
transaction: client title/series_url/cover apply only when the series row
is new, then only the poll may change them (security boundary — the row
is shared and the values are scraped page content). Reads join series
back in, so GET/PUT emit and accept exactly the flat field set they did
before (ADR-0004), asserted by TestFlatWireFieldSet.

The poller walks Series instead of Bookmarks: one fetch per shared
series per due cycle, due queue ordered reader_count DESC then
latest_checked_at ASC, orphaned series never due and never deleted, row
still stamped before the fetch. Batch/stagger/interval unchanged.

Migration 0002 backfills series from existing bookmarks; verified by
TestMigration0002BackfillsExistingBookmarks.
This commit is contained in:
Sulthan Zaki
2026-08-08 07:11:43 +07:00
parent 08749df050
commit a48aba67b8
8 changed files with 756 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/store/store.go
+183 -73
View File
@@ -19,6 +19,11 @@ import (
//
// LastChapter* is the user's read progress; LatestChapter* is the newest
// chapter the site has published, captured opportunistically by the userscript.
//
// Title, SeriesURL, Cover, Kind and LatestChapter* live on the shared Series
// row (ADR-0003) and are joined in on read; Bookmark carries only what differs
// between readers: progress, favourite, lifecycle bucket, updated_at. The wire
// format stays flat regardless — see ADR-0004.
type Bookmark struct {
Key string `json:"key"`
Site string `json:"site"`
@@ -42,6 +47,35 @@ type Bookmark struct {
Kind string `json:"kind"`
}
// Series is one distinct work, shared by every bookmark that tracks it. It is
// keyed (site, series_id) — the pair a bookmark key decomposes into — and
// exists once no matter how many bookmarks point at it (ADR-0003).
//
// Title, SeriesURL and Cover are written once, at creation: a PUT naming an
// existing Series has them ignored, and only the backend's own Poll may change
// them. Kind and the latest-chapter fields are last-write-wins like the
// bookmark's own fields. Never serialized: the wire format is the flat
// Bookmark (ADR-0004).
type Series struct {
Site string
SeriesID string
Title string
SeriesURL string
Cover string
Kind string
LatestChapter string
LatestChapterNum *float64 // nil until first captured
LatestCheckedAt int64 // unix ms; see MarkLatestChecked
// readerCount is the number of bookmarks referencing this series, filled
// only by the due-queue query that orders on it.
readerCount int
}
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>"),
// used by the poller's logs and by tests asserting on the due queue.
func (s Series) Key() string { return s.Site + ":" + s.SeriesID }
// HasNewChapter reports whether the site has published past the read point.
// A nil LatestChapterNum means nothing has been captured yet, which is not the
// same as "nothing new".
@@ -122,10 +156,18 @@ const (
var migrations embed.FS
// bookmarkColumns is the only value ever concatenated into query text. It is a
// compile-time constant; every request value is bound as a parameter.
const bookmarkColumns = `key, site, series_id, title, series_url, cover,
last_chapter, last_chapter_num, last_chapter_url,
favorite, latest_chapter, latest_chapter_num, updated_at, status, kind`
// compile-time constant; every request value is bound as a parameter. The
// series-owned fields are joined in from the series table, in scanBookmark
// order, so the flat Bookmark reads back whole despite the split (ADR-0004).
const bookmarkColumns = `b.key, b.site, b.series_id, s.title, s.series_url, s.cover,
b.last_chapter, b.last_chapter_num, b.last_chapter_url,
b.favorite, s.latest_chapter, s.latest_chapter_num, b.updated_at, b.status, s.kind`
// seriesColumns is the series row in scanSeries order, used by the poller's
// due query. latest_checked_at lives only on series — see MarkLatestChecked
// for why it stays off every client-visible write.
const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at`
// Store is the Postgres-backed bookmark store.
type Store struct {
@@ -237,14 +279,37 @@ func scanBookmark(scan func(...any) error) (Bookmark, error) {
return b, nil
}
// scanSeries reads one row in seriesColumns order, plus the due query's
// reader_count column. latest_chapter_num is NULL until the first capture,
// same as on the bookmark read path.
func scanSeries(scan func(...any) error) (Series, error) {
var (
sr Series
latestChapterNum sql.NullFloat64
)
if err := scan(
&sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover,
&sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt,
&sr.readerCount,
); err != nil {
return Series{}, err
}
if latestChapterNum.Valid {
sr.LatestChapterNum = &latestChapterNum.Float64
}
return sr, nil
}
// Close releases the underlying database handle.
func (s *Store) Close() error { return s.db.Close() }
// List returns every bookmark, newest activity first.
// List returns every bookmark, newest activity first. Series-owned fields are
// joined in, so each Bookmark reads back whole and flat (ADR-0004).
func (s *Store) List() ([]Bookmark, error) {
rows, err := s.db.Query(`SELECT ` + bookmarkColumns + `
FROM bookmarks
ORDER BY updated_at DESC`)
FROM bookmarks b
JOIN series s ON s.site = b.site AND s.series_id = b.series_id
ORDER BY b.updated_at DESC`)
if err != nil {
return nil, fmt.Errorf("query bookmarks: %w", err)
}
@@ -266,7 +331,9 @@ func (s *Store) List() ([]Bookmark, error) {
// the fields they do not touch.
func (s *Store) Get(key string) (Bookmark, bool, error) {
b, err := scanBookmark(s.db.QueryRow(
`SELECT `+bookmarkColumns+` FROM bookmarks WHERE key = $1`, key).Scan)
`SELECT `+bookmarkColumns+` FROM bookmarks b
JOIN series s ON s.site = b.site AND s.series_id = b.series_id
WHERE b.key = $1`, key).Scan)
if errors.Is(err, sql.ErrNoRows) {
return Bookmark{}, false, nil
}
@@ -277,7 +344,15 @@ func (s *Store) Get(key string) (Bookmark, bool, error) {
}
// Upsert inserts or replaces a bookmark by key (last-write-wins) and returns
// the row as actually stored.
// the row as actually stored — one flat object with the series-owned fields
// joined in, exactly as GET reports it (ADR-0004).
//
// The flat body is decomposed across two tables in one transaction. The series
// row is written first (the bookmarks FK requires it to exist), then the
// bookmark row. On the series side, title/series_url/cover are applied only
// when the row is brand new: once a series exists, client-supplied values are
// ignored, because the row is shared and the values are scraped page content —
// see ADR-0003. Kind and the latest-chapter fields are last-write-wins.
//
// b.UpdatedAt is only a candidate: it is applied when the row is new or when
// last_chapter_num changes, and otherwise the stored value is kept. Clients
@@ -296,53 +371,65 @@ func (s *Store) Upsert(b Bookmark) (Bookmark, error) {
latestNum = *b.LatestChapterNum
}
// IS DISTINCT FROM is Postgres's null-safe comparison, and it is what
// implements the ordering rule. Within DO UPDATE, a bare column is the
// stored row and excluded.* is the incoming one; a brand-new key never
// reaches this clause, so it keeps the fresh timestamp from VALUES.
//
// The status and kind columns resolve on the VALUES side, not in the
// conflict clause: excluded.* is the row *after* these expressions are
// evaluated, so a default applied there would look identical to a real
// 'reading' / 'manga' and would overwrite an archived or novel row on
// every PUT from a client that knows nothing about the column. Resolved
// once here, an empty incoming status or kind means "keep what is
// stored", and only a brand-new row falls through to the literal
// default. The subquery runs inside this transaction, so it sees the
// row this statement is about to conflict with.
// The kind column resolves on the VALUES side, not in the conflict clause:
// excluded.* is the row *after* these expressions are evaluated, so a
// default applied there would look identical to a real 'manga' and would
// overwrite a novel series on every PUT from a client that knows nothing
// about the column. Resolved once here, an empty incoming kind means "keep
// what is stored", and only a brand-new row falls through to the literal
// default. The subquery runs inside this transaction, so it sees the row
// this statement is about to conflict with. Same pattern as the status
// COALESCE on the bookmark insert below.
//
// The ::text casts are load-bearing: inside COALESCE/NULLIF there is no
// target column to infer the parameter type from, and Postgres rejects the
// statement rather than guessing.
if _, err := tx.Exec(`
INSERT INTO bookmarks (`+bookmarkColumns+`)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
COALESCE(NULLIF($14::text, ''), (SELECT status FROM bookmarks WHERE key = $1), 'reading'),
COALESCE(NULLIF($15::text, ''), (SELECT kind FROM bookmarks WHERE key = $1), 'manga'))
INSERT INTO series (site, series_id, title, series_url, cover, kind,
latest_chapter, latest_chapter_num)
VALUES ($1, $2, $3, $4, $5,
COALESCE(NULLIF($6::text, ''), (SELECT kind FROM series WHERE site = $1 AND series_id = $2), 'manga'),
$7, $8)
ON CONFLICT (site, series_id) DO UPDATE SET
kind=excluded.kind,
latest_chapter=excluded.latest_chapter,
latest_chapter_num=excluded.latest_chapter_num`,
b.Site, b.SeriesID, b.Title, b.SeriesURL, b.Cover, b.Kind,
b.LatestChapter, latestNum); err != nil {
return Bookmark{}, fmt.Errorf("upsert series for %q: %w", b.Key, err)
}
// IS DISTINCT FROM is Postgres's null-safe comparison, and it is what
// implements the ordering rule. Within DO UPDATE, a bare column is the
// stored row and excluded.* is the incoming one; a brand-new key never
// reaches this clause, so it keeps the fresh timestamp from VALUES.
if _, err := tx.Exec(`
INSERT INTO bookmarks (key, site, series_id, last_chapter, last_chapter_num,
last_chapter_url, favorite, status, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7,
COALESCE(NULLIF($8::text, ''), (SELECT status FROM bookmarks WHERE key = $1), 'reading'),
$9)
ON CONFLICT (key) DO UPDATE SET
site=excluded.site, series_id=excluded.series_id, title=excluded.title,
series_url=excluded.series_url, cover=excluded.cover,
site=excluded.site, series_id=excluded.series_id,
last_chapter=excluded.last_chapter, last_chapter_num=excluded.last_chapter_num,
last_chapter_url=excluded.last_chapter_url,
favorite=excluded.favorite,
latest_chapter=excluded.latest_chapter,
latest_chapter_num=excluded.latest_chapter_num,
status=excluded.status,
kind=excluded.kind,
updated_at=CASE
WHEN bookmarks.last_chapter_num IS DISTINCT FROM excluded.last_chapter_num
THEN excluded.updated_at
ELSE bookmarks.updated_at
END`,
b.Key, b.Site, b.SeriesID, b.Title, b.SeriesURL, b.Cover,
b.Key, b.Site, b.SeriesID,
b.LastChapter, b.LastChapterNum, b.LastChapterURL,
b.Favorite, b.LatestChapter, latestNum, b.UpdatedAt,
b.Status, b.Kind); err != nil {
b.Favorite, b.Status, b.UpdatedAt); err != nil {
return Bookmark{}, fmt.Errorf("upsert %q: %w", b.Key, err)
}
stored, err := scanBookmark(tx.QueryRow(
`SELECT `+bookmarkColumns+` FROM bookmarks WHERE key = $1`, b.Key).Scan)
`SELECT `+bookmarkColumns+` FROM bookmarks b
JOIN series s ON s.site = b.site AND s.series_id = b.series_id
WHERE b.key = $1`, b.Key).Scan)
if err != nil {
return Bookmark{}, fmt.Errorf("read back %q: %w", b.Key, err)
}
@@ -360,71 +447,94 @@ func (s *Store) Delete(key string) error {
return nil
}
// DueForLatestCheck returns bookmarks whose server-side latest-chapter check has
// aged past cutoffMs, least-recently-checked first, at most limit of them.
// DueForLatestCheck returns series whose server-side latest-chapter check has
// aged past cutoffMs, ordered by how many bookmarks reference them (descending)
// then least-recently-checked first, at most limit of them.
//
// Oldest-first is what keeps the poller fair when the backlog outgrows its
// The reader_count ordering is the point of the split (ADR-0003): a series
// shared by several readers is fetched once per due cycle, and the popular
// ones stay freshest while the long tail absorbs any shortfall. Within one
// reader count, oldest-first keeps the poll fair when the backlog outgrows its
// throughput: the most neglected series is always next, so a large collection
// refreshes uniformly slower rather than leaving a tail that never refreshes at
// all. The userscript sorts its own queue the same way (L453).
//
// Bookmarks with no series_url are skipped — there is nothing to fetch, which
// is the same filter the userscript applies at L452.
//
// Finished series are excluded: nothing more is coming, so fetching them only
// burns requests. Archived ones are deliberately still polled — knowing what a
// shelved series is up to is the whole reason for archiving instead of deleting.
func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Bookmark, error) {
rows, err := s.db.Query(`SELECT `+bookmarkColumns+`
FROM bookmarks
WHERE series_url <> ''
AND status IS DISTINCT FROM 'finished'
AND latest_checked_at <= $1
ORDER BY latest_checked_at ASC
// Series with no series_url are skipped — there is nothing to fetch, which is
// the same filter the userscript applies at L452. Series whose only bookmarks
// are finished are skipped too: nothing more is coming, so fetching them only
// burns requests. Archived bookmarks still count — knowing what a shelved
// series is up to is the whole reason for archiving instead of deleting.
// A series with no bookmarks at all never appears: the join excludes it.
func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Series, error) {
rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(b.key) AS reader_count
FROM series s
JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
WHERE s.series_url <> ''
AND s.latest_checked_at <= $1
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover,
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at
HAVING COUNT(b.key) FILTER (WHERE b.status <> 'finished') > 0
ORDER BY reader_count DESC, s.latest_checked_at ASC
LIMIT $2`, cutoffMs, limit)
if err != nil {
return nil, fmt.Errorf("query due bookmarks: %w", err)
return nil, fmt.Errorf("query due series: %w", err)
}
defer rows.Close()
out := []Bookmark{}
out := []Series{}
for rows.Next() {
b, err := scanBookmark(rows.Scan)
sr, err := scanSeries(rows.Scan)
if err != nil {
return nil, fmt.Errorf("scan due bookmark: %w", err)
return nil, fmt.Errorf("scan due series: %w", err)
}
out = append(out, b)
out = append(out, sr)
}
return out, rows.Err()
}
// MarkLatestChecked records that the server looked at key at ts, whatever the
// look turned up. Marking a missing key is not an error: the row may have been
// deleted while a fetch was in flight.
// MarkLatestChecked records that the server looked at a series at ts, whatever
// the look turned up. Marking a missing series is not an error: the row may
// have been orphaned while a fetch was in flight.
//
// This is the one write that does not go through Upsert, and the column is kept
// out of bookmarkColumns on purpose. PUT /bookmarks/{key} decodes a whole
// Bookmark from the client and Upsert writes every column it knows about, so a
// userscript PUT — which has no idea this field exists — would write a zero and
// reset the cooldown, making the poller re-fetch that series every tick for as
// long as the user kept reading it.
func (s *Store) MarkLatestChecked(key string, ts int64) error {
// out of the client-visible read path on purpose. PUT /bookmarks/{key} decodes
// a whole Bookmark from the client and Upsert writes every series column it
// knows about, so a userscript PUT — which has no idea this field exists —
// would write a zero and reset the cooldown, making the poller re-fetch that
// series every tick for as long as the user kept reading it.
func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error {
if _, err := s.db.Exec(
`UPDATE bookmarks SET latest_checked_at = $1 WHERE key = $2`, ts, key); err != nil {
return fmt.Errorf("mark checked %q: %w", key, err)
`UPDATE series SET latest_checked_at = $1 WHERE site = $2 AND series_id = $3`,
ts, site, seriesID); err != nil {
return fmt.Errorf("mark checked %s:%s: %w", site, seriesID, err)
}
return nil
}
// LatestCheckedAt reads the column MarkLatestChecked writes. It exists for
// tests outside this package (the poller's own tests assert on cooldown
// bookkeeping) — see MarkLatestChecked for why the field itself stays off
// Bookmark.
func (s *Store) LatestCheckedAt(key string) (int64, error) {
// bookkeeping) — see MarkLatestChecked for why the field stays off the
// client-visible row.
func (s *Store) LatestCheckedAt(site, seriesID string) (int64, error) {
var ts int64
if err := s.db.QueryRow(
`SELECT latest_checked_at FROM bookmarks WHERE key = $1`, key).Scan(&ts); err != nil {
return 0, fmt.Errorf("latest checked at %q: %w", key, err)
`SELECT latest_checked_at FROM series WHERE site = $1 AND series_id = $2`,
site, seriesID).Scan(&ts); err != nil {
return 0, fmt.Errorf("latest checked at %s:%s: %w", site, seriesID, err)
}
return ts, nil
}
// SetLatestChapter records the newest chapter the poll found on a series page.
// The poller walks Series rather than Bookmarks, so this is a series-level
// write: the row is shared, and updating it once refreshes every bookmark that
// joins to it. Touching a missing series is not an error.
func (s *Store) SetLatestChapter(site, seriesID, label string, num float64) error {
if _, err := s.db.Exec(
`UPDATE series SET latest_chapter = $3, latest_chapter_num = $4
WHERE site = $1 AND series_id = $2`,
site, seriesID, label, num); err != nil {
return fmt.Errorf("set latest chapter %s:%s: %w", site, seriesID, err)
}
return nil
}