From 0a9042dc42db597ff5569b74c96a6a3aad38f112 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sun, 26 Jul 2026 15:30:50 +0700 Subject: [PATCH] feat(store): track when the server last checked a series Adds latest_checked_at plus DueForLatestCheck and MarkLatestChecked. The column is kept out of bookmarkColumns on purpose: PUT /bookmarks decodes a whole Bookmark and Upsert writes every column it knows, so a client PUT would zero the field and defeat the cooldown. --- backend/store.go | 56 ++++++++++++ backend/store_test.go | 194 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) diff --git a/backend/store.go b/backend/store.go index 7588509..d75916c 100644 --- a/backend/store.go +++ b/backend/store.go @@ -59,6 +59,7 @@ CREATE TABLE IF NOT EXISTS bookmarks ( favorite INTEGER NOT NULL DEFAULT 0, latest_chapter TEXT NOT NULL DEFAULT '', latest_chapter_num REAL, + latest_checked_at INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL );` @@ -68,6 +69,10 @@ var addedColumns = []struct{ name, ddl string }{ {"favorite", `ALTER TABLE bookmarks ADD COLUMN favorite INTEGER NOT NULL DEFAULT 0`}, {"latest_chapter", `ALTER TABLE bookmarks ADD COLUMN latest_chapter TEXT NOT NULL DEFAULT ''`}, {"latest_chapter_num", `ALTER TABLE bookmarks ADD COLUMN latest_chapter_num REAL`}, + // When the server last looked at this series, unix ms; 0 means never, and + // sorts first so a new bookmark is picked up on the next tick with no + // special case. Deliberately NOT in bookmarkColumns — see MarkLatestChecked. + {"latest_checked_at", `ALTER TABLE bookmarks ADD COLUMN latest_checked_at INTEGER NOT NULL DEFAULT 0`}, } const bookmarkColumns = `key, site, series_id, title, series_url, cover, @@ -277,3 +282,54 @@ func (s *Store) Delete(key string) error { } return nil } + +// DueForLatestCheck returns bookmarks whose server-side latest-chapter check has +// aged past cutoffMs, least-recently-checked first, at most limit of them. +// +// Oldest-first is what keeps the poller fair when the backlog outgrows its +// throughput: the most neglected series is always next, so a large collection +// refreshes uniformly slower rather than leaving a tail that never refreshes at +// all. The userscript sorts its own queue the same way (L453). +// +// Bookmarks with no series_url are skipped — there is nothing to fetch, which +// is the same filter the userscript applies at L452. +func (s *Store) DueForLatestCheck(cutoffMs int64, limit int) ([]Bookmark, error) { + rows, err := s.db.Query(`SELECT `+bookmarkColumns+` + FROM bookmarks + WHERE series_url IS NOT NULL AND series_url <> '' + AND latest_checked_at <= ? + ORDER BY latest_checked_at ASC + LIMIT ?`, cutoffMs, limit) + if err != nil { + return nil, fmt.Errorf("query due bookmarks: %w", err) + } + defer rows.Close() + + out := []Bookmark{} + for rows.Next() { + b, err := scanBookmark(rows.Scan) + if err != nil { + return nil, fmt.Errorf("scan due bookmark: %w", err) + } + out = append(out, b) + } + return out, rows.Err() +} + +// MarkLatestChecked records that the server looked at key at ts, whatever the +// look turned up. Marking a missing key is not an error: the row may have been +// deleted while a fetch was in flight. +// +// This is the one write that does not go through Upsert, and the column is kept +// out of bookmarkColumns on purpose. PUT /bookmarks/{key} decodes a whole +// Bookmark from the client and Upsert writes every column it knows about, so a +// userscript PUT — which has no idea this field exists — would write a zero and +// reset the cooldown, making the poller re-fetch that series every tick for as +// long as the user kept reading it. +func (s *Store) MarkLatestChecked(key string, ts int64) error { + if _, err := s.db.Exec( + `UPDATE bookmarks SET latest_checked_at = ? WHERE key = ?`, ts, key); err != nil { + return fmt.Errorf("mark checked %q: %w", key, err) + } + return nil +} diff --git a/backend/store_test.go b/backend/store_test.go index 1a442ee..6102db9 100644 --- a/backend/store_test.go +++ b/backend/store_test.go @@ -528,3 +528,197 @@ func TestLoadConfigWebPassword(t *testing.T) { t.Fatalf("WebPassword = %q with the variable unset, want empty", got) } } + +// readLatestCheckedAt reads the column directly. It is deliberately absent from +// Bookmark (see Store.Upsert), so tests cannot assert on it any other way. +func readLatestCheckedAt(t *testing.T, s *Store, key string) int64 { + t.Helper() + var ts int64 + if err := s.db.QueryRow( + `SELECT latest_checked_at FROM bookmarks WHERE key = ?`, key).Scan(&ts); err != nil { + t.Fatalf("read latest_checked_at %q: %v", key, err) + } + return ts +} + +// seedForCheck inserts a bookmark and forces its latest_checked_at. +func seedForCheck(t *testing.T, s *Store, key, seriesURL string, checkedAt int64) { + t.Helper() + if _, err := s.Upsert(Bookmark{ + Key: key, + Site: "asura", + SeriesID: key, + SeriesURL: seriesURL, + UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed %q: %v", key, err) + } + if err := s.MarkLatestChecked(key, checkedAt); err != nil { + t.Fatalf("seed mark %q: %v", key, err) + } +} + +func TestDueForLatestCheck(t *testing.T) { + const hour = int64(3600_000) + now := 10 * hour + + tests := []struct { + name string + checkedAt int64 + seriesURL string + wantDue bool + }{ + {"never checked", 0, "https://asurascans.com/comics/a", true}, + {"checked 30m ago", now - hour/2, "https://asurascans.com/comics/a", false}, + {"checked exactly at cutoff", now - hour, "https://asurascans.com/comics/a", true}, + {"checked 2h ago", now - 2*hour, "https://asurascans.com/comics/a", true}, + {"no series url", 0, "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:x", tt.seriesURL, tt.checkedAt) + + due, err := s.DueForLatestCheck(now-hour, 10) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if got := len(due) == 1; got != tt.wantDue { + t.Fatalf("due=%v, want %v (got %d rows)", got, tt.wantDue, len(due)) + } + }) + } +} + +func TestDueForLatestCheckOldestFirstAndLimited(t *testing.T) { + s := newTestStore(t) + // Insert newest-checked first so a correct ORDER BY has to reverse it. + seedForCheck(t, s, "asura:c", "https://asurascans.com/comics/c", 300) + seedForCheck(t, s, "asura:b", "https://asurascans.com/comics/b", 200) + seedForCheck(t, s, "asura:a", "https://asurascans.com/comics/a", 100) + + due, err := s.DueForLatestCheck(1000, 2) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 2 { + t.Fatalf("got %d rows, want 2 (limit)", len(due)) + } + if due[0].Key != "asura:a" || due[1].Key != "asura:b" { + t.Fatalf("got %q,%q; want asura:a,asura:b (oldest first)", due[0].Key, due[1].Key) + } +} + +func TestMarkLatestChecked(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 0) + + if err := s.MarkLatestChecked("asura:x", 4242); err != nil { + t.Fatalf("MarkLatestChecked: %v", err) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got != 4242 { + t.Fatalf("latest_checked_at = %d, want 4242", got) + } + // A missing key is not an error: the row may have been deleted mid-fetch. + if err := s.MarkLatestChecked("asura:gone", 1); err != nil { + t.Fatalf("MarkLatestChecked on missing key: %v", err) + } +} + +// Upsert must not touch latest_checked_at. If the column ever migrates into +// bookmarkColumns, this fails and the cooldown is silently dead. +func TestUpsertPreservesLatestCheckedAt(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 999) + + b, ok, err := s.Get("asura:x") + if err != nil || !ok { + t.Fatalf("Get: %v ok=%v", err, ok) + } + b.Title = "changed" + if _, err := s.Upsert(b); err != nil { + t.Fatalf("Upsert: %v", err) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got != 999 { + t.Fatalf("latest_checked_at = %d after Upsert, want 999", got) + } +} + +// migrateColumns must be able to bring a database created before this column up +// to date, not just create it fresh. +func TestMigrateAddsLatestCheckedAt(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + + old, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open: %v", err) + } + // A pre-latest_checked_at table, matching the schema as it shipped before. + if _, err := old.Exec(`CREATE TABLE bookmarks ( + key TEXT PRIMARY KEY, site TEXT NOT NULL, series_id TEXT NOT NULL, + title TEXT, series_url TEXT, cover TEXT, + last_chapter TEXT, last_chapter_num REAL, last_chapter_url TEXT, + favorite INTEGER NOT NULL DEFAULT 0, + latest_chapter TEXT NOT NULL DEFAULT '', latest_chapter_num REAL, + updated_at INTEGER NOT NULL)`); err != nil { + t.Fatalf("create old table: %v", err) + } + if _, err := old.Exec( + `INSERT INTO bookmarks (key, site, series_id, series_url, updated_at) + VALUES ('asura:x', 'asura', 'x', 'https://asurascans.com/comics/x', 5)`); err != nil { + t.Fatalf("seed old row: %v", err) + } + if err := old.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + s, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore on pre-existing db: %v", err) + } + t.Cleanup(func() { s.Close() }) + + // The migrated row must default to 0 (never checked) and so be due. + if got := readLatestCheckedAt(t, s, "asura:x"); got != 0 { + t.Fatalf("migrated latest_checked_at = %d, want 0", got) + } + due, err := s.DueForLatestCheck(1000, 10) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 1 { + t.Fatalf("got %d due rows after migration, want 1", len(due)) + } +} + +// A userscript PUT body has no latest_checked_at field. If the column is ever +// moved into bookmarkColumns, this test catches it: the PUT would reset the +// cooldown and the poller would re-fetch that series on every single tick. +func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + store, err := OpenStore(dbPath) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + srv := newRouter(store, testConfig()) + + seedForCheck(t, store, "asura:x", "https://asurascans.com/comics/x", 777) + + // Exactly what the userscript sends: no latest_checked_at key at all. + body := `{"key":"asura:x","site":"asura","series_id":"x", + "series_url":"https://asurascans.com/comics/x", + "last_chapter":"Chapter 5","last_chapter_num":5}` + req := httptest.NewRequest(http.MethodPut, "/bookmarks/asura:x", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+testToken) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + if got := readLatestCheckedAt(t, store, "asura:x"); got != 777 { + t.Fatalf("latest_checked_at = %d after client PUT, want 777 preserved", got) + } +}