Spec #134, all ten tickets. Closes #134. ## What ships The admin surface becomes four bookmarkable addresses behind one nav row, and Lane observability stops dying with the process. - **#138** `/admin` splits into Overview, Lanes, Readers, Series, each a real route with the active tab underlined. - **#139** `poll_passes` and `poll_lanes` land as durable tables with their store surface. - **#140** cross-Series admin read model, with the privacy boundary in the projection: the Reader id that raised a Latest Chapter never leaves the store package. - **#141** the poller records exactly one pass row per exit, with a skip reason and outcome counts. - **#142** Series list: eight hygiene filters, Site and Library narrowing, paging — all of it in the query string, so a filtered list is a bookmark. - **#143** Overview: a three-state verdict line and a stats block where every non-zero figure links to the list that counts it. - **#144** per-Series detail page, keyed by the `site:series_id` composite the rest of the system already uses. - **#145** the Lanes page reads the database; the in-memory Lane state, `web.LaneReporter` and `latest.Status` are deleted. - **#146** Forced Poll: *Check now* stamps `series.force_poll_at` and never commands the poller. - **#147** pause and resume one Site's Lane, with a mandatory 1h/6h/24h expiry. ## Shape of the design Two decisions carry the rest. **Commands go through the database, never at the poller**: both *Check now* and a Lane pause write a row the next pass reads, so they survive a restart and the whole surface stays testable with no poller running. And **pending is derived, never stored** — the request stamp being newer than the check stamp — which self-clears on the check stamp with no second write and no sweeper, because the check stamp is written before the fetch. ADRs: `docs/adr/0012-persisted-lane-state.md`, `docs/adr/0013-commands-through-the-database.md`. ## Verification `go test ./...` green on the merged base (`264839e`), all packages, Docker-backed. `gofmt -l` and `go vet` clean. Every ticket was reviewed on both axes (`cr-spec` + `cr-standards`) before merge. ## Known, non-blocking - **#143** the verdict ignores never-reported Lanes when other Lanes have reported, and the per-Site table lists Sites that have Series rather than the whole registry. The ticket prose asks for eight hygiene figures per Site; the design mock and the landed `.tbl.sites` grid both say six columns, and the mock won. - **#146** two `SeriesPage` scans per press instead of a keyed read — `ponytail:`-commented in-tree with the upgrade path. - **#147** a paused Site with no pass row yet renders no row and so no control, since the Lanes page lists Sites that have passed. - **#141** a sibling browser Lane declining at the top of a pass records as `sidecar-down`. Specified deliberately; the later spec in this series settles it. Reviewed-on: #148 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #148.
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Series filter names (issue #140), ordered permanent-then-fixable — the
|
||||
// repairs nothing will ever undo first, the ones a Poll can make right after.
|
||||
// A name is the repair a row needs, not the SQL that finds it; the values are
|
||||
// the wire form the Series list URL carries (#142). "all" is the absent and
|
||||
// unknown case: every Series.
|
||||
const (
|
||||
SeriesFilterAll = "all"
|
||||
SeriesFilterNoURL = "no_series_url"
|
||||
SeriesFilterNoChapter = "never_read_a_chapter"
|
||||
SeriesFilterNoReaders = "no_readers"
|
||||
SeriesFilterNeverChecked = "never_checked"
|
||||
SeriesFilterStale = "stale"
|
||||
SeriesFilterNoCover = "no_cover"
|
||||
SeriesFilterReaderReport = "reader_report"
|
||||
)
|
||||
|
||||
// SeriesFilter is one named hygiene predicate over the whole library. Site
|
||||
// and Kind narrow the row read; Name picks the predicate; Cutoff is the
|
||||
// staleness boundary the "stale" filter compares against, supplied by the
|
||||
// caller's clock — the store has no clock; Page is 1-based.
|
||||
type SeriesFilter struct {
|
||||
Site string // "" = every Site
|
||||
Kind string // "" = both library buckets' series
|
||||
Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll
|
||||
Cutoff int64 // unix ms; "stale" reads it, the store never does
|
||||
Page int // 1-based page of the row read; default 1
|
||||
}
|
||||
|
||||
// adminSeriesColumns is the owner's library-wide Series projection in
|
||||
// scanAdminSeries order. It is the privacy boundary: a Series' row carries
|
||||
// the Reader id that raised its Latest Chapter (latest_raised_by), and that id
|
||||
// must never leave the store package — so the projection does not select it,
|
||||
// and only the anonymous boolean in raisedByReaderAnswer crosses it.
|
||||
const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at`
|
||||
|
||||
// raisedByReaderAnswer answers "did a Reader's report set this number" without
|
||||
// naming which Reader. Kept apart from adminSeriesColumns so the column list —
|
||||
// the shape scanAdminSeries is fed — stays free of the Sighting-raiser
|
||||
// identity, and the owner learns which rows to act on and nothing about the
|
||||
// Reader behind them.
|
||||
const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader`
|
||||
|
||||
// seriesPageSize is the row read's page length. The tie-break in the query's
|
||||
// ORDER BY is what makes this a stable page boundary — see SeriesPage.
|
||||
const seriesPageSize = 50
|
||||
|
||||
// AdminSeries is one Series as the owner's library-wide view sees it: a
|
||||
// Series-level fact plus an anonymous Reader count. ReaderCount being zero is
|
||||
// the orphan marker. RaisedByReader is the only trace of the Sighting
|
||||
// mechanism here; the Reader id behind it never reaches this type.
|
||||
type AdminSeries struct {
|
||||
Site string
|
||||
SeriesID string
|
||||
Title string
|
||||
SeriesURL string
|
||||
CoverAddress string // "" = no Cover yet
|
||||
Kind string
|
||||
LatestChapter string
|
||||
LatestChapterNum *float64 // nil until first captured
|
||||
LatestCheckedAt int64
|
||||
// ForcePollAt is the owner's "check now" request stamp (issue #146), zero
|
||||
// meaning never asked. Pending is derived, never stored: a request is
|
||||
// pending while ForcePollAt is newer than LatestCheckedAt.
|
||||
ForcePollAt int64
|
||||
ReaderCount int
|
||||
RaisedByReader bool // a Reader's report set LatestChapterNum
|
||||
}
|
||||
|
||||
// SeriesPage is one page of the owner's filtered Series list plus the count
|
||||
// of every Series matching the same filter — a window number, not the page's
|
||||
// len, so the landing page's figure and the list heading come from one query.
|
||||
type SeriesPage struct {
|
||||
Rows []AdminSeries
|
||||
Total int
|
||||
}
|
||||
|
||||
// SiteSeriesShape is one Site's share of the Series matching a filter: how
|
||||
// many, and the manga/novel split. One grouped pass, then library-wide totals
|
||||
// are summed in Go over the rows — the landing page's per-Site table reads
|
||||
// this and never pays for the rows the list discards.
|
||||
type SiteSeriesShape struct {
|
||||
Site string
|
||||
Total int
|
||||
Manga int
|
||||
Novel int
|
||||
}
|
||||
|
||||
// Key returns the canonical identity in bookmark-key form ("<site>:<series_id>").
|
||||
func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID }
|
||||
|
||||
// adminFilter maps a filter's named predicate to its compile-time WHERE and
|
||||
// HAVING clauses and their bound parameters — the name never reaches query
|
||||
// text, and Site and Kind bind as parameters. Shared by the row read and the
|
||||
// per-Site aggregate so the two cannot disagree on what a filter means.
|
||||
//
|
||||
// The WHERE set is: no URL (an empty URL only — the host-failing-the-fetch-
|
||||
// gate case is invisible to SQL, needs the Site registry in Go, and belongs to
|
||||
// a later repair), never-read-a-chapter and never-checked as disjoint halves
|
||||
// (non-zero versus zero check stamp), stale, no cover, and Reader-report.
|
||||
// no_readers is the one HAVING predicate: it is the orphan test, an aggregate
|
||||
// over the LEFT JOIN, where a bare WHERE has no row to test.
|
||||
//
|
||||
// stale is the checked-but-old half of the stamp partition — because the
|
||||
// verdict line wants "not checked in twelve hours" as one figure, and a never
|
||||
// checked Series is already counted on its own "waiting"/never-checked
|
||||
// filter, folding it in would double-report it. The landing page computes the
|
||||
// inclusive number as stale + never-checked.
|
||||
func adminFilter(f SeriesFilter) (where, having string, args []any, err error) {
|
||||
var clauses []string
|
||||
if f.Kind != "" {
|
||||
args = append(args, f.Kind)
|
||||
clauses = append(clauses, "s.kind = $"+strconv.Itoa(len(args)))
|
||||
}
|
||||
switch f.Name {
|
||||
case "", SeriesFilterAll:
|
||||
case SeriesFilterNoURL:
|
||||
clauses = append(clauses, `s.series_url = ''`)
|
||||
case SeriesFilterNoChapter:
|
||||
clauses = append(clauses, `s.latest_checked_at <> 0 AND s.latest_chapter_num IS NULL`)
|
||||
case SeriesFilterNeverChecked:
|
||||
clauses = append(clauses, `s.latest_checked_at = 0`)
|
||||
case SeriesFilterStale:
|
||||
clauses = append(clauses, `s.latest_checked_at > 0 AND s.latest_checked_at < $`+strconv.Itoa(len(args)+1))
|
||||
args = append(args, f.Cutoff)
|
||||
case SeriesFilterNoCover:
|
||||
clauses = append(clauses, `s.cover_address = ''`)
|
||||
case SeriesFilterReaderReport:
|
||||
clauses = append(clauses, `s.latest_raised_by IS NOT NULL`)
|
||||
case SeriesFilterNoReaders:
|
||||
having = `HAVING COUNT(b.reader_id) = 0`
|
||||
default:
|
||||
return "", "", nil, fmt.Errorf("unknown series filter %q", f.Name)
|
||||
}
|
||||
if len(clauses) > 0 {
|
||||
where = "WHERE " + strings.Join(clauses, " AND ")
|
||||
}
|
||||
return where, having, args, nil
|
||||
}
|
||||
|
||||
// SeriesPage returns one page of the Series matching the filter, least
|
||||
// recently checked first. The LEFT JOIN to Bookmarks is what surfaces the
|
||||
// orphans that hygiene has to find — an inner join would hide them, exactly
|
||||
// as the Lane's join does. ReaderCount is a plain count of every Bookmark on
|
||||
// the Series, which knowingly disagrees with the two Lane queries for as long
|
||||
// as the finished lifecycle bucket exists (#140).
|
||||
//
|
||||
// The tie-break is mandatory, not decorative: every unpollable Series shares a
|
||||
// zero check stamp, so ordering on that column alone gives no stable page
|
||||
// boundary and rows would repeat or vanish across pages. (site, series_id) is
|
||||
// the primary key, hence total. The filtered total is a window count in the
|
||||
// same query — window functions run after grouping and before the limit, so
|
||||
// one where-clause cannot disagree with a second copy of itself.
|
||||
func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return SeriesPage{}, err
|
||||
}
|
||||
if f.Page < 1 {
|
||||
f.Page = 1
|
||||
}
|
||||
// Site narrowing is the row read's own; the aggregate must see every Site.
|
||||
if f.Site != "" {
|
||||
args = append(args, f.Site)
|
||||
clause := "s.site = $" + strconv.Itoa(len(args))
|
||||
if where == "" {
|
||||
where = "WHERE " + clause
|
||||
} else {
|
||||
where += " AND " + clause
|
||||
}
|
||||
}
|
||||
base := len(args)
|
||||
args = append(args, seriesPageSize, seriesPageSize*(f.Page-1))
|
||||
rows, err := s.db.Query(`
|
||||
SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`,
|
||||
COUNT(b.reader_id) AS reader_count,
|
||||
COUNT(*) OVER () AS filtered_total
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address,
|
||||
s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at,
|
||||
s.force_poll_at, s.latest_raised_by
|
||||
`+having+`
|
||||
ORDER BY s.latest_checked_at, s.site, s.series_id
|
||||
LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("query series page: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := SeriesPage{}
|
||||
for rows.Next() {
|
||||
a, total, err := scanAdminSeries(rows.Scan)
|
||||
if err != nil {
|
||||
return SeriesPage{}, fmt.Errorf("scan series page: %w", err)
|
||||
}
|
||||
out.Rows = append(out.Rows, a)
|
||||
out.Total = total
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SeriesShapes returns each Site's share of the Series matching the filter,
|
||||
// one grouped pass. Site and Page are row-read concerns and are ignored; the
|
||||
// Landing page reads this per Site and sums the totals in Go for the
|
||||
// library-wide figure.
|
||||
func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) {
|
||||
where, having, args, err := adminFilter(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.db.Query(`
|
||||
SELECT site,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE kind = 'manga') AS manga,
|
||||
COUNT(*) FILTER (WHERE kind = 'novel') AS novel
|
||||
FROM (
|
||||
SELECT s.site, s.kind
|
||||
FROM series s
|
||||
LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id
|
||||
`+where+`
|
||||
GROUP BY s.site, s.series_id, s.kind
|
||||
`+having+`
|
||||
) shape
|
||||
GROUP BY site
|
||||
ORDER BY site`, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query series shapes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []SiteSeriesShape{}
|
||||
for rows.Next() {
|
||||
var sh SiteSeriesShape
|
||||
if err := rows.Scan(&sh.Site, &sh.Total, &sh.Manga, &sh.Novel); err != nil {
|
||||
return nil, fmt.Errorf("scan series shape: %w", err)
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer
|
||||
// order, plus the query's reader_count and filtered_total columns, and returns
|
||||
// the window total alongside the row. latest_chapter_num is NULL until first
|
||||
// captured — the "never read a chapter" state. The Sighting-raiser column is
|
||||
// never among the scanned columns.
|
||||
func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) {
|
||||
var (
|
||||
a AdminSeries
|
||||
latestChapterNum sql.NullFloat64
|
||||
total int
|
||||
)
|
||||
if err := scan(
|
||||
&a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress,
|
||||
&a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt,
|
||||
&a.ForcePollAt,
|
||||
&a.RaisedByReader, &a.ReaderCount, &total,
|
||||
); err != nil {
|
||||
return AdminSeries{}, 0, err
|
||||
}
|
||||
if latestChapterNum.Valid {
|
||||
a.LatestChapterNum = &latestChapterNum.Float64
|
||||
}
|
||||
return a, total, nil
|
||||
}
|
||||
Reference in New Issue
Block a user