From 889f0f3f380262fd23d689d7a2c9462cb8440ed2 Mon Sep 17 00:00:00 2001 From: Sulthan Zaki Date: Sat, 22 Aug 2026 08:27:19 +0700 Subject: [PATCH] Admin dashboard: pages, Lane observability, poll pass log, per-Series intervention (#134) (#148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec #134, all ten tickets. Closes #134. ## What ships The admin surface becomes four bookmarkable addresses behind one nav row, and Lane observability stops dying with the process. - **#138** `/admin` splits into Overview, Lanes, Readers, Series, each a real route with the active tab underlined. - **#139** `poll_passes` and `poll_lanes` land as durable tables with their store surface. - **#140** cross-Series admin read model, with the privacy boundary in the projection: the Reader id that raised a Latest Chapter never leaves the store package. - **#141** the poller records exactly one pass row per exit, with a skip reason and outcome counts. - **#142** Series list: eight hygiene filters, Site and Library narrowing, paging — all of it in the query string, so a filtered list is a bookmark. - **#143** Overview: a three-state verdict line and a stats block where every non-zero figure links to the list that counts it. - **#144** per-Series detail page, keyed by the `site:series_id` composite the rest of the system already uses. - **#145** the Lanes page reads the database; the in-memory Lane state, `web.LaneReporter` and `latest.Status` are deleted. - **#146** Forced Poll: *Check now* stamps `series.force_poll_at` and never commands the poller. - **#147** pause and resume one Site's Lane, with a mandatory 1h/6h/24h expiry. ## Shape of the design Two decisions carry the rest. **Commands go through the database, never at the poller**: both *Check now* and a Lane pause write a row the next pass reads, so they survive a restart and the whole surface stays testable with no poller running. And **pending is derived, never stored** — the request stamp being newer than the check stamp — which self-clears on the check stamp with no second write and no sweeper, because the check stamp is written before the fetch. ADRs: `docs/adr/0012-persisted-lane-state.md`, `docs/adr/0013-commands-through-the-database.md`. ## Verification `go test ./...` green on the merged base (`264839e`), all packages, Docker-backed. `gofmt -l` and `go vet` clean. Every ticket was reviewed on both axes (`cr-spec` + `cr-standards`) before merge. ## Known, non-blocking - **#143** the verdict ignores never-reported Lanes when other Lanes have reported, and the per-Site table lists Sites that have Series rather than the whole registry. The ticket prose asks for eight hygiene figures per Site; the design mock and the landed `.tbl.sites` grid both say six columns, and the mock won. - **#146** two `SeriesPage` scans per press instead of a keyed read — `ponytail:`-commented in-tree with the upgrade path. - **#147** a paused Site with no pass row yet renders no row and so no control, since the Lanes page lists Sites that have passed. - **#141** a sibling browser Lane declining at the top of a pass records as `sidecar-down`. Specified deliberately; the later spec in this series settles it. Reviewed-on: https://gitea.violetcrown.my.id/sulthan/mangaBookmark/pulls/148 Co-authored-by: Sulthan Zaki Co-committed-by: Sulthan Zaki --- backend/AGENTS.md | 23 +- backend/api_test.go | 10 +- backend/cover_test.go | 2 +- backend/internal/latest/acquire_test.go | 27 + backend/internal/latest/poller.go | 317 +++- backend/internal/latest/poller_test.go | 822 ++++++++- backend/internal/latest/sites.go | 19 +- backend/internal/latest/status.go | 79 - backend/internal/store/admin.go | 276 +++ backend/internal/store/admin_test.go | 396 ++++ .../store/migrations/0012_poll_lanes.sql | 6 + .../store/migrations/0013_poll_passes.sql | 16 + .../migrations/0014_admin_read_model.sql | 11 + backend/internal/store/store.go | 285 ++- backend/internal/store/store_test.go | 317 +++- backend/internal/web/admin.go | 155 +- backend/internal/web/admin_lanes.go | 338 ++++ backend/internal/web/admin_overview.go | 197 ++ backend/internal/web/admin_series.go | 398 ++++ backend/internal/web/admin_series_detail.go | 126 ++ backend/internal/web/static/admin.css | 830 +++++++++ backend/internal/web/static/style.css | 69 - backend/internal/web/templates/admin.html | 43 +- backend/internal/web/templates/lanes.html | 82 +- backend/internal/web/templates/overview.html | 14 + .../internal/web/templates/series-detail.html | 35 + .../internal/web/templates/series-list.html | 53 + backend/internal/web/web.go | 22 +- backend/main.go | 38 +- backend/reader_credential_test.go | 6 +- backend/web_test.go | 1630 ++++++++++++++++- docs/adr/0012-persisted-lane-state.md | 44 + .../adr/0013-commands-through-the-database.md | 60 + 33 files changed, 6112 insertions(+), 634 deletions(-) delete mode 100644 backend/internal/latest/status.go create mode 100644 backend/internal/store/admin.go create mode 100644 backend/internal/store/admin_test.go create mode 100644 backend/internal/store/migrations/0012_poll_lanes.sql create mode 100644 backend/internal/store/migrations/0013_poll_passes.sql create mode 100644 backend/internal/store/migrations/0014_admin_read_model.sql create mode 100644 backend/internal/web/admin_lanes.go create mode 100644 backend/internal/web/admin_overview.go create mode 100644 backend/internal/web/admin_series.go create mode 100644 backend/internal/web/admin_series_detail.go create mode 100644 backend/internal/web/static/admin.css create mode 100644 backend/internal/web/templates/overview.html create mode 100644 backend/internal/web/templates/series-detail.html create mode 100644 backend/internal/web/templates/series-list.html create mode 100644 docs/adr/0012-persisted-lane-state.md create mode 100644 docs/adr/0013-commands-through-the-database.md diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 2fd68d6..49d3443 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -31,8 +31,7 @@ image, TLS terminated by the reverse proxy so the service listens plain `:8080`. store must have that `TestMain` or it has no database at all. ### Reader-owned store — `internal/store`, `internal/token` - -Four tables; shape is in the migrations, behaviour in `Store`'s methods. +- The Reader-owned tables are `readers`, `bookmarks`, `series`, and `sessions`; auxiliary `covers`, `poll_lanes`, and `poll_passes` are also defined in the migrations. - **Credentials are derived, never stored.** `token.Token(TOKEN_KEY, discord_id, epoch)` is an HMAC; only its SHA-256 reaches `readers.token_sha256`. So install URLs @@ -135,7 +134,7 @@ unchanged read is exactly the Sighting worth deferring a Poll on. rest. **Refusals and browser loss are Lane-local.** Two `errChallengeHeld` in a pass -stop that Site for `refuseBackoff` while other Lanes continue. An +stop that Site for `RefuseBackoff` while other Lanes continue. An `errBrowserInterrupted` (remote Chrome restarted) sets a shared Poller flag so the *other* browser Lanes skip their passes for the same window — otherwise a restarting Chrome stamps one Series per Lane per pass, burning rests on @@ -253,16 +252,18 @@ keep warning to reinstall on all devices. (`view.Owner = readerID == h.store.OwnerID()`): it gates a link, not an endpoint, so it is a rendering decision a registration-time wrapper cannot express. Do not "unify" it into the gate. -- Lane figures come through the `web.LaneReporter` seam - (`latest.Poller.LaneStatus`), never a table. `main.newRouter` takes the - reporter as an interface and converts a nil `*Poller` to a nil interface — a - typed nil would make the page claim a poller exists. +- The Lanes page reads the pass log, never a running poller: `lanesView()` in + `admin_lanes.go` projects `store.LatestLanePasses()` and + `store.LanePassOutcomes()` (ADR-0012), so a restart answers the instant the + database is up. Browser configuration is a config fact and reachability is + derived from recent browser-Site passes inside `latest.RefuseBackoff` — no + reporter interface exists to fake. - A pass that returns before computing figures (refusal backoff, sidecar down) carries the previous pass's numbers forward rather than recording zeroes. - **`Checked` next to `Due` is what separates a stopped Lane from a quiet one**, so neither may be dropped from the row. - Due-without-Checked is **not** by itself a stall: a browser Lane under both - wake thresholds sets `LaneState.Asleep` and renders "browser asleep", and - never counts toward `Attention`. That is the commonest healthy state for - kagane, comix and novelfull, so spending the stall mark on it would train the - owner to ignore the mark that matters. + wake thresholds records its pass with the `SkipAsleep` skip and renders + "browser asleep", and that never counts toward `Attention`. It is the + commonest healthy state for kagane, comix and novelfull, so spending the + stall mark on it would train the owner to ignore the mark that matters. diff --git a/backend/api_test.go b/backend/api_test.go index beee5a8..1037808 100644 --- a/backend/api_test.go +++ b/backend/api_test.go @@ -48,7 +48,7 @@ func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } func newTestServer(t *testing.T) http.Handler { t.Helper() - return newRouter(newTestStore(t), testConfig(), nil) + return newRouter(newTestStore(t), testConfig()) } func newTestStore(t *testing.T) *store.Store { @@ -599,7 +599,7 @@ func TestLoadConfigDiscord(t *testing.T) { // cooldown and the poller would re-fetch that series on every single tick. func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) { s := newTestStore(t) - srv := newRouter(s, testConfig(), nil) + srv := newRouter(s, testConfig()) seedForCheck(t, s, "asura:x", "https://asurascans.com/comics/x", 777) @@ -627,7 +627,7 @@ func TestPutDoesNotClobberLatestCheckedAt(t *testing.T) { // series stops being due the moment the PUT lands. func TestPutRecordsASighting(t *testing.T) { s := newTestStore(t) - srv := newRouter(s, testConfig(), nil) + srv := newRouter(s, testConfig()) now := time.Now().UnixMilli() hour := time.Hour.Milliseconds() @@ -677,7 +677,7 @@ func TestUserscriptServedWithWebUIDisabled(t *testing.T) { rr := httptest.NewRecorder() req := httptest.NewRequest(http.MethodGet, "/u/"+ownerCredential()+"/manga-bookmark.user.js", nil) - newRouter(s, cfg, nil).ServeHTTP(rr, req) + newRouter(s, cfg).ServeHTTP(rr, req) if rr.Code != http.StatusOK { t.Fatalf("status = %d, want 200", rr.Code) } @@ -699,7 +699,7 @@ func TestNovelUserscriptServed(t *testing.T) { cfg := testConfig() cfg.NovelUserscriptPath = novelPath - srv := newRouter(s, cfg, nil) + srv := newRouter(s, cfg) rr := httptest.NewRecorder() srv.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, diff --git a/backend/cover_test.go b/backend/cover_test.go index 77ea269..9c3c93f 100644 --- a/backend/cover_test.go +++ b/backend/cover_test.go @@ -98,7 +98,7 @@ func TestPublicCoverNeverEchoesNonImage(t *testing.T) { if _, err := db.Exec(`UPDATE covers SET content_type = 'text/html' WHERE address = $1`, address); err != nil { t.Fatalf("poison row: %v", err) } - rr := getCover(t, newRouter(st, testConfig(), nil), "/covers/"+address, nil) + rr := getCover(t, newRouter(st, testConfig()), "/covers/"+address, nil) if rr.Code == http.StatusOK { t.Fatalf("status = 200, want a refusal for a non-image row (body %q)", rr.Body.String()) } diff --git a/backend/internal/latest/acquire_test.go b/backend/internal/latest/acquire_test.go index 9ff6def..4a6cd0c 100644 --- a/backend/internal/latest/acquire_test.go +++ b/backend/internal/latest/acquire_test.go @@ -135,6 +135,33 @@ func TestAcquireFillsChapterAndCoverFromOneFetch(t *testing.T) { } } +// A pause governs the Lane only: a Reader's first bookmark of a Series on a +// paused Site still reads the page, because acquisition is the creation-time +// fetch, not the poll queue (issue #147). +func TestAcquireIgnoresLanePause(t *testing.T) { + s, _ := newTestStore(t) + if err := s.PauseLane("asura", time.Now().Add(6*time.Hour).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + page := &fakeFetcher{body: asuraSeriesAndCoverFixture, status: 200} + covers := &fakeBytesCoverFetcher{body: []byte("cover-bytes"), contentType: "image/jpeg"} + acq := newAcquirer(s, page, covers) + + bookmarkNewSeries(t, s, acquireSeriesURL) + acq.Wait() + + if got := page.callCount(); got != 1 { + t.Fatalf("series page fetches on a paused Site = %d, want 1", got) + } + if got := covers.callCount(); got != 1 { + t.Fatalf("cover fetches = %d, want 1", got) + } + got := readBookmark(t, s, acquireKey) + if got.LatestChapterNum == nil || *got.LatestChapterNum != 181 { + t.Fatalf("LatestChapterNum = %v, want 181", got.LatestChapterNum) + } +} + // A Series that already exists is not re-acquired: no fetch, and the Cover it // already has is left alone. func TestAcquireSkipsAnExistingSeries(t *testing.T) { diff --git a/backend/internal/latest/poller.go b/backend/internal/latest/poller.go index 50741a8..035ff84 100644 --- a/backend/internal/latest/poller.go +++ b/backend/internal/latest/poller.go @@ -5,7 +5,6 @@ import ( "errors" "log" "net/url" - "sort" "sync" "time" @@ -48,19 +47,22 @@ type Poller struct { // same failure-isolated prefetch path. CoverBytesFetch CoverBytesFetcher Now func() time.Time // injected so tests can freeze it + // eligibleCount reports how many of a Site's Series are eligible for + // polling, defaulting to Store.EligibleSeriesCount. Injected so tests can + // fail the count alone: the eligible query shares the due query's tables, + // so no real store failure can reach this path without breaking the due + // query first (issue #141). + eligibleCount func(site string) (int, error) // refuseUntil gates a Site's Lane after it refused twice in one run: no // Series of that Site is attempted again before this time (issue #100). + // The stamp is durable — the pass gate reads it from the store, so a + // restart does not forget the refusal; nothing of it lives in memory. // browserDownAt is when a browser Lane last lost the sidecar; the other - // browser Lanes skip their passes for the next refuseBackoff, so a + // browser Lanes skip their passes for the next RefuseBackoff, so a // restarting Chrome does not stamp one Series per pass per Lane (story 20). mu sync.Mutex - refuseUntil map[string]time.Time browserDownAt time.Time - // laneStates is the owner's page snapshot of each Lane's last pass - // (issue #102), keyed by Site. Guarded by mu; a Site appears only after - // its first pass, so a restart renders "no data yet" rather than zeroes. - laneStates map[string]LaneState // coverWG tracks in-flight cover work. Covers heal in the background so a // slow cover host cannot delay the next Series-page Poll; tests join it // before asserting on cover fetches. @@ -167,14 +169,7 @@ func fetcherFor(site string, browser, tls Fetcher) Fetcher { // laneNames returns every registry Site in the deterministic order both Run // and runOnce iterate: sorted, so lane behaviour and its tests agree on who // runs first. -func laneNames() []string { - names := make([]string, 0, len(sites)) - for name := range sites { - names = append(names, name) - } - sort.Strings(names) - return names -} +func laneNames() []string { return SiteNames() } func (p *Poller) Run(ctx context.Context) { names := laneNames() @@ -216,28 +211,120 @@ func (p *Poller) runOnce(ctx context.Context) { } } +// One skip value per way a Lane Pass can return before its loop (issue #141); +// empty means the pass reached the loop. The values are wire strings — stored +// in poll_passes and read by the Lanes page — so they are stable, not prose. +const ( + // Exported so the web layer renders a skip's reason without retyping the + // wire string (issue #145); the values are storage and page-stable. + SkipPaused = "paused" // the pause row was read at the top + SkipRefusing = "refusing" // refusal backoff + SkipSidecarDown = "sidecar-down" // a sibling browser Lane lost Chrome + SkipNoFetcher = "no-fetcher" // browser Site, no browser configured, no fallback + SkipDueQuery = "due-query" // the due query failed + SkipAsleep = "asleep" // under both browser wake thresholds + SkipEligibleCount = "eligible-count" // the eligible count failed + SkipNothingEligible = "nothing-eligible" // nothing eligible; sleeps a full rest +) + +// readOutcome classifies one Series read for the pass row's outcome counts +// (issue #141). The classification the read already makes is counted, never a +// second taxonomy: refused is the Site holding a challenge, unreachable the +// browser interrupting, noChapter a 200 with real HTML but no chapter links, +// unfetchable the host pin or a missing fetcher, and errors everything else. +type readOutcome int + +const ( + outcomeSuccess readOutcome = iota + outcomeRefused + outcomeUnreachable + outcomeNoChapter + outcomeUnfetchable + outcomeError +) + +// outcomeCounts are the five named outcome counts of one pass. A success +// count is derived, never stored: checked minus the four, with unreachable +// excluded because the sidecar-loss path returns before the checked counter +// increments (issue #141). +type outcomeCounts struct { + refused, unreachable, noChapter, unfetchable, errors int +} + +func (c *outcomeCounts) add(o readOutcome) { + switch o { + case outcomeRefused: + c.refused++ + case outcomeUnreachable: + c.unreachable++ + case outcomeNoChapter: + c.noChapter++ + case outcomeUnfetchable: + c.unfetchable++ + case outcomeError: + c.errors++ + } +} + +// passRecord is what one pass's durable row will be: the skip value and +// outcome counts filled in along the pass's return path. recordPass assembles +// the row, so every exit records exactly once. +type passRecord struct { + site string + ranAt int64 + skip string + counts outcomeCounts +} + +// lanePassRetention is how far back a Lane's pass log is kept. It is not the +// display window: retention is how far back a question can reach, and the +// window is what the owner is shown (issue #139). +const lanePassRetention = 14 * 24 * time.Hour + // runLanePass processes one pass of one Site's Lane: select the due Series, // pace through them, and report how long the Lane should wait before its next // pass. paced spaces consecutive fetches by the Site's effective gap — the // production Lane's rate limit; the deterministic test entry runs back to back. func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time.Duration { now := p.Now() - // Snapshot this pass for the owner's page (issue #102). Recorded on every - // return path, with the figures filled in where the pass computes them. - st := LaneState{Site: name, LastRun: now, Browser: isBrowserSite(name)} - defer func() { p.recordLaneState(st) }() - if until := p.refusalBackoff(name); now.Before(until) { + // Durable pass log (issue #141): one row per exit. The figures are filled + // in as the pass measures them; a pass that returns before measuring + // carries the previous pass's forward inside recordPass. + fig := passFigures{} + rec := passRecord{site: name, ranAt: now.UnixMilli()} + defer func() { p.recordPass(rec, fig) }() + + // One Lane row read at the top of a pass, serving two gates (issue #139). + // Both stamps outlive our process, so the gates read the durable row + // rather than memory: a refusal is the Site's mood and a pause the + // owner's order, and neither is lost to a restart. + pausedUntil, refuseUntil, err := p.Store.LaneGates(name) + if err != nil { + // Fail open: a store that cannot answer the gate cannot record the + // pass either, and one Lane must not stall on its own gate read. + log.Printf("latest poll %s: lane gates: %v", name, err) + } + if pausedUntil > now.UnixMilli() { + // Paused ahead of the refusal check: no Series is touched, so the + // queue stays intact for when the pause lifts (issue #141, #147). + rec.skip = SkipPaused + log.Printf("latest poll %s: paused until %s, skipping pass", name, time.UnixMilli(pausedUntil).Format(time.RFC3339)) + return time.Duration(pausedUntil-now.UnixMilli()) * time.Millisecond + } + if refuseUntil > now.UnixMilli() { // Cooling down after a refusal: do not attempt this Site at all. - return until.Sub(now) + rec.skip = SkipRefusing + return time.Duration(refuseUntil-now.UnixMilli()) * time.Millisecond } if isBrowserSite(name) { - if downFor, down := p.browserDownFor(now); down && downFor < refuseBackoff { + if downFor, down := p.browserDownFor(now); down && downFor < RefuseBackoff { // A sibling browser Lane lost the sidecar within the backoff // window: skip this pass, so a restarting Chrome does not stamp - // this Site's Series one pass at a time. After refuseBackoff the + // this Site's Series one pass at a time. After RefuseBackoff the // flag decays and the Lane probes again (issue #100, story 20). + rec.skip = SkipSidecarDown log.Printf("latest poll %s: browser lane skipping pass (sidecar down %s ago)", name, downFor) - return refuseBackoff - downFor + return RefuseBackoff - downFor } } s := sites[name] @@ -246,24 +333,29 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. // No fetcher at all right now (browser absent, no fallback): every // Series stays unstamped and due, so a browser that appears after a // restart finds its full queue waiting (issue #100). - st.Gap = defaultGap + rec.skip = SkipNoFetcher + fig.Gap = defaultGap return defaultGap } due, err := p.Store.DueForLatestCheck(name, now.Add(-s.Rest).UnixMilli(), now.Add(-sightingCeilingRests*s.Rest).UnixMilli()) if err != nil { + rec.skip = SkipDueQuery log.Printf("latest poll %s: due query: %v", name, err) - st.Gap = defaultGap + fig.Gap = defaultGap return defaultGap } - st.Due = len(due) - if s.Browser != nil && f == p.BrowserFetch && !browserWakeDue(due, now, s.Rest) { + fig.Due = len(due) + if s.Browser != nil && f == p.BrowserFetch && !browserWakeDue(due, now, s.Rest) && !anyForced(due) { // Below both thresholds Chrome stays asleep (ADR-0005 on-demand // browser): waking it for a single Poll would cost a challenge solve - // per request. The Lane still paces at the default gap, which is what - // the owner's page must show rather than a zero. - st.Gap, st.Asleep = defaultGap, true + // per request. A forced Series is the one exception — a human asking + // is not the machine waking itself (issue #146). The Lane still paces + // at the default gap, which is what the owner's page must show rather + // than a zero. + rec.skip = SkipAsleep + fig.Gap = defaultGap return defaultGap } if s.Browser != nil { @@ -276,20 +368,22 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. } } - eligible, err := p.Store.EligibleSeriesCount(name) + eligible, err := p.countEligible(name) if err != nil { + rec.skip = SkipEligibleCount log.Printf("latest poll %s: eligible count: %v", name, err) - st.Gap = defaultGap + fig.Gap = defaultGap return defaultGap } gap, clamped := effectiveGap(s, eligible) - st.Gap, st.Clamped = gap, clamped + fig.Gap, fig.Clamped = gap, clamped if clamped { log.Printf("latest poll %s: gap clamped to %s floor (eligible series=%d)", name, minGap, eligible) } if eligible == 0 { // Nothing to poll for the foreseeable future; sleep a full rest instead // of re-querying every gap. + rec.skip = SkipNothingEligible return s.Rest } @@ -300,7 +394,7 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. } if refusals >= 2 { // This Site refused twice in a row: the remaining Series are left - // unstamped and due, and the Lane waits refuseBackoff before + // unstamped and due, and the Lane waits RefuseBackoff before // trying it again. break } @@ -314,46 +408,98 @@ func (p *Poller) runLanePass(ctx context.Context, name string, paced bool) time. break } } - if err := p.checkOne(ctx, sr); err != nil { - switch { - case errors.Is(err, errChallengeHeld): - refusals++ - case errors.Is(err, errBrowserInterrupted): - p.setBrowserDown(now) - log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, refuseBackoff) - return gap - default: - refusals = 0 - } + outcome := p.checkOne(ctx, sr) + if outcome == outcomeUnreachable { + // The mid-loop browser loss writes an empty skip on purpose: the + // pass returns before the checked counter increments, so its row + // is stall-shaped (due > 0, checked 0, skip ''), and a stall is + // the exact signal this exit produces. A tenth skip value would + // make it legible but is deliberately not invented here. + rec.counts.add(outcome) + p.setBrowserDown(now) + log.Printf("latest poll %s: browser unreachable, browser lanes skipping passes for %s", name, RefuseBackoff) + return gap + } + if outcome == outcomeRefused { + refusals++ } else { refusals = 0 } - st.Checked++ + rec.counts.add(outcome) + fig.Checked++ } - if st.Checked > 0 { - log.Printf("latest poll %s: due=%d checked=%d", name, len(due), st.Checked) + if fig.Checked > 0 { + log.Printf("latest poll %s: due=%d checked=%d", name, len(due), fig.Checked) } if refusals >= 2 { - p.setRefusalBackoff(name, now.Add(refuseBackoff)) - log.Printf("latest poll %s: refused twice this run, waiting %s", name, refuseBackoff) - return refuseBackoff + // The refusal outlives the process: the durable stamp gates a restart, + // so a Site that just told us to back off is not re-probed. + if err := p.Store.SetLaneRefusal(name, now.Add(RefuseBackoff).UnixMilli()); err != nil { + log.Printf("latest poll %s: persist refusal: %v", name, err) + } + log.Printf("latest poll %s: refused twice this run, waiting %s", name, RefuseBackoff) + return RefuseBackoff } return gap } -func (p *Poller) refusalBackoff(name string) time.Time { - p.mu.Lock() - defer p.mu.Unlock() - return p.refuseUntil[name] +// passFigures are the numbers one pass measured for its durable row (issue +// #141): due and checked as the pass saw them, the pace it chose, and whether +// the gap sat on the floor. A pass that returned before measuring keeps the +// previous pass's figures via carry-forward in recordPass; the in-memory +// snapshot those once mirrored into is gone — the page reads the durable row +// now (issue #145). +type passFigures struct { + Due, Checked int + Gap time.Duration + Clamped bool } -func (p *Poller) setRefusalBackoff(name string, until time.Time) { - p.mu.Lock() - defer p.mu.Unlock() - if p.refuseUntil == nil { - p.refuseUntil = make(map[string]time.Time) +// recordPass writes the durable row for one pass (issue #141). Called deferred +// from runLanePass so every return path records exactly one row. A pass that +// never computed its own figures — its gap is zero — carries the previous +// pass's due, gap, clamped and checked forward rather than stating zeroes it +// did not measure; the skip column says why it declined, so the zeroes that +// remain (due-query, no-fetcher) read as explanations rather than +// measurements. +func (p *Poller) recordPass(rec passRecord, fig passFigures) { + row := store.LanePass{ + Site: rec.site, + RanAt: rec.ranAt, + Skip: rec.skip, + Due: fig.Due, + Checked: fig.Checked, + GapMS: fig.Gap.Milliseconds(), + Clamped: fig.Clamped, + Refused: rec.counts.refused, + Unreachable: rec.counts.unreachable, + NoChapter: rec.counts.noChapter, + Unfetchable: rec.counts.unfetchable, + Errors: rec.counts.errors, } - p.refuseUntil[name] = until + if row.GapMS == 0 { + // The pass never computed a gap, so it has no figures of its own: + // carry the previous pass's, in one latest-per-Site read — the + // recorder needs one Site, not six (issue #139). + if prev, ok, err := p.Store.LatestLanePass(rec.site); err != nil { + log.Printf("latest poll %s: previous pass: %v", rec.site, err) + } else if ok { + row.Due, row.Checked = prev.Due, prev.Checked + row.GapMS, row.Clamped = prev.GapMS, prev.Clamped + } + } + if err := p.Store.RecordLanePass(row, rec.ranAt-lanePassRetention.Milliseconds()); err != nil { + log.Printf("latest poll %s: record lane pass: %v", rec.site, err) + } +} + +// countEligible routes the eligible count through the test seam when one is +// set, else the store. +func (p *Poller) countEligible(site string) (int, error) { + if p.eligibleCount != nil { + return p.eligibleCount(site) + } + return p.Store.EligibleSeriesCount(site) } // setBrowserDown records when a browser Lane lost the sidecar. It is Poller @@ -366,7 +512,7 @@ func (p *Poller) setBrowserDown(now time.Time) { // browserDownFor reports how long the sidecar has been down and that it is // down at all — the zero time means never down, which must not read as a -// zero-duration loss. The window decays: once refuseBackoff passes without a +// zero-duration loss. The window decays: once RefuseBackoff passes without a // fresh loss, Lanes probe again. func (p *Poller) browserDownFor(now time.Time) (time.Duration, bool) { p.mu.Lock() @@ -395,6 +541,19 @@ func browserWakeDue(due []store.Series, now time.Time, rest time.Duration) bool return maxSeriesWait(due, now, rest) >= browserWakeAge } +// anyForced reports whether the due list holds a forced Series: one whose +// owner check-now request (issue #146) has not been answered yet. A human +// asking wakes a sleeping Chrome even below the wake thresholds; the request +// itself still ages visibly if the home machine is off. +func anyForced(due []store.Series) bool { + for _, sr := range due { + if sr.Forced { + return true + } + } + return false +} + // maxSeriesWait returns how long the most-overdue of the due Series has been // waiting past its due moment (0 when due is empty). func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.Duration { @@ -409,13 +568,15 @@ func maxSeriesWait(due []store.Series, now time.Time, rest time.Duration) time.D // checkOne re-checks one series. Every failure path here is "log and move on": // the poller is a best-effort enhancement, and no single bad series may stall a -// Lane or take down the process. The returned error is the page read's -// classified outcome so the Lane can tell a refusal from a loss of the -// browser; non-classified failures still return nil-equivalent behaviour. -func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { +// Lane or take down the process. The returned outcome classifies the read for +// the pass row (issue #141), so the Lane can count a refusal, a lost browser, +// a chapter-less page, an unfetchable address or a transport error without +// re-deriving the taxonomy. +func (p *Poller) checkOne(ctx context.Context, sr store.Series) (outcome readOutcome) { defer func() { if r := recover(); r != nil { log.Printf("latest poll %q: recovered from panic: %v", sr.Key(), r) + outcome = outcomeError } }() @@ -427,7 +588,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // the stamp means "attempted", and an untried Series stays due. if err := p.Store.MarkLatestChecked(sr.Site, sr.SeriesID, p.Now().UnixMilli()); err != nil { log.Printf("latest poll %q: mark checked: %v", sr.Key(), err) - return nil + return outcomeError } facts, err := readSeriesPage(ctx, sr.Site, sr.SeriesURL, p.BrowserFetch, p.Fetch) @@ -438,17 +599,23 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // passes the gate is retried at rest pace rather than // hot-looping. log.Printf("latest poll %q: not fetchable: site=%q url=%q", sr.Key(), sr.Site, sr.SeriesURL) - return err + return outcomeUnfetchable case errors.Is(err, errNoFetcher): log.Printf("latest poll %q: no fetcher for site %q", sr.Key(), sr.Site) - return err + return outcomeUnfetchable } // A legacy cover heals independently of the page read: its source may // answer — a CDN — while the origin does not, so a fetch failure does // not skip the heal, matching the order the shared read replaced. p.healCover(ctx, sr) log.Printf("latest poll %q: %v", sr.Key(), err) - return err + if errors.Is(err, errChallengeHeld) { + return outcomeRefused + } + if errors.Is(err, errBrowserInterrupted) { + return outcomeUnreachable + } + return outcomeError } // A legacy cover source is healed independently of the page read. p.healCover(ctx, sr) @@ -459,7 +626,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // Most likely a challenge page or a layout change. Either way the row is // already stamped, so this waits out a rest instead of hot-looping. log.Printf("latest poll %q: no chapter links in %d bytes", sr.Key(), facts.BodyLen) - return nil + return outcomeNoChapter } // The Poll is the oracle for whatever Sighting last raised this Series @@ -472,7 +639,7 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // against the due-query snapshot; a concurrent write in between only costs // one redundant UPDATE of the same absolute value, never a wrong one. if sr.LatestChapterNum != nil && *sr.LatestChapterNum == facts.Latest.Num { - return nil + return outcomeSuccess } // Series-level write: the row is shared, so one update refreshes every @@ -481,10 +648,10 @@ func (p *Poller) checkOne(ctx context.Context, sr store.Series) error { // the list. if err := p.Store.SetLatestChapter(sr.Site, sr.SeriesID, facts.Latest.Label, facts.Latest.Num); err != nil { log.Printf("latest poll %q: set latest chapter: %v", sr.Key(), err) - return nil + return outcomeError } log.Printf("latest poll %q: latest is now %s", sr.Key(), facts.Latest.Label) - return nil + return outcomeSuccess } // judgeSighting settles the Sighting the Series' stored Latest Chapter is owed diff --git a/backend/internal/latest/poller_test.go b/backend/internal/latest/poller_test.go index 72736bb..bf8d7c8 100644 --- a/backend/internal/latest/poller_test.go +++ b/backend/internal/latest/poller_test.go @@ -427,9 +427,12 @@ func TestRunLogsLaneDefaults(t *testing.T) { log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(previous) }) + // The pass log writes through the store, so Run needs a real one — a + // poller without a Store is not a poller. + s, _ := newTestStore(t) ctx, cancel := context.WithCancel(context.Background()) cancel() - (&Poller{Now: func() time.Time { return time.Now() }}).Run(ctx) + (&Poller{Store: s, Now: func() time.Time { return time.Now() }}).Run(ctx) got := logs.String() for _, want := range []string{"6 lanes", "rest=1h0m0s", "gap=10s"} { @@ -1305,7 +1308,7 @@ func TestRunOnceOrdersBySharednessThenAge(t *testing.T) { } // Two refusals in one pass stop the Lane: the remaining Series stay unstamped -// and due, and the Lane backs off for refuseBackoff before trying the Site +// and due, and the Lane backs off for RefuseBackoff before trying the Site // again. One hostile Site burns only its own Lane's budget (issue #100). func TestRunOnceSiteRefusalSkipsRestOfLaneAndBacksOff(t *testing.T) { s, _ := newTestStore(t) @@ -1388,11 +1391,9 @@ func TestBrowserLaneWakeThresholds(t *testing.T) { if got := browser.callCount(); got != 0 { t.Fatalf("browser fetches with 3 freshly-due series = %d, want 0 (Chrome stays asleep)", got) } - // The owner's page reads this state off the snapshot, and Due-without- - // Checked has to be distinguishable there from a Lane that has stopped. - if lane := laneByName(t, p, "kagane"); !lane.Asleep || lane.Due != 3 || lane.Checked != 0 { - t.Fatalf("asleep kagane lane = %+v, want Asleep with 3 due and 0 checked", lane) - } + // The skipped pass still records its row, carrying the due count it never + // read; the Lanes page (issue #145) reads that row — see + // TestRunLanePassRecordsEveryExit/"asleep". // 5 due crosses the count threshold. for i := 3; i < 5; i++ { seed(i) @@ -1401,9 +1402,6 @@ func TestBrowserLaneWakeThresholds(t *testing.T) { if got := browser.callCount(); got != 5 { t.Fatalf("browser fetches with 5 due series = %d, want 5", got) } - if lane := laneByName(t, p, "kagane"); lane.Asleep { - t.Fatalf("woken kagane lane still reports Asleep: %+v", lane) - } // A single long-neglected series wakes the browser by age alone. seedForCheck(t, s, "kagane:ancient", "https://kagane.to/series/ancient", 0) p.runOnce(context.Background()) @@ -1412,19 +1410,6 @@ func TestBrowserLaneWakeThresholds(t *testing.T) { } } -// laneByName pulls one Lane out of the poller's snapshot, failing rather than -// returning a zero LaneState a caller would assert against by accident. -func laneByName(t *testing.T, p *Poller, site string) LaneState { - t.Helper() - for _, lane := range p.LaneStatus().Lanes { - if lane.Site == site { - return lane - } - } - t.Fatalf("no %q lane in the snapshot", site) - return LaneState{} -} - // When one browser Lane loses the sidecar, the round's remaining browser // Lanes are skipped: every fetch would fail anyway, and their Series must not // burn their stamps on a dead Chrome (issue #100). @@ -1456,7 +1441,7 @@ func TestRunOnceUnreachableBrowserStopsBrowserLanes(t *testing.T) { } } - // The shared flag decays after refuseBackoff: the next round probes + // The shared flag decays after RefuseBackoff: the next round probes // again. comix's Series is resting (stamped last round), so the probe // falls to kagane — the only Lane with something due — and its fresh // loss re-gates the Lanes behind it. @@ -1602,19 +1587,21 @@ func TestRunOnceClampWarningNamesTheSite(t *testing.T) { } } -// The owner's admin page (issue #102) reads Lane state out of the poller. -// Before any pass the snapshot is empty — a restart must render "no data -// yet", not zeroes — and each pass records what it saw: the frozen clock, -// the due count and the pace, with refusal backoff derived at snapshot time. -func TestLaneStatus(t *testing.T) { +// The Lanes page (issue #145) reads the durable pass log, so the poller's +// only duty to it is that every return path writes its row; the snapshot it +// used to mirror into memory is gone. A refusing pass still records why it +// declined, and the latest row per Site is what the page renders — nothing +// else is left to assert against here, because the page's seam moved into the +// web layer (web_test.go, seeded-row tests). +func TestPassLogIsTheLanesPagesOnlyWindow(t *testing.T) { s, _ := newTestStore(t) now := time.UnixMilli(5_000_000) p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) - if st := p.LaneStatus(); len(st.Lanes) != 0 { - t.Fatalf("lanes before any pass = %d, want 0 (nothing has run)", len(st.Lanes)) - } else if st.BrowserConfigured || st.BrowserReachable { - t.Fatalf("browser before any pass = configured=%v reachable=%v, want false without a browser fetcher", st.BrowserConfigured, st.BrowserReachable) + // No pass has run: the log is empty, which the page renders as "none + // observed" rather than confident zeroes. + if _, ok, err := s.LatestLanePass("asura"); err != nil || ok { + t.Fatalf("LatestLanePass before any pass = ok %v err %v, want no row", ok, err) } seedForCheck(t, s, "asura:chronicles", "https://asurascans.com/series/chronicles", 0) @@ -1627,65 +1614,726 @@ func TestLaneStatus(t *testing.T) { } p.runOnce(context.Background()) - st := p.LaneStatus() - var asura, kagane LaneState - for _, lane := range st.Lanes { - switch lane.Site { - case "asura": - asura = lane - case "kagane": - kagane = lane - } + // Both lanes wrote their rows: asura the pass it read; kagane the + // mid-loop double-refusal exit, which records an empty skip by design + // with its two refused counts — the durable row is the whole record, and + // the page reads it rather than a snapshot. + asura := latestPassFor(t, s, "asura") + if asura.Due != 1 || asura.Checked != 1 || asura.GapMS == 0 { + t.Fatalf("asura row = %+v, want due 1 checked 1 with the Lane's pace", asura) } - if st.Lanes[0].Site != "asura" { - t.Fatalf("first lane = %q, want asura (snapshot sorted by Site)", st.Lanes[0].Site) - } - if asura.Site == "" { - t.Fatalf("asura missing from snapshot: %+v", st.Lanes) - } - if !asura.LastRun.Equal(now) { - t.Fatalf("asura LastRun = %s, want the frozen clock %s", asura.LastRun, now) - } - if asura.Due != 1 { - t.Fatalf("asura Due = %d, want 1", asura.Due) - } - if asura.Gap == 0 { - t.Fatal("asura Gap = 0, want the Lane's pace") - } - if asura.Browser || asura.Refusing { - t.Fatalf("asura = %+v, want a TLS Lane that is not refusing", asura) - } - if !kagane.Browser || !kagane.Refusing { - t.Fatalf("kagane = %+v, want a browser Lane in refusal backoff", kagane) - } - if !st.BrowserConfigured || !st.BrowserReachable { - t.Fatalf("browser after round = configured=%v reachable=%v, want true/true (sidecar never lost)", st.BrowserConfigured, st.BrowserReachable) - } - if asura.Checked != 1 { - t.Fatalf("asura Checked = %d, want the one Series it read", asura.Checked) + kagane := latestPassFor(t, s, "kagane") + if kagane.Skip != "" || kagane.Refused != 2 || kagane.Due != 2 || kagane.GapMS == 0 { + t.Fatalf("kagane row = %+v, want an empty-skip double-refusal pass with 2 refused", kagane) } - // A pass that declines to look (kagane is now in backoff) must not restate - // the figures it never gathered as zeroes: the last real pass's due count - // and pace stand until a pass replaces them. + // With the refusal now durable, the next pass declines ahead of the loop: + // it records the refusing skip and carries the previous pass's figures + // forward rather than restating zeroes it never gathered (the carry logic + // itself is driven in TestRunLanePassCarryForwardOnlyWhenGapZero). before := kagane - if before.Due == 0 || before.Gap == 0 { - t.Fatalf("kagane after its refusing pass = %+v, want the figures that pass gathered", before) - } + p.Now = func() time.Time { return now.Add(time.Minute) } p.runOnce(context.Background()) - for _, lane := range p.LaneStatus().Lanes { - if lane.Site != "kagane" { - continue - } - if lane.Due != before.Due || lane.Gap != before.Gap { - t.Fatalf("kagane after a skipped pass = due %d gap %s, want the previous pass's %d / %s", - lane.Due, lane.Gap, before.Due, before.Gap) - } + again := latestPassFor(t, s, "kagane") + if again.Skip != SkipRefusing { + t.Fatalf("kagane refusing pass skip = %q, want %q", again.Skip, SkipRefusing) } - - // A lost sidecar reads as unreachable for the same window the Lanes skip. - p.setBrowserDown(now) - if st := p.LaneStatus(); !st.BrowserConfigured || st.BrowserReachable { - t.Fatalf("browser after loss = configured=%v reachable=%v, want true/false", st.BrowserConfigured, st.BrowserReachable) + if again.Due != before.Due || again.GapMS != before.GapMS || again.Checked != before.Checked { + t.Fatalf("kagane after a skipped pass = due %d gap %d, want the previous pass's %d / %d", + again.Due, again.GapMS, before.Due, before.GapMS) + } +} + +// latestPassFor reads a Site's newest durable pass row, failing rather than +// returning a zero LanePass a caller would assert against by accident. +func latestPassFor(t *testing.T, s *store.Store, site string) store.LanePass { + t.Helper() + pass, ok, err := s.LatestLanePass(site) + if err != nil || !ok { + t.Fatalf("LatestLanePass(%s): ok=%v err=%v", site, ok, err) + } + return pass +} + +// countPassRows counts a Site's durable pass rows, for asserting that a pass +// records exactly one. +func countPassRows(t *testing.T, dbURL, site string) int { + t.Helper() + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, site).Scan(&n); err != nil { + t.Fatalf("count passes for %s: %v", site, err) + } + return n +} + +// One durable pass row per exit, with the skip value naming the exit. The +// mid-loop browser-unreachable return also writes one row — but with an empty +// skip, so the row is stall-shaped (due > 0, checked 0, skip ”), matching the +// deliberate absence of a tenth skip value. +func TestRunLanePassRecordsEveryExit(t *testing.T) { + now := time.UnixMilli(5_000_000) + + t.Run("paused", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + p := newTestPoller(t, s, f, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { + t.Fatalf("paused pace = %s, want 30m (sleep until the expiry)", pace) + } + if f.callCount() != 0 { + t.Fatalf("fetches while paused = %d, want 0", f.callCount()) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got != 0 { + t.Fatalf("stamp while paused = %d, want 0 (Series stay due and unstamped)", got) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused { + t.Fatalf("skip = %q, want %q", pass.Skip, SkipPaused) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("refusing", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) + if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + browser := &fakeFetcher{status: 200} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + if pace := p.runLanePass(context.Background(), "kagane", false); pace != 10*time.Minute { + t.Fatalf("refusing pace = %s, want 10m", pace) + } + if browser.callCount() != 0 { + t.Fatalf("fetches while refusing = %d, want 0", browser.callCount()) + } + if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipRefusing { + t.Fatalf("skip = %q, want %q", pass.Skip, SkipRefusing) + } + if got := countPassRows(t, dbURL, "kagane"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("sidecar-down", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) + browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + p.setBrowserDown(now) // a sibling Lane lost Chrome within the backoff window + if pace := p.runLanePass(context.Background(), "kagane", false); pace != RefuseBackoff { + t.Fatalf("sidecar-down pace = %s, want %s", pace, RefuseBackoff) + } + if browser.callCount() != 0 { + t.Fatalf("fetches with the sidecar down = %d, want 0", browser.callCount()) + } + if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipSidecarDown { + t.Fatalf("skip = %q, want %q", pass.Skip, SkipSidecarDown) + } + if got := countPassRows(t, dbURL, "kagane"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("no-fetcher", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap { + t.Fatalf("no-fetcher pace = %s, want %s", pace, defaultGap) + } + pass := latestPassFor(t, s, "comix") + if pass.Skip != SkipNoFetcher || pass.GapMS != defaultGap.Milliseconds() { + t.Fatalf("no-fetcher pass = %+v, want skip %q with its own gap %s", pass, SkipNoFetcher, defaultGap) + } + if got := countPassRows(t, dbURL, "comix"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("due-query", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + // The due query is the pass's first store read after the gates; making + // it fail without touching poll_passes takes its tables away. + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + if _, err := db.Exec(`DROP TABLE bookmarks`); err != nil { + t.Fatalf("drop bookmarks: %v", err) + } + db.Close() + p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { + t.Fatalf("due-query pace = %s, want %s", pace, defaultGap) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipDueQuery { + t.Fatalf("skip = %q, want %q", pass.Skip, SkipDueQuery) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("asleep", func(t *testing.T) { + s, dbURL := newTestStore(t) + for i := 0; i < 3; i++ { + key := fmt.Sprintf("kagane:w%d", i) + seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], now.Add(-62*time.Minute).UnixMilli()) + } + browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + if pace := p.runLanePass(context.Background(), "kagane", false); pace != defaultGap { + t.Fatalf("asleep pace = %s, want %s", pace, defaultGap) + } + if browser.callCount() != 0 { + t.Fatalf("fetches while Chrome is asleep = %d, want 0", browser.callCount()) + } + pass := latestPassFor(t, s, "kagane") + if pass.Skip != SkipAsleep || pass.Due != 3 || pass.GapMS != defaultGap.Milliseconds() { + t.Fatalf("asleep pass = %+v, want skip %q with 3 due and the default gap", pass, SkipAsleep) + } + if got := countPassRows(t, dbURL, "kagane"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("eligible-count", func(t *testing.T) { + s, dbURL := newTestStore(t) + // The eligible query shares the due query's tables, so no real store + // failure reaches it after a successful due read; the seam is how the + // path is driven at all (see Poller.eligibleCount). + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.eligibleCount = func(string) (int, error) { return 0, errors.New("count failed") } + if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { + t.Fatalf("eligible-count pace = %s, want %s", pace, defaultGap) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipEligibleCount { + t.Fatalf("skip = %q, want %q", pass.Skip, SkipEligibleCount) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("nothing-eligible", func(t *testing.T) { + s, dbURL := newTestStore(t) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != sites["asura"].Rest { + t.Fatalf("nothing-eligible pace = %s, want a full rest %s", pace, sites["asura"].Rest) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipNothingEligible { + t.Fatalf("skip = %q, want %q", pass.Skip, SkipNothingEligible) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("reached the loop", func(t *testing.T) { + s, dbURL := newTestStore(t) + const url = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + seedForCheck(t, s, "asura:chronicles-of-the-demon-faction-f886a8af", url, 0) + p := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != defaultGap { + t.Fatalf("loop pace = %s, want %s", pace, defaultGap) + } + pass := latestPassFor(t, s, "asura") + if pass.Skip != "" || pass.Due != 1 || pass.Checked != 1 { + t.Fatalf("loop pass = %+v, want an empty skip with 1 due and 1 checked", pass) + } + if got := countPassRows(t, dbURL, "asura"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) + + t.Run("browser-unreachable mid-loop", func(t *testing.T) { + s, dbURL := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + interrupted := fmt.Errorf("%w: %w", errBrowserInterrupted, errors.New("restart")) + browser := &fakeFetcher{status: 200, err: interrupted} + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = browser + if pace := p.runLanePass(context.Background(), "comix", false); pace != defaultGap { + t.Fatalf("mid-loop pace = %s, want %s", pace, defaultGap) + } + // One row was still written, and it is stall-shaped by construction: + // empty skip with due > 0 and checked 0 because the return precedes + // the checked counter. The assertion below pins that shape precisely. + pass := latestPassFor(t, s, "comix") + if pass.Skip != "" || pass.Unreachable != 1 || pass.Checked != 0 || pass.Due != 1 { + t.Fatalf("mid-loop pass = %+v, want empty skip, unreachable 1, checked 0, due 1", pass) + } + if got := countPassRows(t, dbURL, "comix"); got != 1 { + t.Fatalf("pass rows = %d, want exactly 1", got) + } + }) +} + +// Carry-forward moves verbatim from the in-memory snapshot (issue #141): a +// pass that never computed its own gap carries the previous pass's due, gap, +// clamped and checked forward; a pass with a gap of its own records its own +// figures, zeroes included, beside the skip reason that explains them. +func TestRunLanePassCarryForwardOnlyWhenGapZero(t *testing.T) { + now := time.UnixMilli(5_000_000) + + t.Run("no gap of its own carries the previous pass's figures", func(t *testing.T) { + s, _ := newTestStore(t) + seedForCheck(t, s, "kagane:x", "https://kagane.to/series/x", 0) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{body: kaganeAPIFixture, status: 200} + p.runLanePass(context.Background(), "kagane", false) + first := latestPassFor(t, s, "kagane") + if first.Due != 1 || first.Checked != 1 || first.GapMS == 0 { + t.Fatalf("first pass = %+v, want a measured pass", first) + } + + if err := s.SetLaneRefusal("kagane", now.Add(10*time.Minute).UnixMilli()); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + // The pass row is keyed (site, ran_at), so the second pass needs its + // own timestamp: a minute later is still inside the refusal. + p.Now = func() time.Time { return now.Add(time.Minute) } + p.runLanePass(context.Background(), "kagane", false) + second := latestPassFor(t, s, "kagane") + if second.Skip != SkipRefusing { + t.Fatalf("second pass skip = %q, want %q", second.Skip, SkipRefusing) + } + if second.Due != first.Due || second.Checked != first.Checked || + second.GapMS != first.GapMS || second.Clamped != first.Clamped { + t.Fatalf("carried pass = %+v, want the first pass's figures %+v", second, first) + } + }) + + t.Run("a gap of its own records its own figures", func(t *testing.T) { + s, _ := newTestStore(t) + seedForCheck(t, s, "comix:c", "https://comix.to/title/c", 0) + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{body: comixSeriesFixture, status: 200} + p.runLanePass(context.Background(), "comix", false) + first := latestPassFor(t, s, "comix") + if first.Due != 1 || first.GapMS == 0 { + t.Fatalf("first pass = %+v, want a measured pass", first) + } + + // The no-fetcher exit sets its own gap, so no carry-forward: the due + // count it never gathered records as zero beside its skip reason. A + // minute later gives the second pass its own (site, ran_at) key. + p.BrowserFetch = nil + p.Now = func() time.Time { return now.Add(time.Minute) } + p.runLanePass(context.Background(), "comix", false) + second := latestPassFor(t, s, "comix") + if second.Skip != SkipNoFetcher { + t.Fatalf("second pass skip = %q, want %q", second.Skip, SkipNoFetcher) + } + if second.GapMS != defaultGap.Milliseconds() { + t.Fatalf("second pass gap = %d, want its own %d (not carried)", second.GapMS, defaultGap.Milliseconds()) + } + if second.Due != 0 || second.Checked != 0 { + t.Fatalf("second pass = %+v, want due 0 checked 0 (its own, not the previous pass's)", second) + } + }) +} + +// The pass row's five outcome counts are the classification the Series read +// already makes — never a second taxonomy (issue #141). Success is derived, +// never stored: checked minus the four named counts, unreachable excluded +// because its exit returns before the checked counter increments. +func TestRunLanePassCountsOutcomes(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + const ( + successKey = "asura:chronicles-of-the-demon-faction-f886a8af" + successURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seeds := map[string]string{ + "asura:refused": "https://asurascans.com/series/refused", + "asura:no-chapter": "https://asurascans.com/comics/no-chapter", + "asura:unfetchable": "https://evil.example/x", + "asura:transport": "https://asurascans.com/series/transport", + successKey: successURL, + } + for key, url := range seeds { + seedForCheck(t, s, key, url, 0) + } + + f := &fakeFetcher{perURL: map[string]fakeResponse{ + seeds["asura:refused"]: {status: 403}, + seeds["asura:no-chapter"]: {body: "", status: 200}, + seeds["asura:transport"]: {err: errors.New("dial tcp: refused")}, + successURL: {body: asuraSeriesFixture, status: 200}, + }} + newTestPoller(t, s, f, now).runLanePass(context.Background(), "asura", false) + + pass := latestPassFor(t, s, "asura") + if pass.Skip != "" || pass.Due != 5 || pass.Checked != 5 { + t.Fatalf("pass = %+v, want a full pass over 5 due Series", pass) + } + if pass.Refused != 1 || pass.Unreachable != 0 || pass.NoChapter != 1 || + pass.Unfetchable != 1 || pass.Errors != 1 { + t.Fatalf("outcome counts = refused %d unreachable %d no_chapter %d unfetchable %d errors %d, want 1 0 1 1 1", + pass.Refused, pass.Unreachable, pass.NoChapter, pass.Unfetchable, pass.Errors) + } + if success := pass.Checked - (pass.Refused + pass.NoChapter + pass.Unfetchable + pass.Errors); success != 1 { + t.Fatalf("derived success = %d, want 1", success) + } + // The one genuine read went through: the success Series carries the + // fixture's newest chapter, and none of the four failures do. + b, ok, err := s.Get(s.OwnerID(), successKey) + if err != nil || !ok { + t.Fatalf("Get: %v ok=%v", err, ok) + } + if b.LatestChapterNum == nil || *b.LatestChapterNum != 181 { + t.Fatalf("success Series latest = %v, want 181 (the fixture's newest)", b.LatestChapterNum) + } + for _, key := range []string{"asura:refused", "asura:no-chapter", "asura:unfetchable", "asura:transport"} { + b, _, err := s.Get(s.OwnerID(), key) + if err != nil { + t.Fatalf("Get %s: %v", key, err) + } + if b.LatestChapterNum != nil { + t.Fatalf("%s latest = %v, want nil (no chapter survived a failed read)", key, *b.LatestChapterNum) + } + } +} + +// A refusal is the Site's mood and outlives our process: the durable stamp a +// pass writes is honoured by a freshly constructed poller, which must not +// re-probe the Site inside its backoff (issue #141). +func TestDurableRefusalSurvivesFreshPoller(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + // Four due Series: the Lane refuses twice, stamps those two, and leaves + // the remaining two untried and due — the queue the fresh poller must + // find intact once the durable backoff lifts. + for i := 0; i < 4; i++ { + key := fmt.Sprintf("kagane:s%d", i) + seedForCheck(t, s, key, "https://kagane.to/series/"+key[7:], 0) + } + + p := newTestPoller(t, s, &fakeFetcher{status: 200}, now) + p.BrowserFetch = &fakeFetcher{status: 403} + p.runLanePass(context.Background(), "kagane", false) + if got := p.BrowserFetch.(*fakeFetcher).callCount(); got != 2 { + t.Fatalf("fetches on the refusing pass = %d, want 2 (refused twice)", got) + } + + // A restart: a brand-new poller, no in-memory refusal, same store. The + // durable stamp gates the pass. + fresh := newTestPoller(t, s, &fakeFetcher{status: 200}, now.Add(14*time.Minute)) + fresh.BrowserFetch = &fakeFetcher{status: 403} + fresh.runLanePass(context.Background(), "kagane", false) + if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 0 { + t.Fatalf("fetches by a fresh poller inside the backoff = %d, want 0", got) + } + if pass := latestPassFor(t, s, "kagane"); pass.Skip != SkipRefusing { + t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, SkipRefusing) + } + + // Past the backoff the fresh poller probes again — the two Series the + // original pass never reached, still due with their stamps untouched. + fresh.Now = func() time.Time { return now.Add(16 * time.Minute) } + fresh.runLanePass(context.Background(), "kagane", false) + if got := fresh.BrowserFetch.(*fakeFetcher).callCount(); got != 2 { + t.Fatalf("fetches after the backoff = %d, want 2", got) + } + for i := 2; i < 4; i++ { + if got := readLatestCheckedAt(t, s, fmt.Sprintf("kagane:s%d", i)); got == 0 { + t.Fatalf("kagane:s%d still untried after the backoff", i) + } + } +} + +// The pause is read ahead of the refusal check: a Lane that is both paused +// and inside a refusal backoff records the paused skip value, not the +// refusing one. The pause is the owner's order and outranks the Site's mood +// (issue #147). +func TestPauseGatePrecedesRefusalGate(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + if err := s.SetLaneRefusal("asura", now.Add(10*time.Minute).UnixMilli()); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + p := newTestPoller(t, s, f, now) + if pace := p.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { + t.Fatalf("paused-while-refusing pace = %s, want 30m (the pause's expiry)", pace) + } + if f.callCount() != 0 { + t.Fatalf("fetches while paused and refusing = %d, want 0", f.callCount()) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused { + t.Fatalf("skip = %q, want %q (the pause outranks the refusal)", pass.Skip, SkipPaused) + } +} + +// A pause is a fact about the Site, not about the process: a freshly +// constructed poller against a store holding a pause row stays paused until +// the expiry, then runs the Lane normally. The restart criterion is the +// whole point of writing a row instead of commanding a poller (issue #147). +func TestDurablePauseSurvivesFreshPoller(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + + // A restart: a brand-new poller, no in-memory state, same store. + fresh := newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now) + if pace := fresh.runLanePass(context.Background(), "asura", false); pace != 30*time.Minute { + t.Fatalf("fresh poller's paused pace = %s, want 30m", pace) + } + if pass := latestPassFor(t, s, "asura"); pass.Skip != SkipPaused { + t.Fatalf("fresh poller's pass skip = %q, want %q", pass.Skip, SkipPaused) + } + + // Past the expiry the same fresh poller runs the Lane normally. + fresh.Now = func() time.Time { return now.Add(31 * time.Minute) } + fresh.runLanePass(context.Background(), "asura", false) + if pass := latestPassFor(t, s, "asura"); pass.Skip != "" { + t.Fatalf("pass after the expiry skip = %q, want the loop reached", pass.Skip) + } + if got := readLatestCheckedAt(t, s, "asura:x"); got == 0 { + t.Fatal("the Series was not checked after the pause lifted") + } +} + +// ResumeLane zeroes the pause and the Lane's next pass finds its full queue +// waiting: a pause delays work rather than discarding it, so the due Series +// sit unstamped while paused and are all fetched once the pause lifts +// (issue #147). +func TestResumeLaneRestoresTheQueue(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + for i := 0; i < 3; i++ { + seedForCheck(t, s, fmt.Sprintf("asura:s%d", i), "https://asurascans.com/series/x", 0) + } + if err := s.PauseLane("asura", now.Add(30*time.Minute).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + + f := &fakeFetcher{body: asuraSeriesFixture, status: 200} + p := newTestPoller(t, s, f, now) + p.runLanePass(context.Background(), "asura", false) + if f.callCount() != 0 { + t.Fatalf("fetches while paused = %d, want 0", f.callCount()) + } + for i := 0; i < 3; i++ { + if got := readLatestCheckedAt(t, s, fmt.Sprintf("asura:s%d", i)); got != 0 { + t.Fatalf("asura:s%d stamp while paused = %d, want 0 (due and unstamped)", i, got) + } + } + + if err := s.ResumeLane("asura"); err != nil { + t.Fatalf("ResumeLane: %v", err) + } + before := f.callCount() + p.runLanePass(context.Background(), "asura", false) + if got := f.callCount() - before; got != 3 { + t.Fatalf("fetches after resume = %d, want 3 (the full due queue)", got) + } + for i := 0; i < 3; i++ { + if got := readLatestCheckedAt(t, s, fmt.Sprintf("asura:s%d", i)); got == 0 { + t.Fatalf("asura:s%d still untried after resume", i) + } + } +} + +// RecordLanePass prunes in the same call that inserts, so the retention +// cutoff the recorder passes is observable in what survives: a row just inside +// 14 days behind the poller's clock is kept, one just outside is pruned +// (issue #139, #141). +func TestPassRetentionCutoffIsFourteenDays(t *testing.T) { + s, dbURL := newTestStore(t) + // A real-world clock: the seeded rows sit 14 days back, so they must be + // positive timestamps or the seed's own prune (ran_at < 0) removes them. + now := time.UnixMilli(1_800_000_000_000) + kept := now.Add(-14*24*time.Hour + time.Minute).UnixMilli() + pruned := now.Add(-14*24*time.Hour - time.Minute).UnixMilli() + for _, ranAt := range []int64{kept, pruned} { + if err := s.RecordLanePass(store.LanePass{Site: "asura", RanAt: ranAt}, 0); err != nil { + t.Fatalf("seed pass at %d: %v", ranAt, err) + } + } + + seedForCheck(t, s, "asura:x", "https://asurascans.com/series/x", 0) + newTestPoller(t, s, &fakeFetcher{body: asuraSeriesFixture, status: 200}, now). + runLanePass(context.Background(), "asura", false) + + db, err := sql.Open("pgx", dbURL) + if err != nil { + t.Fatalf("open %s: %v", dbURL, err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1 AND ran_at = $2`, "asura", pruned).Scan(&n); err != nil { + t.Fatalf("count pruned row: %v", err) + } + if n != 0 { + t.Fatalf("row at %d survived, want it pruned (older than 14 days)", pruned) + } + if err := db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&n); err != nil { + t.Fatalf("count passes: %v", err) + } + if n != 2 { + t.Fatalf("passes = %d, want 2 (this pass plus the kept row)", n) + } +} + +// stampRecordingFetcher is a fakeFetcher that records the Series' check stamp +// at call time — the seam that proves the check stamp is written before the +// fetch (issue #146). If the order were swapped, the recorded stamp would be +// the pre-pass value and the ordering assertion would fail. +type stampRecordingFetcher struct { + fakeFetcher + store *store.Store + site string + seriesID string + stampAtCall int64 +} + +func (f *stampRecordingFetcher) Get(ctx context.Context, url string) (string, int, error) { + ts, err := f.store.LatestCheckedAt(f.site, f.seriesID) + if err != nil { + return "", 0, err + } + f.stampAtCall = ts + return f.fakeFetcher.Get(ctx, url) +} + +// The check stamp is written before the fetch is attempted: the fake fetcher +// records the stamp it sees at call time, and it must already be the pass's +// own stamp. The order is load-bearing — a forced request is pending while +// force_poll_at > latest_checked_at, so stamping after the fetch would make a +// failed forced request sticky — and the assertion fails if it is swapped. +func TestCheckStampIsWrittenBeforeFetch(t *testing.T) { + s, _ := newTestStore(t) + const ( + key = "asura:chronicles-of-the-demon-faction-f886a8af" + seriesURL = "https://asurascans.com/comics/chronicles-of-the-demon-faction-f886a8af" + ) + seedForCheck(t, s, key, seriesURL, 0) + now := time.UnixMilli(7_000_000) + rec := &stampRecordingFetcher{ + fakeFetcher: fakeFetcher{body: asuraSeriesFixture, status: 200}, + store: s, site: "asura", seriesID: "chronicles-of-the-demon-faction-f886a8af", + } + newTestPoller(t, s, rec, now).runOnce(context.Background()) + if rec.stampAtCall != now.UnixMilli() { + t.Fatalf("check stamp at fetch time = %d, want %d (the stamp must be written before the fetch)", rec.stampAtCall, now.UnixMilli()) + } +} + +// A forced Series whose attempt fails is no longer pending: the check stamp +// is written before the fetch, so the first attempt ends the pending state +// whatever it returns. A naive implementation (stamp only on success, or +// after the fetch) leaves the request sticky and the row due next pass. +func TestForcedSeriesSelfClearsOnFailedAttempt(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(7_000_000) + const url = "https://asurascans.com/comics/x" + // Freshly checked, so only the force flag makes it due. + seedForCheck(t, s, "asura:x", url, now.Add(-30*time.Minute).UnixMilli()) + if err := s.ForceSeriesPoll("asura", "x", now.UnixMilli()); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + + due, err := s.DueForLatestCheck("asura", now.Add(-time.Hour).UnixMilli(), -1) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 1 || due[0].Key() != "asura:x" || !due[0].Forced { + t.Fatalf("forced row not due and flagged before the attempt: %+v", due) + } + + // The attempt fails, but the attempt still happened: the row is stamped + // and no longer pending. + f := &fakeFetcher{err: errors.New("dial tcp: refused")} + newTestPoller(t, s, f, now).runOnce(context.Background()) + if got := readLatestCheckedAt(t, s, "asura:x"); got != now.UnixMilli() { + t.Fatalf("latest_checked_at = %d, want %d", got, now.UnixMilli()) + } + due, err = s.DueForLatestCheck("asura", now.Add(-time.Hour).UnixMilli(), -1) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 0 { + t.Fatalf("failed attempt left the forced row due: %v", due) + } +} + +// The refusal backoff is a Lane gate, not a Series gate: a forced Series does +// not override a Site that is actively refusing, because hand-forcing a +// request into a refusal only makes it worse (issue #146). +func TestForcedSeriesDoesNotOverrideRefusalBackoff(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + seedForCheck(t, s, "kagane:w", "https://kagane.to/series/w", 0) + if err := s.ForceSeriesPoll("kagane", "w", now.UnixMilli()); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + if err := s.SetLaneRefusal("kagane", now.Add(RefuseBackoff).UnixMilli()); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + + browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} + p := &Poller{ + Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser, + Now: func() time.Time { return now }, + } + p.runOnce(context.Background()) + if got := browser.callCount(); got != 0 { + t.Fatalf("browser fetches through a refusal backoff = %d, want 0", got) + } +} + +// A forced Series wakes a sleeping browser Lane: the wake thresholds exist to +// stop the machine waking itself for one unattended check, and a human asking +// is not that (issue #146). Below both thresholds the Lane still sleeps when +// nothing is forced. +func TestForcedSeriesWakesSleepingBrowser(t *testing.T) { + s, _ := newTestStore(t) + now := time.UnixMilli(5_000_000) + // Freshly due: checked two minutes before the rest elapses, so the wait + // is far below browserWakeAge and the count is under browserWakeCount. + seedForCheck(t, s, "kagane:w1", "https://kagane.to/series/w1", now.Add(-62*time.Minute).UnixMilli()) + seedForCheck(t, s, "kagane:w2", "https://kagane.to/series/w2", now.Add(-62*time.Minute).UnixMilli()) + + browser := &fakeFetcher{body: kaganeAPIFixture, status: 200} + p := &Poller{ + Store: s, Fetch: &fakeFetcher{status: 200}, BrowserFetch: browser, + Now: func() time.Time { return now }, + } + p.runOnce(context.Background()) + if got := browser.callCount(); got != 0 { + t.Fatalf("browser fetches without a forced series = %d, want 0 (Chrome stays asleep)", got) + } + + if err := s.ForceSeriesPoll("kagane", "w1", now.UnixMilli()); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + p.runOnce(context.Background()) + if got := browser.callCount(); got != 2 { + t.Fatalf("browser fetches with a forced series = %d, want 2 (the lane wakes)", got) } } diff --git a/backend/internal/latest/sites.go b/backend/internal/latest/sites.go index 64580b4..3be1f34 100644 --- a/backend/internal/latest/sites.go +++ b/backend/internal/latest/sites.go @@ -401,9 +401,10 @@ const ( // express (docs/research/cloudflare-bot-scoring-and-poll-cadence.md); // below it the Lane is outrunning its own plan and says so loudly. minGap = time.Second - // refuseBackoff is how long a Lane waits after its Site refused twice in - // one run before attempting it again. - refuseBackoff = 15 * time.Minute + // RefuseBackoff is how long a Lane waits after its Site refused twice in + // one run before attempting it again. Exported so the web layer can derive + // browser reachability from the pass log over the same window (issue #145). + RefuseBackoff = 15 * time.Minute // browserWakeCount and browserWakeAge gate a browser Lane's run: five or // more due Series, or any one of them waiting this long, or Chrome stays // asleep (ADR-0005 on-demand browser). @@ -511,6 +512,18 @@ var sites = map[string]site{ }, } +// SiteNames returns every registry Site, sorted. The admin Series list's Site +// select needs the full registry, not just the Sites that have rows, and +// laneNames() is the poller's copy of the same list — both read this. +func SiteNames() []string { + names := make([]string, 0, len(sites)) + for name := range sites { + names = append(names, name) + } + sort.Strings(names) + return names +} + // browserBackedSites is derived from the registry: the Sites whose pages are // read through the browser sidecar. Sorted so callers that range it (the // browser fetcher's dispatch) see a stable order instead of map-iteration diff --git a/backend/internal/latest/status.go b/backend/internal/latest/status.go deleted file mode 100644 index 49328a8..0000000 --- a/backend/internal/latest/status.go +++ /dev/null @@ -1,79 +0,0 @@ -package latest - -import "time" - -// LaneState is the administrative page's view of one Poll Lane (issue #102): -// what the Lane's last pass saw. Due, Gap and Checked are filled in as the -// pass computes them; a pass that returned before reaching a figure (refusal -// backoff, sidecar down) carries the previous pass's figures forward rather -// than overwriting them with zeroes the page would state as fact. -type LaneState struct { - Site string - Due int - LastRun time.Time - Gap time.Duration - // Checked is how many Series this pass actually read. A Lane with Series - // due and nothing checked has stopped working; one with nothing due is - // merely quiet, and the page must not draw the two the same (story 13). - Checked int - Clamped bool - Refusing bool - Browser bool - // Asleep marks a browser Lane whose last pass declined to wake Chrome - // because it was under both wake thresholds (ADR-0005). Due without - // Checked then means "waiting for the group to gather", not "stopped", and - // the page must not draw it as a stall. - Asleep bool -} - -// Status is the owner's page snapshot of the whole poller (issue #102). -type Status struct { - Lanes []LaneState - BrowserConfigured bool - BrowserReachable bool -} - -// LaneStatus returns a copy of the poller's Lane state for the owner's page. -// Only Sites that have completed a pass appear — a restart therefore renders -// "no data yet" instead of confident zeroes — in the same order Run iterates. -// Refusing is derived at snapshot time from the refusal backoff, not stored, -// so a Lane that cooled down between passes reports false without a new pass. -// BrowserReachable mirrors the Lanes' own gate: the sidecar is down only -// within the refuseBackoff window since its last loss. -func (p *Poller) LaneStatus() Status { - p.mu.Lock() - defer p.mu.Unlock() - lanes := make([]LaneState, 0, len(p.laneStates)) - now := p.Now() - for _, name := range laneNames() { - st, ok := p.laneStates[name] - if !ok { - continue - } - st.Refusing = now.Before(p.refuseUntil[name]) - lanes = append(lanes, st) - } - configured := p.BrowserFetch != nil - reachable := configured - if reachable && !p.browserDownAt.IsZero() && now.Sub(p.browserDownAt) < refuseBackoff { - reachable = false - } - return Status{Lanes: lanes, BrowserConfigured: configured, BrowserReachable: reachable} -} - -// recordLaneState stores one Lane's last pass for LaneStatus. Called deferred -// from runLanePass so every return path records, even a pass that refused. -// A pass that never reached the pace (Gap zero) keeps the last pass's figures: -// the Lane's due count and gap did not become zero because this pass declined -// to look, and the row's own marks say why it declined. -func (p *Poller) recordLaneState(st LaneState) { - p.mu.Lock() - defer p.mu.Unlock() - if p.laneStates == nil { - p.laneStates = make(map[string]LaneState) - } - if prev, ok := p.laneStates[st.Site]; ok && st.Gap == 0 { - st.Due, st.Gap, st.Clamped, st.Checked = prev.Due, prev.Gap, prev.Clamped, prev.Checked - } - p.laneStates[st.Site] = st -} diff --git a/backend/internal/store/admin.go b/backend/internal/store/admin.go new file mode 100644 index 0000000..7e78924 --- /dev/null +++ b/backend/internal/store/admin.go @@ -0,0 +1,276 @@ +package store + +import ( + "database/sql" + "fmt" + "strconv" + "strings" +) + +// Series filter names (issue #140), ordered permanent-then-fixable — the +// repairs nothing will ever undo first, the ones a Poll can make right after. +// A name is the repair a row needs, not the SQL that finds it; the values are +// the wire form the Series list URL carries (#142). "all" is the absent and +// unknown case: every Series. +const ( + SeriesFilterAll = "all" + SeriesFilterNoURL = "no_series_url" + SeriesFilterNoChapter = "never_read_a_chapter" + SeriesFilterNoReaders = "no_readers" + SeriesFilterNeverChecked = "never_checked" + SeriesFilterStale = "stale" + SeriesFilterNoCover = "no_cover" + SeriesFilterReaderReport = "reader_report" +) + +// SeriesFilter is one named hygiene predicate over the whole library. Site +// and Kind narrow the row read; Name picks the predicate; Cutoff is the +// staleness boundary the "stale" filter compares against, supplied by the +// caller's clock — the store has no clock; Page is 1-based. +type SeriesFilter struct { + Site string // "" = every Site + Kind string // "" = both library buckets' series + Name string // one of the SeriesFilter* constants; "" = SeriesFilterAll + Cutoff int64 // unix ms; "stale" reads it, the store never does + Page int // 1-based page of the row read; default 1 +} + +// adminSeriesColumns is the owner's library-wide Series projection in +// scanAdminSeries order. It is the privacy boundary: a Series' row carries +// the Reader id that raised its Latest Chapter (latest_raised_by), and that id +// must never leave the store package — so the projection does not select it, +// and only the anonymous boolean in raisedByReaderAnswer crosses it. +const adminSeriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover_address, + s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.force_poll_at` + +// raisedByReaderAnswer answers "did a Reader's report set this number" without +// naming which Reader. Kept apart from adminSeriesColumns so the column list — +// the shape scanAdminSeries is fed — stays free of the Sighting-raiser +// identity, and the owner learns which rows to act on and nothing about the +// Reader behind them. +const raisedByReaderAnswer = `(s.latest_raised_by IS NOT NULL) AS raised_by_reader` + +// seriesPageSize is the row read's page length. The tie-break in the query's +// ORDER BY is what makes this a stable page boundary — see SeriesPage. +const seriesPageSize = 50 + +// AdminSeries is one Series as the owner's library-wide view sees it: a +// Series-level fact plus an anonymous Reader count. ReaderCount being zero is +// the orphan marker. RaisedByReader is the only trace of the Sighting +// mechanism here; the Reader id behind it never reaches this type. +type AdminSeries struct { + Site string + SeriesID string + Title string + SeriesURL string + CoverAddress string // "" = no Cover yet + Kind string + LatestChapter string + LatestChapterNum *float64 // nil until first captured + LatestCheckedAt int64 + // ForcePollAt is the owner's "check now" request stamp (issue #146), zero + // meaning never asked. Pending is derived, never stored: a request is + // pending while ForcePollAt is newer than LatestCheckedAt. + ForcePollAt int64 + ReaderCount int + RaisedByReader bool // a Reader's report set LatestChapterNum +} + +// SeriesPage is one page of the owner's filtered Series list plus the count +// of every Series matching the same filter — a window number, not the page's +// len, so the landing page's figure and the list heading come from one query. +type SeriesPage struct { + Rows []AdminSeries + Total int +} + +// SiteSeriesShape is one Site's share of the Series matching a filter: how +// many, and the manga/novel split. One grouped pass, then library-wide totals +// are summed in Go over the rows — the landing page's per-Site table reads +// this and never pays for the rows the list discards. +type SiteSeriesShape struct { + Site string + Total int + Manga int + Novel int +} + +// Key returns the canonical identity in bookmark-key form (":"). +func (a AdminSeries) Key() string { return a.Site + ":" + a.SeriesID } + +// adminFilter maps a filter's named predicate to its compile-time WHERE and +// HAVING clauses and their bound parameters — the name never reaches query +// text, and Site and Kind bind as parameters. Shared by the row read and the +// per-Site aggregate so the two cannot disagree on what a filter means. +// +// The WHERE set is: no URL (an empty URL only — the host-failing-the-fetch- +// gate case is invisible to SQL, needs the Site registry in Go, and belongs to +// a later repair), never-read-a-chapter and never-checked as disjoint halves +// (non-zero versus zero check stamp), stale, no cover, and Reader-report. +// no_readers is the one HAVING predicate: it is the orphan test, an aggregate +// over the LEFT JOIN, where a bare WHERE has no row to test. +// +// stale is the checked-but-old half of the stamp partition — because the +// verdict line wants "not checked in twelve hours" as one figure, and a never +// checked Series is already counted on its own "waiting"/never-checked +// filter, folding it in would double-report it. The landing page computes the +// inclusive number as stale + never-checked. +func adminFilter(f SeriesFilter) (where, having string, args []any, err error) { + var clauses []string + if f.Kind != "" { + args = append(args, f.Kind) + clauses = append(clauses, "s.kind = $"+strconv.Itoa(len(args))) + } + switch f.Name { + case "", SeriesFilterAll: + case SeriesFilterNoURL: + clauses = append(clauses, `s.series_url = ''`) + case SeriesFilterNoChapter: + clauses = append(clauses, `s.latest_checked_at <> 0 AND s.latest_chapter_num IS NULL`) + case SeriesFilterNeverChecked: + clauses = append(clauses, `s.latest_checked_at = 0`) + case SeriesFilterStale: + clauses = append(clauses, `s.latest_checked_at > 0 AND s.latest_checked_at < $`+strconv.Itoa(len(args)+1)) + args = append(args, f.Cutoff) + case SeriesFilterNoCover: + clauses = append(clauses, `s.cover_address = ''`) + case SeriesFilterReaderReport: + clauses = append(clauses, `s.latest_raised_by IS NOT NULL`) + case SeriesFilterNoReaders: + having = `HAVING COUNT(b.reader_id) = 0` + default: + return "", "", nil, fmt.Errorf("unknown series filter %q", f.Name) + } + if len(clauses) > 0 { + where = "WHERE " + strings.Join(clauses, " AND ") + } + return where, having, args, nil +} + +// SeriesPage returns one page of the Series matching the filter, least +// recently checked first. The LEFT JOIN to Bookmarks is what surfaces the +// orphans that hygiene has to find — an inner join would hide them, exactly +// as the Lane's join does. ReaderCount is a plain count of every Bookmark on +// the Series, which knowingly disagrees with the two Lane queries for as long +// as the finished lifecycle bucket exists (#140). +// +// The tie-break is mandatory, not decorative: every unpollable Series shares a +// zero check stamp, so ordering on that column alone gives no stable page +// boundary and rows would repeat or vanish across pages. (site, series_id) is +// the primary key, hence total. The filtered total is a window count in the +// same query — window functions run after grouping and before the limit, so +// one where-clause cannot disagree with a second copy of itself. +func (s *Store) SeriesPage(f SeriesFilter) (SeriesPage, error) { + where, having, args, err := adminFilter(f) + if err != nil { + return SeriesPage{}, err + } + if f.Page < 1 { + f.Page = 1 + } + // Site narrowing is the row read's own; the aggregate must see every Site. + if f.Site != "" { + args = append(args, f.Site) + clause := "s.site = $" + strconv.Itoa(len(args)) + if where == "" { + where = "WHERE " + clause + } else { + where += " AND " + clause + } + } + base := len(args) + args = append(args, seriesPageSize, seriesPageSize*(f.Page-1)) + rows, err := s.db.Query(` + SELECT `+adminSeriesColumns+`, `+raisedByReaderAnswer+`, + COUNT(b.reader_id) AS reader_count, + COUNT(*) OVER () AS filtered_total + FROM series s + LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id + `+where+` + GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover_address, + s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, + s.force_poll_at, s.latest_raised_by + `+having+` + ORDER BY s.latest_checked_at, s.site, s.series_id + LIMIT $`+strconv.Itoa(base+1)+` OFFSET $`+strconv.Itoa(base+2), args...) + if err != nil { + return SeriesPage{}, fmt.Errorf("query series page: %w", err) + } + defer rows.Close() + + out := SeriesPage{} + for rows.Next() { + a, total, err := scanAdminSeries(rows.Scan) + if err != nil { + return SeriesPage{}, fmt.Errorf("scan series page: %w", err) + } + out.Rows = append(out.Rows, a) + out.Total = total + } + return out, rows.Err() +} + +// SeriesShapes returns each Site's share of the Series matching the filter, +// one grouped pass. Site and Page are row-read concerns and are ignored; the +// Landing page reads this per Site and sums the totals in Go for the +// library-wide figure. +func (s *Store) SeriesShapes(f SeriesFilter) ([]SiteSeriesShape, error) { + where, having, args, err := adminFilter(f) + if err != nil { + return nil, err + } + rows, err := s.db.Query(` + SELECT site, + COUNT(*) AS total, + COUNT(*) FILTER (WHERE kind = 'manga') AS manga, + COUNT(*) FILTER (WHERE kind = 'novel') AS novel + FROM ( + SELECT s.site, s.kind + FROM series s + LEFT JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id + `+where+` + GROUP BY s.site, s.series_id, s.kind + `+having+` + ) shape + GROUP BY site + ORDER BY site`, args...) + if err != nil { + return nil, fmt.Errorf("query series shapes: %w", err) + } + defer rows.Close() + + out := []SiteSeriesShape{} + for rows.Next() { + var sh SiteSeriesShape + if err := rows.Scan(&sh.Site, &sh.Total, &sh.Manga, &sh.Novel); err != nil { + return nil, fmt.Errorf("scan series shape: %w", err) + } + out = append(out, sh) + } + return out, rows.Err() +} + +// scanAdminSeries reads one row in adminSeriesColumns + raisedByReaderAnswer +// order, plus the query's reader_count and filtered_total columns, and returns +// the window total alongside the row. latest_chapter_num is NULL until first +// captured — the "never read a chapter" state. The Sighting-raiser column is +// never among the scanned columns. +func scanAdminSeries(scan func(...any) error) (AdminSeries, int, error) { + var ( + a AdminSeries + latestChapterNum sql.NullFloat64 + total int + ) + if err := scan( + &a.Site, &a.SeriesID, &a.Title, &a.SeriesURL, &a.CoverAddress, + &a.Kind, &a.LatestChapter, &latestChapterNum, &a.LatestCheckedAt, + &a.ForcePollAt, + &a.RaisedByReader, &a.ReaderCount, &total, + ); err != nil { + return AdminSeries{}, 0, err + } + if latestChapterNum.Valid { + a.LatestChapterNum = &latestChapterNum.Float64 + } + return a, total, nil +} diff --git a/backend/internal/store/admin_test.go b/backend/internal/store/admin_test.go new file mode 100644 index 0000000..f5f7a73 --- /dev/null +++ b/backend/internal/store/admin_test.go @@ -0,0 +1,396 @@ +package store + +import ( + "reflect" + "strconv" + "strings" + "testing" +) + +// seriesSeed describes one Series (and optionally its bookmarks) to stand up +// for an admin filter test. Direct SQL, because the filters separate rows the +// Upsert path could not produce together: an orphan has no bookmark, and a +// Reader-raised Latest Chapter needs a Sighting the store does not create. +type seriesSeed struct { + key string + kind string + url string + cover string // cover_address + checkedAt int64 + latestNum *float64 + bookmarks int // readers that hold it; 0 = orphan + raisedBy bool // a Reader's report is attributed as the raiser +} + +// seedAdminSeries inserts one series row and its bookmarks (owner first, then +// fresh readers), with the exact admin-relevant facts a test needs. +func seedAdminSeries(t *testing.T, s *Store, seed seriesSeed) { + t.Helper() + site, seriesID, ok := strings.Cut(seed.key, ":") + if !ok { + t.Fatalf("key %q: no ':' separator", seed.key) + } + if seed.kind == "" { + seed.kind = "manga" + } + var latestChapter any = "" + if seed.latestNum != nil { + latestChapter = "Chapter " + strconv.FormatFloat(*seed.latestNum, 'f', -1, 64) + } + if _, err := s.db.Exec(` + INSERT INTO series (site, series_id, title, kind, series_url, cover_address, + latest_checked_at, latest_chapter, latest_chapter_num) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover, + seed.checkedAt, latestChapter, seed.latestNum); err != nil { + t.Fatalf("seed series %q: %v", seed.key, err) + } + for i := range seed.bookmarks { + var readerID int64 = s.OwnerID() + if i > 0 { + readerID = secondReader(t, s) + } + if _, err := s.db.Exec(` + INSERT INTO bookmarks (reader_id, site, series_id, + last_chapter, last_chapter_num, last_chapter_url, + favorite, status, updated_at) + VALUES ($1, $2, $3, '', 0, '', false, 'reading', $4)`, + readerID, site, seriesID, seed.checkedAt); err != nil { + t.Fatalf("seed bookmark %q: %v", seed.key, err) + } + } + if seed.raisedBy { + if _, err := s.db.Exec( + `UPDATE series SET latest_raised_by = $1 WHERE site = $2 AND series_id = $3`, + s.OwnerID(), site, seriesID); err != nil { + t.Fatalf("seed raised-by %q: %v", seed.key, err) + } + } +} + +func pageKeys(t *testing.T, s *Store, f SeriesFilter) map[string]bool { + t.Helper() + page, err := s.SeriesPage(f) + if err != nil { + t.Fatalf("SeriesPage(%+v): %v", f, err) + } + keys := map[string]bool{} + for _, a := range page.Rows { + keys[a.Key()] = true + } + return keys +} + +// Each filter must return the rows it names and no others, over one shared +// seeded mix where every healthy neighbour is present to be wrongly returned. +// The stale cutoff is 5000: a Series checked at 9000 is current, at 2000 stale. +func TestAdminSeriesFilters(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:healthy", url: "https://asurascans.com/comics/healthy", cover: "aaa", checkedAt: 9000, latestNum: new(10.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:nourl", url: "", cover: "bbb", checkedAt: 9000, latestNum: new(5.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "https://asurascans.com/comics/nochapter", cover: "ccc", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "https://asurascans.com/comics/neverchecked", cover: "ddd", checkedAt: 0, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:orphan", url: "https://asurascans.com/comics/orphan", cover: "eee", checkedAt: 9000, latestNum: new(7.0), bookmarks: 0}) + seedAdminSeries(t, s, seriesSeed{key: "asura:stale", url: "https://asurascans.com/comics/stale", cover: "fff", checkedAt: 2000, latestNum: new(4.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:nocover", url: "https://asurascans.com/comics/nocover", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:report", url: "https://asurascans.com/comics/report", cover: "ggg", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1, raisedBy: true}) + + cases := []struct { + name string + f SeriesFilter + want []string + }{ + {"all", SeriesFilter{}, []string{"asura:healthy", "asura:nourl", "asura:nochapter", "asura:neverchecked", "asura:orphan", "asura:stale", "asura:nocover", "asura:report"}}, + {"no series url", SeriesFilter{Name: SeriesFilterNoURL}, []string{"asura:nourl"}}, + {"never read a chapter", SeriesFilter{Name: SeriesFilterNoChapter}, []string{"asura:nochapter"}}, + {"never checked", SeriesFilter{Name: SeriesFilterNeverChecked}, []string{"asura:neverchecked"}}, + {"no readers", SeriesFilter{Name: SeriesFilterNoReaders}, []string{"asura:orphan"}}, + {"stale", SeriesFilter{Name: SeriesFilterStale, Cutoff: 5000}, []string{"asura:stale"}}, + {"no cover", SeriesFilter{Name: SeriesFilterNoCover}, []string{"asura:nocover"}}, + {"reader report", SeriesFilter{Name: SeriesFilterReaderReport}, []string{"asura:report"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := pageKeys(t, s, tc.f) + want := map[string]bool{} + for _, k := range tc.want { + want[k] = true + } + if len(got) != len(want) { + t.Fatalf("%+v returned %v, want exactly %v", tc.f, got, want) + } + for k := range want { + if !got[k] { + t.Fatalf("%+v dropped %q (got %v)", tc.f, k, got) + } + } + }) + } +} + +// "Never read a chapter" and "never checked" are disjoint by construction: +// the first requires a non-zero check stamp, the second a zero one. Over a +// mix that should satisfy both, no row may be counted twice. +func TestAdminNeverChapterAndNeverCheckedAreDisjoint(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:nochapter", url: "u", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:neverchecked", url: "u", checkedAt: 0, bookmarks: 1}) + // A zero-stamp, no-chapter row is never-checked only: if never-read-a- + // chapter ever lost its non-zero-stamp guard, it would claim this row too + // and the two counts would double-report it. + seedAdminSeries(t, s, seriesSeed{key: "asura:both", url: "u", checkedAt: 0, bookmarks: 1}) + noChapter := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoChapter}) + neverChecked := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNeverChecked}) + for k := range noChapter { + if neverChecked[k] { + t.Fatalf("row %q matches both never-read-a-chapter and never-checked", k) + } + } + if !noChapter["asura:nochapter"] || !neverChecked["asura:neverchecked"] { + t.Fatalf("disjoint split lost its own rows: no-chapter=%v never-checked=%v", noChapter, neverChecked) + } +} + +// Several rows share a zero check stamp, so ordering on latest_checked_at +// alone gives no stable page boundary. The (site, series_id) tie-break must +// make page 2 a strict continuation of page 1: no repeat, no vanishing row. +func TestAdminSeriesPageTieBreakIsStable(t *testing.T) { + s := newTestStore(t) + const total = 53 // > one page, < two (page size 50) + for i := range total { + id := "tie" + strconv.Itoa(i) + seedAdminSeries(t, s, seriesSeed{key: "asura:" + id, url: "u", checkedAt: 0, bookmarks: 1}) + } + // A second Site's zero-stamp row is part of the same all-filter list, and + // must land on a valid page boundary rather than duplicating or dropping + // one of asura's rows: the tie-break is global (site, series_id). + seedAdminSeries(t, s, seriesSeed{key: "demonic:z", url: "u", checkedAt: 0, bookmarks: 1}) + wantTotal := total + 1 + + p1, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll}) + if err != nil { + t.Fatalf("SeriesPage page 1: %v", err) + } + p2, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 2}) + if err != nil { + t.Fatalf("SeriesPage page 2: %v", err) + } + if len(p1.Rows) != seriesPageSize { + t.Fatalf("page 1 has %d rows, want %d", len(p1.Rows), seriesPageSize) + } + seen := map[string]bool{} + for _, a := range append(append([]AdminSeries{}, p1.Rows...), p2.Rows...) { + if seen[a.Key()] { + t.Fatalf("row %q repeats across pages", a.Key()) + } + seen[a.Key()] = true + } + if len(seen) != wantTotal { + t.Fatalf("%d distinct rows across pages, want %d (a row vanished)", len(seen), wantTotal) + } + if p1.Total != wantTotal { + t.Fatalf("page total = %d, want %d (the window count must span pages)", p1.Total, wantTotal) + } + // A page beyond the end is empty, not an error (the list re-reads page 1). + // The window count runs over the rows present in the result set, so an + // overflow page has no rows and therefore no total — the caller must not + // render it, which is exactly why the list re-reads page 1. + pFinal, err := s.SeriesPage(SeriesFilter{Name: SeriesFilterAll, Page: 99}) + if err != nil { + t.Fatalf("SeriesPage beyond end: %v", err) + } + if len(pFinal.Rows) != 0 { + t.Fatalf("page beyond end = %d rows, want 0", len(pFinal.Rows)) + } +} + +// The filtered total is the window number over the same filter the rows use, +// and the per-Site aggregate sums to the same figure — so the landing page's +// count and the list's heading can never disagree, whoever computes them. +func TestAdminTotalAgreesWithRowsAndShapes(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", cover: "a", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:b", url: "u", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "asura:c", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(2.0), raisedBy: true}) + seedAdminSeries(t, s, seriesSeed{key: "demonic:d", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1}) + + filters := []SeriesFilter{ + {}, + {Name: SeriesFilterNoCover}, + {Name: SeriesFilterReaderReport}, + {Name: SeriesFilterNoChapter}, + } + for _, f := range filters { + page, err := s.SeriesPage(f) + if err != nil { + t.Fatalf("SeriesPage(%+v): %v", f, err) + } + want := len(page.Rows) + if f.Page == 0 && want == seriesPageSize { + t.Fatalf("seed produced a full page; bump the seed or drop page size in the test") + } + if page.Total != want { + t.Fatalf("%+v total = %d, want %d (window count disagrees with row count)", f, page.Total, want) + } + shapes, err := s.SeriesShapes(f) + if err != nil { + t.Fatalf("SeriesShapes(%+v): %v", f, err) + } + sum := 0 + for _, sh := range shapes { + sum += sh.Total + } + if sum != want { + t.Fatalf("%+v aggregate sum = %d, want %d (aggregate disagrees with row query)", f, sum, want) + } + } + + // The default filter's aggregate carries the library shape: per-Site + // totals and the manga/novel split, summed in Go for library wide. + shapes, err := s.SeriesShapes(SeriesFilter{}) + if err != nil { + t.Fatalf("SeriesShapes default: %v", err) + } + if len(shapes) != 2 || shapes[0].Site != "asura" || shapes[1].Site != "demonic" { + t.Fatalf("shapes = %+v, want asura then demonic", shapes) + } + if shapes[0].Total != 3 || shapes[0].Manga != 3 || shapes[0].Novel != 0 { + t.Fatalf("asura shape = %+v, want 3 manga, 0 novel", shapes[0]) + } + if shapes[1].Total != 1 || shapes[1].Manga != 0 || shapes[1].Novel != 1 { + t.Fatalf("demonic shape = %+v, want 1 novel", shapes[1]) + } +} + +// Site and Library narrowing stack on a named filter without changing what +// the filter means. +func TestAdminFilterSiteAndKindNarrow(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:aa", url: "u", checkedAt: 9000, bookmarks: 1, latestNum: new(1.0)}) + seedAdminSeries(t, s, seriesSeed{key: "asura:ab", url: "", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "demonic:aa", url: "u", kind: "novel", checkedAt: 9000, bookmarks: 1}) + seedAdminSeries(t, s, seriesSeed{key: "demonic:ab", url: "", kind: "novel", checkedAt: 9000, bookmarks: 1}) + + got := pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Site: "asura"}) + if len(got) != 1 || !got["asura:ab"] { + t.Fatalf("site+nourl = %v, want only asura:ab", got) + } + got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"}) + if len(got) != 1 || !got["demonic:ab"] { + t.Fatalf("kind+nourl = %v, want only demonic:ab", got) + } + got = pageKeys(t, s, SeriesFilter{Name: SeriesFilterAll, Site: "demonic", Kind: "novel"}) + if len(got) != 2 || !got["demonic:aa"] || !got["demonic:ab"] { + t.Fatalf("site+kind+all = %v, want both demonic rows", got) + } + + // The aggregate ignores the Site narrowing (it is per-Site by shape), but + // honours the Library narrowing: asura's missing-URL row is manga, so the + // novel no-URL list is demonic alone. + shapes, err := s.SeriesShapes(SeriesFilter{Name: SeriesFilterNoURL, Kind: "novel"}) + if err != nil { + t.Fatalf("SeriesShapes: %v", err) + } + if len(shapes) != 1 || shapes[0].Site != "demonic" || + shapes[0].Total != 1 || shapes[0].Novel != 1 { + t.Fatalf("novel no-URL aggregate = %+v, want demonic {Total:1 Novel:1}", shapes) + } +} + +// The projection is the privacy boundary: a Series whose Latest Chapter was +// raised by a Reader's report reads back with the anonymous boolean set, not +// with the Reader's id, and no Reader id travels in any returned row. +func TestAdminSeriesReportsAnonymously(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:raised", url: "u", checkedAt: 9000, latestNum: new(9.0), bookmarks: 1, raisedBy: true}) + seedAdminSeries(t, s, seriesSeed{key: "asura:polled", url: "u", checkedAt: 9000, latestNum: new(8.0), bookmarks: 1}) + + page, err := s.SeriesPage(SeriesFilter{}) + if err != nil { + t.Fatalf("SeriesPage: %v", err) + } + byKey := map[string]AdminSeries{} + for _, a := range page.Rows { + byKey[a.Key()] = a + } + if !byKey["asura:raised"].RaisedByReader { + t.Fatal("Reader-raised Series read back RaisedByReader=false") + } + if byKey["asura:polled"].RaisedByReader { + t.Fatal("Poll-raised Series read back RaisedByReader=true") + } +} + +// The privacy test that cannot rot into a template-only guarantee: assert the +// admin column constant does not carry the Sighting-raiser column and that the +// admin row type has no field for it, modelled on the guard on the Bookmark +// column list. +func TestAdminProjectionHidesSightingRaiser(t *testing.T) { + if strings.Contains(adminSeriesColumns, "latest_raised_by") { + t.Fatal("admin column list carries latest_raised_by: the Sighting-raiser id would reach the owner") + } + if _, ok := reflect.TypeOf(AdminSeries{}).FieldByName("LatestRaisedBy"); ok { + t.Fatal("AdminSeries carries a field for the Sighting-raiser id") + } +} + +// An unknown filter name is rejected rather than silently meaning "all" — +// otherwise a mistyped URL would present an empty page as the whole library. +func TestAdminFilterUnknownNameRejected(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:a", url: "u", checkedAt: 9000, bookmarks: 1}) + for name, call := range map[string]func() error{ + "page": func() error { _, err := s.SeriesPage(SeriesFilter{Name: "bogus"}); return err }, + "shape": func() error { _, err := s.SeriesShapes(SeriesFilter{Name: "bogus"}); return err }, + } { + if err := call(); err == nil || !strings.Contains(err.Error(), "unknown series filter") { + t.Fatalf("%s with bogus filter = %v, want unknown-filter error", name, err) + } + } +} + +// ForceSeriesPoll is the idempotent stamp write: a second press overwrites +// the request time, and touching a missing series is not an error. +func TestForceSeriesPollStampsIdempotently(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 9000, bookmarks: 1}) + + if err := s.ForceSeriesPoll("asura", "x", 42); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + if err := s.ForceSeriesPoll("asura", "x", 99); err != nil { + t.Fatalf("ForceSeriesPoll re-stamp: %v", err) + } + // Touching a missing series is not an error: the row may have been + // orphaned, and the caller's read decides what exists. + if err := s.ForceSeriesPoll("asura", "ghost", 99); err != nil { + t.Fatalf("ForceSeriesPoll missing: %v", err) + } + + var got int64 + if err := s.db.QueryRow( + `SELECT force_poll_at FROM series WHERE site = 'asura' AND series_id = 'x'`).Scan(&got); err != nil { + t.Fatalf("read force_poll_at: %v", err) + } + if got != 99 { + t.Fatalf("force_poll_at = %d, want 99 (the later press wins)", got) + } +} + +// The admin projection carries the force stamp so the web layer can derive +// the pending flag without a second read. +func TestAdminSeriesCarriesForcePollAt(t *testing.T) { + s := newTestStore(t) + seedAdminSeries(t, s, seriesSeed{key: "asura:x", url: "u", checkedAt: 1000, bookmarks: 1}) + if err := s.ForceSeriesPoll("asura", "x", 5000); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + + page, err := s.SeriesPage(SeriesFilter{}) + if err != nil { + t.Fatalf("SeriesPage: %v", err) + } + if len(page.Rows) != 1 || page.Rows[0].ForcePollAt != 5000 { + t.Fatalf("row = %+v, want ForcePollAt 5000", page.Rows) + } +} diff --git a/backend/internal/store/migrations/0012_poll_lanes.sql b/backend/internal/store/migrations/0012_poll_lanes.sql new file mode 100644 index 0000000..ff56f12 --- /dev/null +++ b/backend/internal/store/migrations/0012_poll_lanes.sql @@ -0,0 +1,6 @@ +-- One durable state row per Poll Lane. Zero means no pause or refusal is set. +CREATE TABLE poll_lanes ( + site text NOT NULL PRIMARY KEY, + paused_until bigint NOT NULL DEFAULT 0, + refuse_until bigint NOT NULL DEFAULT 0 +); diff --git a/backend/internal/store/migrations/0013_poll_passes.sql b/backend/internal/store/migrations/0013_poll_passes.sql new file mode 100644 index 0000000..e5095ee --- /dev/null +++ b/backend/internal/store/migrations/0013_poll_passes.sql @@ -0,0 +1,16 @@ +-- Append-only Lane Pass log. Timestamps are unix milliseconds from the poller's clock. +CREATE TABLE poll_passes ( + site text NOT NULL, + ran_at bigint NOT NULL, + skip text NOT NULL, + due integer NOT NULL, + checked integer NOT NULL, + gap_ms bigint NOT NULL, + clamped boolean NOT NULL, + refused integer NOT NULL, + unreachable integer NOT NULL, + no_chapter integer NOT NULL, + unfetchable integer NOT NULL, + errors integer NOT NULL, + PRIMARY KEY (site, ran_at) +); diff --git a/backend/internal/store/migrations/0014_admin_read_model.sql b/backend/internal/store/migrations/0014_admin_read_model.sql new file mode 100644 index 0000000..af059e5 --- /dev/null +++ b/backend/internal/store/migrations/0014_admin_read_model.sql @@ -0,0 +1,11 @@ +-- Admin read-model foundation (#140). The Series list's default order is +-- least-recently-checked first, so the table — which has only its primary key +-- today — gets an index that can serve it. A grouped query over a join may +-- ignore the index, so this is a judgement, not a measurement: re-time on real +-- data before adding a second. +CREATE INDEX series_latest_checked_at_idx ON series (latest_checked_at); + +-- force_poll_at is the "ask for one Series to be checked now" stamp (#146). +-- Zero means never forced; nothing reads the column before that ticket wires +-- it, so it lands here unused. +ALTER TABLE series ADD COLUMN force_poll_at bigint NOT NULL DEFAULT 0; diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index f102b63..da4dd0d 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -88,6 +88,39 @@ type Series struct { // readerCount is the number of bookmarks referencing this series, filled // only by the due-queue query that orders on it. readerCount int + // Forced is whether the owner asked for a check now (issue #146): the + // request stamp is newer than the check stamp. Derived in the due query, + // never stored, and the flag that jumps the queue and opens the browser + // wake gate. + Forced bool +} + +// LanePass is one Poll Lane's durable pass snapshot. Pause and refusal stamps +// are joined from poll_lanes on read; they are not pass facts. +type LanePass struct { + Site string + RanAt int64 + Skip string + Due, Checked int + GapMS int64 + Clamped bool + Refused, Unreachable, NoChapter int + Unfetchable, Errors int + PausedUntil, RefuseUntil int64 +} + +// SiteOutcomes is one Site's summed Lane Pass outcomes over a caller-supplied +// window. +type SiteOutcomes struct { + Site string + Refused, Unreachable, NoChapter int + Unfetchable, Errors int +} + +// LanePause is one persisted Lane pause stamp. +type LanePause struct { + Site string + PausedUntil int64 } // Key returns the canonical identity in bookmark-key form (":"), @@ -188,9 +221,9 @@ const ( //go:embed migrations/*.sql var migrations embed.FS -// bookmarkColumns is the only value ever concatenated into query text. It is a -// compile-time constant; every request value is bound as a parameter. The -// series-owned fields are joined in from the series table, in scanBookmark +// These column lists are the only values ever concatenated into query text. +// They are compile-time constants; every request value is bound as a parameter. +// The series-owned fields are joined in from the series table, in scanBookmark // order, so the flat Bookmark reads back whole despite the split (ADR-0004). const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_address, b.last_chapter, b.last_chapter_num, b.last_chapter_url, @@ -202,6 +235,10 @@ const bookmarkColumns = `b.site, b.series_id, s.title, s.series_url, s.cover_add const seriesColumns = `s.site, s.series_id, s.title, s.series_url, s.cover, s.cover_address, s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, s.latest_raised_by` +const lanePassColumns = `p.site, p.ran_at, p.skip, p.due, p.checked, p.gap_ms, p.clamped, + p.refused, p.unreachable, p.no_chapter, p.unfetchable, p.errors, + COALESCE(l.paused_until, 0), COALESCE(l.refuse_until, 0)` + // Owner is the person running the service: the first Reader, seeded at startup // so a fresh deployment has a library before anyone logs in. The seed makes // sure exactly one readers row matches their Discord ID, carrying the SHA-256 @@ -589,8 +626,9 @@ func (s *Store) scanBookmark(scan func(...any) error) (Bookmark, error) { } // scanSeries reads one row in seriesColumns order, plus the due query's -// reader_count column. latest_chapter_num and latest_raised_by are both -// nullable, same as latest_chapter_num on the bookmark read path. +// forced flag and reader_count columns. latest_chapter_num and +// latest_raised_by are both nullable, same as latest_chapter_num on the +// bookmark read path. func scanSeries(scan func(...any) error) (Series, error) { var ( sr Series @@ -600,7 +638,7 @@ func scanSeries(scan func(...any) error) (Series, error) { if err := scan( &sr.Site, &sr.SeriesID, &sr.Title, &sr.SeriesURL, &sr.Cover, &sr.CoverAddress, &sr.Kind, &sr.LatestChapter, &latestChapterNum, &sr.LatestCheckedAt, &latestRaisedBy, - &sr.readerCount, + &sr.Forced, &sr.readerCount, ); err != nil { return Series{}, err } @@ -613,6 +651,18 @@ func scanSeries(scan func(...any) error) (Series, error) { return sr, nil } +func scanLanePass(scan func(...any) error) (LanePass, error) { + var p LanePass + if err := scan( + &p.Site, &p.RanAt, &p.Skip, &p.Due, &p.Checked, &p.GapMS, &p.Clamped, + &p.Refused, &p.Unreachable, &p.NoChapter, &p.Unfetchable, &p.Errors, + &p.PausedUntil, &p.RefuseUntil, + ); err != nil { + return LanePass{}, err + } + return p, nil +} + // Close releases the underlying database handle. func (s *Store) Close() error { return s.db.Close() } @@ -934,6 +984,184 @@ func (s *Store) Delete(readerID int64, key string) error { return nil } +// RecordLanePass appends one pass and prunes every older row in the same +// transaction. retainBefore is supplied by the poller's clock. +func (s *Store) RecordLanePass(p LanePass, retainBefore int64) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin lane pass %s: %w", p.Site, err) + } + defer tx.Rollback() + if _, err := tx.Exec(` + INSERT INTO poll_passes + (site, ran_at, skip, due, checked, gap_ms, clamped, + refused, unreachable, no_chapter, unfetchable, errors) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + p.Site, p.RanAt, p.Skip, p.Due, p.Checked, p.GapMS, p.Clamped, + p.Refused, p.Unreachable, p.NoChapter, p.Unfetchable, p.Errors); err != nil { + return fmt.Errorf("insert lane pass %s at %d: %w", p.Site, p.RanAt, err) + } + if _, err := tx.Exec(`DELETE FROM poll_passes WHERE ran_at < $1`, retainBefore); err != nil { + return fmt.Errorf("prune lane passes before %d: %w", retainBefore, err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit lane pass %s at %d: %w", p.Site, p.RanAt, err) + } + return nil +} + +// LatestLanePass returns the newest pass for one Site, with its current Lane +// state joined on. A Site without a pass has no durable snapshot yet. +func (s *Store) LatestLanePass(site string) (LanePass, bool, error) { + p, err := scanLanePass(s.db.QueryRow(`SELECT `+lanePassColumns+` + FROM poll_passes p + LEFT JOIN poll_lanes l ON l.site = p.site + WHERE p.site = $1 + ORDER BY p.ran_at DESC + LIMIT 1`, site).Scan) + if errors.Is(err, sql.ErrNoRows) { + return LanePass{}, false, nil + } + if err != nil { + return LanePass{}, false, fmt.Errorf("latest lane pass %s: %w", site, err) + } + return p, true, nil +} + +// LatestLanePasses returns the newest pass for each Site, with current Lane +// state joined on. Sites without a pass have no row yet. +func (s *Store) LatestLanePasses() ([]LanePass, error) { + rows, err := s.db.Query(`SELECT ` + lanePassColumns + ` + FROM ( + SELECT DISTINCT ON (site) + site, ran_at, skip, due, checked, gap_ms, clamped, + refused, unreachable, no_chapter, unfetchable, errors + FROM poll_passes + ORDER BY site, ran_at DESC + ) p + LEFT JOIN poll_lanes l ON l.site = p.site + ORDER BY p.site`) + if err != nil { + return nil, fmt.Errorf("query latest lane passes: %w", err) + } + defer rows.Close() + + out := []LanePass{} + for rows.Next() { + p, err := scanLanePass(rows.Scan) + if err != nil { + return nil, fmt.Errorf("scan latest lane pass: %w", err) + } + out = append(out, p) + } + return out, rows.Err() +} + +// LanePassOutcomes sums the named outcomes for each Site at or after since. +// The window boundary is supplied by the caller; the store has no clock. +func (s *Store) LanePassOutcomes(since int64) ([]SiteOutcomes, error) { + rows, err := s.db.Query(` + SELECT site, SUM(refused), SUM(unreachable), SUM(no_chapter), + SUM(unfetchable), SUM(errors) + FROM poll_passes + WHERE ran_at >= $1 + GROUP BY site + ORDER BY site`, since) + if err != nil { + return nil, fmt.Errorf("query lane pass outcomes: %w", err) + } + defer rows.Close() + + out := []SiteOutcomes{} + for rows.Next() { + var outcomes SiteOutcomes + if err := rows.Scan( + &outcomes.Site, &outcomes.Refused, &outcomes.Unreachable, + &outcomes.NoChapter, &outcomes.Unfetchable, &outcomes.Errors, + ); err != nil { + return nil, fmt.Errorf("scan lane pass outcomes: %w", err) + } + out = append(out, outcomes) + } + return out, rows.Err() +} + +// SetLaneRefusal persists a Site's refusal backoff stamp without touching its +// pause. until is supplied by the caller's clock. +func (s *Store) SetLaneRefusal(site string, until int64) error { + if _, err := s.db.Exec(` + INSERT INTO poll_lanes (site, refuse_until) VALUES ($1, $2) + ON CONFLICT (site) DO UPDATE SET refuse_until = EXCLUDED.refuse_until`, site, until); err != nil { + return fmt.Errorf("set lane refusal %s: %w", site, err) + } + return nil +} + +// PauseLane persists a bounded pause. The caller must ensure until is after +// its current timestamp; the store has no clock and rejects only the invalid +// zero and negative sentinels. +func (s *Store) PauseLane(site string, until int64) error { + if until <= 0 { + return fmt.Errorf("pause lane %s: expiry must be positive", site) + } + if _, err := s.db.Exec(` + INSERT INTO poll_lanes (site, paused_until) VALUES ($1, $2) + ON CONFLICT (site) DO UPDATE SET paused_until = EXCLUDED.paused_until`, site, until); err != nil { + return fmt.Errorf("pause lane %s: %w", site, err) + } + return nil +} + +// ResumeLane clears only the pause stamp and keeps the Lane state row, along +// with any refusal stamp already persisted on it. +func (s *Store) ResumeLane(site string) error { + if _, err := s.db.Exec( + `UPDATE poll_lanes SET paused_until = 0 WHERE site = $1`, site); err != nil { + return fmt.Errorf("resume lane %s: %w", site, err) + } + return nil +} + +// PausedLanes returns Lane rows with a nonzero pause stamp. Expiry comparison +// stays with the caller because the store is deliberately clockless. +func (s *Store) PausedLanes() ([]LanePause, error) { + rows, err := s.db.Query(` + SELECT site, paused_until + FROM poll_lanes + WHERE paused_until > 0 + ORDER BY site`) + if err != nil { + return nil, fmt.Errorf("query paused lanes: %w", err) + } + defer rows.Close() + + out := []LanePause{} + for rows.Next() { + var pause LanePause + if err := rows.Scan(&pause.Site, &pause.PausedUntil); err != nil { + return nil, fmt.Errorf("scan paused lane: %w", err) + } + out = append(out, pause) + } + return out, rows.Err() +} + +// LaneGates reads a Site's pause and refusal stamps in one row read — the +// top-of-pass gate the poller uses (issue #141). A missing state row is the +// default: unpaused and not refusing. +func (s *Store) LaneGates(site string) (pausedUntil, refuseUntil int64, err error) { + err = s.db.QueryRow( + `SELECT paused_until, refuse_until FROM poll_lanes WHERE site = $1`, site). + Scan(&pausedUntil, &refuseUntil) + if errors.Is(err, sql.ErrNoRows) { + return 0, 0, nil + } + if err != nil { + return 0, 0, fmt.Errorf("lane gates %s: %w", site, err) + } + return pausedUntil, refuseUntil, nil +} + // DueForLatestCheck returns one Site's series whose server-side // latest-chapter check has aged past cutoffMs, ordered by how many bookmarks // reference them (descending) then least-recently-checked first. One Site per @@ -942,6 +1170,13 @@ func (s *Store) Delete(readerID int64, key string) error { // limit — the Lane's own gap paces the fetches, and the batch size that used // to cap this query is gone with the shared pace. // +// A forced Series (force_poll_at newer than latest_checked_at, issue #146) +// overrides exactly three gates: the rest cutoff, the Sighting-deferral +// clause and the finished-only bucket. It never overrides an empty +// series_url or the Bookmarks join — nothing to fetch, and no consumer for +// the result — so those stay unconditional. Forced rows sort to the front of +// the queue; the reader-count-then-age ordering among the rest is ADR-0003. +// // The reader_count ordering is the point of the split (ADR-0003): a series // shared by several readers is fetched once per due cycle, and the popular // ones stay freshest while the long tail absorbs any shortfall. Within one @@ -970,19 +1205,26 @@ func (s *Store) Delete(readerID int64, key string) error { // allowed to defer at all was settled when the Sighting was recorded — see // RecordSighting. func (s *Store) DueForLatestCheck(site string, cutoffMs, ceilingMs int64) ([]Series, error) { - rows, err := s.db.Query(`SELECT `+seriesColumns+`, COUNT(*) AS reader_count + rows, err := s.db.Query(`SELECT `+seriesColumns+`, + (s.force_poll_at > s.latest_checked_at) AS forced, + COUNT(*) AS reader_count FROM series s JOIN bookmarks b ON b.site = s.site AND b.series_id = s.series_id WHERE s.site = $1 AND s.series_url <> '' - AND s.latest_checked_at <= $2::bigint + AND (s.latest_checked_at <= $2::bigint + OR s.force_poll_at > s.latest_checked_at) GROUP BY s.site, s.series_id, s.title, s.series_url, s.cover, - s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at - HAVING COUNT(*) FILTER (WHERE b.status <> 'finished') > 0 + s.kind, s.latest_chapter, s.latest_chapter_num, s.latest_checked_at, + s.force_poll_at + HAVING (COUNT(*) FILTER (WHERE b.status <> 'finished') > 0 + OR s.force_poll_at > s.latest_checked_at) AND (COUNT(*) > 1 OR s.latest_sighted_at <= $2::bigint - OR s.latest_checked_at <= $3::bigint) - ORDER BY reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs) + OR s.latest_checked_at <= $3::bigint + OR s.force_poll_at > s.latest_checked_at) + ORDER BY (s.force_poll_at > s.latest_checked_at) DESC, + reader_count DESC, s.latest_checked_at ASC`, site, cutoffMs, ceilingMs) if err != nil { return nil, fmt.Errorf("query due series: %w", err) } @@ -1042,6 +1284,25 @@ func (s *Store) MarkLatestChecked(site, seriesID string, ts int64) error { return nil } +// ForceSeriesPoll stamps a Series with the owner's "check now" request +// (issue #146): a fact about the Series the Lane's next pass reads through +// DueForLatestCheck, never a command to the poller — so the request survives +// a restart. Writing again overwrites the request time; the write is +// idempotent. Touching a missing series is not an error: the row may have +// been orphaned, and the caller's read decides what exists. The stamp never +// expires by itself — an unanswered request keeps ageing — and pending is +// derived as force_poll_at > latest_checked_at, which is why the poller's +// check stamp is written before the fetch: the first attempt ends the +// pending state whatever it returns. +func (s *Store) ForceSeriesPoll(site, seriesID string, at int64) error { + if _, err := s.db.Exec( + `UPDATE series SET force_poll_at = $1 WHERE site = $2 AND series_id = $3`, + at, site, seriesID); err != nil { + return fmt.Errorf("force poll %s:%s: %w", site, seriesID, err) + } + return nil +} + // LatestCheckedAt reads the column MarkLatestChecked writes. It exists for // tests outside this package (the poller's own tests assert on rest // bookkeeping) — see MarkLatestChecked for why the field stays off the diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index 1b4a83b..4aa3b9d 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -758,7 +758,7 @@ func TestMigration0008DropsLegacyCoverRows(t *testing.T) { func readSeries(t *testing.T, s *Store, site, seriesID string) Series { t.Helper() sr, err := scanSeries(s.db.QueryRow( - `SELECT `+seriesColumns+`, 0 AS reader_count FROM series s + `SELECT `+seriesColumns+`, false AS forced, 0 AS reader_count FROM series s WHERE s.site = $1 AND s.series_id = $2`, site, seriesID).Scan) if err != nil { t.Fatalf("read series %s:%s: %v", site, seriesID, err) @@ -1584,3 +1584,318 @@ func TestCoverStoreAcceptsAnySourceURL(t *testing.T) { t.Fatalf("rejected cover = found %v, err %v; want missing", ok, err) } } +func TestRecordLanePassPrunesBeforeInsertCutoff(t *testing.T) { + s := newTestStore(t) + for _, pass := range []LanePass{ + {Site: "asura", RanAt: 99}, + {Site: "asura", RanAt: 100}, + } { + if err := s.RecordLanePass(pass, 100); err != nil { + t.Fatalf("RecordLanePass(%d): %v", pass.RanAt, err) + } + } + + if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 200}, 100); err != nil { + t.Fatalf("RecordLanePass(200): %v", err) + } + var count int + if err := s.db.QueryRow(`SELECT count(*) FROM poll_passes WHERE site = $1`, "asura").Scan(&count); err != nil { + t.Fatalf("count passes: %v", err) + } + if count != 2 { + t.Fatalf("retained passes = %d, want 2", count) + } + if _, ok, err := s.LatestLanePass("asura"); err != nil || !ok { + t.Fatalf("LatestLanePass = ok %v, err %v; want latest row", ok, err) + } +} + +func TestLatestLanePassesKeepsNewestPerSiteAndJoinsState(t *testing.T) { + s := newTestStore(t) + for _, pass := range []LanePass{ + {Site: "asura", RanAt: 100, Due: 1}, + {Site: "asura", RanAt: 200, Skip: "due-query", Due: 2, Checked: 3, GapMS: 4000, Clamped: true}, + {Site: "demonic", RanAt: 150, Due: 4}, + } { + if err := s.RecordLanePass(pass, -1); err != nil { + t.Fatalf("RecordLanePass(%s/%d): %v", pass.Site, pass.RanAt, err) + } + } + if err := s.PauseLane("asura", 1234); err != nil { + t.Fatalf("PauseLane: %v", err) + } + if err := s.SetLaneRefusal("asura", 5678); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + + got, err := s.LatestLanePasses() + if err != nil { + t.Fatalf("LatestLanePasses: %v", err) + } + if len(got) != 2 { + t.Fatalf("latest passes = %d, want one per Site", len(got)) + } + bySite := map[string]LanePass{} + for _, pass := range got { + bySite[pass.Site] = pass + } + asura := bySite["asura"] + if asura.RanAt != 200 || asura.Skip != "due-query" || asura.Due != 2 || asura.Checked != 3 || asura.GapMS != 4000 || !asura.Clamped || + asura.PausedUntil != 1234 || asura.RefuseUntil != 5678 { + t.Fatalf("asura latest pass = %+v, want newest pass and joined state", asura) + } + if demonic := bySite["demonic"]; demonic.RanAt != 150 || demonic.Due != 4 { + t.Fatalf("demonic latest pass = %+v, want its only pass", demonic) + } +} + +func TestLanePassOutcomesSumsWindow(t *testing.T) { + s := newTestStore(t) + for _, pass := range []LanePass{ + {Site: "asura", RanAt: 99, Refused: 1, Unreachable: 2, NoChapter: 3, Unfetchable: 4, Errors: 5}, + {Site: "asura", RanAt: 100, Refused: 2, Unreachable: 3, NoChapter: 4, Unfetchable: 5, Errors: 6}, + {Site: "asura", RanAt: 200, Refused: 3, Unreachable: 4, NoChapter: 5, Unfetchable: 6, Errors: 7}, + {Site: "demonic", RanAt: 150, Refused: 8, Unreachable: 9, NoChapter: 10, Unfetchable: 11, Errors: 12}, + } { + if err := s.RecordLanePass(pass, -1); err != nil { + t.Fatalf("RecordLanePass(%s/%d): %v", pass.Site, pass.RanAt, err) + } + } + + got, err := s.LanePassOutcomes(100) + if err != nil { + t.Fatalf("LanePassOutcomes: %v", err) + } + if len(got) != 2 { + t.Fatalf("outcome Sites = %d, want 2", len(got)) + } + bySite := map[string]SiteOutcomes{} + for _, outcomes := range got { + bySite[outcomes.Site] = outcomes + } + if want := (SiteOutcomes{Site: "asura", Refused: 5, Unreachable: 7, NoChapter: 9, Unfetchable: 11, Errors: 13}); bySite["asura"] != want { + t.Fatalf("asura outcomes = %+v, want %+v", bySite["asura"], want) + } + if want := (SiteOutcomes{Site: "demonic", Refused: 8, Unreachable: 9, NoChapter: 10, Unfetchable: 11, Errors: 12}); bySite["demonic"] != want { + t.Fatalf("demonic outcomes = %+v, want %+v", bySite["demonic"], want) + } +} + +func TestLaneGatesReadsOneRow(t *testing.T) { + s := newTestStore(t) + paused, refused, err := s.LaneGates("asura") + if err != nil || paused != 0 || refused != 0 { + t.Fatalf("LaneGates on a missing state row = (%d, %d, %v), want (0, 0, nil)", paused, refused, err) + } + if err := s.PauseLane("asura", 2000); err != nil { + t.Fatalf("PauseLane: %v", err) + } + if err := s.SetLaneRefusal("asura", 3000); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + paused, refused, err = s.LaneGates("asura") + if err != nil || paused != 2000 || refused != 3000 { + t.Fatalf("LaneGates = (%d, %d, %v), want (2000, 3000, nil)", paused, refused, err) + } +} + +func TestLaneStatePauseResumeAndRefusal(t *testing.T) { + s := newTestStore(t) + for _, until := range []int64{0, -1} { + if err := s.PauseLane("asura", until); err == nil { + t.Fatalf("PauseLane(%d) accepted a non-future expiry", until) + } + } + if err := s.PauseLane("asura", 2000); err != nil { + t.Fatalf("PauseLane: %v", err) + } + if err := s.SetLaneRefusal("asura", 3000); err != nil { + t.Fatalf("SetLaneRefusal: %v", err) + } + if err := s.RecordLanePass(LanePass{Site: "asura", RanAt: 1}, -1); err != nil { + t.Fatalf("RecordLanePass: %v", err) + } + pausedUntil, refuseUntil, err := s.LaneGates("asura") + if err != nil || pausedUntil != 2000 || refuseUntil != 3000 { + t.Fatalf("LaneGates = %d/%d, %v; want 2000/3000", pausedUntil, refuseUntil, err) + } + paused, err := s.PausedLanes() + if err != nil { + t.Fatalf("PausedLanes: %v", err) + } + if len(paused) != 1 || paused[0] != (LanePause{Site: "asura", PausedUntil: 2000}) { + t.Fatalf("PausedLanes = %+v, want asura/2000", paused) + } + + if err := s.ResumeLane("asura"); err != nil { + t.Fatalf("ResumeLane: %v", err) + } + latest, ok, err := s.LatestLanePass("asura") + if err != nil || !ok || latest.RefuseUntil != 3000 { + t.Fatalf("latest refusal after resume = %+v, ok=%v, err=%v; want 3000 preserved", latest, ok, err) + } + if got, got2, err := s.LaneGates("asura"); err != nil || got != 0 || got2 != 3000 { + t.Fatalf("LaneGates after resume = %d/%d, %v; want 0/3000", got, got2, err) + } + if paused, err := s.PausedLanes(); err != nil || len(paused) != 0 { + t.Fatalf("PausedLanes after resume = %+v, %v; want empty", paused, err) + } + var rows int + if err := s.db.QueryRow(`SELECT count(*) FROM poll_lanes WHERE site = $1`, "asura").Scan(&rows); err != nil { + t.Fatalf("count lane state: %v", err) + } + if rows != 1 { + t.Fatalf("lane state rows after resume = %d, want 1", rows) + } +} + +// A forced Series is due ahead of the rest cutoff: the request overrides the +// rest gate so the Lane's next pass picks it up however recently it was +// checked. An unforced series under the rest stays out. +func TestDueForLatestCheckForcedOverridesRestCutoff(t *testing.T) { + s := newTestStore(t) + const now = int64(10 * 3600_000) + seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", now-30*60_000) + seedForCheck(t, s, "asura:fresh", "https://asurascans.com/comics/fresh", now-30*60_000) + if err := s.ForceSeriesPoll("asura", "forced", now); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + + due, err := s.DueForLatestCheck("asura", now-3600_000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 1 || due[0].Key() != "asura:forced" { + t.Fatalf("due = %v, want only the forced series", due) + } +} + +// The Sighting-deferral clause holds a solitary series back after a recent +// Reader report; a forced request overrides it and puts the series back on +// the Lane's list. +func TestDueForLatestCheckForcedOverridesSightingDeferral(t *testing.T) { + s := newTestStore(t) + const now = int64(10 * 3600_000) + // One bookmark (so deferral can apply), sighted and checked 10m ago: + // inside the deferral window and under the ceiling. + seedForCheck(t, s, "asura:deferred", "https://asurascans.com/comics/deferred", now-10*60_000) + if _, err := s.db.Exec( + `UPDATE series SET latest_sighted_at = $1 WHERE site = 'asura' AND series_id = 'deferred'`, + now-10*60_000); err != nil { + t.Fatalf("seed sighting: %v", err) + } + + // Unforced: deferred, and under the rest anyway. + due, err := s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 0 { + t.Fatalf("unforced deferred series is due: %v", due) + } + + // Forced: the request overrides the deferral. + if err := s.ForceSeriesPoll("asura", "deferred", now); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + due, err = s.DueForLatestCheck("asura", now-3600_000, now-3*3600_000) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 1 || due[0].Key() != "asura:deferred" { + t.Fatalf("forced deferred series not due: %v", due) + } +} + +// The finished-only bucket excludes a series whose only bookmarks are +// finished; a forced request overrides it — the owner asked, so the Lane +// looks. +func TestDueForLatestCheckForcedOverridesFinishedBucket(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:reading", "https://asurascans.com/comics/reading", 0) + if _, err := s.Upsert(s.OwnerID(), Bookmark{ + Key: "asura:finished", Site: "asura", SeriesID: "finished", + SeriesURL: "https://asurascans.com/comics/finished", + Status: StatusFinished, UpdatedAt: 1000, + }); err != nil { + t.Fatalf("seed finished: %v", err) + } + + due, err := s.DueForLatestCheck("asura", 1000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + for _, sr := range due { + if sr.Key() == "asura:finished" { + t.Fatalf("unforced finished series is due: %v", due) + } + } + + if err := s.ForceSeriesPoll("asura", "finished", 5000); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + due, err = s.DueForLatestCheck("asura", 1000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + got := map[string]bool{} + for _, sr := range due { + got[sr.Key()] = true + } + if !got["asura:reading"] || !got["asura:finished"] { + t.Fatalf("forced finished series not due: %v", due) + } +} + +// A forced Series jumps the queue: it sorts ahead of a more-read series that +// is due by rest, without disturbing the reader-count-then-age tie-break +// among the unforced rows (ADR-0003). +func TestDueForLatestCheckForcedSortsFirst(t *testing.T) { + s := newTestStore(t) + // "popular" has two readers and is long overdue; "forced" has one reader + // and a fresh check stamp. The forced row must come first. + seedForCheck(t, s, "asura:popular", "https://asurascans.com/comics/popular", 100) + seedSecondReader(t, s, "asura:popular:2", "asura", "popular", 1001) + seedForCheck(t, s, "asura:forced", "https://asurascans.com/comics/forced", 900) + if err := s.ForceSeriesPoll("asura", "forced", 5000); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + + due, err := s.DueForLatestCheck("asura", 1000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 2 { + t.Fatalf("due = %d rows, want 2", len(due)) + } + if due[0].Key() != "asura:forced" || due[1].Key() != "asura:popular" { + t.Fatalf("due order = %q, %q; want forced first, then popular", due[0].Key(), due[1].Key()) + } +} + +// A forced Series with no series URL is still not fetched — nothing to fetch — +// and one with no Bookmarks is still excluded by the join. The force flag +// opens the three gates it is allowed to, not the whole query. +func TestDueForLatestCheckForcedDoesNotOverrideURLOrJoin(t *testing.T) { + s := newTestStore(t) + seedForCheck(t, s, "asura:nourl", "", 0) + if err := s.ForceSeriesPoll("asura", "nourl", 5000); err != nil { + t.Fatalf("ForceSeriesPoll: %v", err) + } + if _, err := s.db.Exec(` + INSERT INTO series (site, series_id, title, series_url, cover, kind, + latest_chapter, latest_chapter_num, latest_checked_at, + force_poll_at) + VALUES ('asura', 'orphan', 'Orphan', 'https://asurascans.com/comics/orphan', + '', 'manga', '', NULL, 0, 5000)`); err != nil { + t.Fatalf("seed orphan: %v", err) + } + + due, err := s.DueForLatestCheck("asura", 1000, noCeiling) + if err != nil { + t.Fatalf("DueForLatestCheck: %v", err) + } + if len(due) != 0 { + t.Fatalf("due = %v, want neither the URL-less nor the orphan series", due) + } +} diff --git a/backend/internal/web/admin.go b/backend/internal/web/admin.go index 5ae53e3..ca39e8c 100644 --- a/backend/internal/web/admin.go +++ b/backend/internal/web/admin.go @@ -6,69 +6,27 @@ import ( "strconv" "time" - "bookmarkmanager/backend/internal/latest" "bookmarkmanager/backend/internal/store" ) -// LaneReporter is the administrative page's whole window onto the running -// poller: one snapshot of Poll Lane state, copied out of memory on request. -// The Poller satisfies it in production and a fake with fixed values satisfies -// it in tests, so the page's tests need neither a poller nor a Site. -type LaneReporter interface { - LaneStatus() latest.Status -} +// ownerWindow is the staleness boundary the Series list's "not checked in +// 12h" filter compares against. Declared once; later admin tickets read it. +const ownerWindow = 12 * time.Hour -// adminView is what the administrative page and the roster fragment receive. +// adminView is the shared shell data for an administrative page and the roster +// fragment returned after a Reader action. type adminView struct { + Page string Readers []store.ReaderSummary // OwnerID travels with the roster so it can tell the owner's own row from // the Readers they may act on. - OwnerID int64 - Lanes lanesView -} - -// lanesView is the Lane status block: one row per Site that has run, plus the -// browser fact, which is shared by the three browser Sites rather than held -// once per Site. -type lanesView struct { - Rows []laneRow - // PollerOff means no poller is running at all (disabled by config, or its - // client could not be built). The browser line must not answer "not - // configured" then: the sidecar is not the reason nothing is polled. - PollerOff bool - BrowserConfigured bool - BrowserReachable bool -} - -// laneRow is one Lane formatted for reading rather than for arithmetic: the -// template renders strings and flags, and every judgement about what they mean -// is made here. -type laneRow struct { - Site string - Due int - Ran string - // Checked is how many Series the last pass read. Due without Checked is a - // Lane that has stopped working; the two figures side by side are what - // separate that from a Lane with nothing to do. - Checked int - // Gap is empty when no pass has reached the pace yet, so the row omits the - // figure instead of stating a zero. - Gap string - Clamped bool - Refusing bool - // BrowserLost marks a Lane whose pages can only be read through the - // sidecar while the sidecar is unreachable — including the case where none - // is configured, which stops those Series just as completely. - BrowserLost bool - // Stalled marks a Lane with Series waiting that its last pass did not read - // — the difference between a stopped Lane and a quiet one (story 13). A - // browser Lane holding Chrome asleep under the wake thresholds is neither, - // so it carries Asleep instead and never Stalled. - Stalled bool - Asleep bool - // Attention is the one flag the template colours on, so an unhealthy Lane - // is found at a glance rather than read for. - Attention bool + OwnerID int64 + Lanes lanesView + SeriesList seriesListView + // Detail is the per-Series page data; zero on every other page. + Detail seriesDetailView + // Overview is the landing page data; zero on every other page. + Overview overviewView } // adminRoute pairs a route pattern with its handler so the route list and the @@ -84,6 +42,13 @@ type adminRoute struct { func (h *Handler) adminRoutes() []adminRoute { return []adminRoute{ {"GET /admin", h.admin}, + {"GET /admin/lanes", h.adminLanes}, + {"GET /admin/readers", h.adminReaders}, + {"GET /admin/series", h.adminSeries}, + {"GET /admin/series/{key}", h.adminSeriesDetail}, + {"POST /admin/series/{key}/poll", h.adminSeriesPoll}, + {"POST /admin/lanes/{site}/pause", h.adminLanePause}, + {"POST /admin/lanes/{site}/resume", h.adminLaneResume}, {"GET /ui/admin/lanes", h.uiLanes}, {"POST /readers/{id}/revoke", h.revokeReaderSessions}, {"POST /readers/{id}/clear-marks", h.clearReaderMarks}, @@ -116,78 +81,32 @@ func (h *Handler) requireOwner(next http.HandlerFunc) http.HandlerFunc { }) } -// admin renders the owner's page: the Reader roster and Poll Lane status. +// admin renders the Overview landing page: a verdict line, a stats block +// where every figure is a door into the list it counts, and the per-Site +// library shape table — all read from the database, never from a poller. func (h *Handler) admin(w http.ResponseWriter, r *http.Request) { - readers, err := h.store.Readers() + view, err := h.overviewView() if err != nil { - log.Printf("admin: %v", err) + log.Printf("admin overview: %v", err) http.Error(w, "internal error", http.StatusInternalServerError) return } - h.render(w, http.StatusOK, "admin", adminView{ - Readers: readers, - OwnerID: h.store.OwnerID(), - Lanes: h.lanesView(), - }) + h.renderAdmin(w, adminView{Page: "overview", Overview: view}) } -// uiLanes answers the status block's own refresh. Only the block refreshes on a -// timer; the roster re-renders after an action, as it always has. -func (h *Handler) uiLanes(w http.ResponseWriter, r *http.Request) { - h.render(w, http.StatusOK, "lanes", h.lanesView()) +// adminReaders renders the Reader roster on its own bookmarkable page. +func (h *Handler) adminReaders(w http.ResponseWriter, r *http.Request) { + readers, err := h.store.Readers() + if err != nil { + log.Printf("admin readers: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + h.renderAdmin(w, adminView{Page: "readers", Readers: readers, OwnerID: h.store.OwnerID()}) } -// lanesView copies the poller's snapshot into display form. A nil reporter (no -// poller running) and a poller no Lane has reported to yet are the same thing -// to the page: no data, which it must say rather than draw as confident zeroes -// — an empty page a few seconds after a restart must not read as a stopped one. -func (h *Handler) lanesView() lanesView { - if h.lanes == nil { - return lanesView{PollerOff: true} - } - snap := h.lanes.LaneStatus() - v := lanesView{ - Rows: make([]laneRow, 0, len(snap.Lanes)), - BrowserConfigured: snap.BrowserConfigured, - BrowserReachable: snap.BrowserReachable, - } - now := time.Now() - for _, l := range snap.Lanes { - lost := l.Browser && !snap.BrowserReachable - // Series waiting and none read is the shape of a Lane that has stopped - // working, as distinct from one that is quiet for want of work — or one - // deliberately leaving Chrome asleep until its group gathers. - stalled := l.Due > 0 && l.Checked == 0 && !l.Asleep - gap := "" - if l.Gap > 0 { - gap = l.Gap.Truncate(time.Second).String() - } - v.Rows = append(v.Rows, laneRow{ - Site: l.Site, - Due: l.Due, - Ran: since(now, l.LastRun), - Checked: l.Checked, - Gap: gap, - Clamped: l.Clamped, - Refusing: l.Refusing, - BrowserLost: lost, - Stalled: stalled, - Asleep: l.Asleep, - Attention: l.Clamped || l.Refusing || lost || stalled, - }) - } - return v -} - -// since formats how long ago a Lane last ran, at second resolution: the block -// refreshes every thirty seconds, so anything finer is noise the owner would -// have to ignore. -func since(now, then time.Time) string { - d := now.Sub(then).Truncate(time.Second) - if d < time.Second { - return "just now" - } - return d.String() + " ago" +func (h *Handler) renderAdmin(w http.ResponseWriter, view adminView) { + h.render(w, http.StatusOK, "admin", view) } // revokeReaderSessions logs one Reader out of every browser they are signed in diff --git a/backend/internal/web/admin_lanes.go b/backend/internal/web/admin_lanes.go new file mode 100644 index 0000000..599bb64 --- /dev/null +++ b/backend/internal/web/admin_lanes.go @@ -0,0 +1,338 @@ +package web + +import ( + "fmt" + "log" + "net/http" + "slices" + "time" + + "bookmarkmanager/backend/internal/latest" + "bookmarkmanager/backend/internal/store" +) + +// lanesView is the Lane status block: one row per Site's latest durable pass, +// plus the browser fact derived from that same log. No poller is consulted — +// the page answers from the database, so it is complete thirty seconds after +// a deploy (issue #145). +type lanesView struct { + Rows []laneRow + // PollerOff means latest-chapter polling is switched off in this + // deployment (LATEST_CHAPTER_POLL_ENABLED). It is a config fact, not a + // poller answering "absent": the browser line must not blame the sidecar + // when nothing polls. + PollerOff bool + BrowserConfigured bool + BrowserReachable bool +} + +// laneRow is one Lane formatted for reading rather than for arithmetic: the +// template renders strings and flags, and every judgement about what they +// mean is made here. +type laneRow struct { + Site string + Due int + Checked int + // Gap is the last pass's pace, or "—" when no pass has reached one yet — + // a refused Lane still reports the pace its last real pass chose, so a + // zero here would be a figure the row never measured. + Gap string + Ran string + // Chips are the named outcome counts over the owner's window, in the + // taxonomy's fixed order. Empty writes "none observed". + Chips []chip + HasChips bool + // StatePhrase is the reason this Lane declined to work: a skipped pass's + // own sentence, or the one true stall. Empty means the pass reached its + // loop and read normally. StateGood marks a healthy way to do nothing + // (paused, browser asleep, nothing eligible) rather than a fault. + StatePhrase string + StateGood bool + // Attention is the one flag the template colours on, so a Lane that + // needs the owner is found at a glance rather than read for. + Attention bool + // Paused is the live pause state — the poll_lanes stamp the pass row + // joins on, still in the future — not the pass's skip: the control must + // offer Resume from the moment the owner presses Pause, with no pass + // having run to record it (issue #147). + Paused bool +} + +// chip is one named outcome count over the owner's window. +type chip struct { + Name string + Count int +} + +// adminLanes renders the page that hosts the live Lane fragment. +func (h *Handler) adminLanes(w http.ResponseWriter, r *http.Request) { + h.renderAdmin(w, adminView{Page: "lanes", Lanes: h.lanesView()}) +} + +// uiLanes answers the status block's own refresh. Only the block refreshes on +// a timer; the roster re-renders after an action, as it always has. +func (h *Handler) uiLanes(w http.ResponseWriter, r *http.Request) { + h.render(w, http.StatusOK, "lanes", h.lanesView()) +} + +// pauseDurations are the offered pause lengths, by their wire value. A fixed +// allow-list rather than time.ParseDuration: the unoffered value must be +// refused, and a permissive parser turns the offered set into "anything Go +// can read" (issue #147). +var pauseDurations = map[string]time.Duration{ + "1h": time.Hour, + "6h": 6 * time.Hour, + "24h": 24 * time.Hour, +} + +// laneSite reads the Site a lane route names, answering the request itself +// when it is not a registry Site. The path value is client-supplied, so it +// is checked against the registry before it reaches the store. +func laneSite(w http.ResponseWriter, r *http.Request) (string, bool) { + site := r.PathValue("site") + if !slices.Contains(latest.SiteNames(), site) { + http.Error(w, "unknown site", http.StatusBadRequest) + return "", false + } + return site, true +} + +// adminLanePause writes a bounded pause for one Site and answers with the +// freshly rendered Lanes block, so the figures describe the state after the +// press. The pause is a fact about the Site — the Lane's next pass reads it +// from the durable row, never from this process — so it survives a restart. +// The owner gate is the route's, not this handler's; the body is capped like +// the API path caps its bodies; the Site and the duration are validated +// here, before the store sees them (issue #147). +func (h *Handler) adminLanePause(w http.ResponseWriter, r *http.Request) { + site, ok := laneSite(w, r) + if !ok { + return + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<16) + if err := r.ParseForm(); err != nil { + http.Error(w, "invalid form", http.StatusBadRequest) + return + } + d, ok := pauseDurations[r.PostFormValue("duration")] + if !ok { + http.Error(w, "unknown pause duration", http.StatusBadRequest) + return + } + if err := h.store.PauseLane(site, time.Now().Add(d).UnixMilli()); err != nil { + log.Printf("pause lane %s: %v", site, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + h.render(w, http.StatusOK, "lanes", h.lanesView()) +} + +// adminLaneResume zeroes one Site's pause and answers with the freshly +// rendered Lanes block. Resume is the reversal of a bounded pause, so it +// fires instantly with no confirm row (issue #147). +func (h *Handler) adminLaneResume(w http.ResponseWriter, r *http.Request) { + site, ok := laneSite(w, r) + if !ok { + return + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<16) + if err := r.ParseForm(); err != nil { + http.Error(w, "invalid form", http.StatusBadRequest) + return + } + if err := h.store.ResumeLane(site); err != nil { + log.Printf("resume lane %s: %v", site, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + h.render(w, http.StatusOK, "lanes", h.lanesView()) +} + +// lanesView builds the Lane status block from the durable pass log. Both +// reads are the store's latest-per-Site projection, so the page's seam is a +// seeded row rather than a fake poller; errors degrade to the empty state and +// are logged, never shown to the owner in detail. +func (h *Handler) lanesView() lanesView { + v := lanesView{ + PollerOff: !h.pollerEnabled, + BrowserConfigured: h.browserConfigured, + } + passes, err := h.store.LatestLanePasses() + if err != nil { + log.Printf("admin lanes: latest passes: %v", err) + // No evidence of a lost sidecar reads as reachable, per the same rule + // browserReachable applies: a store failure must not condemn the + // browser. The empty table already says no Lane has recorded a pass. + v.BrowserReachable = true + return v + } + now := time.Now() + outcomes, err := h.store.LanePassOutcomes(now.Add(-ownerWindow).UnixMilli()) + if err != nil { + // The rows are complete without the chips, so a failed outcome sum + // must not blank the table into "no data yet" — that is the confident + // wrong statement the page exists to avoid. Every row renders "none + // observed" instead, which is honest. + log.Printf("admin lanes: outcomes: %v", err) + outcomes = nil + } + bySite := make(map[string]store.SiteOutcomes, len(outcomes)) + for _, o := range outcomes { + bySite[o.Site] = o + } + v.Rows = make([]laneRow, 0, len(passes)) + v.BrowserReachable = browserReachable(passes, now) + for _, p := range passes { + v.Rows = append(v.Rows, buildLaneRow(p, bySite[p.Site], now)) + } + return v +} + +// browserReachable derives the sidecar's reachability from the pass log: a +// browser Site is down when its latest pass inside the refusal backoff is a +// sidecar loss, a missing fetcher, or an interrupted read. Only browser Sites +// ever produce those signals, so no Site registry leaks into the web layer. +// A configured browser with no such evidence reads as reachable; an unset +// BROWSER_WS_URL degrades identically to a browser that is down. +func browserReachable(passes []store.LanePass, now time.Time) bool { + backoff := latest.RefuseBackoff + for _, p := range passes { + ran := time.UnixMilli(p.RanAt) + if now.Sub(ran) >= backoff || ran.After(now) { + continue + } + if p.Skip == latest.SkipSidecarDown || p.Skip == latest.SkipNoFetcher || p.Unreachable > 0 { + return false + } + } + return true +} + +// buildLaneRow turns one Site's latest pass and window outcome sums into the +// row the template prints. The skip column is the authority on why a pass did +// nothing; the outcomes render named and unlinked, because the pass row holds +// counts and never identities. +func buildLaneRow(p store.LanePass, o store.SiteOutcomes, now time.Time) laneRow { + row := laneRow{ + Site: p.Site, + Due: p.Due, + Checked: p.Checked, + Gap: "—", + Ran: since(now, time.UnixMilli(p.RanAt)), + } + if p.GapMS > 0 { + row.Gap = (time.Duration(p.GapMS) * time.Millisecond).Truncate(time.Second).String() + } + row.Chips = outcomeChips(o) + row.HasChips = len(row.Chips) > 0 + row.StatePhrase, row.StateGood, row.Attention = laneState(p, now) + row.Paused = time.UnixMilli(p.PausedUntil).After(now) + return row +} + +// outcomeChips lists a Site's nonzero window sums in the taxonomy's fixed +// order, so the chips never reorder as the window changes. None observed is +// written by the template, not drawn as a confident zero count. +func outcomeChips(o store.SiteOutcomes) []chip { + fixed := []struct { + name string + count int + }{ + {"refused", o.Refused}, + {"unreachable", o.Unreachable}, + {"no chapter", o.NoChapter}, + {"unfetchable", o.Unfetchable}, + {"errors", o.Errors}, + } + var out []chip + for _, f := range fixed { + if f.count > 0 { + out = append(out, chip{Name: f.name, Count: f.count}) + } + } + return out +} + +// laneState renders the reason a Lane's last pass did nothing, in one sentence +// per skip value with the one true stall kept apart from every Lane that +// declined and said why. Good states — a pause, a sleeping browser, nothing +// eligible — carry no Attention: the mark must stay spendable on the faults +// that actually need the owner. +func laneState(p store.LanePass, now time.Time) (phrase string, good, attention bool) { + // The pause phrase reads the live poll_lanes stamp the pass row joins + // on, not the pass's skip: the owner's press must render as paused on + // the very answer it gets, with no pass having run to record it. The + // pause is a fact about the Site, and the join delivers it (issue #147). + if pausedUntil := time.UnixMilli(p.PausedUntil); pausedUntil.After(now) { + phrase = "paused · resumes in " + humanDuration(pausedUntil.Sub(now)) + good = true + return phrase, good, attention + } + switch p.Skip { + case latest.SkipPaused: + // A paused pass whose stamp has since lapsed: the Lane still + // declined with a reason, so it is never the one true stall. + phrase = "paused · resumes in " + humanDuration(time.UnixMilli(p.PausedUntil).Sub(now)) + good = true + case latest.SkipRefusing: + phrase = "refusing" + if until := time.UnixMilli(p.RefuseUntil); until.After(now) { + phrase += " · backs off until " + until.Format("15:04") + } + attention = true + case latest.SkipSidecarDown, latest.SkipNoFetcher: + // Known false positive shipped per spec: a sibling Lane's Chrome loss + // stamps this Site too, and the enum deliberately has no tenth value + // to separate it (issue #141). Render it as written. + phrase = "no browser" + attention = true + case latest.SkipAsleep: + phrase = "browser asleep" + good = true + case latest.SkipDueQuery: + phrase = "due query failed" + attention = true + case latest.SkipEligibleCount: + phrase = "eligible count failed" + attention = true + case latest.SkipNothingEligible: + phrase = "nothing eligible" + good = true + } + if phrase == "" && p.Due > 0 && p.Checked == 0 { + // The one true stall: the pass reached its loop, Series were waiting, + // and none were read. Every skip above is a Lane that said why. + phrase = "not checking" + attention = true + } + return phrase, good, attention +} + +// humanDuration renders a positive duration compactly for a "resumes in" clue +// at the pause and refusal scales — minutes under an hour, then h and h+m. +func humanDuration(d time.Duration) string { + d = d.Round(time.Minute) + if d <= 0 { + return "soon" + } + if d < time.Hour { + return fmt.Sprintf("%dm", int(d/time.Minute)) + } + h := int(d / time.Hour) + if m := int(d%time.Hour) / int(time.Minute); m == 0 { + return fmt.Sprintf("%dh", h) + } else { + return fmt.Sprintf("%dh%dm", h, m) + } +} + +// since formats how long ago a Lane last ran, at second resolution: the block +// refreshes every thirty seconds, so anything finer is noise the owner would +// have to ignore. +func since(now, then time.Time) string { + d := now.Sub(then).Truncate(time.Second) + if d < time.Second { + return "just now" + } + return d.String() + " ago" +} diff --git a/backend/internal/web/admin_overview.go b/backend/internal/web/admin_overview.go new file mode 100644 index 0000000..1851a17 --- /dev/null +++ b/backend/internal/web/admin_overview.go @@ -0,0 +1,197 @@ +package web + +import ( + "fmt" + "time" + + "bookmarkmanager/backend/internal/store" +) + +// overviewView is the Overview landing page's data: one verdict line, the +// hygiene and library stats blocks, and the per-Site library shape table. +// Every judgement — the verdict state, which figures link, what a Lane's +// state means — is made here; the template only prints. +type overviewView struct { + // Verdict is the attention phrase that leads the page. + Verdict string + // HasCounts is false on a virgin pass log: the waiting figure would be a + // confident zero, and "nothing has happened" must not render as health. + HasCounts bool + // Waiting is the sum of Due over the latest pass per Site. + Waiting int + // Unchecked is the number of Series not checked in the window, computed + // as stale + never_checked: a never-checked Series is already counted on + // its own filter, and the verdict wants the inclusive number. + Unchecked int + // Hygiene is the seven problem filters in the Series list's own render + // order; Library is the library split plus the roster. Every figure is a + // door into the list that counts it, except a zero. + Hygiene []fig + Library []fig + // Sites is the per-Site library shape table, one row per Site with any + // Series, in the store's Site order. + Sites []siteRow +} + +// fig is one stats figure: its label, the list it counts, and the count +// itself. Href empty means the count is zero: a measured zero is a real +// figure that stays on the page, but it is not a door, because following it +// lands on an empty list. +type fig struct { + Label string + Href string + Count int +} + +// siteRow is one Site's share of the library: the Series total and the three +// hygiene counts the per-Site table carries, each a door to the list narrowed +// to that Site, plus the Lane state phrase derived from its latest pass. The +// table is library shape only — the Poll outcome sums live on the Lanes page. +type siteRow struct { + Site string + SiteHref string + Figs []fig + // State is the Lane's own sentence; "" means the last pass read normally. + // StateGood / StateBad pick the ok / bad second class. + State string + StateGood bool + StateBad bool +} + +// overviewView assembles the landing page from the store's read model: one +// SeriesShapes pass per filter summed in Go (the shipped surface offers eight +// grouped passes, not a stats query — #140), the pass log's latest pass per +// Site, and the roster. A failure in any read is a 500 with a logged reason, +// never a page of silent zeroes. +func (h *Handler) overviewView() (overviewView, error) { + now := time.Now() + cutoff := now.Add(-ownerWindow).UnixMilli() + + shapes := make(map[string][]store.SiteSeriesShape, len(seriesFilterOrder)) + totals := make(map[string]int, len(seriesFilterOrder)) + for _, name := range seriesFilterOrder { + rows, err := h.store.SeriesShapes(store.SeriesFilter{Name: name, Cutoff: cutoff}) + if err != nil { + return overviewView{}, err + } + shapes[name] = rows + for _, sh := range rows { + totals[name] += sh.Total + } + } + + passes, err := h.store.LatestLanePasses() + if err != nil { + return overviewView{}, err + } + readers, err := h.store.Readers() + if err != nil { + return overviewView{}, err + } + + view := overviewView{Waiting: waiting(passes)} + view.Unchecked = totals[store.SeriesFilterStale] + totals[store.SeriesFilterNeverChecked] + view.Verdict, view.HasCounts = overviewVerdict(passes, now) + + // The seven problem filters, in seriesFilterOrder's permanent-then-fixable + // order; the All filter's count belongs to the Library block, not to a + // "hygiene" figure. + hygiene := make([]fig, 0, len(seriesFilterOrder)-1) + for _, name := range seriesFilterOrder[1:] { + hygiene = append(hygiene, door(seriesFilterLabels[name], totals[name], seriesListHref(name, "", "", 0))) + } + view.Hygiene = hygiene + + var manga, novel int + for _, sh := range shapes[store.SeriesFilterAll] { + manga += sh.Manga + novel += sh.Novel + } + view.Library = []fig{ + door("Series", totals[store.SeriesFilterAll], seriesListHref("", "", "", 0)), + door("Manga", manga, seriesListHref("", "", store.KindManga, 0)), + door("Novels", novel, seriesListHref("", "", store.KindNovel, 0)), + door("Readers", len(readers), "/admin/readers"), + } + + // One row per Site with any Series, from the All shapes; the hygiene + // counts come from the same per-Site projection so the table cannot + // disagree with the library-wide figures above it. + siteCounts := make(map[string]map[string]int, len(shapes)) + for name, rows := range shapes { + m := make(map[string]int, len(rows)) + for _, sh := range rows { + m[sh.Site] = sh.Total + } + siteCounts[name] = m + } + passBySite := make(map[string]store.LanePass, len(passes)) + for _, p := range passes { + passBySite[p.Site] = p + } + view.Sites = make([]siteRow, 0, len(shapes[store.SeriesFilterAll])) + for _, sh := range shapes[store.SeriesFilterAll] { + row := siteRow{ + Site: sh.Site, + SiteHref: seriesListHref("", sh.Site, "", 0), + Figs: []fig{ + door("", sh.Total, seriesListHref("", sh.Site, "", 0)), + door("", siteCounts[store.SeriesFilterNoCover][sh.Site], seriesListHref(store.SeriesFilterNoCover, sh.Site, "", 0)), + door("", siteCounts[store.SeriesFilterNeverChecked][sh.Site], seriesListHref(store.SeriesFilterNeverChecked, sh.Site, "", 0)), + door("", siteCounts[store.SeriesFilterStale][sh.Site], seriesListHref(store.SeriesFilterStale, sh.Site, "", 0)), + }, + } + if p, ok := passBySite[sh.Site]; ok { + row.State, row.StateGood, row.StateBad = laneState(p, now) + } else { + row.State = "no pass yet" + } + view.Sites = append(view.Sites, row) + } + return view, nil +} + +// door is one figure with its door: the list that counts it. A measured zero +// is still a real figure, but the door closes — following it would land on an +// empty list. The count is written once so the figure and what it links to +// cannot drift apart. +func door(label string, count int, href string) fig { + if count == 0 { + href = "" + } + return fig{Label: label, Href: href, Count: count} +} + +// overviewVerdict decides the landing page's one line from the latest pass +// per Site: no passes at all is "no Lane has reported yet" — never confident +// zeroes; otherwise the count of Lanes whose last pass needs the owner, or +// "all lanes healthy". The count comes from the same laneState judgement the +// Lanes page colours on, so the two pages cannot disagree on what a fault is. +func overviewVerdict(passes []store.LanePass, now time.Time) (phrase string, counts bool) { + if len(passes) == 0 { + return "no Lane has reported yet", false + } + attention := 0 + for _, p := range passes { + if _, _, attn := laneState(p, now); attn { + attention++ + } + } + if attention == 0 { + return "all lanes healthy", true + } + if attention == 1 { + return "1 lane needs a look", true + } + return fmt.Sprintf("%d lanes need a look", attention), true +} + +// waiting sums Due over the latest pass per Site: how many Series the Lanes +// found waiting, from the durable log rather than a running poller. +func waiting(passes []store.LanePass) int { + n := 0 + for _, p := range passes { + n += p.Due + } + return n +} diff --git a/backend/internal/web/admin_series.go b/backend/internal/web/admin_series.go new file mode 100644 index 0000000..e9099ad --- /dev/null +++ b/backend/internal/web/admin_series.go @@ -0,0 +1,398 @@ +package web + +import ( + "fmt" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "bookmarkmanager/backend/internal/latest" + "bookmarkmanager/backend/internal/store" +) + +// seriesPageSize matches the store's row-read page length: the pager's range +// must agree with the LIMIT the store applies or the "of N" figure describes +// the wrong page. The store does not export it (#140). +const seriesPageSize = 50 + +// seriesFilterLabels names every hygiene filter for the Series list select, +// keyed by the wire constant the URL carries. The render order is +// seriesFilterOrder; the labels are read by later admin tickets too, so the +// map and the constants cannot drift apart. +var seriesFilterLabels = map[string]string{ + store.SeriesFilterAll: "All series", + store.SeriesFilterNoURL: "No series URL", + store.SeriesFilterNoChapter: "Never read a chapter", + store.SeriesFilterNoReaders: "No Readers", + store.SeriesFilterNeverChecked: "Never checked", + store.SeriesFilterStale: "Not checked in 12h", + store.SeriesFilterNoCover: "No cover", + store.SeriesFilterReaderReport: "Latest from a Reader", +} + +// seriesFilterOrder is the select's render order: All first, then the +// permanent repairs, then the fixable ones (issue #140). +var seriesFilterOrder = []string{ + store.SeriesFilterAll, + store.SeriesFilterNoURL, + store.SeriesFilterNoChapter, + store.SeriesFilterNoReaders, + store.SeriesFilterNeverChecked, + store.SeriesFilterStale, + store.SeriesFilterNoCover, + store.SeriesFilterReaderReport, +} + +// seriesListView is the Series list page's data. The template renders strings +// and flags, and every judgement about what a value means is made here. +type seriesListView struct { + Filters []seriesFilterOption + Sites []string + Site string // "" = every Site + Kind string // "" = both libraries + FilterLabel string + Rows []seriesRowView + Total int + // KindBoth / KindManga / KindNovel are the Library segment links, and + // PrevHref / NextHref the pager's, all carrying the active filter, Site + // and Kind so narrowing never drops state. + KindBoth string + KindManga string + KindNovel string + PrevHref string + NextHref string + Range string +} + +// seriesFilterOption is one entry of the Show select: its wire value, its +// rendered label with the library-wide count, and whether it is the active +// filter. +type seriesFilterOption struct { + Name string + Label string + Count int + Selected bool +} + +// seriesRowView is one Series row formatted for the template. Band carries +// the alternating row tint by class rather than nth-of-type, so the confirm +// rows later tickets add are row siblings without breaking the alternation. +// Attention tints the title patina: a row with any hygiene chip needs one. +// +// CanPoll is the Check now control's visibility: absent on a Series with no +// page to fetch and on an orphan, so the owner is never offered a button that +// can never do anything. Pending is derived — the request stamp is newer than +// the check stamp — and Requested is its ageing label. +type seriesRowView struct { + Key string + Title string + Site string + Ch string // chapter number; "—" until first captured + Age string // checked age; "never" until first check + Readers int + Notes []string // chips, capped at two + More int // chips past the cap, rendered as a +N tail + Band bool + Attention bool + CanPoll bool + Pending bool + Requested string // "requested 3m ago", rendered only while pending +} + +// adminSeries renders the filterable, bookmarkable Series list: filter, Site, +// Library and page all live in the query string, so the list's state is an +// address rather than a click path. +func (h *Handler) adminSeries(w http.ResponseWriter, r *http.Request) { + view, err := h.seriesListView(r) + if err != nil { + log.Printf("admin series: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + h.renderAdmin(w, adminView{Page: "series", SeriesList: view}) +} + +// adminSeriesPoll is the Check now action: it stamps the Series' force_poll_at +// and answers with the freshly rendered row, so the figures describe the +// state after the press. The control never commands the poller — the request +// is a fact about the Series, and the Lane's next pass reads it through +// DueForLatestCheck (ADR-0013). The owner gate is the route's, not this +// handler's; the body is capped like the API path caps its bodies; the key is +// validated here — a malformed key is a 400 and an unknown one a 404. +func (h *Handler) adminSeriesPoll(w http.ResponseWriter, r *http.Request) { + site, seriesID, ok := strings.Cut(r.PathValue("key"), ":") + if !ok || site == "" || seriesID == "" { + http.Error(w, "bad series key", http.StatusBadRequest) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<16) + if err := r.ParseForm(); err != nil { + http.Error(w, "invalid form", http.StatusBadRequest) + return + } + if _, found, err := h.adminSeriesByKey(site, seriesID); err != nil { + log.Printf("series poll %s: %v", site+":"+seriesID, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } else if !found { + http.NotFound(w, r) + return + } + if err := h.store.ForceSeriesPoll(site, seriesID, time.Now().UnixMilli()); err != nil { + log.Printf("series poll %s: %v", site+":"+seriesID, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + // Re-read after the stamp: the answer must describe the state after the + // press. The detail page's control swaps its meta in place and the list + // row's swaps the row; htmx names an id target in HX-Target, so the + // response matches the surface it came from. The row's band parity travels + // with the press (hx-vals), so the swap keeps the zebra alternation. + a, found, err := h.adminSeriesByKey(site, seriesID) + if err != nil { + log.Printf("series poll %s: %v", site+":"+seriesID, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if !found { + http.NotFound(w, r) + return + } + if r.Header.Get("HX-Target") == "detail-meta" { + h.render(w, http.StatusOK, "series-detail-meta", h.seriesDetailView(a)) + return + } + band := 0 + if r.PostFormValue("band") == "1" { + band = 1 + } + h.render(w, http.StatusOK, "series-row", seriesRow(a, band, time.Now())) +} + +// seriesListView assembles one Series list view from the request's query +// string. An unknown filter value is the absent All case, never an error: the +// select's options are not the only way this URL can be reached. +func (h *Handler) seriesListView(r *http.Request) (seriesListView, error) { + q := r.URL.Query() + filter := q.Get("filter") + if _, ok := seriesFilterLabels[filter]; !ok { + filter = store.SeriesFilterAll + } + site := q.Get("site") + kind := q.Get("kind") + if kind != store.KindManga && kind != store.KindNovel { + kind = "" + } + page := 1 + if p, err := strconv.Atoi(q.Get("page")); err == nil && p > 1 { + page = p + } + + sf := store.SeriesFilter{ + Site: site, + Kind: kind, + Name: filter, + Cutoff: time.Now().Add(-ownerWindow).UnixMilli(), + Page: page, + } + data, err := h.store.SeriesPage(sf) + if err != nil { + return seriesListView{}, err + } + // A page past the end is not an empty list: the store's window count runs + // over the rows the result set carries, so an overflow page reports zero + // rows and zero total, and the list re-reads at page 1 to know the truth. + if len(data.Rows) == 0 && page > 1 { + page = 1 + sf.Page = 1 + data, err = h.store.SeriesPage(sf) + if err != nil { + return seriesListView{}, err + } + } + + view := seriesListView{ + Site: site, + Kind: kind, + FilterLabel: seriesFilterLabels[filter], + Rows: make([]seriesRowView, 0, len(data.Rows)), + Total: data.Total, + Sites: latest.SiteNames(), + } + now := time.Now() + for i, a := range data.Rows { + view.Rows = append(view.Rows, seriesRow(a, i, now)) + } + view.Filters, err = h.seriesFilterOptions(filter, sf.Cutoff) + if err != nil { + return seriesListView{}, err + } + view.KindBoth = seriesListHref(filter, site, "", 0) + view.KindManga = seriesListHref(filter, site, store.KindManga, 0) + view.KindNovel = seriesListHref(filter, site, store.KindNovel, 0) + if page > 1 { + view.PrevHref = seriesListHref(filter, site, kind, page-1) + } + if last := (data.Total + seriesPageSize - 1) / seriesPageSize; page < last { + view.NextHref = seriesListHref(filter, site, kind, page+1) + } + view.Range = pagerRange(data.Total, len(data.Rows), page) + return view, nil +} + +// seriesFilterOptions renders every hygiene filter with its library-wide +// count, one SeriesShapes pass per filter summed in Go — the shipped surface +// offers eight grouped passes, not a single stats query (#140). The counts +// are library-wide because the select sits next to the Site narrowing and +// must not shift as the owner narrows the list itself. Cutoff travels with +// the stale filter, or its count would always be zero. +func (h *Handler) seriesFilterOptions(selected string, cutoff int64) ([]seriesFilterOption, error) { + out := make([]seriesFilterOption, 0, len(seriesFilterOrder)) + for _, name := range seriesFilterOrder { + shapes, err := h.store.SeriesShapes(store.SeriesFilter{Name: name, Cutoff: cutoff}) + if err != nil { + return nil, err + } + count := 0 + for _, sh := range shapes { + count += sh.Total + } + out = append(out, seriesFilterOption{ + Name: name, + Label: seriesFilterLabels[name], + Count: count, + Selected: name == selected, + }) + } + return out, nil +} + +// seriesRow shapes one store row for the template, capping its chips at two +// plus a +N tail; attention marks a row that carries any. +// pollState derives the Check now control and the pending marker (issue +// #146), shared by the list row and the detail page: CanPoll is false on a +// Series with no page to fetch and on an orphan, so the owner is never +// offered a button that can never do anything. Pending is derived — the +// request stamp is newer than the check stamp — and requested is its ageing +// label, which never expires. +func pollState(a store.AdminSeries, now time.Time) (canPoll, pending bool, requested string) { + canPoll = a.SeriesURL != "" && a.ReaderCount > 0 + if a.ForcePollAt > a.LatestCheckedAt { + pending = true + requested = requestedAge(now, a.ForcePollAt) + } + return canPoll, pending, requested +} + +func seriesRow(a store.AdminSeries, i int, now time.Time) seriesRowView { + canPoll, pending, requested := pollState(a, now) + row := seriesRowView{ + Key: a.Key(), + Site: a.Site, + Title: a.Title, + Readers: a.ReaderCount, + Band: i%2 == 1, + CanPoll: canPoll, + Pending: pending, + Requested: requested, + } + if a.LatestChapterNum != nil { + row.Ch = strconv.FormatFloat(*a.LatestChapterNum, 'f', -1, 64) + } else { + row.Ch = "—" + } + row.Age = checkedAge(now, a.LatestCheckedAt) + notes := seriesNotes(a, now) + if n := len(notes); n > 2 { + row.Notes, row.More = notes[:2], n-2 + } else { + row.Notes = notes + } + row.Attention = len(notes) > 0 + return row +} + +// seriesNotes are a row's hygiene chips in the design's order: no URL, no +// cover, orphan, stale, reader sighting. +func seriesNotes(a store.AdminSeries, now time.Time) []string { + notes := []string{} + if a.SeriesURL == "" { + notes = append(notes, "no URL") + } + if a.CoverAddress == "" { + notes = append(notes, "no cover") + } + if a.ReaderCount == 0 { + notes = append(notes, "orphan") + } + if a.LatestCheckedAt > 0 && a.LatestCheckedAt < now.Add(-ownerWindow).UnixMilli() { + notes = append(notes, "stale") + } + if a.RaisedByReader { + notes = append(notes, "reader sighting") + } + return notes +} + +// checkedAge formats how long ago a Series was last checked, at the +// granularity the list reads at — minutes, hours, days. Zero means never. +func checkedAge(now time.Time, ts int64) string { + if ts == 0 { + return "never" + } + d := now.Sub(time.UnixMilli(ts)) + switch { + case d < time.Hour: + m := int(d / time.Minute) + if m < 1 { + m = 1 + } + return fmt.Sprintf("%dm ago", m) + case d < 24*time.Hour: + return fmt.Sprintf("%dh ago", int(d/time.Hour)) + default: + return fmt.Sprintf("%dd ago", int(d/(24*time.Hour))) + } +} + +// requestedAge is the pending marker's text: how long ago the owner asked, +// and nothing about when the request will run — the page does not know when a +// sleeping browser will wake (issue #146). An unanswered request ages forever; +// there is no expiry. +func requestedAge(now time.Time, ts int64) string { + return "requested " + checkedAge(now, ts) +} + +// pagerRange is the pager's "1–50 of 120" line. The template renders the +// pager only over rows (the empty state replaces it), so it is never asked +// to describe an empty list. +func pagerRange(total, rows, page int) string { + from := (page-1)*seriesPageSize + 1 + return fmt.Sprintf("%d–%d of %d", from, from+rows-1, total) +} + +// seriesListHref is one Series list address carrying the filter, Site, Kind +// and page. The All filter and page 1 are the absent cases and stay out of +// the URL, so the default address is the shortest one. +func seriesListHref(filter, site, kind string, page int) string { + q := url.Values{} + if filter != "" && filter != store.SeriesFilterAll { + q.Set("filter", filter) + } + if site != "" { + q.Set("site", site) + } + if kind != "" { + q.Set("kind", kind) + } + if page > 1 { + q.Set("page", strconv.Itoa(page)) + } + if len(q) == 0 { + return "/admin/series" + } + return "/admin/series?" + q.Encode() +} diff --git a/backend/internal/web/admin_series_detail.go b/backend/internal/web/admin_series_detail.go new file mode 100644 index 0000000..74810aa --- /dev/null +++ b/backend/internal/web/admin_series_detail.go @@ -0,0 +1,126 @@ +package web + +import ( + "log" + "net/http" + "strconv" + "strings" + "time" + + "bookmarkmanager/backend/internal/store" +) + +// seriesDetailView is one Series' page as the owner sees it: strings and +// flags, every judgement made here, the template left to print. ReaderCount +// is the only figure that crosses the privacy boundary — the owner learns how +// many Readers hold the Series, never which Reader reads what. +type seriesDetailView struct { + Key string // ":", the page's address and the Series' identity + Site string + Kind string + // Title, Cover and Chapter come from the shared Series row; the Cover is + // the wire URL of the stored bytes, "" before any exist. + Title string + Cover string + Chapter string // Latest Chapter number, or "—" before the first capture + Checked string // how long ago the poller last checked, or "never" + Readers int + + // Marks, one per hygiene fact, rendered only while it holds. + Unpollable bool // no SeriesURL to fetch + NoCover bool + Orphan bool // no Reader holds the Series + SightingRaised bool // a Reader's Sighting set the Latest Chapter + + // Poll is the Check now control and the pending marker (issue #146): the + // same derivation and visibility as the list row. CanPoll is false on a + // Series with no page to fetch and on an orphan; Pending is derived — + // the request stamp is newer than the check stamp — and Requested is its + // ageing label. + CanPoll bool + Pending bool + Requested string +} + +// adminSeriesDetail renders one Series' page, keyed by the composite +// ":" the list row already shows. The row is read through +// the list's own SeriesPage read narrowed to the key's Site: the admin +// projection is the privacy boundary, and a dedicated single-row read would +// be a second definition of it. +func (h *Handler) adminSeriesDetail(w http.ResponseWriter, r *http.Request) { + site, seriesID, ok := strings.Cut(r.PathValue("key"), ":") + if !ok || site == "" || seriesID == "" { + http.NotFound(w, r) + return + } + a, found, err := h.adminSeriesByKey(site, seriesID) + if err != nil { + log.Printf("series detail %s: %v", site+":"+seriesID, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if !found { + http.NotFound(w, r) + return + } + h.renderAdmin(w, adminView{Page: "series-detail", Detail: h.seriesDetailView(a)}) +} + +// adminSeriesByKey reads one Series through the list's own SeriesPage read +// narrowed to the key's Site: the admin projection is the privacy boundary, +// and a dedicated single-row read would be a second definition of it. Absence +// is reported with found=false, never an error. +// ponytail: a page scan per keyed read, one query per page of the Site's rows +// up to the window total; a keyed read alongside SeriesPage when the library +// outgrows the page size. +func (h *Handler) adminSeriesByKey(site, seriesID string) (store.AdminSeries, bool, error) { + seen := 0 + for page := 1; ; page++ { + p, err := h.store.SeriesPage(store.SeriesFilter{Site: site, Page: page}) + if err != nil { + return store.AdminSeries{}, false, err + } + seen += len(p.Rows) + for i := range p.Rows { + if p.Rows[i].SeriesID == seriesID { + return p.Rows[i], true, nil + } + } + if seen >= p.Total { + break + } + } + return store.AdminSeries{}, false, nil +} + +// seriesDetailView shapes one AdminSeries row for display: every judgement in +// Go, the template left to print strings and flags. +func (h *Handler) seriesDetailView(a store.AdminSeries) seriesDetailView { + canPoll, pending, requested := pollState(a, time.Now()) + v := seriesDetailView{ + Key: a.Key(), + Site: a.Site, + Kind: a.Kind, + Title: a.Title, + Cover: h.store.CoverWireURL(a.CoverAddress), + Readers: a.ReaderCount, + Unpollable: a.SeriesURL == "", + NoCover: a.CoverAddress == "", + Orphan: a.ReaderCount == 0, + SightingRaised: a.RaisedByReader, + CanPoll: canPoll, + Pending: pending, + Requested: requested, + } + if a.LatestChapterNum == nil { + v.Chapter = "—" + } else { + v.Chapter = strconv.FormatFloat(*a.LatestChapterNum, 'f', -1, 64) + } + if a.LatestCheckedAt == 0 { + v.Checked = "never" + } else { + v.Checked = since(time.Now(), time.UnixMilli(a.LatestCheckedAt)) + } + return v +} diff --git a/backend/internal/web/static/admin.css b/backend/internal/web/static/admin.css new file mode 100644 index 0000000..eb54abd --- /dev/null +++ b/backend/internal/web/static/admin.css @@ -0,0 +1,830 @@ +:root { + --measure-wide: 1080px; +} + +@media (prefers-color-scheme: light) { + :root { + --measure-wide: 1080px; + } +} + +.admin-sheet { + max-width: var(--measure-wide); +} + +.admin-sheet .brand em { + color: var(--patina); +} + +.admin-sheet .brand .mark > g > g:last-child { + stroke: var(--patina); +} + +.topbar-actions { + display: flex; + align-items: center; + gap: 18px; + margin-left: auto; +} + +.navrow { + display: flex; + gap: 18px; + padding: 2px 20px 0; + overflow-x: auto; + overflow-y: hidden; + scrollbar-width: none; + border-bottom: 1px solid var(--rule); +} + +.navrow::-webkit-scrollbar { + display: none; +} + +.navrow a { + flex: none; + display: flex; + align-items: center; + min-width: 44px; + padding: 8px 0 12px; + color: var(--mute); + font: 400 17px var(--font-display); + white-space: nowrap; +} + +.navrow a:hover { + color: var(--paper-dim); +} + +.navrow a.active { + color: var(--paper); + border-bottom: 2px solid var(--paper); + margin-bottom: -1px; +} + +.admin-page { + padding: 0 20px 40px; +} + +.admin-page > .sec, +.admin-sheet .readers h2, +.admin-sheet .lanes h2 { + position: relative; + margin: 0; + padding: 26px 0 8px; + font: 500 11px/1 var(--font-mono); + letter-spacing: .2em; + text-transform: uppercase; + color: var(--mute-2); +} + +.admin-page > .sec::before, +.admin-sheet .readers h2::before, +.admin-sheet .lanes h2::before { + content: ""; + position: absolute; + top: 14px; + left: 0; + width: 34px; + height: 2px; + background: var(--patina); +} + +.admin-sheet .readers, +.admin-sheet .lanes { + margin: 0; + padding: 0 0 16px; + border-bottom: none; +} + +.admin-sheet .readerlist, +.admin-sheet .lanelist { + margin: 0; + padding: 0; + list-style: none; +} + +.admin-sheet .readerlist li, +.admin-sheet .lanelist li { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 4px 16px; + min-height: 48px; + padding: 10px 0; + border-top: 1px solid var(--rule); +} + +.admin-sheet .reader-actions { + display: flex; + flex: 0 0 auto; + gap: 18px; + margin-left: auto; + white-space: nowrap; +} + +.admin-sheet .reader-actions .ghost, +.admin-sheet .c-act .ghost { + font-size: 12px; + color: var(--patina); +} + +.admin-sheet .reader-actions .ghost.danger, +.admin-sheet .c-act .ghost.danger { + color: var(--danger); +} + +.admin-sheet .readerlist form { + margin: 0; +} + +.admin-sheet .lane-browser { + padding: 12px 0 0; +} + +.admin-sheet .ghost.danger { + color: var(--danger); +} + +.admin-sheet .ghost.danger:hover { + color: var(--danger); + border-bottom-color: var(--danger); +} + +.admin-sheet .reader-id { + font: 500 15px/1.5 var(--font-mono); + letter-spacing: .01em; + color: var(--paper); +} + +.admin-sheet .reader-sessions, +.admin-sheet .reader-sightings, +.admin-sheet .reader-blocked, +.admin-sheet .lane-fact, +.admin-sheet .lane-mark { + font: 500 13px/1.4 var(--font-mono); + letter-spacing: .06em; + text-transform: uppercase; +} + +.admin-sheet .reader-sessions { + color: var(--paper-dim); +} + +.admin-sheet .reader-sightings, +.admin-sheet .lane-fact { + color: var(--mute); +} + +.admin-sheet .reader-blocked, +.admin-sheet .lane-mark { + color: var(--patina); +} + +.admin-sheet .lane-site { + font: 400 19px/1.2 var(--font-display); + color: var(--paper-dim); +} + +.admin-sheet .lanelist li.attention .lane-site { + color: var(--danger); +} + +/* A single grid keeps row rules continuous; cell padding supplies gutters. */ +.admin-sheet .sechead { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 20px; + flex-wrap: wrap; +} + +.admin-sheet .sechead .statusline { + padding: 0 0 8px; + font-size: 11px; + letter-spacing: .14em; + text-transform: uppercase; +} + +.admin-sheet .statusline { + margin: 0; + padding: 0 0 10px; + font: 500 12px/1 var(--font-mono); + letter-spacing: .04em; + color: var(--mute-2); +} + +/* The verdict line is set in the data face, not the display face: it is + three counts, not a page title. The judgement is the only bright thing. */ +.admin-sheet .verdict { + padding: 16px 0 12px; + border-bottom: 1px solid var(--rule); + font: 500 15px/1.6 var(--font-mono); + letter-spacing: .04em; + color: var(--mute); +} + +.admin-sheet .verdict .attn { + color: var(--patina); +} + +.admin-sheet .verdict .counts b { + color: var(--paper); + font-weight: 500; +} + +.admin-sheet .tbl { + display: grid; + grid-template-columns: minmax(240px, 1fr) 156px 92px 110px 76px minmax(150px, 220px) 140px; + column-gap: 0; + font-variant-numeric: tabular-nums; +} + +.admin-sheet .tbl .thead { + display: contents; +} + +.admin-sheet .tbl .thead > * { + padding: 10px 14px 8px 0; + border-bottom: 1px solid var(--rule); + font: 500 12px/1 var(--font-mono); + letter-spacing: .12em; + text-transform: uppercase; + color: var(--mute-2); +} + +.admin-sheet .tbl .trow { + display: contents; +} + +.admin-sheet .tbl .trow > * { + padding: 11px 14px 11px 0; + border-bottom: 1px solid var(--rule); + font: 500 15px/1.5 var(--font-mono); + letter-spacing: .01em; + color: var(--paper-dim); +} + +.admin-sheet .tbl .thead > *:last-child, +.admin-sheet .tbl .trow > *:last-child { + padding-right: 0; +} + +.admin-sheet .tbl .c-title, +.admin-sheet .tbl .c-site { + font: 400 18px/1.35 var(--font-display); + letter-spacing: 0; + color: var(--paper); +} + +.admin-sheet .tbl .c-title a:hover, +.admin-sheet .tbl .c-act .ghost:hover { + color: var(--patina); +} + +.admin-sheet .tbl .trow.attention .c-title { + color: var(--patina); +} + +.admin-sheet .tbl .c-ch, +.admin-sheet .tbl .c-rd { + text-align: right; + padding-right: 26px; +} + +.admin-sheet .tbl .trow .c-note .mark { + margin-right: 8px; +} + +.admin-sheet .tbl .c-act { + text-align: right; +} + +.admin-sheet .tbl .c-act .ghost + .ghost { + margin-left: 12px; +} + +.admin-sheet .tbl .trow > .confirm-row { + grid-column: 1 / -1; + padding: 10px 12px; + border-bottom: none; +} + +.admin-sheet .stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(232px, 1fr)); + margin: 4px 0 0; + border-bottom: 1px solid var(--rule); +} + +.admin-sheet .stat { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 16px; + padding: 11px 24px 11px 0; +} + +.admin-sheet .stat .lbl { + font: 500 13px/1.3 var(--font-mono); + letter-spacing: .1em; + text-transform: uppercase; + color: var(--mute-2); +} + +.admin-sheet .stat .fig { + font: 500 19px/1 var(--font-mono); + font-variant-numeric: tabular-nums; + color: var(--paper); +} + +.admin-sheet .stat .fig.zero { + color: var(--mute); +} + +.admin-sheet .stat a.fig { + /* The hover rule below sets the border on :hover only, which shifts the + row a pixel on hover; the transparent baseline holds the layout. */ + border-bottom: 1px solid transparent; +} + +.admin-sheet .stat a.fig:hover { + color: var(--patina); + border-bottom: 1px solid var(--patina); +} + +.admin-sheet .mark, +.admin-sheet .mark-faint { + font: 500 13px/1 var(--font-mono); + letter-spacing: .06em; + text-transform: uppercase; + white-space: nowrap; +} + +.admin-sheet .mark { + color: var(--patina); +} + +.admin-sheet .mark-faint { + color: var(--mute-2); +} + +.admin-sheet .mark.mark-strong { + font-size: 13px; + letter-spacing: .14em; + color: var(--patina); +} + +.admin-sheet .mark.mark-strong::before { + content: ""; + display: inline-block; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--patina); + margin-right: 8px; + vertical-align: .08em; +} + +.admin-sheet .mark.bad { + color: var(--danger); +} + +.admin-sheet .tbl.sites { + grid-template-columns: 170px repeat(4, 106px) minmax(180px, 1fr); +} + +.admin-sheet .tbl.sites .c-site, +.admin-sheet .tbl.lanes .c-site { + font: 400 18px/1.35 var(--font-display); + letter-spacing: 0; + color: var(--paper); +} + +.admin-sheet .tbl.sites .c-state { + color: var(--patina); + white-space: nowrap; +} + +.admin-sheet .tbl.sites .c-state.bad, +.admin-sheet .tbl.lanes .c-skip .bad, +.admin-sheet .tbl.lanes .trow.attention .c-site { + color: var(--danger); +} + +.admin-sheet .tbl.lanes .c-skip .ok { + color: var(--patina); +} + +.admin-sheet .tbl.sites .thead > *:nth-child(n+2):nth-child(-n+5), +.admin-sheet .tbl.sites .trow > *:nth-child(n+2):nth-child(-n+5) { + padding-right: 0; + text-align: center; +} + +.admin-sheet .tbl.lanes { + grid-template-columns: 150px 62px 88px 66px 118px minmax(0, 1fr) 168px; +} + +.admin-sheet .tbl.lanes .c-skip { + white-space: normal; +} + +.admin-sheet .tbl.lanes .c-skip > * { + white-space: nowrap; +} + +.admin-sheet .tbl.lanes .c-ctrl { + text-align: right; +} + +.admin-sheet .tbl.lanes .thead > *:nth-child(n+2):nth-child(-n+4), +.admin-sheet .tbl.lanes .trow > *:nth-child(n+2):nth-child(-n+4) { + padding-right: 26px; + text-align: right; +} + +.admin-sheet .tbl.series { + grid-template-columns: 150px 84px 104px 76px minmax(120px, 1fr) 212px; + row-gap: 4px; +} + +.admin-sheet .tbl.series .thead > *:first-child { + padding-left: 20px; +} + +.admin-sheet .tbl.series .thead > *:last-child { + padding-right: 20px; +} + +/* The site cell's colour is a class, never an inline style: site is + client-supplied and unvalidated, and a hostile value reaching a CSS + context would render ZgotmplZ, while an unknown class degrades to the + unstyled cell. */ +.admin-sheet .tbl .c-site.site-asura { + color: var(--asura); +} + +.admin-sheet .tbl .c-site.site-demonic { + color: var(--demonic); +} + +.admin-sheet .tbl .c-site.site-comix { + color: var(--comix); +} + +.admin-sheet .tbl .c-site.site-kagane { + color: var(--kagane); +} + +.admin-sheet .tbl .c-site.site-novelfull { + color: var(--novelfull); +} + +.admin-sheet .tbl .c-site.site-lightnovelworld { + color: var(--lightnovelworld); +} + +.admin-sheet .tbl.series .trow { + display: grid; + grid-column: 1 / -1; + grid-template-columns: subgrid; + padding: 13px 20px 14px; +} + +.admin-sheet .tbl.series .trow.band { + background: var(--hover); +} + +.admin-sheet .tbl.series .trow > * { + padding: 0 14px 0 0; + border-bottom: none; +} + +.admin-sheet .tbl.series .c-title { + display: flex; + grid-column: 1 / -1; + align-items: baseline; + gap: 16px; + padding: 0 0 4px; +} + +.admin-sheet .tbl.series .c-title .mark { + margin-left: auto; +} + +.admin-sheet .filterbar { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 14px 22px; + padding: 16px 0 14px; + border-bottom: 1px solid var(--rule); +} + +.admin-sheet .fsel { + display: flex; + align-items: baseline; + gap: 10px; +} + +.admin-sheet .fsel > span { + font: 500 11px/1 var(--font-mono); + letter-spacing: .14em; + text-transform: uppercase; + color: var(--mute-2); +} + +.admin-sheet .fsel select, +.admin-sheet .pausebar select { + color-scheme: dark; + background: var(--ink); + border: none; + border-bottom: 1px solid var(--field-line); + color: var(--paper); + font: 400 16px/1.4 var(--font-display); + padding: 4px 4px 5px 0; +} + +.admin-sheet .fsel select:hover, +.admin-sheet .pausebar select:hover { + border-bottom-color: var(--patina); +} + +.admin-sheet .fsel select option, +.admin-sheet .pausebar select option { + background: var(--ink); + color: var(--paper); +} + +.admin-sheet .segrow { + display: inline-flex; + gap: 2px; +} + +.admin-sheet .segrow a { + padding: 6px 11px 7px; + border-bottom: 2px solid transparent; + color: var(--mute); + font: 500 14px/1 var(--font-mono); +} + +.admin-sheet .segrow a.active { + border-bottom-color: var(--patina); + color: var(--patina); +} + +.admin-sheet .listhead { + display: flex; + align-items: baseline; + gap: 12px; + padding: 16px 0 4px; + color: var(--paper); + font: 400 18px/1.2 var(--font-display); +} + +.admin-sheet .listhead .lbl { + color: var(--mute); +} + +.admin-sheet .listhead .lbl em { + color: var(--patina); + font-style: normal; +} + +.admin-sheet .pager { + display: flex; + align-items: center; + gap: 14px; + padding: 14px 0 0; + color: var(--mute-2); + font: 500 11px/1 var(--font-mono); + letter-spacing: .14em; + text-transform: uppercase; +} + +.admin-sheet .pager .pg { + color: var(--paper-dim); +} + +.admin-sheet .pager .pg:hover { + color: var(--paper); +} + +.admin-sheet .pager .pg.disabled { + color: var(--faint); + pointer-events: none; +} + +.admin-sheet .empty { + padding: 28px 0; + text-align: center; +} + +.admin-sheet .empty strong { + color: var(--paper); + font: 400 20px var(--font-display); +} + +.admin-sheet .empty p { + margin: 6px 0 0; + font: 14px/1.5 var(--font-body); + color: var(--mute); +} + +.admin-sheet .confirm-row { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 12px; + background: var(--danger-wash); +} + +.admin-sheet .confirm-row span { + flex: 1 1 16ch; + color: var(--danger-soft); + font: 400 15px/1.3 var(--font-display); +} + +.admin-sheet .confirm-row div { + display: flex; + flex: none; + gap: 12px; + margin-left: auto; +} + +.admin-sheet .detail-back { + display: inline-block; + margin: 18px 0 0; +} + +.admin-sheet .detail-title { + margin: 10px 0 2px; + color: var(--paper); + font: 400 28px/1.25 var(--font-display); +} + +.admin-sheet .detail-key { + margin: 0; + color: var(--mute-2); + font: 500 11px/1.4 var(--font-mono); + letter-spacing: .08em; +} + +.admin-sheet .detail-meta { + display: flex; + flex-wrap: wrap; + gap: 4px 14px; + margin: 10px 0 0; + color: var(--mute-2); + font: 500 12px/1.5 var(--font-mono); + letter-spacing: .08em; + text-transform: uppercase; +} + +.admin-sheet .cover { + width: 160px; + aspect-ratio: 3 / 4; + display: flex; + align-items: center; + justify-content: center; + margin: 18px 0 4px; + background: var(--hatch); + color: var(--mute-2); + font: 500 10px/1 var(--font-mono); + letter-spacing: .2em; + text-transform: uppercase; +} + +.admin-sheet .detail-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0 28px; +} + +.admin-sheet .dform { + padding: 14px 0 0; +} + +.admin-sheet .dform h3 { + margin: 0 0 4px; + color: var(--mute-2); + font: 500 10px/1 var(--font-mono); + letter-spacing: .2em; + text-transform: uppercase; +} + +.admin-sheet .dform .field { + display: flex; + gap: 10px; + margin-top: 8px; +} + +.admin-sheet .pausebar { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; +} + +@media (max-width: 719px) { + .admin-sheet .topbar { + flex-wrap: wrap; + row-gap: 12px; + } + + .admin-sheet .brand { + flex: 1 1 100%; + } + + .admin-sheet .topbar-actions { + margin-left: auto; + } +} + +@media (max-width: 1019px) { + .admin-sheet .tbl.lanes { + display: block; + } + + .admin-sheet .tbl.lanes .thead { + display: none; + } + + .admin-sheet .tbl.lanes .trow { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 4px 16px; + padding: 12px 0; + border-bottom: 1px solid var(--rule); + } + + .admin-sheet .tbl.lanes .trow > * { + padding: 0; + border-bottom: none; + text-align: left; + } + + .admin-sheet .tbl.lanes .c-site { + width: 100%; + padding-bottom: 2px; + } + + .admin-sheet .tbl.lanes .c-ctrl { + margin-left: auto; + text-align: right; + } +} + +@media (max-width: 899px) { + .admin-sheet .tbl, + .admin-sheet .tbl.lanes { + display: block; + } + + .admin-sheet .tbl .thead { + display: none; + } + + .admin-sheet .tbl .trow { + display: flex; + flex-wrap: wrap; + gap: 2px 10px; + padding: 11px 0; + border-bottom: 1px solid var(--rule); + } + + .admin-sheet .tbl .trow > * { + padding: 0; + border-bottom: none; + text-align: left; + } + + .admin-sheet .tbl .c-ch, + .admin-sheet .tbl .c-rd { + padding-right: 0; + text-align: left; + } + + .admin-sheet .tbl .c-title { + width: 100%; + padding-bottom: 2px; + } + + .admin-sheet .tbl.series .trow { + display: flex; + } + + .admin-sheet .tbl.series .trow > * { + padding: 0; + } + + .admin-sheet .detail-grid { + grid-template-columns: 1fr; + } + + .admin-sheet .tbl .c-act { + margin-left: auto; + } +} diff --git a/backend/internal/web/static/style.css b/backend/internal/web/static/style.css index 4a1ba46..91c1526 100644 --- a/backend/internal/web/static/style.css +++ b/backend/internal/web/static/style.css @@ -296,69 +296,6 @@ button { cursor: pointer; } letter-spacing: .04em; } -/* ---- admin page: two sections on the same measured sheet, no cards ---- - The reading page is a list of series; this is a list of facts. Both are - sheets of hairline-separated rows, so the roster keeps the shape it had as - a fold-out and the Lane block copies it. */ -.readers, .lanes { margin: 0 20px; padding: 12px 0 16px; border-bottom: 1px solid var(--rule); } -.readers h2, .lanes h2 { - margin: 0; - padding: 8px 0; - font: 500 10px/1 var(--font-mono); - letter-spacing: .2em; - text-transform: uppercase; - color: var(--mute-2); -} -.readerlist, .lanelist { margin: 0; padding: 0; list-style: none; } -.readerlist li, .lanelist li { - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 4px 16px; - min-height: 44px; - border-top: 1px solid var(--rule); -} -.reader-actions { display: flex; gap: 18px; margin-left: auto; } -.readerlist form { margin: 0; } -.reader-id { - font: 500 13px/1.4 var(--font-mono); - letter-spacing: .04em; - color: var(--paper); -} -.reader-sessions { - font: 500 10px/1 var(--font-mono); - letter-spacing: .14em; - text-transform: uppercase; - color: var(--mute); -} -.reader-sightings, .lane-fact { - font: 500 10px/1 var(--font-mono); - letter-spacing: .14em; - text-transform: uppercase; - color: var(--mute-2); -} -/* Two states the owner is meant to find rather than read for: a Reader whose - reports no longer defer a Poll, and a Lane that is not keeping its promise. - Both wear --patina — never ember, which means one thing, and never danger, - which is destruction. */ -.reader-blocked, .lane-mark { - font: 500 10px/1 var(--font-mono); - letter-spacing: .14em; - text-transform: uppercase; - color: var(--patina); -} -.lane-site { - font: 400 19px/1.2 var(--font-display); - color: var(--paper-dim); -} -/* The whole row leans patina when the Lane needs attention, so the scan is one - pass down the left edge rather than a read of every mark. */ -.lanelist li.attention .lane-site { color: var(--patina); } -.lane-browser { padding: 12px 0 0; } -/* Revocation cuts someone off, so it wears --danger. Ember stays reserved for - the new-chapter signal. */ -.ghost.danger { color: var(--danger); } -.ghost.danger:hover { color: var(--danger); border-bottom-color: var(--danger); } .chrome { display: flex; flex-direction: column; } @@ -643,9 +580,6 @@ button { cursor: pointer; } box-shadow: inset 0 -2px 0 var(--ember); } .topbar form { margin-left: 18px; } -/* The admin page's topbar has no switch to fill the middle, so its back link - keeps company with Log out at the right edge instead of floating centre. */ -.topbar .back { margin-left: auto; } /* At phone width brand + switch + Log out do not fit on one line, so the switch takes its own row under the wordmark rather than pushing Log out off-screen. */ @@ -655,9 +589,6 @@ button { cursor: pointer; } .libswitch { order: 3; margin-left: 0; } .libswitch a { flex: 1; text-align: center; padding: 8px 14px; } .topbar form { margin-left: 12px; } - /* The admin page has no switch to take the second row, so its brand claims - the first outright and the back link keeps Log out company below. */ - .topbar:has(.back) .brand { flex: 1 1 100%; } } /* ---- action strip: full-width on a phone, hairline-divided cells ---- */ diff --git a/backend/internal/web/templates/admin.html b/backend/internal/web/templates/admin.html index f2f8e0d..79e287d 100644 --- a/backend/internal/web/templates/admin.html +++ b/backend/internal/web/templates/admin.html @@ -1,7 +1,5 @@ -{{/* The owner's administrative page: everything that reaches past one Reader, - at its own address so it can be bookmarked rather than hunted for inside - the reading page. Owner-only at route registration (requireOwner), which - is why nothing in here re-tests who is asking. */}} +{{/* Every owner-only address shares this shell; page content stays behind its + bookmarkable route so the active tab survives a reload. */}} {{define "admin"}} @@ -12,27 +10,40 @@ BookmarkManager — Admin + -
+

{{template "mark" .}}BookmarkManager

- {{/* Back to the library, no switch: this page belongs to neither library, - and the ember-lit switch says which library you are reading. */}} - Library -
- -
+ + Library +
+ +
+
- {{/* The live region wraps the swapped block rather than being it: the - refresh replaces the section wholesale, and a region recreated on every - update is never announced. */}} -
{{template "lanes" .Lanes}}
+ - {{template "readers" .}} +
+ {{if eq .Page "lanes"}} +
{{template "lanes" .Lanes}}
+ {{else if eq .Page "readers"}} + {{template "readers" .}} + {{else if eq .Page "series"}} + {{template "series-list" .SeriesList}} + {{else if eq .Page "series-detail"}} + {{template "series-detail" .Detail}} + {{else}}{{template "overview" .Overview}}{{end}} +
diff --git a/backend/internal/web/templates/lanes.html b/backend/internal/web/templates/lanes.html index 32a3976..c4a359e 100644 --- a/backend/internal/web/templates/lanes.html +++ b/backend/internal/web/templates/lanes.html @@ -1,42 +1,60 @@ -{{/* Poll Lane status: one row per Site, refreshing itself so a run can be - watched rather than sampled by reloading. The refresh is one attribute on - the fragment root and the endpoint answers with this same fragment, so the - swap replaces the element that asked for it. +{{/* Poll Lane status: one row per Site's latest durable pass, refreshing + itself so a run can be watched rather than sampled by reloading. The + refresh is one attribute on the fragment root and the endpoint answers + with this same fragment, so the swap replaces the element that asked. - Every figure here is read out of the running poller, never out of a table: - a Site absent from Rows has not completed a pass since the last restart, - which the empty state must say — zeroes would read as a stopped Lane. */}} + Every figure is read from poll_passes, never from a running poller: a + restart answers from the database the moment it is up (issue #145). The + browser fact is a deployment-config fact plus a reachability derived from + the pass log; the cause chips and the state phrase are decided in Go, + this template only prints them. */}} {{define "lanes"}}
-

Poll Lanes

+
+

Poll Lanes

+

+ {{if .PollerOff}}Polling: off + {{else}}Browser: {{if not .BrowserConfigured}}not configured{{else if .BrowserReachable}}reachable{{else}}unreachable{{end}}{{end}} +

+
{{if .Rows}} -
    +
    +
    + SiteDueCheckedGap + Last passOutcomes · state +
    {{range .Rows}} - - {{.Site}} - {{.Due}} due - {{.Checked}} checked - ran {{.Ran}} - {{if .Gap}}gap {{.Gap}}{{end}} - {{if .Clamped}}gap at floor{{end}} - {{if .Refusing}}refusing{{end}} - {{if .BrowserLost}}no browser{{end}} - {{if .Stalled}}not checking{{end}} - {{if .Asleep}}browser asleep{{end}} - +
    + {{.Site}} + {{.Due}} + {{.Checked}} + {{.Gap}} + ran {{.Ran}} + {{if .HasChips}}{{range $i, $c := .Chips}}{{if $i}} · {{end}}{{$c.Name}} {{$c.Count}}{{end}}{{else}}none observed{{end}}{{if .StatePhrase}} · {{.StatePhrase}}{{end}} + {{/* The pause control lives in the one slot the design leaves for it: + a running Lane offers the three durations and Pause; a paused Lane + offers Resume in the same place. Pause is not destruction — it + takes nothing away and reverses in one press — so neither wears a + confirm row or the danger accent. The form wraps the select so the + offered duration travels with the press. */}} + {{if .Paused}} +
    + +
    +     {{else}} +
    + + +
    +     {{end}}     +
    {{end}} -
+
{{else}} -

No data yet — no Lane has completed a pass since the - backend started.

+

No data yet — no Lane has recorded a pass.

{{end}} -

- {{if .PollerOff}}Polling is switched off in this deployment: no Lane runs, - and Latest Chapter comes from the userscripts alone. - {{else}}Browser sidecar: - {{if not .BrowserConfigured}}not configured — comix, kagane and novelfull - pages are not fetched through it{{else if .BrowserReachable}}reachable - {{else}}unreachable{{end}}.{{end}}

-{{end}} +{{end}} \ No newline at end of file diff --git a/backend/internal/web/templates/overview.html b/backend/internal/web/templates/overview.html new file mode 100644 index 0000000..701772d --- /dev/null +++ b/backend/internal/web/templates/overview.html @@ -0,0 +1,14 @@ +{{/* The Overview landing page: one verdict line leading, then a stats block + where every figure is a door into the list that counts it, and the + per-Site library shape table. Every judgement — the verdict state, which + figures link, what a Lane's state means — is made in Go; this template + only prints. */}} +{{define "overview"}} +

{{.Verdict}} {{if .HasCounts}}· {{.Waiting}} series waiting · {{.Unchecked}} unchecked over 12h{{end}}

+

Hygiene

+
{{range .Hygiene}}
{{.Label}}{{if .Href}}{{.Count}}{{else}}{{.Count}}{{end}}
{{end}}
+

Library

+
{{range .Library}}
{{.Label}}{{if .Href}}{{.Count}}{{else}}{{.Count}}{{end}}
{{end}}
+

Sites · last 12h

+
SiteSeriesNo coverNever chkStaleState
{{range .Sites}}
{{.Site}}{{range .Figs}}{{if .Href}}{{.Count}}{{else}}{{.Count}}{{end}}{{end}}{{.State}}
{{end}}
+{{end}} diff --git a/backend/internal/web/templates/series-detail.html b/backend/internal/web/templates/series-detail.html new file mode 100644 index 0000000..3866e16 --- /dev/null +++ b/backend/internal/web/templates/series-detail.html @@ -0,0 +1,35 @@ +{{/* Per-Series page: one address per Series, keyed ":" so the + list row is one hop from it. Everything here is a Series-level fact plus + the anonymous Reader count. The Check now control lands in its own .dform + below the (empty) .detail-grid; the pending marker rides the meta line + with the other marks. */}} +{{define "series-detail"}} +← Series +

{{.Title}}

+

{{.Key}} · {{.Site}} · {{.Kind}}

+{{if .Cover}}
+{{else}}
{{end}} +{{template "series-detail-meta" .}} +
+{{if .CanPoll}} +
+
Check now
+
+{{end}} +{{end}} + +{{/* series-detail-meta is the meta line, and the answer a Check now press on + the detail page swaps into its place: the same marks, re-rendered after + the stamp so the pending marker shows. */}} +{{define "series-detail-meta"}} +
+ ch {{.Chapter}} + checked {{.Checked}} + {{.Readers}} readers + {{if .Pending}}{{.Requested}}{{end}} + {{if .Unpollable}}unpollable{{end}} + {{if .NoCover}}no cover{{end}} + {{if .Orphan}}orphan{{end}} + {{if .SightingRaised}}sighting-raised{{end}} +
+{{end}} diff --git a/backend/internal/web/templates/series-list.html b/backend/internal/web/templates/series-list.html new file mode 100644 index 0000000..c2cce35 --- /dev/null +++ b/backend/internal/web/templates/series-list.html @@ -0,0 +1,53 @@ +{{/* The Series list: every Series across every Reader's library, filtered by + one hygiene rule and narrowed by Site and Library. Filter, Site, Library + and page all live in the query string, so the list's state is an address + that can be bookmarked: the two selects submit the GET form, and the + Library segment links and the pager preserve the filter and Site. */}} +{{define "series-list"}} +
+ + + + + both + manga + novels + +
+
{{.Total}} series · {{.FilterLabel}}
+{{if .Rows}} +
+
SiteChCheckedReadersNotes
+ {{range .Rows}}{{template "series-row" .}}{{end}} +
+
+ {{if .PrevHref}}‹ prev{{else}}‹ prev{{end}} + {{.Range}} + {{if .NextHref}}next ›{{else}}next ›{{end}} +
+{{else}} +
No series

Nothing matches {{.FilterLabel}}.

+{{end}} +{{end}} + +{{/* series-row is one Series list row, and the answer a Check now press swaps + into the row's place (hx-target="closest .trow"): it must render the + pending marker the press created. The control is absent on a Series with + no page to fetch and on an orphan, so the owner is never offered a button + that can never do anything. */}} +{{define "series-row"}} +
+ {{.Title}}{{if .Pending}}{{.Requested}}{{end}} + {{.Site}} + {{.Ch}} + {{.Age}} + {{.Readers}} + {{range .Notes}}{{.}}{{end}}{{if .More}}+{{.More}}{{end}} + {{if .CanPoll}}Check now{{end}} +
+{{end}} diff --git a/backend/internal/web/web.go b/backend/internal/web/web.go index e1f3a69..367df34 100644 --- a/backend/internal/web/web.go +++ b/backend/internal/web/web.go @@ -50,9 +50,14 @@ type Handler struct { // httpClient is the plain stdlib client that talks to Discord. It is not // an injected interface: tests point APIBase at a stub server instead. httpClient *http.Client - // lanes is the Poll Lane snapshot source the administrative page reads. - // Nil is a running deployment with no poller, not a bug. - lanes LaneReporter + // pollerEnabled reports whether latest-chapter polling is switched on in + // this deployment (LATEST_CHAPTER_POLL_ENABLED) and browserConfigured + // whether a browser sidecar is configured (BROWSER_WS_URL set). Both are + // deployment facts resolved by the composition root; the Lanes page (issue + // #145) reports them from config and derives reachability from the pass + // log rather than from whether a poller goroutine happened to start. + pollerEnabled bool + browserConfigured bool } // listView is what every list-rendering template receives. @@ -110,9 +115,11 @@ type loginView struct { // New parses every template up front so a broken one kills the process at // startup rather than the first request that touches it. // -// lanes is the administrative page's window onto the running Poller; nil means -// nothing is polling, which the page reports rather than hides. -func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string, lanes LaneReporter) (*Handler, error) { +// pollerEnabled and browserConfigured are deployment facts the composition +// root resolves from LATEST_CHAPTER_POLL_ENABLED and BROWSER_WS_URL: the Lanes +// page (issue #145) reports them and derives browser reachability from the +// pass log, so no running poller is wired through here at all. +func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, novelPath string, pollerEnabled, browserConfigured bool) (*Handler, error) { tmpl, err := template.ParseFS(templateFS, "templates/*.html") if err != nil { return nil, err @@ -127,7 +134,8 @@ func New(s *store.Store, discord DiscordConfig, tokenKey []byte, mangaPath, nove states: newOAuthStates(), limiter: session.NewLoginLimiter(), httpClient: &http.Client{Timeout: discordTimeout}, - lanes: lanes, + pollerEnabled: pollerEnabled, + browserConfigured: browserConfigured, }, nil } diff --git a/backend/main.go b/backend/main.go index bebff8d..629e1cf 100644 --- a/backend/main.go +++ b/backend/main.go @@ -54,6 +54,11 @@ type Config struct { // /u/{token}/novel-bookmark.user.js. Same bindmount, second script: the // two libraries are separate installs. NovelUserscriptPath string + // BrowserWSURL is the CDP websocket the poller's browser Sites read + // through. Set means a browser sidecar is configured in this deployment — + // the Lanes page reports the fact and derives reachability from the pass + // log rather than asking the poller (issue #145). + BrowserWSURL string // LatestPoll configures the background latest-chapter fetcher. LatestPoll LatestPoll } @@ -108,6 +113,7 @@ func loadConfig() Config { OwnerDiscordID: os.Getenv("OWNER_DISCORD_ID"), UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"), NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"), + BrowserWSURL: os.Getenv("BROWSER_WS_URL"), LatestPoll: loadLatestPoll(), } c.Discord = web.DiscordConfig{ @@ -130,9 +136,10 @@ func loadConfig() Config { // preflight OPTIONS short-circuits before auth; /bookmarks* is auth-protected, // /healthz is public. // -// lanes may be nil — polling disabled, or its client could not be built. The -// admin page reports that rather than pretending Lanes exist. -func newRouter(s *store.Store, cfg Config, lanes web.LaneReporter) http.Handler { +// The web layer learns the deployment's poller and browser config from cfg — +// nothing of the running poller is wired through here; the Lanes page reads +// the database (issue #145). +func newRouter(s *store.Store, cfg Config) http.Handler { mux := http.NewServeMux() h := &api.Handler{Store: s} mux.HandleFunc("GET /healthz", api.Healthz) @@ -161,9 +168,12 @@ func newRouter(s *store.Store, cfg Config, lanes web.LaneReporter) http.Handler mux.Handle("/bookmarks/", auth) // The browser UI is always registered; signing in is Discord OAuth, so - // there is no password to forget and no gate to leave unset. + // there is no password to forget and no gate to leave unset. The poller + // and browser facts are config, not the poller's: the Lanes page reads + // the pass log and reports the deployment as configured. wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey), - cfg.UserscriptPath, cfg.NovelUserscriptPath, lanes) + cfg.UserscriptPath, cfg.NovelUserscriptPath, + cfg.LatestPoll.Enabled, strings.TrimSpace(cfg.BrowserWSURL) != "") if err != nil { log.Fatalf("web handler: %v", err) } @@ -241,7 +251,7 @@ func main() { var browser latest.Fetcher pollCtx, stopPoll := context.WithCancel(context.Background()) defer stopPoll() - if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" { + if ws := strings.TrimSpace(cfg.BrowserWSURL); ws != "" { bf, err := latest.NewBrowserFetcher(ws) if err != nil { log.Printf("browser fetcher disabled: %v", err) @@ -278,17 +288,14 @@ func main() { } s.OnSeriesCreated = acq.Acquire } - // A nil *Poller must not become a non-nil interface holding a nil pointer: - // the admin page tests the reporter for nil to decide whether anything is - // polling at all. - var lanes web.LaneReporter - if poller := startLatestPoller(pollCtx, s, cfg.LatestPoll, browser); poller != nil { - lanes = poller - } + // The poller's only connection to the web layer is the database now: it is + // started for its own sake, and the Lanes page reads the pass rows it + // records (issue #145). + startLatestPoller(pollCtx, s, cfg.LatestPoll, browser) srv := &http.Server{ Addr: ":" + cfg.Port, - Handler: newRouter(s, cfg, lanes), + Handler: newRouter(s, cfg), ReadHeaderTimeout: 10 * time.Second, } @@ -336,7 +343,8 @@ func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetch // HTTP client cannot be built. Any problem here is logged and skipped: this // feature going missing degrades the service to userscript-only latest-chapter // tracking, which is exactly how it behaved before. It returns the running -// Poller, or nil when there is none — the admin page's Lane status reads it. +// Poller, or nil when there is none; the caller starts it for its own sake — +// the Lanes page reads the pass log, so no return value is wired anywhere. func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) *latest.Poller { if !cfg.Enabled { log.Println("latest-chapter poller: disabled by config") diff --git a/backend/reader_credential_test.go b/backend/reader_credential_test.go index 03d2a85..e66970c 100644 --- a/backend/reader_credential_test.go +++ b/backend/reader_credential_test.go @@ -49,7 +49,7 @@ func withBody(req *http.Request, body string) *http.Request { // A refused credential is refused however plausible it looks: only a hash the // readers table holds authenticates anything. func TestUnknownCredentialRejected(t *testing.T) { - srv := newRouter(newTestStore(t), testConfig(), nil) + srv := newRouter(newTestStore(t), testConfig()) rr := httptest.NewRecorder() srv.ServeHTTP(rr, credRequest(http.MethodGet, "/bookmarks", readerCredential("never-registered"))) @@ -69,7 +69,7 @@ func TestUnknownCredentialRejected(t *testing.T) { func TestPerReaderIsolation(t *testing.T) { s := newTestStore(t) registerReader(t, s, "other-reader") - srv := newRouter(s, testConfig(), nil) + srv := newRouter(s, testConfig()) ownerKey := "asura:solo" putBookmark(t, srv, ownerKey, store.Bookmark{ @@ -267,7 +267,7 @@ func TestRotateCredentialViaWebUI(t *testing.T) { } cfg := testConfig() cfg.UserscriptPath = path - srv := newRouter(s, cfg, nil) + srv := newRouter(s, cfg) oldCred := ownerCredential() rr := httptest.NewRecorder() diff --git a/backend/web_test.go b/backend/web_test.go index baf61c7..3d701f5 100644 --- a/backend/web_test.go +++ b/backend/web_test.go @@ -1,6 +1,7 @@ package main import ( + "crypto/sha256" "database/sql" "encoding/json" "fmt" @@ -11,6 +12,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strconv" "strings" "testing" @@ -28,17 +30,13 @@ import ( const testOwnerID = "owner-snowflake" // newWebTestServer returns the full router plus the store behind it, so tests -// can seed rows and assert on what the handlers wrote back. An optional lane -// reporter stands in for the running poller; omitted means none is running, -// which is what every test that is not about the admin page wants. -func newWebTestServer(t *testing.T, cfg Config, lanes ...web.LaneReporter) (http.Handler, *store.Store) { +// can seed rows and assert on what the handlers wrote back. The Lanes page +// reads the pass log (issue #145), so a test seeds store rows rather than +// standing in for a poller. +func newWebTestServer(t *testing.T, cfg Config) (http.Handler, *store.Store) { t.Helper() st := newTestStore(t) - var reporter web.LaneReporter - if len(lanes) > 0 { - reporter = lanes[0] - } - return newRouter(st, cfg, reporter), st + return newRouter(st, cfg), st } // sessionCookie mints a live session row for the owner and returns the cookie @@ -149,12 +147,12 @@ func discordConfig(stubURL string) web.DiscordConfig { // oauthWebTestServer returns the full router, its store, and a Discord stub // wired as the configured API — the starting point for sign-in tests. -func oauthWebTestServer(t *testing.T, lanes ...web.LaneReporter) (http.Handler, *store.Store, *discordStub) { +func oauthWebTestServer(t *testing.T) (http.Handler, *store.Store, *discordStub) { t.Helper() stub, srv := newDiscordStub(t) cfg := testConfig() cfg.Discord = discordConfig(srv.URL) - router, st := newWebTestServer(t, cfg, lanes...) + router, st := newWebTestServer(t, cfg) return router, st, stub } @@ -418,7 +416,7 @@ func TestDiscordLoginRefusesNonMember(t *testing.T) { cfg.Discord = discordConfig(srv.URL) cfg.Discord.RequiredRole = tc.require st := newTestStore(t) - router := newRouter(st, cfg, nil) + router := newRouter(st, cfg) rr := completeSignIn(t, router, startSignIn(t, router)) if rr.Code != http.StatusForbidden { @@ -634,16 +632,41 @@ func TestOwnerRevokesAnotherReadersSessions(t *testing.T) { } } -// fakeLanes is the admin page's poller stand-in: one fixed snapshot, so the -// page's tests need neither a poller nor a Site. -type fakeLanes struct{ status latest.Status } +// seedPass writes one durable pass row — the Lanes page's whole seam. +// The page renders from the database with no poller running at all (issue +// #145), so a test seeds rows instead of constructing a fake reporter. +func seedPass(t *testing.T, st *store.Store, p store.LanePass) { + t.Helper() + if err := st.RecordLanePass(p, -1); err != nil { + t.Fatalf("seed pass %s: %v", p.Site, err) + } +} -func (f fakeLanes) LaneStatus() latest.Status { return f.status } +// lanesConfig returns a config with latest-chapter polling switched on, so +// the Lanes page's statusline speaks about the browser rather than about +// polling being off. +func lanesConfig() Config { + cfg := testConfig() + cfg.LatestPoll.Enabled = true + return cfg +} -// The roster moved off the reading page onto its own address: the owner gets a -// link, everyone else gets nothing, and the page itself lists every Reader with -// the counters and the two controls. -func TestAdminPageCarriesRosterAndOwnerLink(t *testing.T) { +// lanesBody fetches the Lanes fragment as the owner and returns the body. +func lanesBody(t *testing.T, router http.Handler, st *store.Store) string { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil) + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("GET /ui/admin/lanes status = %d, want 200", rr.Code) + } + return rr.Body.String() +} + +// Every admin address carries the same navigation, while the roster only lives +// on its own page and the other pages keep their shells independent. +func TestAdminPagesCarrySharedNavigation(t *testing.T) { router, st, _ := oauthWebTestServer(t) theirCookie := signInCookie(t, router) ownerCookie := sessionCookie(t, st) @@ -654,12 +677,11 @@ func TestAdminPageCarriesRosterAndOwnerLink(t *testing.T) { router.ServeHTTP(rr, req) body := rr.Body.String() if strings.Contains(body, `id="readers"`) { - t.Error("the reading page still carries the roster; it belongs on /admin") + t.Error("the reading page still carries the roster; it belongs on /admin/readers") } if !strings.Contains(body, `href="/admin"`) { t.Error("the owner's reading page offers no link to the admin page") } - req = httptest.NewRequest(http.MethodGet, "/", nil) req.AddCookie(theirCookie) rr = httptest.NewRecorder() @@ -668,21 +690,55 @@ func TestAdminPageCarriesRosterAndOwnerLink(t *testing.T) { t.Error("a non-owner was offered the admin link") } - req = httptest.NewRequest(http.MethodGet, "/admin", nil) + for _, page := range []struct { + path string + name string + }{ + {"/admin", "Overview"}, + {"/admin/lanes", "Lanes"}, + {"/admin/readers", "Readers"}, + {"/admin/series", "Series"}, + } { + t.Run(page.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, page.path, nil) + req.AddCookie(ownerCookie) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("GET %s status = %d, want 200", page.path, rr.Code) + } + body := rr.Body.String() + if !strings.Contains(body, `class="topbar-actions"`) { + t.Errorf("%s has no topbar action cluster:\n%s", page.path, body) + } + if !strings.Contains(body, `aria-label="Admin pages"`) { + t.Errorf("%s has no admin navigation:\n%s", page.path, body) + } + if strings.Count(body, `aria-current="page"`) != 1 { + t.Errorf("%s has %d active admin tabs, want 1:\n%s", page.path, strings.Count(body, `aria-current="page"`), body) + } + if !strings.Contains(body, page.name) { + t.Errorf("%s does not name its active page %q:\n%s", page.path, page.name, body) + } + if !strings.Contains(body, `href="/static/admin.css"`) { + t.Errorf("%s does not load the admin foundation stylesheet", page.path) + } + if page.name == "Lanes" && strings.Count(body, `hx-trigger="every 30s"`) != 1 { + t.Errorf("%s has %d Lane timers, want exactly 1", page.path, strings.Count(body, `hx-trigger="every 30s"`)) + } + }) + } + + req = httptest.NewRequest(http.MethodGet, "/admin/readers", nil) req.AddCookie(ownerCookie) rr = httptest.NewRecorder() router.ServeHTTP(rr, req) - if rr.Code != http.StatusOK { - t.Fatalf("GET /admin status = %d, want 200", rr.Code) - } body = rr.Body.String() for _, want := range []string{`id="readers"`, testOwnerID, "Revoke sessions", "Clear marks", "confirmed"} { if !strings.Contains(body, want) { - t.Errorf("admin page lacks %q:\n%s", want, body) + t.Errorf("readers page lacks %q:\n%s", want, body) } } - // Exactly one revocable row: the other Reader's. The owner's own row carries - // the same session count and no button. if n := strings.Count(body, "/revoke"); n != 1 { t.Fatalf("roster has %d revoke controls, want 1 (the owner's own row must have none):\n%s", n, body) } @@ -707,6 +763,7 @@ func TestAdminRoutesAreOwnerOnly(t *testing.T) { t.Fatalf("route pattern %q has no method", pattern) } path = strings.Replace(path, "{id}", target, 1) + path = strings.Replace(path, "{site}", "asura", 1) for _, tc := range []struct { name string @@ -737,32 +794,22 @@ func TestAdminRoutesAreOwnerOnly(t *testing.T) { } } -// The Lane block reports what the poller says, and marks the Lanes that need -// attention — a clamped gap, a refusal, a Site whose pages can only be read -// through a sidecar that is not there, and a Lane with Series waiting that its -// last pass did not read. +// The Lane block reports what the pass log says, and marks the Lanes that +// need attention: a Site whose pages can only be read through a sidecar that +// is not there, and a Lane with Series waiting that its last pass did not +// read. Rows come from seeded database rows — no poller runs anywhere. func TestAdminPageShowsLaneStatus(t *testing.T) { - lanes := fakeLanes{latest.Status{ - Lanes: []latest.LaneState{ - {Site: "asura", Due: 12, Checked: 12, LastRun: time.Now().Add(-90 * time.Second), Gap: 40 * time.Second}, - {Site: "kagane", Due: 3, Checked: 3, LastRun: time.Now().Add(-time.Minute), Gap: time.Minute, Browser: true}, - {Site: "demonic", Due: 400, Checked: 400, LastRun: time.Now(), Gap: 8 * time.Second, Clamped: true}, - {Site: "comix", Due: 7, LastRun: time.Now(), Gap: time.Minute, Browser: true}, - }, - BrowserConfigured: true, - BrowserReachable: true, - }} - router, st, _ := oauthWebTestServer(t, lanes) + cfg := lanesConfig() + cfg.BrowserWSURL = "ws://browser:9222" + router, st := newWebTestServer(t, cfg) + now := time.Now() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.Add(-90 * time.Second).UnixMilli(), Due: 12, Checked: 12, GapMS: 40_000}) + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.Add(-time.Minute).UnixMilli(), Due: 3, Checked: 3, GapMS: 60_000}) + seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Due: 400, Checked: 400, GapMS: 8_000}) + seedPass(t, st, store.LanePass{Site: "comix", RanAt: now.UnixMilli(), Due: 7, Checked: 0, GapMS: 60_000}) - req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil) - req.AddCookie(sessionCookie(t, st)) - rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) - if rr.Code != http.StatusOK { - t.Fatalf("GET /ui/admin/lanes status = %d, want 200", rr.Code) - } - body := rr.Body.String() - for _, want := range []string{"asura", "kagane", "12 due", "12 checked", "gap 40s", "ran 1m30s ago", "gap at floor", "not checking", "reachable"} { + body := lanesBody(t, router, st) + for _, want := range []string{"asura", "kagane", "demonic", "comix", "40s", "ran 1m30s ago", "not checking", "none observed", "reachable"} { if !strings.Contains(body, want) { t.Errorf("lane status lacks %q:\n%s", want, body) } @@ -778,78 +825,75 @@ func TestAdminPageShowsLaneStatus(t *testing.T) { // A browser Lane under both wake thresholds holds Chrome asleep (ADR-0005), so // Series due with none checked is the design working, not a stopped Lane. The -// two must not render the same mark: "not checking" is the owner's cue to go -// looking, and spending it on the commonest healthy browser-Lane state trains -// them to ignore it. +// pass row records it with the asleep skip, and the page must render its +// sleeping sentence with no stall mark and no attention. func TestAsleepBrowserLaneIsNotMarkedStalled(t *testing.T) { - lanes := fakeLanes{latest.Status{ - Lanes: []latest.LaneState{ - {Site: "kagane", Due: 1, LastRun: time.Now(), Gap: 10 * time.Second, Browser: true, Asleep: true}, - }, - BrowserConfigured: true, - BrowserReachable: true, - }} - router, st, _ := oauthWebTestServer(t, lanes) + cfg := lanesConfig() + cfg.BrowserWSURL = "ws://browser:9222" + router, st := newWebTestServer(t, cfg) + seedPass(t, st, store.LanePass{ + Site: "kagane", RanAt: time.Now().UnixMilli(), + Skip: latest.SkipAsleep, Due: 1, Checked: 0, GapMS: 10_000, + }) - req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil) - req.AddCookie(sessionCookie(t, st)) - rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) - body := rr.Body.String() + body := lanesBody(t, router, st) if strings.Contains(body, "not checking") { t.Errorf("an asleep browser Lane is marked as stalled:\n%s", body) } if !strings.Contains(body, "browser asleep") { t.Errorf("an asleep browser Lane says nothing about why it read nothing:\n%s", body) } - if strings.Contains(body, `class="attention"`) { + if strings.Contains(body, `class="trow attention"`) { t.Errorf("an asleep browser Lane is coloured as unhealthy:\n%s", body) } } // A Lane whose pass never reached a figure must not have that figure drawn as -// a zero: a refusing Lane still reports the due count and gap its last real -// pass saw, and a Lane that has never reached one omits it entirely. +// a zero: a refusing Lane that has never gathered figures draws "—" for the +// gap rather than stating a zero it did not measure. func TestLaneStatusOmitsUnknownGap(t *testing.T) { - lanes := fakeLanes{latest.Status{ - Lanes: []latest.LaneState{{Site: "comix", LastRun: time.Now(), Refusing: true, Browser: true}}, - BrowserConfigured: true, - BrowserReachable: true, - }} - router, st, _ := oauthWebTestServer(t, lanes) + cfg := lanesConfig() + cfg.BrowserWSURL = "ws://browser:9222" + st, dsn := newTestStoreURL(t) + router := newRouter(st, cfg) + now := time.Now() + oldNow := now.Add(-time.Minute).UnixMilli() + seedPass(t, st, store.LanePass{Site: "comix", RanAt: oldNow, Skip: latest.SkipRefusing}) + // Refusal expiry lives on the Lane (poll_lanes), not the pass row, so it + // must be seeded there for the backs-off-until clause to render. + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + if _, err := db.Exec(`INSERT INTO poll_lanes (site, refuse_until) VALUES ($1, $2)`, "comix", now.Add(10*time.Minute).UnixMilli()); err != nil { + t.Fatalf("seed lane refusal: %v", err) + } - req := httptest.NewRequest(http.MethodGet, "/ui/admin/lanes", nil) - req.AddCookie(sessionCookie(t, st)) - rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) - body := rr.Body.String() - if strings.Contains(body, "gap 0s") { + body := lanesBody(t, router, st) + if strings.Contains(body, `>0s<`) { t.Errorf("a Lane with no pace yet states a zero gap:\n%s", body) } - if !strings.Contains(body, "refusing") { - t.Errorf("a refusing Lane is not marked as such:\n%s", body) + if !strings.Contains(body, "refusing · backs off until") { + t.Errorf("a refusing Lane is not marked with its backoff time:\n%s", body) } } -// No poller and a poller that has not finished a pass both render "no data -// yet" rather than zeroes that read as a stopped backend — but they are not -// the same fact, so the page must not blame the sidecar when nothing polls. +// Polling switched off and a browser not configured are different facts, and +// the page must not blame the sidecar when nothing polls. Neither is a poller +// running — the Lanes page answers entirely from config and the pass log. func TestAdminPageWithoutAPollerSaysSo(t *testing.T) { for _, tc := range []struct { name string - lanes []web.LaneReporter + cfg Config want, unwant string }{ - {"no poller", nil, "Polling is switched off", "not configured"}, - {"poller, no pass yet", []web.LaneReporter{fakeLanes{}}, "not configured", "Polling is switched off"}, + {"polling switched off", testConfig(), `Polling: off`, "not configured"}, + {"browser not configured", lanesConfig(), "not configured", "Polling is switched off"}, } { t.Run(tc.name, func(t *testing.T) { - router, st, _ := oauthWebTestServer(t, tc.lanes...) - req := httptest.NewRequest(http.MethodGet, "/admin", nil) - req.AddCookie(sessionCookie(t, st)) - rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) - body := rr.Body.String() + router, st := newWebTestServer(t, tc.cfg) + body := lanesBody(t, router, st) if !strings.Contains(body, "No data yet") { t.Errorf("admin page with no Lane data does not say so:\n%s", body) } @@ -863,12 +907,334 @@ func TestAdminPageWithoutAPollerSaysSo(t *testing.T) { } } +// Restart survival is the point of the durable lane state: rows seeded into +// poll_passes render with no poller running anywhere, complete thirty seconds +// after a deploy. This is the test that proves the in-memory path is gone. +func TestLanesRenderFromSeededRowsAfterRestart(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + now := time.Now() + // A pass a minute ago, another three hours ago: the latest per Site wins. + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.Add(-3 * time.Hour).UnixMilli(), Due: 1, Checked: 1, GapMS: 10_000}) + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.Add(-time.Minute).UnixMilli(), Due: 5, Checked: 5, GapMS: 20_000}) + + body := lanesBody(t, router, st) + if !strings.Contains(body, `class="c-site">asura`) { + t.Fatalf("asura row missing from a restart-read database:\n%s", body) + } + // The fragment carries its own single timer and answers the swap it asked + // for, so the page keeps refreshing against the database. + if !strings.Contains(body, `hx-get="/ui/admin/lanes"`) || !strings.Contains(body, `hx-trigger="every 30s"`) { + t.Errorf("Lanes fragment lost its self-refresh:\n%s", body) + } +} + +// A skip reason is the whole difference between a Lane resting and a Lane +// stuck: a skipped pass says why it declined, and the one true stall — empty +// skip with Series due and none read — is the only thing that draws the fault. +func TestSkipReasonIsNotAStall(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + now := time.Now() + // Asleep: due but none checked, with a skip that says why — no stall, no + // attention. + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.UnixMilli(), Skip: latest.SkipAsleep, Due: 2, Checked: 0, GapMS: 10_000}) + // The true stall: reached the loop, Series waiting, none read. + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Due: 3, Checked: 0, GapMS: 20_000}) + + body := lanesBody(t, router, st) + if !strings.Contains(body, "browser asleep") { + t.Errorf("the asleep row does not say why it declined:\n%s", body) + } + if !strings.Contains(body, "not checking") { + t.Errorf("the true stall is not rendered as a fault:\n%s", body) + } + // Exactly the stall row wears attention; the asleep row never does. + if strings.Count(body, `class="trow attention"`) != 1 { + t.Errorf("attention is on %d rows, want exactly the stall:\n%s", strings.Count(body, `class="trow attention"`), body) + } +} + +// The five outcome counts render named — the page never prints the word +// "failures" — and a Site with none observed says so rather than drawing a +// blank cell. +func TestNamedOutcomeChips(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + now := time.Now() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Refused: 5, NoChapter: 2, Errors: 1}) + seedPass(t, st, store.LanePass{Site: "demonic", RanAt: now.UnixMilli(), Unreachable: 3, Unfetchable: 4}) + seedPass(t, st, store.LanePass{Site: "comix", RanAt: now.UnixMilli()}) + + body := lanesBody(t, router, st) + for _, want := range []string{"refused 5", "no chapter 2", "errors 1", "unreachable 3", "unfetchable 4"} { + if !strings.Contains(body, want) { + t.Errorf("lane chips lack %q:\n%s", want, body) + } + } + if strings.Contains(body, "failures") { + t.Errorf("the page prints the forbidden word \"failures\":\n%s", body) + } + // comix has no outcomes in the window: it must say none observed, and the + // phrase must not be blank. + if !strings.Contains(body, "none observed") { + t.Errorf("a Site with no outcomes does not say none observed:\n%s", body) + } +} + +// Browser configuration is a deployment fact and reachability is derived from +// the latest browser-Site passes inside the refusal backoff — no poller in any +// of the three. +func TestBrowserConfigAndReachabilityDerived(t *testing.T) { + now := time.Now() + + t.Run("not configured", func(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.UnixMilli(), Skip: latest.SkipNoFetcher}) + body := lanesBody(t, router, st) + if !strings.Contains(body, `mark-faint">not configured`) || strings.Contains(body, "unreachable") { + t.Errorf("unset BROWSER_WS_URL must read as not configured:\n%s", body) + } + }) + + t.Run("unreachable from recent sidecar-down", func(t *testing.T) { + cfg := lanesConfig() + cfg.BrowserWSURL = "ws://browser:9222" + router, st := newWebTestServer(t, cfg) + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.Add(-time.Minute).UnixMilli(), Skip: latest.SkipSidecarDown}) + body := lanesBody(t, router, st) + if !strings.Contains(body, `bad">unreachable`) { + t.Errorf("recent sidecar-down passes must read as unreachable:\n%s", body) + } + }) + + t.Run("reachable from clean passes", func(t *testing.T) { + cfg := lanesConfig() + cfg.BrowserWSURL = "ws://browser:9222" + router, st := newWebTestServer(t, cfg) + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.Add(-time.Minute).UnixMilli(), Checked: 3}) + body := lanesBody(t, router, st) + if !strings.Contains(body, `mark-strong">reachable`) { + t.Errorf("clean browser passes must read as reachable:\n%s", body) + } + }) + + t.Run("sidecar-down outside the backoff is reachable again", func(t *testing.T) { + cfg := lanesConfig() + cfg.BrowserWSURL = "ws://browser:9222" + router, st := newWebTestServer(t, cfg) + seedPass(t, st, store.LanePass{Site: "kagane", RanAt: now.Add(-30 * time.Minute).UnixMilli(), Skip: latest.SkipSidecarDown}) + body := lanesBody(t, router, st) + if !strings.Contains(body, `mark-strong">reachable`) { + t.Errorf("a sidecar-down older than the backoff must read as reachable:\n%s", body) + } + }) +} + +// Pressing Pause writes a future expiry at the offered length and answers +// with the freshly rendered Lanes block, so the row the owner just pressed +// reads as paused with its remaining time and offers Resume — with no poller +// having run at all. The pause is a fact about the Site, never a command to +// a process (issue #147). +func TestLanePauseRoundTrip(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + // A stall-shaped pass — due but none checked, no skip — so the test + // proves the pause renders over it as paused, never as the one true + // stall. + seedPass(t, st, store.LanePass{Site: "asura", RanAt: time.Now().UnixMilli(), Due: 3, Checked: 0, GapMS: 10_000}) + + for _, tc := range []struct{ duration, phrase string }{ + {"1h", "paused · resumes in 1h"}, + {"6h", "paused · resumes in 6h"}, + {"24h", "paused · resumes in 24h"}, + } { + t.Run(tc.duration, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/admin/lanes/asura/pause", + strings.NewReader("duration="+tc.duration)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("POST pause status = %d, want 200 (body %s)", rr.Code, rr.Body.String()) + } + body := rr.Body.String() + if !strings.Contains(body, `id="lanes"`) { + t.Errorf("pause response is not the Lanes block:\n%s", body) + } + if !strings.Contains(body, tc.phrase) { + t.Errorf("pause response does not render %q:\n%s", tc.phrase, body) + } + if !strings.Contains(body, `hx-post="/admin/lanes/asura/resume"`) { + t.Errorf("pause response does not offer Resume for asura:\n%s", body) + } + if strings.Contains(body, `class="trow attention"`) { + t.Errorf("a paused Lane is marked for attention:\n%s", body) + } + if strings.Contains(body, "not checking") { + t.Errorf("a paused Lane reads as the one true stall:\n%s", body) + } + // The expiry is a future fact about the Site, at the offered length. + paused, _, err := st.LaneGates("asura") + if err != nil { + t.Fatalf("LaneGates: %v", err) + } + d, _ := time.ParseDuration(tc.duration) + want := time.Now().Add(d).UnixMilli() + if paused < want-time.Minute.Milliseconds() || paused > want+time.Minute.Milliseconds() { + t.Errorf("pause expiry = %d, want now+%s (%d, within a minute)", paused, tc.duration, want) + } + }) + } +} + +// A missing duration and any value outside the offered set are refused with +// 400 and nothing is written: a permissive parser would turn the offered set +// into "anything Go can read", and an unoffered pause is a silent outage the +// owner left behind (issue #147). +func TestLanePauseRejectsBadDurations(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + seedPass(t, st, store.LanePass{Site: "asura", RanAt: time.Now().UnixMilli()}) + + for _, tc := range []struct{ name, body string }{ + {"missing", ""}, + {"empty value", "duration="}, + {"unoffered", "duration=2h"}, + {"zero", "duration=0h"}, + {"absurd", "duration=999h"}, + {"not a duration", "duration=six"}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/admin/lanes/asura/pause", strings.NewReader(tc.body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusBadRequest { + t.Errorf("pause body %q status = %d, want 400", tc.body, rr.Code) + } + }) + } + paused, _, err := st.LaneGates("asura") + if err != nil { + t.Fatalf("LaneGates: %v", err) + } + if paused != 0 { + t.Errorf("a rejected pause still wrote expiry %d", paused) + } +} + +// The Site in the path is client-supplied, so it is checked against the +// registry before the store sees it: an unknown Site is a 400 on both action +// routes, not a write (issue #147). +func TestLaneActionsRejectUnknownSite(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + for _, tc := range []struct{ path, body string }{ + {"/admin/lanes/notasite/pause", "duration=6h"}, + {"/admin/lanes/notasite/resume", ""}, + } { + req := httptest.NewRequest(http.MethodPost, tc.path, strings.NewReader(tc.body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusBadRequest { + t.Errorf("POST %s status = %d, want 400", tc.path, rr.Code) + } + } +} + +// Pressing Resume zeroes the pause and answers with the freshly rendered +// block: the same slot now offers the duration select and Pause, and the +// paused phrase is gone. The queue half of resume lives in the poller tests +// (issue #147). +func TestLaneResumeRoundTrip(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + seedPass(t, st, store.LanePass{Site: "asura", RanAt: time.Now().UnixMilli(), Due: 2, Checked: 2, GapMS: 10_000}) + if err := st.PauseLane("asura", time.Now().Add(6*time.Hour).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/admin/lanes/asura/resume", nil) + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("POST resume status = %d, want 200 (body %s)", rr.Code, rr.Body.String()) + } + body := rr.Body.String() + if !strings.Contains(body, `hx-post="/admin/lanes/asura/pause"`) || !strings.Contains(body, ">Pause") { + t.Errorf("resume response does not offer Pause again:\n%s", body) + } + if strings.Contains(body, "Resume") || strings.Contains(body, "paused") { + t.Errorf("resume response still reads as paused:\n%s", body) + } + paused, _, err := st.LaneGates("asura") + if err != nil { + t.Fatalf("LaneGates: %v", err) + } + if paused != 0 { + t.Errorf("resume left pause expiry %d, want 0", paused) + } +} + +// Form bodies on both action routes are capped the way the API path caps +// them: an oversized body is a 400, not a memory grant, and nothing is +// written (issue #147). +func TestLaneActionsCapBody(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + seedPass(t, st, store.LanePass{Site: "asura", RanAt: time.Now().UnixMilli()}) + big := strings.Repeat("a", 1<<17) // 128 KiB, over the 64 KiB cap + for _, path := range []string{"/admin/lanes/asura/pause", "/admin/lanes/asura/resume"} { + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(big)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + if rr.Code != http.StatusBadRequest { + t.Errorf("oversized body on %s status = %d, want 400", path, rr.Code) + } + } + paused, _, err := st.LaneGates("asura") + if err != nil { + t.Fatalf("LaneGates: %v", err) + } + if paused != 0 { + t.Errorf("an oversized body still paused the Lane (expiry %d)", paused) + } +} + +// A pause renders as paused, never as stalled: the owner's own act must not +// be reported back as a fault. The stamp lives on poll_lanes and the pass +// rows join it, so a pause seeded straight into the store — a restart, no +// poller anywhere — renders its patina phrase with no attention flag and no +// stall mark (issue #147). +func TestPausedLaneRendersAsPausedNotStalled(t *testing.T) { + router, st := newWebTestServer(t, lanesConfig()) + now := time.Now() + seedPass(t, st, store.LanePass{Site: "asura", RanAt: now.UnixMilli(), Due: 3, Checked: 0, GapMS: 10_000}) + if err := st.PauseLane("asura", now.Add(6*time.Hour).UnixMilli()); err != nil { + t.Fatalf("PauseLane: %v", err) + } + + body := lanesBody(t, router, st) + if !strings.Contains(body, `class="ok">paused · resumes in 6h<`) { + t.Errorf("a paused Lane does not render the patina phrase:\n%s", body) + } + if strings.Contains(body, `class="trow attention"`) { + t.Errorf("a paused Lane is marked for attention:\n%s", body) + } + if strings.Contains(body, "not checking") { + t.Errorf("a paused Lane reads as the one true stall:\n%s", body) + } + if strings.Contains(body, `class="bad"`) { + t.Errorf("a paused Lane wears the fault accent:\n%s", body) + } +} + // A Reader past the disagreement threshold is rendered as blocked, and // clearing their marks both zeroes the counters and lifts the block in the // roster the response carries back. func TestOwnerClearsReaderMarks(t *testing.T) { st, dsn := newTestStoreURL(t) - router := newRouter(st, testConfig(), nil) + router := newRouter(st, testConfig()) cookie := sessionCookie(t, st) // The counters are filled by issue #103; until it lands the only way to // stand a marked Reader up is to write the columns directly. @@ -881,7 +1247,7 @@ func TestOwnerClearsReaderMarks(t *testing.T) { t.Fatalf("mark reader: %v", err) } - req := httptest.NewRequest(http.MethodGet, "/admin", nil) + req := httptest.NewRequest(http.MethodGet, "/admin/readers", nil) req.AddCookie(cookie) rr := httptest.NewRecorder() router.ServeHTTP(rr, req) @@ -1808,3 +2174,1067 @@ func TestNovelNewTabFallsBackToAll(t *testing.T) { t.Fatal("novel tab=new should render the novel All list") } } + +// seriesRowSeed is one series row (and optionally its bookmarks) for the +// Series list tests. Seeded with direct SQL because the store's own surface +// cannot produce an orphan series or a Reader-raised Latest Chapter — the +// same reason the store's admin tests seed this way. +type seriesRowSeed struct { + key string + kind string + url string + cover string // cover_address + checkedAt int64 + latestNum *float64 + bookmarks int // readers that hold it; 0 = orphan + raisedBy bool // a Reader's report is attributed as the raiser +} + +// seedSeriesRow inserts one series row and its bookmarks (owner first, then +// fresh readers) with the exact admin-relevant facts a test needs. +func seedSeriesRow(t *testing.T, st *store.Store, db *sql.DB, seed seriesRowSeed) { + t.Helper() + site, seriesID, ok := strings.Cut(seed.key, ":") + if !ok { + t.Fatalf("key %q: no ':' separator", seed.key) + } + if seed.kind == "" { + seed.kind = store.KindManga + } + var latestChapter any = "" + if seed.latestNum != nil { + latestChapter = "Chapter " + strconv.FormatFloat(*seed.latestNum, 'f', -1, 64) + } + if _, err := db.Exec(` + INSERT INTO series (site, series_id, title, kind, series_url, cover_address, + latest_checked_at, latest_chapter, latest_chapter_num) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + site, seriesID, "Title of "+seed.key, seed.kind, seed.url, seed.cover, + seed.checkedAt, latestChapter, seed.latestNum); err != nil { + t.Fatalf("seed series %q: %v", seed.key, err) + } + for i := range seed.bookmarks { + var readerID int64 = st.OwnerID() + if i > 0 { + readerID = seedReader(t, st) + } + if _, err := db.Exec(` + INSERT INTO bookmarks (reader_id, site, series_id, + last_chapter, last_chapter_num, last_chapter_url, + favorite, status, updated_at) + VALUES ($1, $2, $3, '', 0, '', false, 'reading', $4)`, + readerID, site, seriesID, seed.checkedAt); err != nil { + t.Fatalf("seed bookmark %q: %v", seed.key, err) + } + } + if seed.raisedBy { + if _, err := db.Exec( + `UPDATE series SET latest_raised_by = $1 WHERE site = $2 AND series_id = $3`, + st.OwnerID(), site, seriesID); err != nil { + t.Fatalf("seed raised-by %q: %v", seed.key, err) + } + } +} + +// seedReader mints a fresh Reader for a second bookmark, so a series can carry +// a Reader count above one. +func seedReader(t *testing.T, st *store.Store) int64 { + t.Helper() + discordID := "seed-" + strconv.FormatInt(time.Now().UnixNano(), 10) + id, err := st.EnsureReader(discordID, [32]byte{}) + if err != nil { + t.Fatalf("EnsureReader: %v", err) + } + return id +} + +// adminSeriesPage drives one Series list request as the owner and returns the +// rendered body, failing the test on anything but a 200. +func adminSeriesPage(t *testing.T, srv http.Handler, st *store.Store, query string) string { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/admin/series"+query, nil) + req.AddCookie(sessionCookie(t, st)) + rr := httptest.NewRecorder() + srv.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("GET /admin/series%s status = %d, want 200", query, rr.Code) + } + return rr.Body.String() +} + +// The filter select reaches the store as the wire constant and the heading +// states the same total the rows render: ?filter=no_cover renders only the +// no-cover row, its option label carries its library-wide count, and an +// unknown filter value is the absent All case, never an error. +func TestSeriesListFilterWiring(t *testing.T) { + st, dsn := newTestStoreURL(t) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open %s: %v", dsn, err) + } + defer db.Close() + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:healthy", url: "https://asurascans.com/comics/healthy", cover: "aaa", checkedAt: time.Now().UnixMilli(), latestNum: floatPtr(10), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:nocover", url: "https://asurascans.com/comics/nocover", checkedAt: time.Now().UnixMilli(), latestNum: floatPtr(3), bookmarks: 1}) + seedSeriesRow(t, st, db, seriesRowSeed{key: "asura:stale", url: "https://asurascans.com/comics/stale", cover: "bbb", checkedAt: time.Now().Add(-24 * time.Hour).UnixMilli(), latestNum: floatPtr(4), bookmarks: 1}) + srv := newRouter(st, testConfig()) + + body := adminSeriesPage(t, srv, st, "?filter=no_cover") + if !strings.Contains(body, "Title of asura:nocover") { + t.Errorf("no_cover list misses its row:\n%s", body) + } + if strings.Contains(body, "Title of asura:healthy") || strings.Contains(body, "Title of asura:stale") { + t.Errorf("no_cover list renders a covered row:\n%s", body) + } + if !strings.Contains(body, "1 series") || !strings.Contains(body, "No cover") { + t.Errorf("no_cover heading lacks the filtered count and name:\n%s", body) + } + if !strings.Contains(body, "No cover (1)") { + t.Errorf("the filter option label lacks its count:\n%s", body) + } + if !strings.Contains(body, `