Files
mangaBookmark/backend/main.go
T
sulthan 90ce14aec6 feat: one Poll Lane per Site, replacing the shared pace (#100)
Each Site runs its own poll goroutine, paced by Rest and Gap from the
registry (sites.go) instead of the shared cooldown/interval/stagger/batch
config. Rest is enforced by the due query's WHERE clause; the Lane sleeps
its effective gap between fetches (hour/eligible, floored at 1s).

Lane-local failures: two refusals stop that Site for 15m, a lost browser
stops only the round's remaining browser Lanes, and cover heals moved to
background goroutines so a slow CDN cannot consume a Lane's gap. The five
LATEST_CHAPTER_POLL_* pace env vars are gone; only the kill switch
remains.
2026-08-16 13:04:30 +07:00

347 lines
12 KiB
Go

package main
import (
"context"
"errors"
"log"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
"bookmarkmanager/backend/internal/api"
"bookmarkmanager/backend/internal/httpmw"
"bookmarkmanager/backend/internal/latest"
"bookmarkmanager/backend/internal/store"
"bookmarkmanager/backend/internal/token"
"bookmarkmanager/backend/internal/userscript"
"bookmarkmanager/backend/internal/web"
)
// Config holds all runtime settings, sourced from environment variables.
type Config struct {
// TokenKey derives every Reader's userscript credential (internal/token).
// Required: without it no install URL can ever be built.
TokenKey string
AllowedOrigins []string
// DatabaseURL is the Postgres connection URL; required, no default,
// because a wrong guess would silently start on an empty database.
DatabaseURL string
// CoverDir is the filesystem volume for immutable cover bytes. Required:
// serving a stored address without durable bytes would be worse than a
// startup failure.
CoverDir string
// PublicBaseURL is the origin this deployment answers on, e.g.
// "https://bookmarks.example.com". Required: cover URLs go out absolute
// because the userscript renders them on third-party origins, where a
// relative path would resolve against the Site (ADR-0007), and there is
// no way to guess it from a request the poller never sees.
PublicBaseURL string
Port string
// OwnerDiscordID identifies the seeded owner Reader (issue #22). Required:
// bookmarks are scoped to a Reader, and a fresh deployment needs one
// before anybody logs in. The owner is also the only Reader who can revoke
// another Reader's sessions.
OwnerDiscordID string
// Discord is the OAuth application the browser UI signs in with.
Discord web.DiscordConfig
// UserscriptPath is the file served at /u/{token}/manga-bookmark.user.js.
// Supplied by a bindmount so the script can be edited without a rebuild.
UserscriptPath string
// NovelUserscriptPath is the file served at
// /u/{token}/novel-bookmark.user.js. Same bindmount, second script: the
// two libraries are separate installs.
NovelUserscriptPath string
// LatestPoll configures the background latest-chapter fetcher.
LatestPoll LatestPoll
}
// LatestPoll configures the background latest-chapter poller.
//
// Only the kill switch lives here. Pace is per Site — rest time and gap are
// registry properties (internal/latest/sites.go, issue #100), because each
// Lane has to be able to differ from the others. The five environment
// settings that used to size a shared pace (cooldown, browser cooldown,
// interval, stagger, batch) are gone with it: no deployed .env may carry them.
type LatestPoll struct {
Enabled bool
}
func envOr(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
// envBool reads a boolean env var. Anything unrecognised falls back to def.
func envBool(key string, def bool) bool {
switch v := strings.ToLower(strings.TrimSpace(os.Getenv(key))); v {
case "":
return def
case "0", "false", "no", "off":
return false
case "1", "true", "yes", "on":
return true
default:
log.Printf("config: %s=%q is not a boolean, using %v", key, v, def)
return def
}
}
// loadLatestPoll reads the poller's settings. The pace knobs that used to be
// clamped here are registry properties now (issue #100), so there is nothing
// left to clamp.
func loadLatestPoll() LatestPoll {
return LatestPoll{Enabled: envBool("LATEST_CHAPTER_POLL_ENABLED", true)}
}
func loadConfig() Config {
c := Config{
TokenKey: os.Getenv("TOKEN_KEY"),
DatabaseURL: os.Getenv("DATABASE_URL"),
CoverDir: os.Getenv("COVER_DIR"),
PublicBaseURL: os.Getenv("PUBLIC_BASE_URL"),
Port: envOr("PORT", "8080"),
OwnerDiscordID: os.Getenv("OWNER_DISCORD_ID"),
UserscriptPath: envOr("USERSCRIPT_PATH", "/userscript/manga-bookmark.user.js"),
NovelUserscriptPath: envOr("NOVEL_USERSCRIPT_PATH", "/userscript/novel-bookmark.user.js"),
LatestPoll: loadLatestPoll(),
}
c.Discord = web.DiscordConfig{
ClientID: os.Getenv("DISCORD_CLIENT_ID"),
ClientSecret: os.Getenv("DISCORD_CLIENT_SECRET"),
GuildID: os.Getenv("DISCORD_GUILD_ID"),
RequiredRole: os.Getenv("DISCORD_REQUIRED_ROLE"),
APIBase: envOr("DISCORD_API_BASE", "https://discord.com/api/v10"),
RedirectURI: os.Getenv("DISCORD_REDIRECT_URI"),
}
for _, o := range strings.Split(os.Getenv("ALLOWED_ORIGINS"), ",") {
if o = strings.TrimSpace(o); o != "" {
c.AllowedOrigins = append(c.AllowedOrigins, o)
}
}
return c
}
// newRouter wires routes and middleware. CORS is the outermost layer so
// preflight OPTIONS short-circuits before auth; /bookmarks* is auth-protected,
// /healthz is public.
func newRouter(s *store.Store, cfg Config) http.Handler {
mux := http.NewServeMux()
h := &api.Handler{Store: s}
mux.HandleFunc("GET /healthz", api.Healthz)
// Public: cover bytes are rendered by the userscript on origins that may
// not send our credentials, and the address is the hash of a URL the Site
// already publishes (ADR-0007).
mux.HandleFunc("GET /covers/{address}", h.Cover)
// Outside httpmw.Auth (the updater sends no Authorization header) and
// outside the web UI's Discord auth (the script must be installable
// without a browser session). The path segment carries the credential
// instead, and the script is rendered with the resolved Reader's
// credential substituted in.
mux.HandleFunc("GET /u/{token}/manga-bookmark.user.js",
userscript.Handler(s, cfg.UserscriptPath))
mux.HandleFunc("GET /u/{token}/novel-bookmark.user.js",
userscript.Handler(s, cfg.NovelUserscriptPath))
protected := http.NewServeMux()
protected.HandleFunc("GET /bookmarks", h.List)
protected.HandleFunc("PUT /bookmarks/{key}", h.Put)
protected.HandleFunc("DELETE /bookmarks/{key}", h.Delete)
auth := httpmw.Auth(s, protected)
mux.Handle("/bookmarks", auth)
mux.Handle("/bookmarks/", auth)
// The browser UI is always registered; signing in is Discord OAuth, so
// there is no password to forget and no gate to leave unset.
wh, err := web.New(s, cfg.Discord, []byte(cfg.TokenKey),
cfg.UserscriptPath, cfg.NovelUserscriptPath)
if err != nil {
log.Fatalf("web handler: %v", err)
}
wh.Register(mux)
return httpmw.CORS(cfg.AllowedOrigins, httpmw.Gzip(guardEmptyUserscriptToken(mux)))
}
// guardEmptyUserscriptToken heads off ServeMux's own path-cleaning redirect:
// an empty {token} segment makes the request path "/u//manga-bookmark.user.js",
// and ServeMux 307s that to "/u/manga-bookmark.user.js" before pattern
// matching ever runs. The endpoint's contract is 404 for any wrong token,
// including this one, so catch it ahead of the mux.
func guardEmptyUserscriptToken(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/u//") {
http.NotFound(w, r)
return
}
next.ServeHTTP(w, r)
})
}
func main() {
cfg := loadConfig()
if cfg.TokenKey == "" {
log.Fatal("TOKEN_KEY is required")
}
if cfg.OwnerDiscordID == "" {
log.Fatal("OWNER_DISCORD_ID is required")
}
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required")
}
if cfg.CoverDir == "" {
log.Fatal("COVER_DIR is required")
}
if cfg.PublicBaseURL == "" {
log.Fatal("PUBLIC_BASE_URL is required")
}
// The web UI signs in through Discord, so a deployment without the OAuth
// application is misconfigured rather than passwordless.
for key, v := range map[string]string{
"DISCORD_CLIENT_ID": cfg.Discord.ClientID,
"DISCORD_CLIENT_SECRET": cfg.Discord.ClientSecret,
"DISCORD_GUILD_ID": cfg.Discord.GuildID,
"DISCORD_REDIRECT_URI": cfg.Discord.RedirectURI,
} {
if v == "" {
log.Fatalf("%s is required", key)
}
}
// The owner's userscript credential is derived from TOKEN_KEY at epoch 0
// (internal/token); the readers row carries its SHA-256, not the
// credential itself.
owner := store.Owner{
DiscordID: cfg.OwnerDiscordID,
TokenHash: token.Hash(token.Token([]byte(cfg.TokenKey), cfg.OwnerDiscordID, 0)),
}
s, err := store.Open(cfg.DatabaseURL, owner, cfg.CoverDir, cfg.PublicBaseURL)
if err != nil {
log.Fatalf("open store: %v", err)
}
defer s.Close()
// The poller is off the request path entirely: if it cannot start, the
// service still serves bookmarks and the userscript still captures latest
// chapters on its own.
//
// One headless browser serves both consumers that need a Cloudflare
// challenge cleared: the poller's kagane/novelfull page fetches and
// kagane's cover bytes. Optional — unset leaves kagane unpolled and its
// Covers blank until the bytes exist.
var browser latest.Fetcher
pollCtx, stopPoll := context.WithCancel(context.Background())
defer stopPoll()
if ws := strings.TrimSpace(os.Getenv("BROWSER_WS_URL")); ws != "" {
bf, err := latest.NewBrowserFetcher(ws)
if err != nil {
log.Printf("browser fetcher disabled: %v", err)
} else {
browser = bf
context.AfterFunc(pollCtx, bf.Close)
log.Printf("browser fetcher at %s", ws)
}
}
// A Series nobody had bookmarked before gets its Latest Chapter and its
// Cover from one fetch, at creation, instead of waiting out a poll queue
// ordered by Reader count. Off the write path: the hook returns as soon
// as the goroutine is started.
var tlsFetch latest.Fetcher
if f, err := latest.NewTLSFetcher(); err != nil {
log.Printf("creation-time acquisition: plain-TLS Sites disabled, cannot build client: %v", err)
} else {
tlsFetch = f
}
var browserCover latest.BrowserCoverFetcher
if b, ok := browser.(latest.BrowserCoverFetcher); ok {
browserCover = b
}
// The Acquirer must survive a TLS client failure: kagane needs only the
// sidecar, and novelfull degrades to whatever is left.
if tlsFetch != nil || browser != nil {
acq := &latest.Acquirer{
Store: s,
Fetch: tlsFetch,
BrowserFetch: browser,
BrowserCoverFetch: browserCover,
Covers: latest.NewCoverFetcher(),
Ctx: pollCtx,
}
s.OnSeriesCreated = acq.Acquire
}
startLatestPoller(pollCtx, s, cfg.LatestPoll, browser)
srv := &http.Server{
Addr: ":" + cfg.Port,
Handler: newRouter(s, cfg),
ReadHeaderTimeout: 10 * time.Second,
}
go func() {
// The connection URL carries a password, so it stays out of the log.
log.Printf("listening on :%s (origins=%v)", cfg.Port, cfg.AllowedOrigins)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("serve: %v", err)
}
}()
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
<-stop
log.Println("shutting down")
// Stop polling before draining requests, so an in-flight series fetch does
// not hold the process open past the shutdown deadline.
stopPoll()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
log.Printf("shutdown: %v", err)
}
}
// newLatestPoller wires the fetcher seams into the poller. Pace is registry
// property, not config (issue #100), so there are no knobs to pass through.
func newLatestPoller(s *store.Store, cfg LatestPoll, fetch, browser latest.Fetcher) *latest.Poller {
var covers latest.BrowserCoverFetcher
if f, ok := browser.(latest.BrowserCoverFetcher); ok {
covers = f
}
return &latest.Poller{
Store: s,
Fetch: fetch,
BrowserFetch: browser,
CoverFetch: covers,
CoverBytesFetch: latest.NewCoverFetcher(),
Now: time.Now,
}
}
// startLatestPoller launches the background poller unless it is disabled or its
// HTTP client cannot be built. Any problem here is logged and skipped: this
// feature going missing degrades the service to userscript-only latest-chapter
// tracking, which is exactly how it behaved before.
func startLatestPoller(ctx context.Context, s *store.Store, cfg LatestPoll, browser latest.Fetcher) {
if !cfg.Enabled {
log.Println("latest-chapter poller: disabled by config")
return
}
f, err := latest.NewTLSFetcher()
if err != nil {
log.Printf("latest-chapter poller: disabled, cannot build client: %v", err)
return
}
// Nil browser: sites behind a JavaScript challenge are simply not polled,
// and their latest_chapter comes from the userscript alone — which is how
// the service behaved before the sidecar existed.
p := newLatestPoller(s, cfg, f, browser)
go p.Run(ctx)
}