diff --git a/backend/go.mod b/backend/go.mod index 56a710f..f4fb0e1 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,10 +1,12 @@ module mangabm/backend -go 1.24.1 +go 1.26 require ( github.com/bogdanfinn/fhttp v0.6.8 github.com/bogdanfinn/tls-client v1.15.1 + github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f + github.com/chromedp/chromedp v0.16.0 modernc.org/sqlite v1.34.4 ) @@ -15,7 +17,12 @@ require ( github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect github.com/bogdanfinn/utls v1.7.7-barnius // indirect github.com/bogdanfinn/websocket v1.5.5-barnius // indirect + github.com/chromedp/sysutil v1.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/klauspost/compress v1.18.2 // indirect @@ -26,7 +33,7 @@ require ( github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.39.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.32.0 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.55.3 // indirect diff --git a/backend/go.sum b/backend/go.sum index cc6daad..ef779f6 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -14,10 +14,24 @@ github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5Zl github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg= github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI= github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0= +github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk= +github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -26,10 +40,14 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -56,8 +74,8 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= diff --git a/backend/internal/latest/browser.go b/backend/internal/latest/browser.go new file mode 100644 index 0000000..81c2707 --- /dev/null +++ b/backend/internal/latest/browser.go @@ -0,0 +1,146 @@ +package latest + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "regexp" + "sync" + "time" + + "github.com/chromedp/cdproto/runtime" + "github.com/chromedp/chromedp" +) + +// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed +// challenge clears in a few seconds when it clears at all; anything longer is a +// challenge that is not going to pass, and the caller's cooldown was already +// stamped before this ran. +const challengeTimeout = 45 * time.Second + +var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`) + +// BrowserFetcher retrieves pages through a remote headless Chrome over the +// DevTools Protocol. +// +// It exists for one reason: kagane.to sits behind a Cloudflare JavaScript +// challenge. Verified 2026-08-03 from the deployment host, plain HTTP and +// bogdanfinn/tls-client with a Chrome_133 profile both get 403 with +// cf-mitigated: challenge on every path, including the API, robots.txt and +// images. Clearing it requires executing the challenge script, which only a +// real browser does. +// +// The request is made *inside* the page rather than by extracting cf_clearance +// and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent +// and often the TLS fingerprint, so replaying it means keeping three things in +// sync that break silently and separately. The browser's own cookie jar +// persists across polls, so the challenge is solved once every few hours. +type BrowserFetcher struct { + allocCtx context.Context + cancel context.CancelFunc + // One page at a time: caps the sidecar's memory and keeps series from + // sharing page state. + mu sync.Mutex +} + +var _ Fetcher = (*BrowserFetcher)(nil) + +// NewBrowserFetcher connects to a headless-shell over CDP. wsURL is the +// container's websocket endpoint, e.g. ws://headless-shell:9222. +// +// NoModifyURL is load-bearing: /json/version advertises a +// webSocketDebuggerUrl pointing at 127.0.0.1, which is meaningless from another +// container, and without this option chromedp follows it and hangs. +func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) { + if wsURL == "" { + return nil, fmt.Errorf("empty browser websocket url") + } + ctx, cancel := chromedp.NewRemoteAllocator( + context.Background(), wsURL, chromedp.NoModifyURL) + return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil +} + +func (f *BrowserFetcher) Close() { + f.cancel() +} + +// Get navigates to seriesURL, lets any challenge resolve, then reads the site's +// JSON API from inside the page so the request carries the clearance cookie. +// The returned body is API JSON, which is what latestChapterFrom's kagane case +// expects — it is not HTML. +func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) { + apiURL, ok := kaganeAPIURL(seriesURL) + if !ok { + return "", 0, fmt.Errorf("not a fetchable kagane series url: %q", seriesURL) + } + + f.mu.Lock() + defer f.mu.Unlock() + + ctx, cancel := context.WithTimeout(ctx, challengeTimeout) + defer cancel() + // A fresh tab per fetch, closed on return, so one wedged page cannot + // poison later polls. + tabCtx, cancelTab := chromedp.NewContext(f.allocCtx) + defer cancelTab() + // Bind the caller's deadline to the tab. + tabCtx, cancelDeadline := context.WithCancel(tabCtx) + defer cancelDeadline() + go func() { + <-ctx.Done() + cancelDeadline() + }() + + var body string + err := chromedp.Run(tabCtx, + chromedp.Navigate(seriesURL), + // The challenge reloads the page itself when it passes; waiting for the + // site's own root element is what tells us we are through it. + chromedp.WaitReady("body", chromedp.ByQuery), + chromedp.Evaluate( + `fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`, + &body, + awaitPromise, + ), + ) + if err != nil { + return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err) + } + if body == "" { + // Challenge still up, or the API refused. Indistinguishable from here + // and handled identically by the caller. + return "", 403, nil + } + return body, 200, nil +} + +// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its +// chapter list. Returning false for anything else is a second line of defence +// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and +// series_url is client-supplied, so the host is pinned here too. +func kaganeAPIURL(seriesURL string) (string, bool) { + u, err := url.Parse(seriesURL) + if err != nil || u.Scheme != "https" || u.Hostname() != "kagane.to" { + return "", false + } + m := kaganeSeriesRe.FindStringSubmatch(u.Path) + if m == nil { + return "", false + } + return "https://kagane.to/api/v2/series/" + m[1], true +} + +// awaitPromise makes Evaluate resolve the promise rather than returning a +// serialised Promise object. +func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams { + return p.WithAwaitPromise(true) +} + +// jsString renders s as a JavaScript string literal for embedding in an +// Evaluate expression. The URL is host-pinned by kaganeAPIURL before it gets +// here, but quoting it properly is what keeps that guarantee intact. +func jsString(s string) string { + b, _ := json.Marshal(s) + return string(b) +} diff --git a/backend/internal/latest/browser_test.go b/backend/internal/latest/browser_test.go new file mode 100644 index 0000000..fa91a3a --- /dev/null +++ b/backend/internal/latest/browser_test.go @@ -0,0 +1,38 @@ +package latest + +import "testing" + +func TestKaganeAPIURL(t *testing.T) { + const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b" + tests := []struct { + name string + seriesURL string + want string + wantOK bool + }{ + { + name: "series page maps to its API endpoint", + seriesURL: "https://kagane.to/series/" + uuid, + want: "https://kagane.to/api/v2/series/" + uuid, + wantOK: true, + }, + { + name: "trailing slash is tolerated", + seriesURL: "https://kagane.to/series/" + uuid + "/", + want: "https://kagane.to/api/v2/series/" + uuid, + wantOK: true, + }, + {"not a series path", "https://kagane.to/search", "", false}, + {"foreign host", "https://evil.example/series/" + uuid, "", false}, + {"garbage", "://", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := kaganeAPIURL(tt.seriesURL) + if ok != tt.wantOK || got != tt.want { + t.Errorf("kaganeAPIURL(%q) = %q, %v; want %q, %v", + tt.seriesURL, got, ok, tt.want, tt.wantOK) + } + }) + } +}