147 lines
5.0 KiB
Go
147 lines
5.0 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"regexp"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/chromedp/cdproto/runtime"
|
|
"github.com/chromedp/chromedp"
|
|
)
|
|
|
|
// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
|
|
// challenge clears in a few seconds when it clears at all; anything longer is a
|
|
// challenge that is not going to pass, and the caller's cooldown was already
|
|
// stamped before this ran.
|
|
const challengeTimeout = 45 * time.Second
|
|
|
|
var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
|
|
|
// BrowserFetcher retrieves pages through a remote headless Chrome over the
|
|
// DevTools Protocol.
|
|
//
|
|
// It exists for one reason: kagane.to sits behind a Cloudflare JavaScript
|
|
// challenge. Verified 2026-08-03 from the deployment host, plain HTTP and
|
|
// bogdanfinn/tls-client with a Chrome_133 profile both get 403 with
|
|
// cf-mitigated: challenge on every path, including the API, robots.txt and
|
|
// images. Clearing it requires executing the challenge script, which only a
|
|
// real browser does.
|
|
//
|
|
// The request is made *inside* the page rather than by extracting cf_clearance
|
|
// and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent
|
|
// and often the TLS fingerprint, so replaying it means keeping three things in
|
|
// sync that break silently and separately. The browser's own cookie jar
|
|
// persists across polls, so the challenge is solved once every few hours.
|
|
type BrowserFetcher struct {
|
|
allocCtx context.Context
|
|
cancel context.CancelFunc
|
|
// One page at a time: caps the sidecar's memory and keeps series from
|
|
// sharing page state.
|
|
mu sync.Mutex
|
|
}
|
|
|
|
var _ Fetcher = (*BrowserFetcher)(nil)
|
|
|
|
// NewBrowserFetcher connects to a headless-shell over CDP. wsURL is the
|
|
// container's websocket endpoint, e.g. ws://headless-shell:9222.
|
|
//
|
|
// NoModifyURL is load-bearing: /json/version advertises a
|
|
// webSocketDebuggerUrl pointing at 127.0.0.1, which is meaningless from another
|
|
// container, and without this option chromedp follows it and hangs.
|
|
func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
|
|
if wsURL == "" {
|
|
return nil, fmt.Errorf("empty browser websocket url")
|
|
}
|
|
ctx, cancel := chromedp.NewRemoteAllocator(
|
|
context.Background(), wsURL, chromedp.NoModifyURL)
|
|
return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil
|
|
}
|
|
|
|
func (f *BrowserFetcher) Close() {
|
|
f.cancel()
|
|
}
|
|
|
|
// Get navigates to seriesURL, lets any challenge resolve, then reads the site's
|
|
// JSON API from inside the page so the request carries the clearance cookie.
|
|
// The returned body is API JSON, which is what latestChapterFrom's kagane case
|
|
// expects — it is not HTML.
|
|
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
|
apiURL, ok := kaganeAPIURL(seriesURL)
|
|
if !ok {
|
|
return "", 0, fmt.Errorf("not a fetchable kagane series url: %q", seriesURL)
|
|
}
|
|
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
|
|
defer cancel()
|
|
// A fresh tab per fetch, closed on return, so one wedged page cannot
|
|
// poison later polls.
|
|
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
|
|
defer cancelTab()
|
|
// Bind the caller's deadline to the tab.
|
|
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
|
|
defer cancelDeadline()
|
|
go func() {
|
|
<-ctx.Done()
|
|
cancelDeadline()
|
|
}()
|
|
|
|
var body string
|
|
err := chromedp.Run(tabCtx,
|
|
chromedp.Navigate(seriesURL),
|
|
// The challenge reloads the page itself when it passes; waiting for the
|
|
// site's own root element is what tells us we are through it.
|
|
chromedp.WaitReady("body", chromedp.ByQuery),
|
|
chromedp.Evaluate(
|
|
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
|
|
&body,
|
|
awaitPromise,
|
|
),
|
|
)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
|
|
}
|
|
if body == "" {
|
|
// Challenge still up, or the API refused. Indistinguishable from here
|
|
// and handled identically by the caller.
|
|
return "", 403, nil
|
|
}
|
|
return body, 200, nil
|
|
}
|
|
|
|
// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
|
|
// chapter list. Returning false for anything else is a second line of defence
|
|
// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
|
|
// series_url is client-supplied, so the host is pinned here too.
|
|
func kaganeAPIURL(seriesURL string) (string, bool) {
|
|
u, err := url.Parse(seriesURL)
|
|
if err != nil || u.Scheme != "https" || u.Hostname() != "kagane.to" {
|
|
return "", false
|
|
}
|
|
m := kaganeSeriesRe.FindStringSubmatch(u.Path)
|
|
if m == nil {
|
|
return "", false
|
|
}
|
|
return "https://kagane.to/api/v2/series/" + m[1], true
|
|
}
|
|
|
|
// awaitPromise makes Evaluate resolve the promise rather than returning a
|
|
// serialised Promise object.
|
|
func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
|
|
return p.WithAwaitPromise(true)
|
|
}
|
|
|
|
// jsString renders s as a JavaScript string literal for embedding in an
|
|
// Evaluate expression. The URL is host-pinned by kaganeAPIURL before it gets
|
|
// here, but quoting it properly is what keeps that guarantee intact.
|
|
func jsString(s string) string {
|
|
b, _ := json.Marshal(s)
|
|
return string(b)
|
|
}
|