feat(latest): add chromedp browser fetcher for challenge-gated sites
This commit is contained in:
@@ -0,0 +1,146 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/runtime"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// challengeTimeout bounds one navigate-and-solve. A Cloudflare managed
|
||||
// challenge clears in a few seconds when it clears at all; anything longer is a
|
||||
// challenge that is not going to pass, and the caller's cooldown was already
|
||||
// stamped before this ran.
|
||||
const challengeTimeout = 45 * time.Second
|
||||
|
||||
var kaganeSeriesRe = regexp.MustCompile(`^/series/([0-9a-f-]{36})/?$`)
|
||||
|
||||
// BrowserFetcher retrieves pages through a remote headless Chrome over the
|
||||
// DevTools Protocol.
|
||||
//
|
||||
// It exists for one reason: kagane.to sits behind a Cloudflare JavaScript
|
||||
// challenge. Verified 2026-08-03 from the deployment host, plain HTTP and
|
||||
// bogdanfinn/tls-client with a Chrome_133 profile both get 403 with
|
||||
// cf-mitigated: challenge on every path, including the API, robots.txt and
|
||||
// images. Clearing it requires executing the challenge script, which only a
|
||||
// real browser does.
|
||||
//
|
||||
// The request is made *inside* the page rather than by extracting cf_clearance
|
||||
// and replaying it through TLSFetcher. That cookie is bound to IP, User-Agent
|
||||
// and often the TLS fingerprint, so replaying it means keeping three things in
|
||||
// sync that break silently and separately. The browser's own cookie jar
|
||||
// persists across polls, so the challenge is solved once every few hours.
|
||||
type BrowserFetcher struct {
|
||||
allocCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
// One page at a time: caps the sidecar's memory and keeps series from
|
||||
// sharing page state.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
var _ Fetcher = (*BrowserFetcher)(nil)
|
||||
|
||||
// NewBrowserFetcher connects to a headless-shell over CDP. wsURL is the
|
||||
// container's websocket endpoint, e.g. ws://headless-shell:9222.
|
||||
//
|
||||
// NoModifyURL is load-bearing: /json/version advertises a
|
||||
// webSocketDebuggerUrl pointing at 127.0.0.1, which is meaningless from another
|
||||
// container, and without this option chromedp follows it and hangs.
|
||||
func NewBrowserFetcher(wsURL string) (*BrowserFetcher, error) {
|
||||
if wsURL == "" {
|
||||
return nil, fmt.Errorf("empty browser websocket url")
|
||||
}
|
||||
ctx, cancel := chromedp.NewRemoteAllocator(
|
||||
context.Background(), wsURL, chromedp.NoModifyURL)
|
||||
return &BrowserFetcher{allocCtx: ctx, cancel: cancel}, nil
|
||||
}
|
||||
|
||||
func (f *BrowserFetcher) Close() {
|
||||
f.cancel()
|
||||
}
|
||||
|
||||
// Get navigates to seriesURL, lets any challenge resolve, then reads the site's
|
||||
// JSON API from inside the page so the request carries the clearance cookie.
|
||||
// The returned body is API JSON, which is what latestChapterFrom's kagane case
|
||||
// expects — it is not HTML.
|
||||
func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int, error) {
|
||||
apiURL, ok := kaganeAPIURL(seriesURL)
|
||||
if !ok {
|
||||
return "", 0, fmt.Errorf("not a fetchable kagane series url: %q", seriesURL)
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, challengeTimeout)
|
||||
defer cancel()
|
||||
// A fresh tab per fetch, closed on return, so one wedged page cannot
|
||||
// poison later polls.
|
||||
tabCtx, cancelTab := chromedp.NewContext(f.allocCtx)
|
||||
defer cancelTab()
|
||||
// Bind the caller's deadline to the tab.
|
||||
tabCtx, cancelDeadline := context.WithCancel(tabCtx)
|
||||
defer cancelDeadline()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cancelDeadline()
|
||||
}()
|
||||
|
||||
var body string
|
||||
err := chromedp.Run(tabCtx,
|
||||
chromedp.Navigate(seriesURL),
|
||||
// The challenge reloads the page itself when it passes; waiting for the
|
||||
// site's own root element is what tells us we are through it.
|
||||
chromedp.WaitReady("body", chromedp.ByQuery),
|
||||
chromedp.Evaluate(
|
||||
`fetch(`+jsString(apiURL)+`).then(r => r.ok ? r.text() : "")`,
|
||||
&body,
|
||||
awaitPromise,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("browser fetch %q: %w", seriesURL, err)
|
||||
}
|
||||
if body == "" {
|
||||
// Challenge still up, or the API refused. Indistinguishable from here
|
||||
// and handled identically by the caller.
|
||||
return "", 403, nil
|
||||
}
|
||||
return body, 200, nil
|
||||
}
|
||||
|
||||
// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
|
||||
// chapter list. Returning false for anything else is a second line of defence
|
||||
// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
|
||||
// series_url is client-supplied, so the host is pinned here too.
|
||||
func kaganeAPIURL(seriesURL string) (string, bool) {
|
||||
u, err := url.Parse(seriesURL)
|
||||
if err != nil || u.Scheme != "https" || u.Hostname() != "kagane.to" {
|
||||
return "", false
|
||||
}
|
||||
m := kaganeSeriesRe.FindStringSubmatch(u.Path)
|
||||
if m == nil {
|
||||
return "", false
|
||||
}
|
||||
return "https://kagane.to/api/v2/series/" + m[1], true
|
||||
}
|
||||
|
||||
// awaitPromise makes Evaluate resolve the promise rather than returning a
|
||||
// serialised Promise object.
|
||||
func awaitPromise(p *runtime.EvaluateParams) *runtime.EvaluateParams {
|
||||
return p.WithAwaitPromise(true)
|
||||
}
|
||||
|
||||
// jsString renders s as a JavaScript string literal for embedding in an
|
||||
// Evaluate expression. The URL is host-pinned by kaganeAPIURL before it gets
|
||||
// here, but quoting it properly is what keeps that guarantee intact.
|
||||
func jsString(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package latest
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestKaganeAPIURL(t *testing.T) {
|
||||
const uuid = "019f84bc-9ba0-7ed9-86f5-8b905ec7c28b"
|
||||
tests := []struct {
|
||||
name string
|
||||
seriesURL string
|
||||
want string
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "series page maps to its API endpoint",
|
||||
seriesURL: "https://kagane.to/series/" + uuid,
|
||||
want: "https://kagane.to/api/v2/series/" + uuid,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "trailing slash is tolerated",
|
||||
seriesURL: "https://kagane.to/series/" + uuid + "/",
|
||||
want: "https://kagane.to/api/v2/series/" + uuid,
|
||||
wantOK: true,
|
||||
},
|
||||
{"not a series path", "https://kagane.to/search", "", false},
|
||||
{"foreign host", "https://evil.example/series/" + uuid, "", false},
|
||||
{"garbage", "://", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := kaganeAPIURL(tt.seriesURL)
|
||||
if ok != tt.wantOK || got != tt.want {
|
||||
t.Errorf("kaganeAPIURL(%q) = %q, %v; want %q, %v",
|
||||
tt.seriesURL, got, ok, tt.want, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user