Series URL repair: owner-typed, gated by the poller's own fetch gate (#151)
This commit is contained in:
@@ -124,7 +124,7 @@ func (f *BrowserFetcher) Get(ctx context.Context, seriesURL string) (string, int
|
||||
// request must be made from inside the page so it carries the clearance
|
||||
// cookie, and the API is the only place the list exists. Refusing any other
|
||||
// address is the per-Site half of the SSRF gate, kept deliberately behind
|
||||
// fetchableSeriesURL (see browserRead.Read).
|
||||
// FetchableSeriesURL (see browserRead.Read).
|
||||
func kaganeRead(seriesURL string, out *string) (chromedp.Action, bool) {
|
||||
apiURL, ok := kaganeAPIURL(seriesURL)
|
||||
if !ok {
|
||||
@@ -326,7 +326,7 @@ func (f *BrowserFetcher) run(ctx context.Context, target string, read chromedp.A
|
||||
|
||||
// kaganeAPIURL maps a stored series_url to the JSON endpoint carrying its
|
||||
// chapter list. Returning false for anything else is a second line of defence
|
||||
// behind fetchableSeriesURL: a headless browser is a strong SSRF primitive and
|
||||
// behind FetchableSeriesURL: a headless browser is a strong SSRF primitive and
|
||||
// series_url is client-supplied, so the host is pinned here too.
|
||||
func kaganeAPIURL(seriesURL string) (string, bool) {
|
||||
u, err := url.Parse(seriesURL)
|
||||
|
||||
@@ -174,7 +174,7 @@ func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte,
|
||||
return body, contentType, nil
|
||||
}
|
||||
|
||||
// This gate deliberately differs from fetchableSeriesURL: cover hosts are
|
||||
// This gate deliberately differs from FetchableSeriesURL: cover hosts are
|
||||
// site-independent CDNs, so a Site host allowlist would reject valid covers.
|
||||
func (f *TLSCoverFetcher) validateURL(ctx context.Context, u *url.URL) error {
|
||||
if u == nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
|
||||
|
||||
@@ -710,7 +710,7 @@ func (p *Poller) waitCovers() {
|
||||
p.coverWG.Wait()
|
||||
}
|
||||
|
||||
// fetchableSeriesURL reports whether site is a Site the registry knows and
|
||||
// FetchableSeriesURL reports whether site is a Site the registry knows and
|
||||
// seriesURL is safe to hand to a fetcher: an https URL whose host matches the
|
||||
// Site's pinned hostname exactly. series_url comes from client-supplied PUT
|
||||
// bodies, so this is a defence against the poller being used to probe
|
||||
@@ -719,7 +719,11 @@ func (p *Poller) waitCovers() {
|
||||
// browser Site guards a control that executes JavaScript and carries cookies,
|
||||
// a parser Site guards a wasted request — but the rule is one rule, from the
|
||||
// registry.
|
||||
func fetchableSeriesURL(site, seriesURL string) bool {
|
||||
//
|
||||
// The owner's series URL repair (issue #151) is a second caller: the web
|
||||
// layer validates with this same gate before storing a repair, so there is
|
||||
// never a second copy of it.
|
||||
func FetchableSeriesURL(site, seriesURL string) bool {
|
||||
s, known := sites[site]
|
||||
if !known {
|
||||
return false
|
||||
|
||||
@@ -588,8 +588,8 @@ func TestFetchableSeriesURL(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := fetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
||||
t.Errorf("fetchableSeriesURL(%q, %q) = %v, want %v",
|
||||
if got := FetchableSeriesURL(tt.site, tt.seriesURL); got != tt.want {
|
||||
t.Errorf("FetchableSeriesURL(%q, %q) = %v, want %v",
|
||||
tt.site, tt.seriesURL, got, tt.want)
|
||||
}
|
||||
})
|
||||
@@ -1019,8 +1019,8 @@ func TestFetchableSeriesURLPinsNovelHosts(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := fetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
||||
t.Fatalf("fetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
||||
if got := FetchableSeriesURL(tc.site, tc.url); got != tc.want {
|
||||
t.Fatalf("FetchableSeriesURL(%q, %q) = %v, want %v", tc.site, tc.url, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ var (
|
||||
// its own network position to whatever URL a token-holder writes, including
|
||||
// link-local/internal addresses or non-https schemes.
|
||||
func readSeriesPage(ctx context.Context, site, seriesURL string, browser, tls Fetcher) (seriesRead, error) {
|
||||
if !fetchableSeriesURL(site, seriesURL) {
|
||||
if !FetchableSeriesURL(site, seriesURL) {
|
||||
return seriesRead{}, fmt.Errorf("%w: site=%q url=%q", errNotFetchable, site, seriesURL)
|
||||
}
|
||||
f := fetcherFor(site, browser, tls)
|
||||
|
||||
@@ -46,7 +46,7 @@ type site struct {
|
||||
type browserRead struct {
|
||||
// Read builds the tab read for seriesURL, refusing (false) an address
|
||||
// this Site will not open in a browser — the per-Site half of the SSRF
|
||||
// gate, kept deliberately behind fetchableSeriesURL: a headless browser
|
||||
// gate, kept deliberately behind FetchableSeriesURL: a headless browser
|
||||
// executes JavaScript and carries cookies, and series_url is
|
||||
// client-supplied.
|
||||
Read func(seriesURL string, out *string) (chromedp.Action, bool)
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestSmokeLnwCommentBoundary(t *testing.T) {
|
||||
if seriesURL == "" {
|
||||
t.Skip("SMOKE_LNW_SERIES_URL unset")
|
||||
}
|
||||
if !fetchableSeriesURL("lightnovelworld", seriesURL) {
|
||||
if !FetchableSeriesURL("lightnovelworld", seriesURL) {
|
||||
t.Fatalf("%q is not a fetchable lightnovelworld series URL", seriesURL)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user