Add gated cover byte fetcher (#66)
## Summary Adds a plain-TLS cover byte fetcher with a destination-class SSRF gate and wires public cover sources through the content-addressed filesystem store. ## Changes - Resolve hostnames before connecting; refuse non-HTTPS, loopback, private, link-local, unique-local, CGNAT, credentials, and mixed public/private DNS answers. - Re-check every redirect and resolve/classify again at dial time to close DNS rebinding. - Reuse `maxBodyBytes`; reject oversized responses and non-image content types before persistence. - Add generic `Store.GetCover`/`PutCover` source-URL storage while preserving the browser-backed kagane path. - Keep cover prefetch failures isolated from chapter polling. - Add observable tests for TLS, no-connection refusals, all refused address classes, redirect blocking, streaming body caps, non-image rejection, content-addressed persistence, DNS rebinding, and poller routing. ## Verification - `go test -count=1 ./...` - `go vet ./...` Both pass. No test touches the live network. Closes #57 Reviewed-on: #66 Co-authored-by: Sulthan Zaki <sultankiki05@gmail.com> Co-committed-by: Sulthan Zaki <sultankiki05@gmail.com>
This commit was merged in pull request #66.
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
package latest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bookmarkmanager/backend/internal/store"
|
||||
)
|
||||
|
||||
// CoverBytesFetcher retrieves one cover from its source URL. The caller owns
|
||||
// persistence; this seam keeps network policy independent from the store.
|
||||
type CoverBytesFetcher interface {
|
||||
Fetch(ctx context.Context, sourceURL string) (body []byte, contentType string, err error)
|
||||
}
|
||||
|
||||
// CoverResolver resolves a host before any connection is attempted. Tests
|
||||
// inject it to exercise hostile DNS results without touching the live network.
|
||||
type CoverResolver func(context.Context, string) ([]netip.Addr, error)
|
||||
|
||||
// TLSCoverFetcher retrieves image bytes with the standard HTTPS client. Unlike
|
||||
// TLSFetcher, it does not need a browser fingerprint: cover hosts are public
|
||||
// CDNs and the response is accepted only after the destination gate passes.
|
||||
type TLSCoverFetcher struct {
|
||||
client *http.Client
|
||||
resolve CoverResolver
|
||||
}
|
||||
|
||||
var _ CoverBytesFetcher = (*TLSCoverFetcher)(nil)
|
||||
|
||||
const coverRequestTimeout = 30 * time.Second
|
||||
|
||||
var carrierGradeNAT = netip.MustParsePrefix("100.64.0.0/10")
|
||||
|
||||
// NewCoverFetcher builds the production cover client with the real resolver.
|
||||
func NewCoverFetcher() *TLSCoverFetcher {
|
||||
return NewCoverFetcherWithResolver(nil)
|
||||
}
|
||||
|
||||
// NewCoverFetcherWithResolver builds a cover client using resolve, or the real
|
||||
// system resolver when resolve is nil.
|
||||
func NewCoverFetcherWithResolver(resolve CoverResolver) *TLSCoverFetcher {
|
||||
if resolve == nil {
|
||||
resolve = defaultCoverResolver
|
||||
}
|
||||
return newCoverFetcher(newCoverHTTPClient(resolve), resolve)
|
||||
}
|
||||
|
||||
func newCoverFetcher(client *http.Client, resolve CoverResolver) *TLSCoverFetcher {
|
||||
f := &TLSCoverFetcher{client: client, resolve: resolve}
|
||||
client.CheckRedirect = func(req *http.Request, _ []*http.Request) error {
|
||||
if err := f.validateURL(req.Context(), req.URL); err != nil {
|
||||
return fmt.Errorf("redirect destination: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func defaultCoverResolver(ctx context.Context, host string) ([]netip.Addr, error) {
|
||||
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
||||
}
|
||||
|
||||
func newCoverHTTPClient(resolve CoverResolver) *http.Client {
|
||||
base, ok := http.DefaultTransport.(*http.Transport)
|
||||
if !ok {
|
||||
base = &http.Transport{}
|
||||
}
|
||||
transport := base.Clone()
|
||||
// A proxy would make the dial target the proxy rather than the cover host,
|
||||
// defeating destination classification. Cover fetching is direct by design.
|
||||
transport.Proxy = nil
|
||||
dialer := &net.Dialer{}
|
||||
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("split cover address %q: %w", address, err)
|
||||
}
|
||||
addrs, err := resolveCoverHost(ctx, host, resolve)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if !publicCoverAddress(addr) {
|
||||
return nil, fmt.Errorf("cover host resolves to refused address %s", addr)
|
||||
}
|
||||
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(addr.String(), port))
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("cover host %q has no reachable address", host)
|
||||
}
|
||||
return &http.Client{Transport: transport, Timeout: coverRequestTimeout}
|
||||
}
|
||||
|
||||
func (f *TLSCoverFetcher) Fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
|
||||
u, err := url.Parse(sourceURL)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("parse cover URL: %w", err)
|
||||
}
|
||||
if err := f.validateURL(ctx, u); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("build cover request: %w", err)
|
||||
}
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("fetch cover: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", fmt.Errorf("fetch cover: status %d", resp.StatusCode)
|
||||
}
|
||||
contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
|
||||
if err != nil || !store.IsCoverContentType(contentType) {
|
||||
return nil, "", fmt.Errorf("fetch cover: unsupported content type %q", resp.Header.Get("Content-Type"))
|
||||
}
|
||||
if resp.ContentLength > maxBodyBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes+1))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("read cover: %w", err)
|
||||
}
|
||||
if len(body) > maxBodyBytes {
|
||||
return nil, "", fmt.Errorf("fetch cover: response exceeds %d bytes", maxBodyBytes)
|
||||
}
|
||||
return body, contentType, nil
|
||||
}
|
||||
|
||||
// This gate deliberately differs from fetchableSeriesURL: cover hosts are
|
||||
// site-independent CDNs, so a Site host allowlist would reject valid covers.
|
||||
func (f *TLSCoverFetcher) validateURL(ctx context.Context, u *url.URL) error {
|
||||
if u == nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
|
||||
return errors.New("cover URL must use HTTPS without credentials")
|
||||
}
|
||||
host := u.Hostname()
|
||||
if host == "" {
|
||||
return errors.New("cover URL has no host")
|
||||
}
|
||||
addrs, err := resolveCoverHost(ctx, host, f.resolve)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve cover host %q: %w", host, err)
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return fmt.Errorf("resolve cover host %q: no addresses", host)
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if !publicCoverAddress(addr) {
|
||||
return fmt.Errorf("cover host %q resolves to refused address %s", host, addr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveCoverHost(ctx context.Context, host string, resolve CoverResolver) ([]netip.Addr, error) {
|
||||
if literal, err := netip.ParseAddr(host); err == nil {
|
||||
return []netip.Addr{literal.Unmap()}, nil
|
||||
}
|
||||
return resolve(ctx, strings.TrimSuffix(host, "."))
|
||||
}
|
||||
|
||||
func publicCoverAddress(addr netip.Addr) bool {
|
||||
addr = addr.Unmap()
|
||||
return addr.IsValid() && addr.IsGlobalUnicast() &&
|
||||
!addr.IsLoopback() && !addr.IsPrivate() && !addr.IsLinkLocalUnicast() &&
|
||||
!carrierGradeNAT.Contains(addr)
|
||||
}
|
||||
Reference in New Issue
Block a user