sulthan
28f2d32e45
Five env vars, all with defaults sized for this deployment (84 series at a 1h cadence). Bad values log and fall back rather than failing startup, and the cooldown is floored at 15m. Fetches go through tls-client with a Chrome profile as defence in depth.
77 lines
2.6 KiB
Go
77 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
|
|
fhttp "github.com/bogdanfinn/fhttp"
|
|
tls_client "github.com/bogdanfinn/tls-client"
|
|
"github.com/bogdanfinn/tls-client/profiles"
|
|
)
|
|
|
|
// maxBodyBytes caps what a single series page can cost in memory. Real pages
|
|
// measured 100-400 KB on 2026-07-26, so this is roughly 10x headroom and mostly
|
|
// guards against a proxy handing back something enormous.
|
|
const maxBodyBytes = 4 << 20
|
|
|
|
// chromeUA matches the client profile below. A Chrome fingerprint paired with a
|
|
// non-Chrome user agent is itself a signal.
|
|
const chromeUA = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 " +
|
|
"(KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36"
|
|
|
|
// tlsFetcher fetches series pages with a Chrome TLS fingerprint.
|
|
//
|
|
// Plain net/http was verified working against both sites on 2026-07-26, so this
|
|
// is not fixing an observed block — it is deliberate defence-in-depth against a
|
|
// future fingerprint-based one, chosen up front rather than reacted to later.
|
|
// The library is pure Go, so CGO_ENABLED=0, the static binary, and the
|
|
// distroless image are all unaffected.
|
|
type tlsFetcher struct {
|
|
client tls_client.HttpClient
|
|
}
|
|
|
|
var _ fetcher = (*tlsFetcher)(nil)
|
|
|
|
func newTLSFetcher() (*tlsFetcher, error) {
|
|
c, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(),
|
|
tls_client.WithTimeoutSeconds(30),
|
|
tls_client.WithClientProfile(profiles.Chrome_133),
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("new tls client: %w", err)
|
|
}
|
|
return &tlsFetcher{client: c}, nil
|
|
}
|
|
|
|
// Get fetches url and returns the body and status. Redirects are followed: the
|
|
// demonic chapter anchors are a redirect form, and asura has moved domains
|
|
// before.
|
|
func (f *tlsFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
|
req, err := fhttp.NewRequest(fhttp.MethodGet, url, nil)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("build request %q: %w", url, err)
|
|
}
|
|
req = req.WithContext(ctx)
|
|
// Header order is part of what is being fingerprinted, so it is stated
|
|
// explicitly instead of left to Go's map iteration order.
|
|
req.Header = fhttp.Header{
|
|
"user-agent": {chromeUA},
|
|
"accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
|
|
"accept-language": {"en-US,en;q=0.9"},
|
|
fhttp.HeaderOrderKey: {"user-agent", "accept", "accept-language"},
|
|
}
|
|
|
|
resp, err := f.client.Do(req)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("get %q: %w", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
|
|
if err != nil {
|
|
return "", resp.StatusCode, fmt.Errorf("read %q: %w", url, err)
|
|
}
|
|
return string(body), resp.StatusCode, nil
|
|
}
|