sulthan
ecce453d13
The Poll scoped the lightnovelworld scan to the slug derived from the stored address, so a Series publishing chapters under more than one Chapter Slug froze its Latest Chapter. The branch now matches any chapter-shaped address on the host (lnwChapterRe) and scans only the body before the wpdiscuz thread marker (wpd-threads); a body without the marker is skipped and logged with its length, never scanned whole. The size-cap comment now states the measured ~3.5x headroom.
78 lines
2.7 KiB
Go
78 lines
2.7 KiB
Go
package latest
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
|
|
fhttp "github.com/bogdanfinn/fhttp"
|
|
tls_client "github.com/bogdanfinn/tls-client"
|
|
"github.com/bogdanfinn/tls-client/profiles"
|
|
)
|
|
|
|
// maxBodyBytes caps what a single series page can cost in memory. Real pages
|
|
// measured 100-400 KB on 2026-07-26; lightnovelworld runs larger — 685 KB and
|
|
// 1.18 MB measured 2026-08-11 — so the headroom there is roughly 3.5x, and the
|
|
// cap mostly guards against a proxy handing back something enormous.
|
|
const maxBodyBytes = 4 << 20
|
|
|
|
// chromeUA matches the client profile below. A Chrome fingerprint paired with a
|
|
// non-Chrome user agent is itself a signal.
|
|
const chromeUA = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 " +
|
|
"(KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36"
|
|
|
|
// TLSFetcher fetches series pages with a Chrome TLS fingerprint.
|
|
//
|
|
// Plain net/http was verified working against both sites on 2026-07-26, so this
|
|
// is not fixing an observed block — it is deliberate defence-in-depth against a
|
|
// future fingerprint-based one, chosen up front rather than reacted to later.
|
|
// The library is pure Go, so CGO_ENABLED=0, the static binary, and the
|
|
// distroless image are all unaffected.
|
|
type TLSFetcher struct {
|
|
client tls_client.HttpClient
|
|
}
|
|
|
|
var _ Fetcher = (*TLSFetcher)(nil)
|
|
|
|
func NewTLSFetcher() (*TLSFetcher, error) {
|
|
c, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(),
|
|
tls_client.WithTimeoutSeconds(30),
|
|
tls_client.WithClientProfile(profiles.Chrome_133),
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("new tls client: %w", err)
|
|
}
|
|
return &TLSFetcher{client: c}, nil
|
|
}
|
|
|
|
// Get fetches url and returns the body and status. Redirects are followed: the
|
|
// demonic chapter anchors are a redirect form, and asura has moved domains
|
|
// before.
|
|
func (f *TLSFetcher) Get(ctx context.Context, url string) (string, int, error) {
|
|
req, err := fhttp.NewRequest(fhttp.MethodGet, url, nil)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("build request %q: %w", url, err)
|
|
}
|
|
req = req.WithContext(ctx)
|
|
// Header order is part of what is being fingerprinted, so it is stated
|
|
// explicitly instead of left to Go's map iteration order.
|
|
req.Header = fhttp.Header{
|
|
"user-agent": {chromeUA},
|
|
"accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
|
|
"accept-language": {"en-US,en;q=0.9"},
|
|
fhttp.HeaderOrderKey: {"user-agent", "accept", "accept-language"},
|
|
}
|
|
|
|
resp, err := f.client.Do(req)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("get %q: %w", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
|
|
if err != nil {
|
|
return "", resp.StatusCode, fmt.Errorf("read %q: %w", url, err)
|
|
}
|
|
return string(body), resp.StatusCode, nil
|
|
}
|