okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
11.8 KiB336 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"errors"
7	"fmt"
8	"net"
9	"net/http"
10	"net/url"
11	"strings"
12	"time"
13
14	"github.com/chromedp/cdproto/emulation"
15	"github.com/chromedp/cdproto/network"
16	"github.com/chromedp/cdproto/page"
17	"github.com/chromedp/chromedp"
18)
19
20type Browser struct {
21	path     string
22	network  *Network
23	slots    chan struct{}
24	stealth  bool
25	headless bool
26	waitMS   time.Duration
27}
28
29// renderOptions are the per-render browser controls.
30type renderOptions struct {
31	stealth         bool
32	waitForSelector string
33	waitMS          time.Duration
34	// proxy is the upstream proxy this render egresses through, chosen once per
35	// render so a session does not move between addresses mid-load.
36	proxy *url.URL
37}
38
39// blockedBrowserResources are the resources an extractor never reads: images,
40// media, fonts, and analytics. Blocking them at the network layer takes real
41// time off page loads. These match the reference implementation's list.
42var blockedBrowserResources = []string{
43	"*.png", "*.jpg", "*.jpeg", "*.webp", "*.gif", "*.avif", "*.svg", "*.ico",
44	"*.mp4", "*.webm", "*.mp3", "*.wav", "*.woff", "*.woff2", "*.ttf", "*.otf",
45	"*://*.googletagmanager.com/*", "*://*.google-analytics.com/*", "*://*.doubleclick.net/*",
46	"*://*.googlesyndication.com/*", "*://*.facebook.net/*", "*://*.hotjar.com/*", "*://*.segment.io/*",
47}
48
49// kagiResultsSelector matches the streaming results container on a Kagi page.
50const kagiResultsSelector = ".search-result, .__srgi"
51
52type browserPage struct {
53	HTML, Title, URL string
54	TextLength       int
55}
56
57func jsLiteral(s string) string {
58	b, err := json.Marshal(s)
59	if err != nil {
60		return `""`
61	}
62	return string(b)
63}
64
65func browserSnapshot(ctx context.Context) (p browserPage, err error) {
66	err = chromedp.Run(ctx, chromedp.Evaluate(`({html: document.documentElement?.outerHTML || "",
67title: document.title, url: location.href, textLength: document.body?.innerText.trim().length || 0})`, &p))
68	return
69}
70
71func isAccessChallenge(title, html string) bool {
72	title, html = strings.ToLower(title), strings.ToLower(html)
73	if strings.Contains(title, "just a moment") || strings.Contains(title, "attention required") {
74		return true
75	}
76	for _, marker := range []string{"cf-challenge", "cf-browser-verification", "cf-chl-", `id="challenge-form"`, `id='challenge-form'`} {
77		if strings.Contains(html, marker) {
78			return true
79		}
80	}
81	return false
82}
83
84// Phase deadlines are soft: keep the tab alive so callers can inspect its DOM.
85// The caller's cancellation/deadline remains a hard limit, including while polling.
86func browserWait(ctx context.Context, cap, interval time.Duration, ready func(context.Context) (bool, error)) error {
87	wait, cancel := context.WithTimeout(ctx, cap)
88	defer cancel()
89	ticks := time.NewTicker(interval)
90	defer ticks.Stop()
91	for {
92		done, err := ready(wait)
93		if ctx.Err() != nil {
94			return ctx.Err()
95		}
96		if wait.Err() != nil {
97			return nil
98		}
99		if err != nil || done {
100			return err
101		}
102		select {
103		case <-wait.Done():
104			return ctx.Err()
105		case <-ticks.C:
106		}
107	}
108}
109
110// browserReady waits for the page to settle — readyState complete and body
111// text length stable across two consecutive polls — or, when a selector is
112// given, for pages that stream their content after the load event.
113func browserReady(ctx context.Context, selector string, waitMS time.Duration) error {
114	if selector != "" {
115		cap := waitMS
116		if cap <= 0 || cap > 15*time.Second {
117			cap = 8 * time.Second
118		}
119		var found bool
120		return browserWait(ctx, cap, 250*time.Millisecond, func(ctx context.Context) (bool, error) {
121			err := chromedp.Run(ctx, chromedp.Evaluate(`!!document.querySelector(`+jsLiteral(selector)+`)`, &found))
122			return found, err
123		})
124	}
125	cap := waitMS
126	if cap <= 0 {
127		cap = 1200 * time.Millisecond
128	}
129	if cap > 10*time.Second {
130		cap = 10 * time.Second
131	}
132	started := time.Now()
133	last, stable := -1, 0
134	return browserWait(ctx, cap, 150*time.Millisecond, func(ctx context.Context) (bool, error) {
135		var length int
136		err := chromedp.Run(ctx, chromedp.Evaluate(`document.readyState === "complete" && document.body ? document.body.innerText.length : -1`, &length))
137		if length >= 0 && length == last {
138			stable++
139		} else {
140			last, stable = length, 0
141		}
142		return stable >= 2 && time.Since(started) >= 450*time.Millisecond, err
143	})
144}
145
146func renderBrowserPage(ctx context.Context, raw string, o renderOptions) (browserPage, error) {
147	// Chrome was already allocated on the parent context. Cancelling only this
148	// navigation wait must not kill the browser or discard a usable partial DOM.
149	nav, cancel := context.WithTimeout(ctx, 10*time.Second)
150	response, err := chromedp.RunResponse(nav, chromedp.Navigate(raw))
151	timedOut := errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil
152	cancel()
153	if err != nil && !timedOut {
154		return browserPage{}, fmt.Errorf("browser navigation failed: %w", err)
155	}
156	if err := browserReady(ctx, o.waitForSelector, o.waitMS); err != nil {
157		return browserPage{}, fmt.Errorf("browser readiness failed: %w", err)
158	}
159	p, err := browserSnapshot(ctx)
160	if err != nil {
161		return p, err
162	}
163	if response != nil && response.Status >= 400 && !isAccessChallenge(p.Title, p.HTML) {
164		return browserPage{}, failure(int(response.Status), "CRAWL_HTTP_ERROR", fmt.Sprintf("browser page returned http %d", response.Status))
165	}
166	if isAccessChallenge(p.Title, p.HTML) {
167		err = browserWait(ctx, 10*time.Second, 500*time.Millisecond, func(ctx context.Context) (bool, error) {
168			var err error
169			p, err = browserSnapshot(ctx)
170			return !isAccessChallenge(p.Title, p.HTML), err
171		})
172		if err != nil {
173			return browserPage{}, err
174		}
175		// A poll cancelled at the phase boundary must not replace the last
176		// observed challenge with a zero-value page and appear to resolve it.
177		p, err = browserSnapshot(ctx)
178		if err != nil {
179			return p, err
180		}
181		if isAccessChallenge(p.Title, p.HTML) {
182			return browserPage{}, failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve")
183		}
184		if err := browserReady(ctx, o.waitForSelector, o.waitMS); err != nil {
185			return browserPage{}, err
186		}
187		p, err = browserSnapshot(ctx)
188		if err != nil {
189			return p, err
190		}
191	}
192	if timedOut && p.TextLength == 0 {
193		return browserPage{}, failure(504, "REQUEST_TIMEOUT", "browser navigation did not complete and rendered no readable content")
194	}
195	if isAccessChallenge(p.Title, p.HTML) {
196		return browserPage{}, failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve")
197	}
198	return p, nil
199}
200
201// stealthActions present a consistent Chrome identity at the network layer and
202// install the anti-fingerprinting patches before any page script runs.
203//
204// The user agent is overridden here rather than only through the launch flag
205// because the launch flag alone leaves HeadlessChrome in place for some
206// request paths. The client-hints metadata is deliberately NOT overridden:
207// Sec-CH-UA-* is opt-in per origin through Accept-CH, so a browser that
208// volunteers those headers unrequested is anomalous, and setting the metadata
209// makes Chromium do exactly that.
210func (b *Browser) stealthActions(o renderOptions) []chromedp.Action {
211	if !o.stealth {
212		return nil
213	}
214	identity := hostIdentity()
215	return []chromedp.Action{
216		chromedp.ActionFunc(func(ctx context.Context) error {
217			return emulation.SetUserAgentOverride(b.userAgent()).
218				WithAcceptLanguage("en-US,en;q=0.9").
219				WithPlatform(identity.platform).
220				Do(ctx)
221		}),
222		chromedp.ActionFunc(func(ctx context.Context) error {
223			_, err := page.AddScriptToEvaluateOnNewDocument(stealthSnapshot(identity)).Do(ctx)
224			return err
225		}),
226	}
227}
228
229func (b *Browser) userAgent() string {
230	return chromeUserAgent(probeChromeVersion(b.path))
231}
232
233// execOptions configures the browser process itself. Flags are applied in
234// order, so a later option overrides an earlier one with the same name.
235//
236// Stealth owns the entire impersonation surface, including the user agent
237// string. Cloaking only part of it would be worse than cloaking none: a Linux
238// user agent on a macOS host next to a MacIntel navigator.platform is a
239// contradiction no real browser produces. With stealth off the browser reports
240// its true identity.
241func (b *Browser) execOptions(raw string, o renderOptions, proxyAddr string) []chromedp.ExecAllocatorOption {
242	opts := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.ExecPath(b.path))
243	if !b.headless {
244		opts = append(opts, chromedp.Flag("headless", false))
245	}
246	if o.stealth {
247		// chromedp enables --enable-automation by default, which turns
248		// navigator.webdriver on and shows an automation infobar. Clearing it
249		// is the single highest-value evasion; the injected script and the
250		// client-hints override cover the rest.
251		opts = append(opts,
252			chromedp.Flag("enable-automation", false),
253			chromedp.Flag("disable-blink-features", "AutomationControlled"),
254			chromedp.Flag("disable-infobars", true),
255			chromedp.WindowSize(windowSize(raw)),
256			chromedp.UserAgent(b.userAgent()),
257		)
258	}
259	opts = append(opts,
260		chromedp.ProxyServer("http://"+proxyAddr),
261		chromedp.Flag("proxy-bypass-list", "<-loopback>"),
262		chromedp.Flag("disable-quic", true),
263		chromedp.Flag("force-webrtc-ip-handling-policy", "disable_non_proxied_udp"),
264		chromedp.Flag("disk-cache-size", "1"),
265		chromedp.Flag("media-cache-size", "1"),
266	)
267	return opts
268}
269
270// renderPage drives one browser tab end to end. Every render is isolated in
271// its own process so no cookies, storage, or cache survive between requests.
272func (b *Browser) renderPage(ctx context.Context, raw string, o renderOptions, kagi bool, session, token string) (browserPage, error) {
273	if b.path == "" {
274		return browserPage{}, fmt.Errorf("browser unavailable; configure BROWSER_PATH")
275	}
276	if _, err := parseWebURL(raw); err != nil {
277		return browserPage{}, err
278	}
279	select {
280	case b.slots <- struct{}{}:
281		defer func() { <-b.slots }()
282	case <-ctx.Done():
283		return browserPage{}, ctx.Err()
284	}
285	ln, err := net.Listen("tcp", "127.0.0.1:0")
286	if err != nil {
287		return browserPage{}, err
288	}
289	proxy := &http.Server{Handler: &guard{net: b.network, upstream: o.proxy}, ReadHeaderTimeout: 5 * time.Second}
290	go proxy.Serve(ln)
291	defer proxy.Close()
292	alloc, cancelAlloc := chromedp.NewExecAllocator(ctx, b.execOptions(raw, o, ln.Addr().String())...)
293	defer cancelAlloc()
294	tab, cancelTab := chromedp.NewContext(alloc)
295	defer cancelTab()
296	tab, cancelTimeout := context.WithTimeout(tab, 30*time.Second)
297	defer cancelTimeout()
298	actions := []chromedp.Action{network.Enable(), network.SetBlockedURLs(blockedBrowserResources)}
299	actions = append(actions, b.stealthActions(o)...)
300	if kagi {
301		actions = append(actions, network.SetCookie("kagi_session", session).WithDomain("kagi.com").WithPath("/").WithSecure(true), network.SetCookie("kagi_search", token).WithDomain("kagi.com").WithPath("/").WithSecure(true))
302	}
303	if err = chromedp.Run(tab, actions...); err != nil {
304		return browserPage{}, fmt.Errorf("browser setup failed: %w", err)
305	}
306	return renderBrowserPage(tab, raw, o)
307}
308
309// render navigates to raw and returns the resulting HTML and final URL. Kagi
310// search pages get session cookies and a results-selector wait, because their
311// results stream in after the load event.
312func (b *Browser) render(ctx context.Context, raw string, kagi bool, session, token string) (string, string, error) {
313	o := renderOptions{stealth: true, waitMS: b.waitMS, proxy: b.network.pickProxy()}
314	if kagi {
315		o.waitForSelector = kagiResultsSelector
316	}
317	p, err := b.renderPage(ctx, raw, o, kagi, session, token)
318	return p.HTML, p.URL, err
319}
320
321// renderDocument renders a page for the document pipeline using the configured
322// stealth posture and the render's selected egress proxy.
323func (b *Browser) renderDocument(ctx context.Context, raw string, proxy *url.URL) (browserPage, error) {
324	return b.renderPage(ctx, raw, renderOptions{stealth: b.stealth, waitMS: b.waitMS, proxy: proxy}, false, "", "")
325}
326
327func newBrowser(cfg Config, network *Network) *Browser {
328	return &Browser{
329		path:     cfg.BrowserPath,
330		network:  network,
331		slots:    make(chan struct{}, 2),
332		stealth:  cfg.Stealth,
333		headless: cfg.Headless,
334		waitMS:   cfg.RenderWaitMS,
335	}
336}