package app import ( "context" "encoding/json" "errors" "fmt" "net" "net/http" "net/url" "strings" "time" "github.com/chromedp/cdproto/emulation" "github.com/chromedp/cdproto/network" "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" ) type Browser struct { path string network *Network slots chan struct{} stealth bool headless bool waitMS time.Duration } // renderOptions are the per-render browser controls. type renderOptions struct { stealth bool waitForSelector string waitMS time.Duration // proxy is the upstream proxy this render egresses through, chosen once per // render so a session does not move between addresses mid-load. proxy *url.URL } // blockedBrowserResources are the resources an extractor never reads: images, // media, fonts, and analytics. Blocking them at the network layer takes real // time off page loads. These match the reference implementation's list. var blockedBrowserResources = []string{ "*.png", "*.jpg", "*.jpeg", "*.webp", "*.gif", "*.avif", "*.svg", "*.ico", "*.mp4", "*.webm", "*.mp3", "*.wav", "*.woff", "*.woff2", "*.ttf", "*.otf", "*://*.googletagmanager.com/*", "*://*.google-analytics.com/*", "*://*.doubleclick.net/*", "*://*.googlesyndication.com/*", "*://*.facebook.net/*", "*://*.hotjar.com/*", "*://*.segment.io/*", } // kagiResultsSelector matches the streaming results container on a Kagi page. const kagiResultsSelector = ".search-result, .__srgi" type browserPage struct { HTML, Title, URL string TextLength int } func jsLiteral(s string) string { b, err := json.Marshal(s) if err != nil { return `""` } return string(b) } func browserSnapshot(ctx context.Context) (p browserPage, err error) { err = chromedp.Run(ctx, chromedp.Evaluate(`({html: document.documentElement?.outerHTML || "", title: document.title, url: location.href, textLength: document.body?.innerText.trim().length || 0})`, &p)) return } func isAccessChallenge(title, html string) bool { title, html = strings.ToLower(title), strings.ToLower(html) if strings.Contains(title, "just a moment") || strings.Contains(title, "attention required") { return true } for _, marker := range []string{"cf-challenge", "cf-browser-verification", "cf-chl-", `id="challenge-form"`, `id='challenge-form'`} { if strings.Contains(html, marker) { return true } } return false } // Phase deadlines are soft: keep the tab alive so callers can inspect its DOM. // The caller's cancellation/deadline remains a hard limit, including while polling. func browserWait(ctx context.Context, cap, interval time.Duration, ready func(context.Context) (bool, error)) error { wait, cancel := context.WithTimeout(ctx, cap) defer cancel() ticks := time.NewTicker(interval) defer ticks.Stop() for { done, err := ready(wait) if ctx.Err() != nil { return ctx.Err() } if wait.Err() != nil { return nil } if err != nil || done { return err } select { case <-wait.Done(): return ctx.Err() case <-ticks.C: } } } // browserReady waits for the page to settle — readyState complete and body // text length stable across two consecutive polls — or, when a selector is // given, for pages that stream their content after the load event. func browserReady(ctx context.Context, selector string, waitMS time.Duration) error { if selector != "" { cap := waitMS if cap <= 0 || cap > 15*time.Second { cap = 8 * time.Second } var found bool return browserWait(ctx, cap, 250*time.Millisecond, func(ctx context.Context) (bool, error) { err := chromedp.Run(ctx, chromedp.Evaluate(`!!document.querySelector(`+jsLiteral(selector)+`)`, &found)) return found, err }) } cap := waitMS if cap <= 0 { cap = 1200 * time.Millisecond } if cap > 10*time.Second { cap = 10 * time.Second } started := time.Now() last, stable := -1, 0 return browserWait(ctx, cap, 150*time.Millisecond, func(ctx context.Context) (bool, error) { var length int err := chromedp.Run(ctx, chromedp.Evaluate(`document.readyState === "complete" && document.body ? document.body.innerText.length : -1`, &length)) if length >= 0 && length == last { stable++ } else { last, stable = length, 0 } return stable >= 2 && time.Since(started) >= 450*time.Millisecond, err }) } func renderBrowserPage(ctx context.Context, raw string, o renderOptions) (browserPage, error) { // Chrome was already allocated on the parent context. Cancelling only this // navigation wait must not kill the browser or discard a usable partial DOM. nav, cancel := context.WithTimeout(ctx, 10*time.Second) response, err := chromedp.RunResponse(nav, chromedp.Navigate(raw)) timedOut := errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil cancel() if err != nil && !timedOut { return browserPage{}, fmt.Errorf("browser navigation failed: %w", err) } if err := browserReady(ctx, o.waitForSelector, o.waitMS); err != nil { return browserPage{}, fmt.Errorf("browser readiness failed: %w", err) } p, err := browserSnapshot(ctx) if err != nil { return p, err } if response != nil && response.Status >= 400 && !isAccessChallenge(p.Title, p.HTML) { return browserPage{}, failure(int(response.Status), "CRAWL_HTTP_ERROR", fmt.Sprintf("browser page returned http %d", response.Status)) } if isAccessChallenge(p.Title, p.HTML) { err = browserWait(ctx, 10*time.Second, 500*time.Millisecond, func(ctx context.Context) (bool, error) { var err error p, err = browserSnapshot(ctx) return !isAccessChallenge(p.Title, p.HTML), err }) if err != nil { return browserPage{}, err } // A poll cancelled at the phase boundary must not replace the last // observed challenge with a zero-value page and appear to resolve it. p, err = browserSnapshot(ctx) if err != nil { return p, err } if isAccessChallenge(p.Title, p.HTML) { return browserPage{}, failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve") } if err := browserReady(ctx, o.waitForSelector, o.waitMS); err != nil { return browserPage{}, err } p, err = browserSnapshot(ctx) if err != nil { return p, err } } if timedOut && p.TextLength == 0 { return browserPage{}, failure(504, "REQUEST_TIMEOUT", "browser navigation did not complete and rendered no readable content") } if isAccessChallenge(p.Title, p.HTML) { return browserPage{}, failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve") } return p, nil } // stealthActions present a consistent Chrome identity at the network layer and // install the anti-fingerprinting patches before any page script runs. // // The user agent is overridden here rather than only through the launch flag // because the launch flag alone leaves HeadlessChrome in place for some // request paths. The client-hints metadata is deliberately NOT overridden: // Sec-CH-UA-* is opt-in per origin through Accept-CH, so a browser that // volunteers those headers unrequested is anomalous, and setting the metadata // makes Chromium do exactly that. func (b *Browser) stealthActions(o renderOptions) []chromedp.Action { if !o.stealth { return nil } identity := hostIdentity() return []chromedp.Action{ chromedp.ActionFunc(func(ctx context.Context) error { return emulation.SetUserAgentOverride(b.userAgent()). WithAcceptLanguage("en-US,en;q=0.9"). WithPlatform(identity.platform). Do(ctx) }), chromedp.ActionFunc(func(ctx context.Context) error { _, err := page.AddScriptToEvaluateOnNewDocument(stealthSnapshot(identity)).Do(ctx) return err }), } } func (b *Browser) userAgent() string { return chromeUserAgent(probeChromeVersion(b.path)) } // execOptions configures the browser process itself. Flags are applied in // order, so a later option overrides an earlier one with the same name. // // Stealth owns the entire impersonation surface, including the user agent // string. Cloaking only part of it would be worse than cloaking none: a Linux // user agent on a macOS host next to a MacIntel navigator.platform is a // contradiction no real browser produces. With stealth off the browser reports // its true identity. func (b *Browser) execOptions(raw string, o renderOptions, proxyAddr string) []chromedp.ExecAllocatorOption { opts := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.ExecPath(b.path)) if !b.headless { opts = append(opts, chromedp.Flag("headless", false)) } if o.stealth { // chromedp enables --enable-automation by default, which turns // navigator.webdriver on and shows an automation infobar. Clearing it // is the single highest-value evasion; the injected script and the // client-hints override cover the rest. opts = append(opts, chromedp.Flag("enable-automation", false), chromedp.Flag("disable-blink-features", "AutomationControlled"), chromedp.Flag("disable-infobars", true), chromedp.WindowSize(windowSize(raw)), chromedp.UserAgent(b.userAgent()), ) } opts = append(opts, chromedp.ProxyServer("http://"+proxyAddr), chromedp.Flag("proxy-bypass-list", "<-loopback>"), chromedp.Flag("disable-quic", true), chromedp.Flag("force-webrtc-ip-handling-policy", "disable_non_proxied_udp"), chromedp.Flag("disk-cache-size", "1"), chromedp.Flag("media-cache-size", "1"), ) return opts } // renderPage drives one browser tab end to end. Every render is isolated in // its own process so no cookies, storage, or cache survive between requests. func (b *Browser) renderPage(ctx context.Context, raw string, o renderOptions, kagi bool, session, token string) (browserPage, error) { if b.path == "" { return browserPage{}, fmt.Errorf("browser unavailable; configure BROWSER_PATH") } if _, err := parseWebURL(raw); err != nil { return browserPage{}, err } select { case b.slots <- struct{}{}: defer func() { <-b.slots }() case <-ctx.Done(): return browserPage{}, ctx.Err() } ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return browserPage{}, err } proxy := &http.Server{Handler: &guard{net: b.network, upstream: o.proxy}, ReadHeaderTimeout: 5 * time.Second} go proxy.Serve(ln) defer proxy.Close() alloc, cancelAlloc := chromedp.NewExecAllocator(ctx, b.execOptions(raw, o, ln.Addr().String())...) defer cancelAlloc() tab, cancelTab := chromedp.NewContext(alloc) defer cancelTab() tab, cancelTimeout := context.WithTimeout(tab, 30*time.Second) defer cancelTimeout() actions := []chromedp.Action{network.Enable(), network.SetBlockedURLs(blockedBrowserResources)} actions = append(actions, b.stealthActions(o)...) if kagi { actions = append(actions, network.SetCookie("kagi_session", session).WithDomain("kagi.com").WithPath("/").WithSecure(true), network.SetCookie("kagi_search", token).WithDomain("kagi.com").WithPath("/").WithSecure(true)) } if err = chromedp.Run(tab, actions...); err != nil { return browserPage{}, fmt.Errorf("browser setup failed: %w", err) } return renderBrowserPage(tab, raw, o) } // render navigates to raw and returns the resulting HTML and final URL. Kagi // search pages get session cookies and a results-selector wait, because their // results stream in after the load event. func (b *Browser) render(ctx context.Context, raw string, kagi bool, session, token string) (string, string, error) { o := renderOptions{stealth: true, waitMS: b.waitMS, proxy: b.network.pickProxy()} if kagi { o.waitForSelector = kagiResultsSelector } p, err := b.renderPage(ctx, raw, o, kagi, session, token) return p.HTML, p.URL, err } // renderDocument renders a page for the document pipeline using the configured // stealth posture and the render's selected egress proxy. func (b *Browser) renderDocument(ctx context.Context, raw string, proxy *url.URL) (browserPage, error) { return b.renderPage(ctx, raw, renderOptions{stealth: b.stealth, waitMS: b.waitMS, proxy: proxy}, false, "", "") } func newBrowser(cfg Config, network *Network) *Browser { return &Browser{ path: cfg.BrowserPath, network: network, slots: make(chan struct{}, 2), stealth: cfg.Stealth, headless: cfg.Headless, waitMS: cfg.RenderWaitMS, } }