okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"errors" 7"fmt" 8"net" 9"net/http" 10"net/url" 11"strings" 12"time" 13 14"github.com/chromedp/cdproto/emulation" 15"github.com/chromedp/cdproto/network" 16"github.com/chromedp/cdproto/page" 17"github.com/chromedp/chromedp" 18) 19 20type Browser struct { 21path string 22network * Network 23slots chan struct {} 24stealth bool 25headless bool 26waitMS time.Duration 27} 28 29// renderOptions are the per-render browser controls. 30type renderOptions struct { 31stealth bool 32waitForSelector string 33waitMS time.Duration 34// proxy is the upstream proxy this render egresses through, chosen once per 35// render so a session does not move between addresses mid-load. 36proxy * url.URL 37} 38 39// blockedBrowserResources are the resources an extractor never reads: images, 40// media, fonts, and analytics. Blocking them at the network layer takes real 41// time off page loads. These match the reference implementation's list. 42var blockedBrowserResources = []string { 43"*.png" ,"*.jpg" ,"*.jpeg" ,"*.webp" ,"*.gif" ,"*.avif" ,"*.svg" ,"*.ico" , 44"*.mp4" ,"*.webm" ,"*.mp3" ,"*.wav" ,"*.woff" ,"*.woff2" ,"*.ttf" ,"*.otf" , 45"*://*.googletagmanager.com/*" ,"*://*.google-analytics.com/*" ,"*://*.doubleclick.net/*" , 46"*://*.googlesyndication.com/*" ,"*://*.facebook.net/*" ,"*://*.hotjar.com/*" ,"*://*.segment.io/*" , 47} 48 49// kagiResultsSelector matches the streaming results container on a Kagi page. 50const kagiResultsSelector = ".search-result, .__srgi" 51 52type browserPage struct { 53HTML ,Title ,URL string 54TextLength int 55} 56 57func jsLiteral (s string )string { 58b ,err := json .Marshal (s ) 59if err != nil { 60return `""` 61 } 62return string (b ) 63} 64 65func browserSnapshot (ctx context.Context ) (p browserPage ,err error ) { 66err = chromedp .Run (ctx ,chromedp .Evaluate (`({html: document.documentElement?.outerHTML || "", 67title: document.title, url: location.href, textLength: document.body?.innerText.trim().length || 0})` ,& p )) 68return 69} 70 71func isAccessChallenge (title ,html string )bool { 72title ,html = strings .ToLower (title ),strings .ToLower (html ) 73if strings .Contains (title ,"just a moment" )|| strings .Contains (title ,"attention required" ) { 74return true 75 } 76for _ ,marker := range []string {"cf-challenge" ,"cf-browser-verification" ,"cf-chl-" ,`id="challenge-form"` ,`id='challenge-form'` } { 77if strings .Contains (html ,marker ) { 78return true 79 } 80 } 81return false 82} 83 84// Phase deadlines are soft: keep the tab alive so callers can inspect its DOM. 85// The caller's cancellation/deadline remains a hard limit, including while polling. 86func browserWait (ctx context.Context ,cap ,interval time.Duration ,ready func (context.Context ) (bool ,error ))error { 87wait ,cancel := context .WithTimeout (ctx ,cap ) 88defer cancel () 89ticks := time .NewTicker (interval ) 90defer ticks .Stop () 91for { 92done ,err := ready (wait ) 93if ctx .Err ()!= nil { 94return ctx .Err () 95 } 96if wait .Err ()!= nil { 97return nil 98 } 99if err != nil || done { 100return err 101 } 102select { 103case <- wait .Done (): 104return ctx .Err () 105case <- ticks .C : 106 } 107 } 108} 109 110// browserReady waits for the page to settle — readyState complete and body 111// text length stable across two consecutive polls — or, when a selector is 112// given, for pages that stream their content after the load event. 113func browserReady (ctx context.Context ,selector string ,waitMS time.Duration )error { 114if selector != "" { 115cap := waitMS 116if cap <= 0 || cap > 15 * time .Second { 117cap = 8 * time .Second 118 } 119var found bool 120return browserWait (ctx ,cap ,250 * time .Millisecond ,func (ctx context.Context ) (bool ,error ) { 121err := chromedp .Run (ctx ,chromedp .Evaluate (`!!document.querySelector(` + jsLiteral (selector )+ `)` ,& found )) 122return found ,err 123 }) 124 } 125cap := waitMS 126if cap <= 0 { 127cap = 1200 * time .Millisecond 128 } 129if cap > 10 * time .Second { 130cap = 10 * time .Second 131 } 132started := time .Now () 133last ,stable := - 1 ,0 134return browserWait (ctx ,cap ,150 * time .Millisecond ,func (ctx context.Context ) (bool ,error ) { 135var length int 136err := chromedp .Run (ctx ,chromedp .Evaluate (`document.readyState === "complete" && document.body ? document.body.innerText.length : -1` ,& length )) 137if length >= 0 && length == last { 138stable ++ 139 }else { 140last ,stable = length ,0 141 } 142return stable >= 2 && time .Since (started )>= 450 * time .Millisecond ,err 143 }) 144} 145 146func renderBrowserPage (ctx context.Context ,raw string ,o renderOptions ) (browserPage ,error ) { 147// Chrome was already allocated on the parent context. Cancelling only this 148// navigation wait must not kill the browser or discard a usable partial DOM. 149nav ,cancel := context .WithTimeout (ctx ,10 * time .Second ) 150response ,err := chromedp .RunResponse (nav ,chromedp .Navigate (raw )) 151timedOut := errors .Is (err ,context .DeadlineExceeded )&& ctx .Err ()== nil 152cancel () 153if err != nil && ! timedOut { 154return browserPage {},fmt .Errorf ("browser navigation failed: %w" ,err ) 155 } 156if err := browserReady (ctx ,o .waitForSelector ,o .waitMS );err != nil { 157return browserPage {},fmt .Errorf ("browser readiness failed: %w" ,err ) 158 } 159p ,err := browserSnapshot (ctx ) 160if err != nil { 161return p ,err 162 } 163if response != nil && response .Status >= 400 && ! isAccessChallenge (p .Title ,p .HTML ) { 164return browserPage {},failure (int (response .Status ),"CRAWL_HTTP_ERROR" ,fmt .Sprintf ("browser page returned http %d" ,response .Status )) 165 } 166if isAccessChallenge (p .Title ,p .HTML ) { 167err = browserWait (ctx ,10 * time .Second ,500 * time .Millisecond ,func (ctx context.Context ) (bool ,error ) { 168var err error 169p ,err = browserSnapshot (ctx ) 170return ! isAccessChallenge (p .Title ,p .HTML ),err 171 }) 172if err != nil { 173return browserPage {},err 174 } 175// A poll cancelled at the phase boundary must not replace the last 176// observed challenge with a zero-value page and appear to resolve it. 177p ,err = browserSnapshot (ctx ) 178if err != nil { 179return p ,err 180 } 181if isAccessChallenge (p .Title ,p .HTML ) { 182return browserPage {},failure (403 ,"CRAWL_CHALLENGE" ,"browser access challenge did not resolve" ) 183 } 184if err := browserReady (ctx ,o .waitForSelector ,o .waitMS );err != nil { 185return browserPage {},err 186 } 187p ,err = browserSnapshot (ctx ) 188if err != nil { 189return p ,err 190 } 191 } 192if timedOut && p .TextLength == 0 { 193return browserPage {},failure (504 ,"REQUEST_TIMEOUT" ,"browser navigation did not complete and rendered no readable content" ) 194 } 195if isAccessChallenge (p .Title ,p .HTML ) { 196return browserPage {},failure (403 ,"CRAWL_CHALLENGE" ,"browser access challenge did not resolve" ) 197 } 198return p ,nil 199} 200 201// stealthActions present a consistent Chrome identity at the network layer and 202// install the anti-fingerprinting patches before any page script runs. 203// 204// The user agent is overridden here rather than only through the launch flag 205// because the launch flag alone leaves HeadlessChrome in place for some 206// request paths. The client-hints metadata is deliberately NOT overridden: 207// Sec-CH-UA-* is opt-in per origin through Accept-CH, so a browser that 208// volunteers those headers unrequested is anomalous, and setting the metadata 209// makes Chromium do exactly that. 210func (b * Browser )stealthActions (o renderOptions ) []chromedp.Action { 211if ! o .stealth { 212return nil 213 } 214identity := hostIdentity () 215return []chromedp.Action { 216chromedp .ActionFunc (func (ctx context.Context )error { 217return emulation .SetUserAgentOverride (b .userAgent ()). 218WithAcceptLanguage ("en-US,en;q=0.9" ). 219WithPlatform (identity .platform ). 220Do (ctx ) 221 }), 222chromedp .ActionFunc (func (ctx context.Context )error { 223_ ,err := page .AddScriptToEvaluateOnNewDocument (stealthSnapshot (identity )).Do (ctx ) 224return err 225 }), 226 } 227} 228 229func (b * Browser )userAgent ()string { 230return chromeUserAgent (probeChromeVersion (b .path )) 231} 232 233// execOptions configures the browser process itself. Flags are applied in 234// order, so a later option overrides an earlier one with the same name. 235// 236// Stealth owns the entire impersonation surface, including the user agent 237// string. Cloaking only part of it would be worse than cloaking none: a Linux 238// user agent on a macOS host next to a MacIntel navigator.platform is a 239// contradiction no real browser produces. With stealth off the browser reports 240// its true identity. 241func (b * Browser )execOptions (raw string ,o renderOptions ,proxyAddr string ) []chromedp.ExecAllocatorOption { 242opts := append (chromedp .DefaultExecAllocatorOptions [:],chromedp .ExecPath (b .path )) 243if ! b .headless { 244opts = append (opts ,chromedp .Flag ("headless" ,false )) 245 } 246if o .stealth { 247// chromedp enables --enable-automation by default, which turns 248// navigator.webdriver on and shows an automation infobar. Clearing it 249// is the single highest-value evasion; the injected script and the 250// client-hints override cover the rest. 251opts = append (opts , 252chromedp .Flag ("enable-automation" ,false ), 253chromedp .Flag ("disable-blink-features" ,"AutomationControlled" ), 254chromedp .Flag ("disable-infobars" ,true ), 255chromedp .WindowSize (windowSize (raw )), 256chromedp .UserAgent (b .userAgent ()), 257 ) 258 } 259opts = append (opts , 260chromedp .ProxyServer ("http://" + proxyAddr ), 261chromedp .Flag ("proxy-bypass-list" ,"<-loopback>" ), 262chromedp .Flag ("disable-quic" ,true ), 263chromedp .Flag ("force-webrtc-ip-handling-policy" ,"disable_non_proxied_udp" ), 264chromedp .Flag ("disk-cache-size" ,"1" ), 265chromedp .Flag ("media-cache-size" ,"1" ), 266 ) 267return opts 268} 269 270// renderPage drives one browser tab end to end. Every render is isolated in 271// its own process so no cookies, storage, or cache survive between requests. 272func (b * Browser )renderPage (ctx context.Context ,raw string ,o renderOptions ,kagi bool ,session ,token string ) (browserPage ,error ) { 273if b .path == "" { 274return browserPage {},fmt .Errorf ("browser unavailable; configure BROWSER_PATH" ) 275 } 276if _ ,err := parseWebURL (raw );err != nil { 277return browserPage {},err 278 } 279select { 280case b .slots <- struct {}{}: 281defer func () {<- b .slots }() 282case <- ctx .Done (): 283return browserPage {},ctx .Err () 284 } 285ln ,err := net .Listen ("tcp" ,"127.0.0.1:0" ) 286if err != nil { 287return browserPage {},err 288 } 289proxy := & http.Server {Handler :& guard {net :b .network ,upstream :o .proxy },ReadHeaderTimeout :5 * time .Second } 290go proxy .Serve (ln ) 291defer proxy .Close () 292alloc ,cancelAlloc := chromedp .NewExecAllocator (ctx ,b .execOptions (raw ,o ,ln .Addr ().String ())... ) 293defer cancelAlloc () 294tab ,cancelTab := chromedp .NewContext (alloc ) 295defer cancelTab () 296tab ,cancelTimeout := context .WithTimeout (tab ,30 * time .Second ) 297defer cancelTimeout () 298actions := []chromedp.Action {network .Enable (),network .SetBlockedURLs (blockedBrowserResources )} 299actions = append (actions ,b .stealthActions (o )... ) 300if kagi { 301actions = append (actions ,network .SetCookie ("kagi_session" ,session ).WithDomain ("kagi.com" ).WithPath ("/" ).WithSecure (true ),network .SetCookie ("kagi_search" ,token ).WithDomain ("kagi.com" ).WithPath ("/" ).WithSecure (true )) 302 } 303if err = chromedp .Run (tab ,actions ... );err != nil { 304return browserPage {},fmt .Errorf ("browser setup failed: %w" ,err ) 305 } 306return renderBrowserPage (tab ,raw ,o ) 307} 308 309// render navigates to raw and returns the resulting HTML and final URL. Kagi 310// search pages get session cookies and a results-selector wait, because their 311// results stream in after the load event. 312func (b * Browser )render (ctx context.Context ,raw string ,kagi bool ,session ,token string ) (string ,string ,error ) { 313o := renderOptions {stealth :true ,waitMS :b .waitMS ,proxy :b .network .pickProxy ()} 314if kagi { 315o .waitForSelector = kagiResultsSelector 316 } 317p ,err := b .renderPage (ctx ,raw ,o ,kagi ,session ,token ) 318return p .HTML ,p .URL ,err 319} 320 321// renderDocument renders a page for the document pipeline using the configured 322// stealth posture and the render's selected egress proxy. 323func (b * Browser )renderDocument (ctx context.Context ,raw string ,proxy * url.URL ) (browserPage ,error ) { 324return b .renderPage (ctx ,raw ,renderOptions {stealth :b .stealth ,waitMS :b .waitMS ,proxy :proxy },false ,"" ,"" ) 325} 326 327func newBrowser (cfg Config ,network * Network )* Browser { 328return & Browser { 329path :cfg .BrowserPath , 330network :network , 331slots :make (chan struct {},2 ), 332stealth :cfg .Stealth , 333headless :cfg .Headless , 334waitMS :cfg .RenderWaitMS , 335 } 336}