okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"fmt" 7"os" 8"path/filepath" 9"runtime" 10"strings" 11"testing" 12"time" 13 14"github.com/chromedp/chromedp" 15) 16 17func TestWindowSizeIsStablePerURLAndWithinBounds (t * testing.T ) { 18for _ ,raw := range []string {"https://a.test/" ,"https://b.test/x" ,"https://c.test/y?z=1" } { 19w1 ,h1 := windowSize (raw ) 20w2 ,h2 := windowSize (raw ) 21if w1 != w2 || h1 != h2 { 22t .Fatalf ("windowSize(%q) is not deterministic: %dx%d then %dx%d" ,raw ,w1 ,h1 ,w2 ,h2 ) 23 } 24if w1 < 1200 || w1 > 1359 || h1 < 800 || h1 > 959 { 25t .Fatalf ("windowSize(%q) = %dx%d outside the intended range" ,raw ,w1 ,h1 ) 26 } 27 } 28} 29 30// The presented identity must belong to the host the browser actually runs on. 31// Claiming another platform is what bot detection looks for, so the user agent 32// and navigator.platform must agree with each other and with this host. 33func TestHostIdentityIsCoherent (t * testing.T ) { 34v := chromeVersion {major :"153" } 35identity := hostIdentity () 36ua := chromeUserAgent (v ) 37 38if ! strings .Contains (ua ,"Chrome/153.0.0.0 Safari/537.36" ) { 39t .Fatalf ("user agent does not carry the probed major version: %q" ,ua ) 40 } 41if strings .Contains (ua ,"Headless" ) { 42t .Fatalf ("user agent announces headless mode: %q" ,ua ) 43 } 44// The platform claim and the user agent must describe one machine. 45wantPlatform := hostUserAgentFragment () 46if wantPlatform == "" { 47t .Skipf ("no identity expectations for %s" ,runtime .GOOS ) 48 } 49if ! strings .Contains (ua ,wantPlatform ) { 50t .Fatalf ("user agent %q does not describe this %s host (want %q)" ,ua ,runtime .GOOS ,wantPlatform ) 51 } 52// macOS reports a real Apple GPU in headless mode, so replacing the WebGL 53// strings there would contradict the rest of the fingerprint. 54if identity .platform == "" { 55t .Fatal ("identity has no navigator.platform" ) 56 } 57if runtime .GOOS == "darwin" && identity .spoofGPU { 58t .Fatal ("macOS identity spoofs the GPU string over a real Apple GPU" ) 59 } 60if runtime .GOOS != "darwin" && ! identity .spoofGPU { 61t .Fatalf ("%s identity leaves the SwiftShader renderer string in place" ,runtime .GOOS ) 62 } 63} 64 65// The injected script must not carry the WebGL replacement when the host 66// reports a real GPU, and must carry it when the host renders through 67// SwiftShader. 68func TestStealthSnapshotOmitsGPUPatchesForRealGPUs (t * testing.T ) { 69real := stealthSnapshot (browserIdentity {platform :"MacIntel" }) 70if strings .Contains (real ,"0x9246" ) { 71t .Fatal ("script patches WebGL for an identity that reports a real GPU" ) 72 } 73if ! strings .Contains (real ,"webdriver" )|| ! strings .Contains (real ,"outerHeight" ) { 74t .Fatal ("script dropped the platform-neutral evasions" ) 75 } 76swift := stealthSnapshot (browserIdentity {platform :"Linux x86_64" ,spoofGPU :true }) 77if ! strings .Contains (swift ,"0x9246" ) { 78t .Fatal ("script left the SwiftShader renderer string in place" ) 79 } 80if ! strings .Contains (swift ,"outerHeight" ) { 81t .Fatal ("script dropped the trailing evasions" ) 82 } 83} 84 85// The client-hints metadata override must not come back: forcing Sec-CH-UA-* 86// headers that a real browser only sends when an origin opts in is what reddit 87// treated as automation. 88func TestNoClientHintsOverrideInStealthActions (t * testing.T ) { 89if strings .Contains (stealthTestSource (t ),"WithUserAgentMetadata" ) { 90t .Fatal ("stealth path overrides Sec-CH-UA client hints" ) 91 } 92} 93 94// hostUserAgentFragment is the part of a Chrome user agent that identifies one 95// operating system. Tests use it to assert the identity matches the host. 96func hostUserAgentFragment ()string { 97return map [string ]string { 98"darwin" :"Macintosh; Intel Mac OS X" , 99"windows" :"Windows NT 10.0; Win64; x64" , 100"linux" :"X11; Linux x86_64" , 101 }[runtime .GOOS ] 102} 103 104func stealthTestSource (t * testing.T )string { 105t .Helper () 106source ,err := os .ReadFile ("browser.go" ) 107if err != nil { 108t .Fatal (err ) 109 } 110return string (source ) 111} 112 113// TestLiveStealthFingerprint asserts real Chrome presents a plausible desktop 114// browser to page scripts instead of announcing automation. It drives Chrome, 115// so it is opt-in like the other browser tests. 116func TestLiveStealthFingerprint (t * testing.T ) { 117if os .Getenv ("LIVE_BROWSER" )!= "1" { 118t .Skip ("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome" ) 119 } 120if os .Getenv ("BROWSER_PATH" )== "" { 121t .Fatal ("BROWSER_PATH is required" ) 122 } 123for _ ,stealth := range []bool {true ,false } { 124name := "stealth off" 125if stealth { 126name = "stealth on" 127 } 128t .Run (name ,func (t * testing.T ) { 129observed := observeFingerprint (t ,stealth ) 130if ! stealth { 131t .Logf ("baseline: webdriver=%v ua=%q" ,observed ["webdriver" ],observed ["userAgent" ]) 132return 133 } 134if observed ["webdriver" ]!= false { 135t .Errorf ("navigator.webdriver = %v, want false" ,observed ["webdriver" ]) 136 } 137ua ,_ := observed ["userAgent" ].(string ) 138if ! strings .Contains (ua ,"Chrome/" )|| strings .Contains (ua ,"Headless" ) { 139t .Errorf ("user agent is not desktop Chrome: %q" ,ua ) 140 } 141// The user agent must describe the host the browser runs on. 142if fragment := hostUserAgentFragment ();fragment != "" && ! strings .Contains (ua ,fragment ) { 143t .Errorf ("user agent %q does not describe this %s host (want %q)" ,ua ,runtime .GOOS ,fragment ) 144 } 145if observed ["vendor" ]!= "Google Inc." { 146t .Errorf ("navigator.vendor = %v" ,observed ["vendor" ]) 147 } 148if ! strings .Contains (fmt .Sprint (observed ["webdriverGetter" ]),"[native code]" ) { 149t .Errorf ("webdriver getter does not stringify as native: %v" ,observed ["webdriverGetter" ]) 150 } 151if observed ["chromeRuntime" ]!= true { 152t .Error ("window.chrome.runtime is missing" ) 153 } 154rendered ,_ := observed ["webglRenderer" ].(string ) 155if strings .Contains (rendered ,"SwiftShader" ) { 156t .Errorf ("webgl renderer is the software rasterizer: %q" ,rendered ) 157 } 158if runtime .GOOS == "darwin" && ! strings .Contains (rendered ,"Apple" ) { 159t .Errorf ("macOS host does not report its real Apple GPU: %q" ,rendered ) 160 } 161if runtime .GOOS != "darwin" && ! strings .Contains (rendered ,"Intel" ) { 162t .Errorf ("SwiftShader host was not given a desktop adapter string: %q" ,rendered ) 163 } 164if langs := fmt .Sprint (observed ["languages" ]);! strings .Contains (langs ,"en-US" ) { 165t .Errorf ("navigator.languages = %v" ,observed ["languages" ]) 166 } 167if plugins ,ok := observed ["plugins" ].(float64 );! ok || plugins != 5 { 168t .Errorf ("navigator.plugins.length = %v, want 5" ,observed ["plugins" ]) 169 } 170 }) 171 } 172} 173 174// observeFingerprint renders a blank page through the production browser setup 175// and reads back the surface a bot-detection script inspects. 176func observeFingerprint (t * testing.T ,stealth bool )map [string ]any { 177t .Helper () 178path ,err := filepath .Abs (os .Getenv ("BROWSER_PATH" )) 179if err != nil { 180t .Fatal (err ) 181 } 182b := & Browser {path :path ,network :newNetwork (true ),slots :make (chan struct {},1 ),stealth :stealth ,headless :true ,waitMS :1200 * time .Millisecond } 183defer b .network .transport .CloseIdleConnections () 184ctx ,cancel := context .WithTimeout (context .Background (),60 * time .Second ) 185defer cancel () 186 187alloc ,cancelAlloc := chromedp .NewExecAllocator (ctx ,b .execOptions ("https://fingerprint.test/" ,renderOptions {stealth :stealth },"127.0.0.1:1" )... ) 188defer cancelAlloc () 189tab ,cancelTab := chromedp .NewContext (alloc ) 190defer cancelTab () 191 192const probe = `(() => { 193const d = Object.getOwnPropertyDescriptor(Navigator.prototype, 'webdriver'); 194const gl = document.createElement('canvas').getContext('webgl'); 195const dbg = gl && gl.getExtension('WEBGL_debug_renderer_info'); 196return JSON.stringify({ 197webdriver: navigator.webdriver, 198vendor: navigator.vendor, 199languages: navigator.languages, 200plugins: navigator.plugins.length, 201userAgent: navigator.userAgent, 202webdriverGetter: d && d.get ? Function.prototype.toString.call(d.get) : null, 203chromeRuntime: !!(window.chrome && window.chrome.runtime), 204webglRenderer: gl ? gl.getParameter(dbg ? dbg.UNMASKED_RENDERER_WEBGL : 0x9246) : null, 205}); 206})()` 207 208var raw string 209actions := b .stealthActions (renderOptions {stealth :stealth }) 210actions = append (actions , 211chromedp .Navigate ("data:text/html,<html><body><h1>probe</h1></body></html>" ), 212chromedp .Evaluate (probe ,& raw ), 213 ) 214if err := chromedp .Run (tab ,actions ... );err != nil { 215t .Fatalf ("fingerprint probe failed: %v" ,err ) 216 } 217var out map [string ]any 218if err := json .Unmarshal ([]byte (raw ),& out );err != nil { 219t .Fatalf ("decode fingerprint %q: %v" ,raw ,err ) 220 } 221return out 222}