okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"crypto/sha256" 5"encoding/binary" 6"fmt" 7"log/slog" 8"os/exec" 9"regexp" 10"runtime" 11"strings" 12"sync" 13) 14 15// chromeVersion identifies the browser being driven by its major version, which 16// is all the user agent needs. 17type chromeVersion struct { 18major string 19} 20 21// pinnedChromeVersion is the cloaking fallback when no browser binary can be 22// probed. It matches the pinned version used by the reference implementation. 23var pinnedChromeVersion = chromeVersion {major :"150" } 24 25// chromeVersionCandidates are tried in order when BROWSER_PATH is unset or 26// cannot be executed. The Linux entries are the ones that matter in 27// containers and on servers, where the service actually runs. 28var chromeVersionCandidates = []string { 29"chromium" , 30"chromium-browser" , 31"google-chrome" , 32"google-chrome-stable" , 33} 34 35var chromeVersionPattern = regexp .MustCompile (`\d+(?:\.\d+)*` ) 36 37var ( 38chromeVersionMu sync.Mutex 39chromeVersionCache = map [string ]chromeVersion {} 40) 41 42// probeChromeVersion reports the version of the configured browser, falling 43// back to a pinned version when nothing can be executed. The answer for a 44// given path cannot change while the process runs and probing spawns a 45// process, so results are cached per path. 46func probeChromeVersion (path string )chromeVersion { 47chromeVersionMu .Lock () 48defer chromeVersionMu .Unlock () 49if cached ,ok := chromeVersionCache [path ];ok { 50return cached 51 } 52detected := detectChromeVersion (path ) 53chromeVersionCache [path ]= detected 54return detected 55} 56 57func detectChromeVersion (path string )chromeVersion { 58var candidates []string 59if path != "" { 60candidates = append (candidates ,path ) 61 } 62candidates = append (candidates ,chromeVersionCandidates ... ) 63for _ ,candidate := range candidates { 64out ,err := exec .Command (candidate ,"--version" ).Output () 65if err != nil { 66continue 67 } 68if found := chromeVersionPattern .FindString (string (out ));found != "" { 69major ,_ ,_ := strings .Cut (found ,"." ) 70return chromeVersion {major :major } 71 } 72 } 73// Cloaking as a version the browser is not is worse than saying nothing, 74// so make the degraded case visible. 75slog .Warn ("could not probe the browser version; falling back to pinned Chrome cloaking" ,"browser_path" ,path ,"pinned_major" ,pinnedChromeVersion .major ) 76return pinnedChromeVersion 77} 78 79// browserIdentity is the platform shape this service presents to pages. 80// 81// It follows the host the browser actually runs on. Claiming a different 82// platform than the host is what bot detection looks for: an Apple GPU, Mac 83// fonts, and a macOS TLS fingerprint alongside a Linux user agent describe a 84// machine that cannot exist. The stealth Chromium builds handle this the same 85// way, running macOS as a native Mac browser rather than spoofing one. 86type browserIdentity struct { 87userAgent string // template; %s is the Chrome major version 88platform string // navigator.platform 89spoofGPU bool // replace the WebGL vendor and renderer strings 90} 91 92func hostIdentity ()browserIdentity { 93switch runtime .GOOS { 94case "darwin" : 95// Headless Chrome on macOS already reports the machine's real Apple 96// GPU, so the WebGL strings are left alone here. 97return browserIdentity { 98userAgent :"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36" , 99platform :"MacIntel" , 100 } 101case "windows" : 102return browserIdentity { 103userAgent :"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36" , 104platform :"Win32" , 105spoofGPU :true , 106 } 107default : 108return browserIdentity { 109userAgent :"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36" , 110platform :"Linux x86_64" , 111spoofGPU :true , 112 } 113 } 114} 115 116// chromeUserAgent is the host's Chrome user agent with the probed major 117// version, so the browser does not announce itself as HeadlessChrome. 118func chromeUserAgent (v chromeVersion )string { 119return fmt .Sprintf (hostIdentity ().userAgent ,v .major ) 120} 121 122// windowSize derives a viewport that is stable for a URL but varies between 123// them, so every request is not rendered at one machine-identifiable size. 124func windowSize (raw string ) (width ,height int ) { 125sum := sha256 .Sum256 ([]byte (raw )) 126seed := binary .BigEndian .Uint64 (sum [:8 ]) 127return 1200 + int (seed % 160 ),800 + int ((seed >> 8 )% 160 ) 128} 129 130// stealthScriptHead is the platform-neutral part of the script injected into 131// every new document before any page script runs. It removes the automation 132// fingerprints that bot detection reads directly, and makes every patch 133// stringify like a native implementation so Function.prototype.toString cannot 134// reveal it. 135// 136// Each section is isolated: this executes in the context of arbitrary, 137// untrusted pages, and one failing patch must never break the rest. 138const stealthScriptHead = ` 139(() => { 140// Make overridden functions stringify like native implementations; 141// Function.prototype.toString is the standard bot-detection tell. 142const nativeToString = Function.prototype.toString; 143const spoofedSources = new WeakMap(); 144const asNative = (fn, source) => { spoofedSources.set(fn, source); return fn; }; 145const nativeGetter = (name) => 'function get ' + name + '() { [native code] }'; 146const nativeFunc = (name) => 'function ' + name + '() { [native code] }'; 147Function.prototype.toString = asNative(function toString() { 148const spoofed = spoofedSources.get(this); 149return spoofed !== undefined ? spoofed : nativeToString.call(this); 150}, 'function toString() { [native code] }'); 151const define = (object, prop, descriptor) => { 152try { Object.defineProperty(object, prop, { configurable: true, ...descriptor }); } catch (_) {} 153}; 154 155// navigator.webdriver is true under automation; real Chrome exposes false. 156define(Navigator.prototype, 'webdriver', { 157get: asNative(() => false, nativeGetter('webdriver')), 158}); 159define(Navigator.prototype, 'vendor', { 160get: asNative(() => 'Google Inc.', nativeGetter('vendor')), 161}); 162define(Navigator.prototype, 'languages', { 163get: asNative(() => ['en-US', 'en'], nativeGetter('languages')), 164}); 165 166// Plugin list: empty under automation, five PDF-related entries in real 167// Chrome. Mid-fidelity PluginArray with named access and iteration. 168try { 169const mimeTypesData = [ 170{ type: 'application/pdf', suffixes: 'pdf', description: '' }, 171{ type: 'text/pdf', suffixes: 'pdf', description: '' }, 172]; 173const pluginsData = [ 174['PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'], 175['Chrome PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'], 176['Chromium PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'], 177['Microsoft Edge PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'], 178['WebKit built-in PDF', 'Portable Document Format', 'internal-pdf-viewer'], 179]; 180const mimeTypes = mimeTypesData.map((m) => { 181const mt = { ...m, enabledPlugin: null }; 182mt.enabledPlugin = mt; 183return mt; 184}); 185const plugins = pluginsData.map(([name, description, filename]) => { 186const plugin = { 187name, description, filename, 188length: mimeTypes.length, 189item: (i) => mimeTypes[i] || null, 190namedItem: (n) => mimeTypes.find((m) => m.type === n) || null, 191}; 192mimeTypes.forEach((m, i) => { plugin[i] = m; plugin[m.type] = m; }); 193return plugin; 194}); 195const pluginArray = { 196length: plugins.length, 197item: asNative((i) => plugins[i] || null, nativeFunc('item')), 198namedItem: asNative((n) => plugins.find((p) => p.name === n) || null, nativeFunc('namedItem')), 199refresh: asNative(() => undefined, nativeFunc('refresh')), 200[Symbol.iterator]: function* () { yield* plugins; }, 201}; 202plugins.forEach((p, i) => { pluginArray[i] = p; pluginArray[p.name] = p; }); 203define(Navigator.prototype, 'plugins', { get: asNative(() => pluginArray, nativeGetter('plugins')) }); 204const mimeTypeArray = { 205length: mimeTypes.length, 206item: asNative((i) => mimeTypes[i] || null, nativeFunc('item')), 207namedItem: asNative((n) => mimeTypes.find((m) => m.type === n) || null, nativeFunc('namedItem')), 208[Symbol.iterator]: function* () { yield* mimeTypes; }, 209}; 210define(Navigator.prototype, 'mimeTypes', { get: asNative(() => mimeTypeArray, nativeGetter('mimeTypes')) }); 211} catch (_) {} 212 213// window.chrome with the surface feature-detection scripts probe. 214try { 215if (!window.chrome) window.chrome = {}; 216window.chrome.app = window.chrome.app || { 217isInstalled: false, 218InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, 219RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }, 220getDetails: asNative(() => null, nativeFunc('getDetails')), 221getIsInstalled: asNative(() => false, nativeFunc('getIsInstalled')), 222installState: asNative((cb) => cb('not_installed'), nativeFunc('installState')), 223runningState: asNative(() => 'cannot_run', nativeFunc('runningState')), 224}; 225window.chrome.runtime = window.chrome.runtime || { 226OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' }, 227OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' }, 228PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, 229PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, 230PlatformOs: { ANDROID: 'android', CROS: 'cros', FUCHSIA: 'fuchsia', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' }, 231RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' }, 232connect: asNative(() => { throw new Error('Could not establish connection.'); }, nativeFunc('connect')), 233sendMessage: asNative(() => { throw new Error('Could not establish connection.'); }, nativeFunc('sendMessage')), 234id: undefined, 235}; 236window.chrome.csi = window.chrome.csi || asNative(function csi() { 237const now = Date.now(); 238return { onloadT: now, pageT: 400 + Math.random() * 800, startE: now - 500, tran: 15 }; 239}, nativeFunc('csi')); 240window.chrome.loadTimes = window.chrome.loadTimes || asNative(function loadTimes() { 241const now = Date.now() / 1000; 242return { 243commitLoadTime: now - 0.4, connectionInfo: 'h2', finishDocumentLoadTime: now, 244finishLoadTime: now + 0.1, firstPaintAfterLoadTime: 0, firstPaintTime: now - 0.2, 245navigationType: 'Other', npnNegotiatedProtocol: 'h2', requestTime: now - 0.6, 246startLoadTime: now - 0.6, wasAlternateProtocolAvailable: false, 247wasFetchedViaSpdy: true, wasNpnNegotiated: true, 248}; 249}, nativeFunc('loadTimes')); 250} catch (_) {} 251 252// Notification permission query must agree with Notification.permission. 253try { 254const originalQuery = window.Permissions && window.Permissions.prototype.query; 255if (originalQuery) { 256window.Permissions.prototype.query = asNative(function query(parameters) { 257if (parameters && parameters.name === 'notifications') { 258return Promise.resolve({ 259state: Notification.permission, 260onchange: null, 261addEventListener() {}, removeEventListener() {}, 262removeListener() {}, addListener() {}, 263dispatchEvent: () => false, 264}); 265} 266return originalQuery.call(this, parameters); 267}, nativeFunc('query')); 268} 269} catch (_) {} 270 271` 272 273// stealthGPUPatches replace the WebGL vendor and renderer strings. Headless 274// Linux and Windows render through SwiftShader, whose renderer string no real 275// user browser produces, so it is replaced with a common desktop adapter. 276// 277// This is deliberately not applied on macOS, where headless Chrome already 278// reports the machine's real Apple GPU and claiming Intel graphics would 279// contradict every other signal the page can read. 280const stealthGPUPatches = ` 281try { 282const patchGetParameter = (proto) => { 283if (!proto) return; 284const original = proto.getParameter; 285proto.getParameter = asNative(function getParameter(parameter) { 286if (parameter === 0x9245) return 'Google Inc. (Intel)'; 287if (parameter === 0x9246) { 288return 'ANGLE (Intel, Mesa Intel(R) UHD Graphics 630 (CFL GT2), OpenGL 4.6 (Core Profile) Mesa 23.2.1)'; 289} 290return original.call(this, parameter); 291}, nativeFunc('getParameter')); 292}; 293patchGetParameter(window.WebGLRenderingContext && window.WebGLRenderingContext.prototype); 294patchGetParameter(window.WebGL2RenderingContext && window.WebGL2RenderingContext.prototype); 295} catch (_) {} 296` 297 298// stealthScriptTail closes the injected script. 299const stealthScriptTail = ` 300// Headless windows have no browser chrome; fake sane outer dimensions. 301define(window, 'outerWidth', { get: asNative(() => window.innerWidth, nativeGetter('outerWidth')) }); 302define(window, 'outerHeight', { get: asNative(() => window.innerHeight + 85, nativeGetter('outerHeight')) }); 303})(); 304` 305 306// stealthSnapshot builds the script to inject for a host identity. 307func stealthSnapshot (identity browserIdentity )string { 308if identity .spoofGPU { 309return stealthScriptHead + stealthGPUPatches + stealthScriptTail 310 } 311return stealthScriptHead + stealthScriptTail 312}