okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
14.0 KiB312 linesraw
1package app
2
3import (
4	"crypto/sha256"
5	"encoding/binary"
6	"fmt"
7	"log/slog"
8	"os/exec"
9	"regexp"
10	"runtime"
11	"strings"
12	"sync"
13)
14
15// chromeVersion identifies the browser being driven by its major version, which
16// is all the user agent needs.
17type chromeVersion struct {
18	major string
19}
20
21// pinnedChromeVersion is the cloaking fallback when no browser binary can be
22// probed. It matches the pinned version used by the reference implementation.
23var pinnedChromeVersion = chromeVersion{major: "150"}
24
25// chromeVersionCandidates are tried in order when BROWSER_PATH is unset or
26// cannot be executed. The Linux entries are the ones that matter in
27// containers and on servers, where the service actually runs.
28var chromeVersionCandidates = []string{
29	"chromium",
30	"chromium-browser",
31	"google-chrome",
32	"google-chrome-stable",
33}
34
35var chromeVersionPattern = regexp.MustCompile(`\d+(?:\.\d+)*`)
36
37var (
38	chromeVersionMu    sync.Mutex
39	chromeVersionCache = map[string]chromeVersion{}
40)
41
42// probeChromeVersion reports the version of the configured browser, falling
43// back to a pinned version when nothing can be executed. The answer for a
44// given path cannot change while the process runs and probing spawns a
45// process, so results are cached per path.
46func probeChromeVersion(path string) chromeVersion {
47	chromeVersionMu.Lock()
48	defer chromeVersionMu.Unlock()
49	if cached, ok := chromeVersionCache[path]; ok {
50		return cached
51	}
52	detected := detectChromeVersion(path)
53	chromeVersionCache[path] = detected
54	return detected
55}
56
57func detectChromeVersion(path string) chromeVersion {
58	var candidates []string
59	if path != "" {
60		candidates = append(candidates, path)
61	}
62	candidates = append(candidates, chromeVersionCandidates...)
63	for _, candidate := range candidates {
64		out, err := exec.Command(candidate, "--version").Output()
65		if err != nil {
66			continue
67		}
68		if found := chromeVersionPattern.FindString(string(out)); found != "" {
69			major, _, _ := strings.Cut(found, ".")
70			return chromeVersion{major: major}
71		}
72	}
73	// Cloaking as a version the browser is not is worse than saying nothing,
74	// so make the degraded case visible.
75	slog.Warn("could not probe the browser version; falling back to pinned Chrome cloaking", "browser_path", path, "pinned_major", pinnedChromeVersion.major)
76	return pinnedChromeVersion
77}
78
79// browserIdentity is the platform shape this service presents to pages.
80//
81// It follows the host the browser actually runs on. Claiming a different
82// platform than the host is what bot detection looks for: an Apple GPU, Mac
83// fonts, and a macOS TLS fingerprint alongside a Linux user agent describe a
84// machine that cannot exist. The stealth Chromium builds handle this the same
85// way, running macOS as a native Mac browser rather than spoofing one.
86type browserIdentity struct {
87	userAgent string // template; %s is the Chrome major version
88	platform  string // navigator.platform
89	spoofGPU  bool   // replace the WebGL vendor and renderer strings
90}
91
92func hostIdentity() browserIdentity {
93	switch runtime.GOOS {
94	case "darwin":
95		// Headless Chrome on macOS already reports the machine's real Apple
96		// GPU, so the WebGL strings are left alone here.
97		return browserIdentity{
98			userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36",
99			platform:  "MacIntel",
100		}
101	case "windows":
102		return browserIdentity{
103			userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36",
104			platform:  "Win32",
105			spoofGPU:  true,
106		}
107	default:
108		return browserIdentity{
109			userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36",
110			platform:  "Linux x86_64",
111			spoofGPU:  true,
112		}
113	}
114}
115
116// chromeUserAgent is the host's Chrome user agent with the probed major
117// version, so the browser does not announce itself as HeadlessChrome.
118func chromeUserAgent(v chromeVersion) string {
119	return fmt.Sprintf(hostIdentity().userAgent, v.major)
120}
121
122// windowSize derives a viewport that is stable for a URL but varies between
123// them, so every request is not rendered at one machine-identifiable size.
124func windowSize(raw string) (width, height int) {
125	sum := sha256.Sum256([]byte(raw))
126	seed := binary.BigEndian.Uint64(sum[:8])
127	return 1200 + int(seed%160), 800 + int((seed>>8)%160)
128}
129
130// stealthScriptHead is the platform-neutral part of the script injected into
131// every new document before any page script runs. It removes the automation
132// fingerprints that bot detection reads directly, and makes every patch
133// stringify like a native implementation so Function.prototype.toString cannot
134// reveal it.
135//
136// Each section is isolated: this executes in the context of arbitrary,
137// untrusted pages, and one failing patch must never break the rest.
138const stealthScriptHead = `
139(() => {
140    // Make overridden functions stringify like native implementations;
141    // Function.prototype.toString is the standard bot-detection tell.
142    const nativeToString = Function.prototype.toString;
143    const spoofedSources = new WeakMap();
144    const asNative = (fn, source) => { spoofedSources.set(fn, source); return fn; };
145    const nativeGetter = (name) => 'function get ' + name + '() { [native code] }';
146    const nativeFunc = (name) => 'function ' + name + '() { [native code] }';
147    Function.prototype.toString = asNative(function toString() {
148        const spoofed = spoofedSources.get(this);
149        return spoofed !== undefined ? spoofed : nativeToString.call(this);
150    }, 'function toString() { [native code] }');
151    const define = (object, prop, descriptor) => {
152        try { Object.defineProperty(object, prop, { configurable: true, ...descriptor }); } catch (_) {}
153    };
154
155    // navigator.webdriver is true under automation; real Chrome exposes false.
156    define(Navigator.prototype, 'webdriver', {
157        get: asNative(() => false, nativeGetter('webdriver')),
158    });
159    define(Navigator.prototype, 'vendor', {
160        get: asNative(() => 'Google Inc.', nativeGetter('vendor')),
161    });
162    define(Navigator.prototype, 'languages', {
163        get: asNative(() => ['en-US', 'en'], nativeGetter('languages')),
164    });
165
166    // Plugin list: empty under automation, five PDF-related entries in real
167    // Chrome. Mid-fidelity PluginArray with named access and iteration.
168    try {
169        const mimeTypesData = [
170            { type: 'application/pdf', suffixes: 'pdf', description: '' },
171            { type: 'text/pdf', suffixes: 'pdf', description: '' },
172        ];
173        const pluginsData = [
174            ['PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'],
175            ['Chrome PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'],
176            ['Chromium PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'],
177            ['Microsoft Edge PDF Viewer', 'Portable Document Format', 'internal-pdf-viewer'],
178            ['WebKit built-in PDF', 'Portable Document Format', 'internal-pdf-viewer'],
179        ];
180        const mimeTypes = mimeTypesData.map((m) => {
181            const mt = { ...m, enabledPlugin: null };
182            mt.enabledPlugin = mt;
183            return mt;
184        });
185        const plugins = pluginsData.map(([name, description, filename]) => {
186            const plugin = {
187                name, description, filename,
188                length: mimeTypes.length,
189                item: (i) => mimeTypes[i] || null,
190                namedItem: (n) => mimeTypes.find((m) => m.type === n) || null,
191            };
192            mimeTypes.forEach((m, i) => { plugin[i] = m; plugin[m.type] = m; });
193            return plugin;
194        });
195        const pluginArray = {
196            length: plugins.length,
197            item: asNative((i) => plugins[i] || null, nativeFunc('item')),
198            namedItem: asNative((n) => plugins.find((p) => p.name === n) || null, nativeFunc('namedItem')),
199            refresh: asNative(() => undefined, nativeFunc('refresh')),
200            [Symbol.iterator]: function* () { yield* plugins; },
201        };
202        plugins.forEach((p, i) => { pluginArray[i] = p; pluginArray[p.name] = p; });
203        define(Navigator.prototype, 'plugins', { get: asNative(() => pluginArray, nativeGetter('plugins')) });
204        const mimeTypeArray = {
205            length: mimeTypes.length,
206            item: asNative((i) => mimeTypes[i] || null, nativeFunc('item')),
207            namedItem: asNative((n) => mimeTypes.find((m) => m.type === n) || null, nativeFunc('namedItem')),
208            [Symbol.iterator]: function* () { yield* mimeTypes; },
209        };
210        define(Navigator.prototype, 'mimeTypes', { get: asNative(() => mimeTypeArray, nativeGetter('mimeTypes')) });
211    } catch (_) {}
212
213    // window.chrome with the surface feature-detection scripts probe.
214    try {
215        if (!window.chrome) window.chrome = {};
216        window.chrome.app = window.chrome.app || {
217            isInstalled: false,
218            InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
219            RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' },
220            getDetails: asNative(() => null, nativeFunc('getDetails')),
221            getIsInstalled: asNative(() => false, nativeFunc('getIsInstalled')),
222            installState: asNative((cb) => cb('not_installed'), nativeFunc('installState')),
223            runningState: asNative(() => 'cannot_run', nativeFunc('runningState')),
224        };
225        window.chrome.runtime = window.chrome.runtime || {
226            OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
227            OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
228            PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
229            PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
230            PlatformOs: { ANDROID: 'android', CROS: 'cros', FUCHSIA: 'fuchsia', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
231            RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' },
232            connect: asNative(() => { throw new Error('Could not establish connection.'); }, nativeFunc('connect')),
233            sendMessage: asNative(() => { throw new Error('Could not establish connection.'); }, nativeFunc('sendMessage')),
234            id: undefined,
235        };
236        window.chrome.csi = window.chrome.csi || asNative(function csi() {
237            const now = Date.now();
238            return { onloadT: now, pageT: 400 + Math.random() * 800, startE: now - 500, tran: 15 };
239        }, nativeFunc('csi'));
240        window.chrome.loadTimes = window.chrome.loadTimes || asNative(function loadTimes() {
241            const now = Date.now() / 1000;
242            return {
243                commitLoadTime: now - 0.4, connectionInfo: 'h2', finishDocumentLoadTime: now,
244                finishLoadTime: now + 0.1, firstPaintAfterLoadTime: 0, firstPaintTime: now - 0.2,
245                navigationType: 'Other', npnNegotiatedProtocol: 'h2', requestTime: now - 0.6,
246                startLoadTime: now - 0.6, wasAlternateProtocolAvailable: false,
247                wasFetchedViaSpdy: true, wasNpnNegotiated: true,
248            };
249        }, nativeFunc('loadTimes'));
250    } catch (_) {}
251
252    // Notification permission query must agree with Notification.permission.
253    try {
254        const originalQuery = window.Permissions && window.Permissions.prototype.query;
255        if (originalQuery) {
256            window.Permissions.prototype.query = asNative(function query(parameters) {
257                if (parameters && parameters.name === 'notifications') {
258                    return Promise.resolve({
259                        state: Notification.permission,
260                        onchange: null,
261                        addEventListener() {}, removeEventListener() {},
262                        removeListener() {}, addListener() {},
263                        dispatchEvent: () => false,
264                    });
265                }
266                return originalQuery.call(this, parameters);
267            }, nativeFunc('query'));
268        }
269    } catch (_) {}
270
271`
272
273// stealthGPUPatches replace the WebGL vendor and renderer strings. Headless
274// Linux and Windows render through SwiftShader, whose renderer string no real
275// user browser produces, so it is replaced with a common desktop adapter.
276//
277// This is deliberately not applied on macOS, where headless Chrome already
278// reports the machine's real Apple GPU and claiming Intel graphics would
279// contradict every other signal the page can read.
280const stealthGPUPatches = `
281    try {
282        const patchGetParameter = (proto) => {
283            if (!proto) return;
284            const original = proto.getParameter;
285            proto.getParameter = asNative(function getParameter(parameter) {
286                if (parameter === 0x9245) return 'Google Inc. (Intel)';
287                if (parameter === 0x9246) {
288                    return 'ANGLE (Intel, Mesa Intel(R) UHD Graphics 630 (CFL GT2), OpenGL 4.6 (Core Profile) Mesa 23.2.1)';
289                }
290                return original.call(this, parameter);
291            }, nativeFunc('getParameter'));
292        };
293        patchGetParameter(window.WebGLRenderingContext && window.WebGLRenderingContext.prototype);
294        patchGetParameter(window.WebGL2RenderingContext && window.WebGL2RenderingContext.prototype);
295    } catch (_) {}
296`
297
298// stealthScriptTail closes the injected script.
299const stealthScriptTail = `
300    // Headless windows have no browser chrome; fake sane outer dimensions.
301    define(window, 'outerWidth', { get: asNative(() => window.innerWidth, nativeGetter('outerWidth')) });
302    define(window, 'outerHeight', { get: asNative(() => window.innerHeight + 85, nativeGetter('outerHeight')) });
303})();
304`
305
306// stealthSnapshot builds the script to inject for a host identity.
307func stealthSnapshot(identity browserIdentity) string {
308	if identity.spoofGPU {
309		return stealthScriptHead + stealthGPUPatches + stealthScriptTail
310	}
311	return stealthScriptHead + stealthScriptTail
312}