okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
8.2 KiB222 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"fmt"
7	"os"
8	"path/filepath"
9	"runtime"
10	"strings"
11	"testing"
12	"time"
13
14	"github.com/chromedp/chromedp"
15)
16
17func TestWindowSizeIsStablePerURLAndWithinBounds(t *testing.T) {
18	for _, raw := range []string{"https://a.test/", "https://b.test/x", "https://c.test/y?z=1"} {
19		w1, h1 := windowSize(raw)
20		w2, h2 := windowSize(raw)
21		if w1 != w2 || h1 != h2 {
22			t.Fatalf("windowSize(%q) is not deterministic: %dx%d then %dx%d", raw, w1, h1, w2, h2)
23		}
24		if w1 < 1200 || w1 > 1359 || h1 < 800 || h1 > 959 {
25			t.Fatalf("windowSize(%q) = %dx%d outside the intended range", raw, w1, h1)
26		}
27	}
28}
29
30// The presented identity must belong to the host the browser actually runs on.
31// Claiming another platform is what bot detection looks for, so the user agent
32// and navigator.platform must agree with each other and with this host.
33func TestHostIdentityIsCoherent(t *testing.T) {
34	v := chromeVersion{major: "153"}
35	identity := hostIdentity()
36	ua := chromeUserAgent(v)
37
38	if !strings.Contains(ua, "Chrome/153.0.0.0 Safari/537.36") {
39		t.Fatalf("user agent does not carry the probed major version: %q", ua)
40	}
41	if strings.Contains(ua, "Headless") {
42		t.Fatalf("user agent announces headless mode: %q", ua)
43	}
44	// The platform claim and the user agent must describe one machine.
45	wantPlatform := hostUserAgentFragment()
46	if wantPlatform == "" {
47		t.Skipf("no identity expectations for %s", runtime.GOOS)
48	}
49	if !strings.Contains(ua, wantPlatform) {
50		t.Fatalf("user agent %q does not describe this %s host (want %q)", ua, runtime.GOOS, wantPlatform)
51	}
52	// macOS reports a real Apple GPU in headless mode, so replacing the WebGL
53	// strings there would contradict the rest of the fingerprint.
54	if identity.platform == "" {
55		t.Fatal("identity has no navigator.platform")
56	}
57	if runtime.GOOS == "darwin" && identity.spoofGPU {
58		t.Fatal("macOS identity spoofs the GPU string over a real Apple GPU")
59	}
60	if runtime.GOOS != "darwin" && !identity.spoofGPU {
61		t.Fatalf("%s identity leaves the SwiftShader renderer string in place", runtime.GOOS)
62	}
63}
64
65// The injected script must not carry the WebGL replacement when the host
66// reports a real GPU, and must carry it when the host renders through
67// SwiftShader.
68func TestStealthSnapshotOmitsGPUPatchesForRealGPUs(t *testing.T) {
69	real := stealthSnapshot(browserIdentity{platform: "MacIntel"})
70	if strings.Contains(real, "0x9246") {
71		t.Fatal("script patches WebGL for an identity that reports a real GPU")
72	}
73	if !strings.Contains(real, "webdriver") || !strings.Contains(real, "outerHeight") {
74		t.Fatal("script dropped the platform-neutral evasions")
75	}
76	swift := stealthSnapshot(browserIdentity{platform: "Linux x86_64", spoofGPU: true})
77	if !strings.Contains(swift, "0x9246") {
78		t.Fatal("script left the SwiftShader renderer string in place")
79	}
80	if !strings.Contains(swift, "outerHeight") {
81		t.Fatal("script dropped the trailing evasions")
82	}
83}
84
85// The client-hints metadata override must not come back: forcing Sec-CH-UA-*
86// headers that a real browser only sends when an origin opts in is what reddit
87// treated as automation.
88func TestNoClientHintsOverrideInStealthActions(t *testing.T) {
89	if strings.Contains(stealthTestSource(t), "WithUserAgentMetadata") {
90		t.Fatal("stealth path overrides Sec-CH-UA client hints")
91	}
92}
93
94// hostUserAgentFragment is the part of a Chrome user agent that identifies one
95// operating system. Tests use it to assert the identity matches the host.
96func hostUserAgentFragment() string {
97	return map[string]string{
98		"darwin":  "Macintosh; Intel Mac OS X",
99		"windows": "Windows NT 10.0; Win64; x64",
100		"linux":   "X11; Linux x86_64",
101	}[runtime.GOOS]
102}
103
104func stealthTestSource(t *testing.T) string {
105	t.Helper()
106	source, err := os.ReadFile("browser.go")
107	if err != nil {
108		t.Fatal(err)
109	}
110	return string(source)
111}
112
113// TestLiveStealthFingerprint asserts real Chrome presents a plausible desktop
114// browser to page scripts instead of announcing automation. It drives Chrome,
115// so it is opt-in like the other browser tests.
116func TestLiveStealthFingerprint(t *testing.T) {
117	if os.Getenv("LIVE_BROWSER") != "1" {
118		t.Skip("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome")
119	}
120	if os.Getenv("BROWSER_PATH") == "" {
121		t.Fatal("BROWSER_PATH is required")
122	}
123	for _, stealth := range []bool{true, false} {
124		name := "stealth off"
125		if stealth {
126			name = "stealth on"
127		}
128		t.Run(name, func(t *testing.T) {
129			observed := observeFingerprint(t, stealth)
130			if !stealth {
131				t.Logf("baseline: webdriver=%v ua=%q", observed["webdriver"], observed["userAgent"])
132				return
133			}
134			if observed["webdriver"] != false {
135				t.Errorf("navigator.webdriver = %v, want false", observed["webdriver"])
136			}
137			ua, _ := observed["userAgent"].(string)
138			if !strings.Contains(ua, "Chrome/") || strings.Contains(ua, "Headless") {
139				t.Errorf("user agent is not desktop Chrome: %q", ua)
140			}
141			// The user agent must describe the host the browser runs on.
142			if fragment := hostUserAgentFragment(); fragment != "" && !strings.Contains(ua, fragment) {
143				t.Errorf("user agent %q does not describe this %s host (want %q)", ua, runtime.GOOS, fragment)
144			}
145			if observed["vendor"] != "Google Inc." {
146				t.Errorf("navigator.vendor = %v", observed["vendor"])
147			}
148			if !strings.Contains(fmt.Sprint(observed["webdriverGetter"]), "[native code]") {
149				t.Errorf("webdriver getter does not stringify as native: %v", observed["webdriverGetter"])
150			}
151			if observed["chromeRuntime"] != true {
152				t.Error("window.chrome.runtime is missing")
153			}
154			rendered, _ := observed["webglRenderer"].(string)
155			if strings.Contains(rendered, "SwiftShader") {
156				t.Errorf("webgl renderer is the software rasterizer: %q", rendered)
157			}
158			if runtime.GOOS == "darwin" && !strings.Contains(rendered, "Apple") {
159				t.Errorf("macOS host does not report its real Apple GPU: %q", rendered)
160			}
161			if runtime.GOOS != "darwin" && !strings.Contains(rendered, "Intel") {
162				t.Errorf("SwiftShader host was not given a desktop adapter string: %q", rendered)
163			}
164			if langs := fmt.Sprint(observed["languages"]); !strings.Contains(langs, "en-US") {
165				t.Errorf("navigator.languages = %v", observed["languages"])
166			}
167			if plugins, ok := observed["plugins"].(float64); !ok || plugins != 5 {
168				t.Errorf("navigator.plugins.length = %v, want 5", observed["plugins"])
169			}
170		})
171	}
172}
173
174// observeFingerprint renders a blank page through the production browser setup
175// and reads back the surface a bot-detection script inspects.
176func observeFingerprint(t *testing.T, stealth bool) map[string]any {
177	t.Helper()
178	path, err := filepath.Abs(os.Getenv("BROWSER_PATH"))
179	if err != nil {
180		t.Fatal(err)
181	}
182	b := &Browser{path: path, network: newNetwork(true), slots: make(chan struct{}, 1), stealth: stealth, headless: true, waitMS: 1200 * time.Millisecond}
183	defer b.network.transport.CloseIdleConnections()
184	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
185	defer cancel()
186
187	alloc, cancelAlloc := chromedp.NewExecAllocator(ctx, b.execOptions("https://fingerprint.test/", renderOptions{stealth: stealth}, "127.0.0.1:1")...)
188	defer cancelAlloc()
189	tab, cancelTab := chromedp.NewContext(alloc)
190	defer cancelTab()
191
192	const probe = `(() => {
193  const d = Object.getOwnPropertyDescriptor(Navigator.prototype, 'webdriver');
194  const gl = document.createElement('canvas').getContext('webgl');
195  const dbg = gl && gl.getExtension('WEBGL_debug_renderer_info');
196  return JSON.stringify({
197    webdriver: navigator.webdriver,
198    vendor: navigator.vendor,
199    languages: navigator.languages,
200    plugins: navigator.plugins.length,
201    userAgent: navigator.userAgent,
202    webdriverGetter: d && d.get ? Function.prototype.toString.call(d.get) : null,
203    chromeRuntime: !!(window.chrome && window.chrome.runtime),
204    webglRenderer: gl ? gl.getParameter(dbg ? dbg.UNMASKED_RENDERER_WEBGL : 0x9246) : null,
205  });
206})()`
207
208	var raw string
209	actions := b.stealthActions(renderOptions{stealth: stealth})
210	actions = append(actions,
211		chromedp.Navigate("data:text/html,<html><body><h1>probe</h1></body></html>"),
212		chromedp.Evaluate(probe, &raw),
213	)
214	if err := chromedp.Run(tab, actions...); err != nil {
215		t.Fatalf("fingerprint probe failed: %v", err)
216	}
217	var out map[string]any
218	if err := json.Unmarshal([]byte(raw), &out); err != nil {
219		t.Fatalf("decode fingerprint %q: %v", raw, err)
220	}
221	return out
222}