okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
10.3 KiB312 linesraw
1package app
2
3import (
4	"context"
5	"errors"
6	"fmt"
7	"io"
8	"net"
9	"net/http"
10	"net/http/httptest"
11	"net/url"
12	"os"
13	"path/filepath"
14	"strings"
15	"sync/atomic"
16	"testing"
17	"time"
18)
19
20// The render path reports a thin page with a distinct tag, because that tag is
21// the only thing that triggers the plain-fetch retry. A navigation failure or an
22// access challenge must not be retried: those are real diagnoses.
23func TestOnlyIncompleteRenderTriggersStaticRetry(t *testing.T) {
24	incomplete := verifyContent("too short", "https://example.test/")
25	var api *APIError
26	if !isIncompleteRender(incomplete) {
27		t.Fatalf("thin content was not reported as incomplete: %v", incomplete)
28	}
29	if !errors.As(incomplete, &api) || api.Status != 502 || api.Tag != incompleteTag {
30		t.Fatalf("incomplete render has the wrong contract: %+v", api)
31	}
32
33	rich := strings.Repeat("real page content ", 20)
34	if err := verifyContent(rich, "https://example.test/"); err != nil {
35		t.Fatalf("content above the floor was rejected: %v", err)
36	}
37
38	for _, terminal := range []error{
39		failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve"),
40		failure(502, "CRAWL_HTTP_ERROR", "browser page returned http 502"),
41		failure(504, "REQUEST_TIMEOUT", "navigation did not complete"),
42		fmt.Errorf("browser navigation failed: connection reset"),
43	} {
44		if isIncompleteRender(terminal) {
45			t.Errorf("terminal error would be retried as a thin render: %v", terminal)
46		}
47	}
48}
49
50// The floor counts non-whitespace characters, so a page padded with layout
51// whitespace cannot pass as content.
52func TestVerifyContentCountsNonWhitespaceOnly(t *testing.T) {
53	if err := verifyContent(strings.Repeat("\n \t", 500), "https://example.test/"); !isIncompleteRender(err) {
54		t.Fatalf("whitespace-only page passed the floor: %v", err)
55	}
56	// Exactly at the floor is content; one below is not.
57	if err := verifyContent(strings.Repeat("x", minMeaningfulChars), "https://example.test/"); err != nil {
58		t.Fatalf("content at the floor was rejected: %v", err)
59	}
60	if err := verifyContent(strings.Repeat("x", minMeaningfulChars-1), "https://example.test/"); !isIncompleteRender(err) {
61		t.Fatalf("content one below the floor was accepted: %v", err)
62	}
63}
64
65// A .pdf URL never goes to the browser: a render cannot add text to a document
66// that has no DOM.
67func TestPDFURLsSkipTheBrowser(t *testing.T) {
68	pdf := fixturePDF(t, "report.pdf")
69	hits := atomic.Int32{}
70	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
71		hits.Add(1)
72		w.Header().Set("Content-Type", "application/pdf")
73		w.Write(pdf)
74	}))
75	defer srv.Close()
76
77	n := newNetwork(true)
78	defer n.transport.CloseIdleConnections()
79	// A browser path that would fail loudly if it were used.
80	f := &Fetcher{net: n, browser: &Browser{path: "/nonexistent/chrome"}, maxBytes: 1 << 20}
81	d, err := f.fetch(context.Background(), srv.URL+"/report.pdf", nil)
82	if err != nil {
83		t.Fatalf("pdf fetch failed: %v", err)
84	}
85	if hits.Load() == 0 {
86		t.Fatal("pdf was not fetched over http")
87	}
88	if !strings.Contains(d.Markdown, "evidence") {
89		t.Fatalf("unexpected pdf text: %.80q", d.Markdown)
90	}
91}
92
93// The fallback ordering still holds when no browser is configured: every fetch
94// takes the plain path.
95func TestNoBrowserUsesStaticOnly(t *testing.T) {
96	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
97		fmt.Fprint(w, `<html><body><main>`+strings.Repeat("static page content ", 20)+`</main></body></html>`)
98	}))
99	defer srv.Close()
100
101	n := newNetwork(true)
102	defer n.transport.CloseIdleConnections()
103	f := &Fetcher{net: n, maxBytes: 1 << 20}
104	if f.canRender() {
105		t.Fatal("fetcher without a browser claims it can render")
106	}
107	d, err := f.fetch(context.Background(), srv.URL, nil)
108	if err != nil {
109		t.Fatal(err)
110	}
111	if !strings.Contains(d.Markdown, "static page content") {
112		t.Fatalf("unexpected document: %.80q", d.Markdown)
113	}
114}
115
116// A plain fetch presents the same identity the browser does.
117func TestStaticFetchUserAgentMatchesBrowser(t *testing.T) {
118	var got string
119	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
120		got = r.Header.Get("User-Agent")
121		fmt.Fprint(w, `<html><body><main>`+strings.Repeat("page text here ", 20)+`</main></body></html>`)
122	}))
123	defer srv.Close()
124
125	n := newNetwork(true)
126	defer n.transport.CloseIdleConnections()
127	f := &Fetcher{net: n, maxBytes: 1 << 20}
128	if _, err := f.fetch(context.Background(), srv.URL, nil); err != nil {
129		t.Fatal(err)
130	}
131	if !strings.Contains(got, "Chrome/") || strings.Contains(got, "Headless") {
132		t.Fatalf("plain fetch announced an unusual identity: %q", got)
133	}
134}
135
136// An oversized declared length is rejected before the body is read.
137func TestOversizedContentLengthRejectedEarly(t *testing.T) {
138	body := strings.Repeat("x", 8192)
139	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
140		w.Header().Set("Content-Length", fmt.Sprint(len(body)))
141		w.Write([]byte(body))
142	}))
143	defer srv.Close()
144
145	n := newNetwork(true)
146	defer n.transport.CloseIdleConnections()
147	f := &Fetcher{net: n, maxBytes: 1024}
148	if _, err := f.fetchStatic(context.Background(), srv.URL, nil); err == nil {
149		t.Fatal("oversized response was accepted")
150	}
151}
152
153// The proxy pool rotates, and an empty pool selects a direct connection.
154func TestProxyPoolRotation(t *testing.T) {
155	n := newNetwork(true)
156	if n.pickProxy() != nil {
157		t.Fatal("empty pool did not select a direct connection")
158	}
159
160	a, _ := url.Parse("http://proxy-a.test:8080")
161	b, _ := url.Parse("http://proxy-b.test:8080")
162	pooled := &Network{proxies: []*url.URL{a, b}}
163	seen := []string{}
164	for i := 0; i < 4; i++ {
165		seen = append(seen, pooled.pickProxy().Host)
166	}
167	want := []string{"proxy-a.test:8080", "proxy-b.test:8080", "proxy-a.test:8080", "proxy-b.test:8080"}
168	for i := range want {
169		if seen[i] != want[i] {
170			t.Fatalf("rotation order = %v, want %v", seen, want)
171		}
172	}
173}
174
175// A configured pool must actually carry the traffic: this runs a real CONNECT
176// proxy and asserts both that the origin was reached and that it saw the proxy.
177func TestProxyPoolRoutesStaticFetchThroughProxy(t *testing.T) {
178	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
179		fmt.Fprint(w, `<html><body><main>`+strings.Repeat("proxied page content ", 20)+`</main></body></html>`)
180	}))
181	defer origin.Close()
182
183	tunnels := atomic.Int32{}
184	proxy := startConnectProxy(t, &tunnels)
185	defer proxy.Close()
186
187	proxyURL, _ := url.Parse(proxy.URL)
188	n := newNetworkWithPool(true, []*url.URL{proxyURL})
189	defer n.closeIdle()
190
191	f := &Fetcher{net: n, maxBytes: 1 << 20}
192	d, err := f.fetchStatic(context.Background(), origin.URL, n.pickProxy())
193	if err != nil {
194		t.Fatalf("proxied fetch failed: %v", err)
195	}
196	if !strings.Contains(d.Markdown, "proxied page content") {
197		t.Fatalf("unexpected document: %.80q", d.Markdown)
198	}
199	if tunnels.Load() == 0 {
200		t.Fatal("the request did not go through the configured proxy")
201	}
202}
203
204// A proxy that refuses the tunnel surfaces as a fetch failure, not a hang.
205func TestProxyRefusalIsReported(t *testing.T) {
206	refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
207		w.WriteHeader(http.StatusForbidden)
208	}))
209	defer refusing.Close()
210
211	proxyURL, _ := url.Parse(refusing.URL)
212	n := newNetworkWithPool(true, []*url.URL{proxyURL})
213	defer n.closeIdle()
214
215	f := &Fetcher{net: n, maxBytes: 1 << 20}
216	if _, err := f.fetchStatic(context.Background(), "https://example.test/", n.pickProxy()); err == nil {
217		t.Fatal("a refusing proxy was treated as a successful fetch")
218	}
219}
220
221// TestLiveFetchRendersBeforeFetching proves the render path is primary: the
222// page's text exists only after its script runs, and the plain fetch cannot see
223// it.
224func TestLiveFetchRendersBeforeFetching(t *testing.T) {
225	if os.Getenv("LIVE_BROWSER") != "1" {
226		t.Skip("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome")
227	}
228	if os.Getenv("BROWSER_PATH") == "" {
229		t.Fatal("BROWSER_PATH is required")
230	}
231	page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
232		w.Header().Set("Content-Type", "text/html")
233		fmt.Fprint(w, `<html><body><main>loading</main><script>
234			const parts = [];
235			for (let i = 0; i < 30; i++) parts.push('rendered sentence number ' + i + '.');
236			document.querySelector('main').textContent = parts.join(' ');
237		</script></body></html>`)
238	}))
239	defer page.Close()
240
241	path, err := filepath.Abs(os.Getenv("BROWSER_PATH"))
242	if err != nil {
243		t.Fatal(err)
244	}
245	n := newNetwork(true)
246	defer n.transport.CloseIdleConnections()
247	b := &Browser{path: path, network: n, slots: make(chan struct{}, 1), stealth: true, headless: true, waitMS: 1200 * time.Millisecond}
248	f := &Fetcher{net: n, browser: b, maxBytes: 5 << 20}
249
250	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
251	defer cancel()
252	d, err := f.fetch(ctx, page.URL, nil)
253	if err != nil {
254		t.Fatal(err)
255	}
256	if !strings.Contains(d.Markdown, "rendered sentence number 29") {
257		t.Fatalf("script-generated content was not captured: %.200q", d.Markdown)
258	}
259}
260
261// startConnectProxy runs a minimal forward proxy: CONNECT tunnels are
262// established directly, and absolute-URI requests are forwarded. That mirrors
263// what a real proxy offers and covers both transport paths.
264func startConnectProxy(t *testing.T, tunnels *atomic.Int32) *httptest.Server {
265	t.Helper()
266	forwarder := &http.Transport{}
267	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
268		if r.Method != http.MethodConnect {
269			tunnels.Add(1)
270			out := r.Clone(r.Context())
271			out.RequestURI = ""
272			res, err := forwarder.RoundTrip(out)
273			if err != nil {
274				w.WriteHeader(http.StatusBadGateway)
275				return
276			}
277			defer res.Body.Close()
278			for key, values := range res.Header {
279				for _, v := range values {
280					w.Header().Add(key, v)
281				}
282			}
283			w.WriteHeader(res.StatusCode)
284			io.Copy(w, res.Body)
285			return
286		}
287		upstream, err := net.DialTimeout("tcp", r.Host, 10*time.Second)
288		if err != nil {
289			w.WriteHeader(http.StatusBadGateway)
290			return
291		}
292		defer upstream.Close()
293		hijacker, ok := w.(http.Hijacker)
294		if !ok {
295			w.WriteHeader(http.StatusInternalServerError)
296			return
297		}
298		client, buffered, err := hijacker.Hijack()
299		if err != nil {
300			return
301		}
302		defer client.Close()
303		tunnels.Add(1)
304		fmt.Fprint(buffered, "HTTP/1.1 200 Connection Established\r\n\r\n")
305		buffered.Flush()
306
307		done := make(chan struct{}, 2)
308		go func() { io.Copy(upstream, buffered); done <- struct{}{} }()
309		go func() { io.Copy(client, upstream); done <- struct{}{} }()
310		<-done
311	}))
312}