package app import ( "context" "errors" "fmt" "io" "net" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" ) // The render path reports a thin page with a distinct tag, because that tag is // the only thing that triggers the plain-fetch retry. A navigation failure or an // access challenge must not be retried: those are real diagnoses. func TestOnlyIncompleteRenderTriggersStaticRetry(t *testing.T) { incomplete := verifyContent("too short", "https://example.test/") var api *APIError if !isIncompleteRender(incomplete) { t.Fatalf("thin content was not reported as incomplete: %v", incomplete) } if !errors.As(incomplete, &api) || api.Status != 502 || api.Tag != incompleteTag { t.Fatalf("incomplete render has the wrong contract: %+v", api) } rich := strings.Repeat("real page content ", 20) if err := verifyContent(rich, "https://example.test/"); err != nil { t.Fatalf("content above the floor was rejected: %v", err) } for _, terminal := range []error{ failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve"), failure(502, "CRAWL_HTTP_ERROR", "browser page returned http 502"), failure(504, "REQUEST_TIMEOUT", "navigation did not complete"), fmt.Errorf("browser navigation failed: connection reset"), } { if isIncompleteRender(terminal) { t.Errorf("terminal error would be retried as a thin render: %v", terminal) } } } // The floor counts non-whitespace characters, so a page padded with layout // whitespace cannot pass as content. func TestVerifyContentCountsNonWhitespaceOnly(t *testing.T) { if err := verifyContent(strings.Repeat("\n \t", 500), "https://example.test/"); !isIncompleteRender(err) { t.Fatalf("whitespace-only page passed the floor: %v", err) } // Exactly at the floor is content; one below is not. if err := verifyContent(strings.Repeat("x", minMeaningfulChars), "https://example.test/"); err != nil { t.Fatalf("content at the floor was rejected: %v", err) } if err := verifyContent(strings.Repeat("x", minMeaningfulChars-1), "https://example.test/"); !isIncompleteRender(err) { t.Fatalf("content one below the floor was accepted: %v", err) } } // A .pdf URL never goes to the browser: a render cannot add text to a document // that has no DOM. func TestPDFURLsSkipTheBrowser(t *testing.T) { pdf := fixturePDF(t, "report.pdf") hits := atomic.Int32{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) w.Header().Set("Content-Type", "application/pdf") w.Write(pdf) })) defer srv.Close() n := newNetwork(true) defer n.transport.CloseIdleConnections() // A browser path that would fail loudly if it were used. f := &Fetcher{net: n, browser: &Browser{path: "/nonexistent/chrome"}, maxBytes: 1 << 20} d, err := f.fetch(context.Background(), srv.URL+"/report.pdf", nil) if err != nil { t.Fatalf("pdf fetch failed: %v", err) } if hits.Load() == 0 { t.Fatal("pdf was not fetched over http") } if !strings.Contains(d.Markdown, "evidence") { t.Fatalf("unexpected pdf text: %.80q", d.Markdown) } } // The fallback ordering still holds when no browser is configured: every fetch // takes the plain path. func TestNoBrowserUsesStaticOnly(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `
`+strings.Repeat("static page content ", 20)+`
`) })) defer srv.Close() n := newNetwork(true) defer n.transport.CloseIdleConnections() f := &Fetcher{net: n, maxBytes: 1 << 20} if f.canRender() { t.Fatal("fetcher without a browser claims it can render") } d, err := f.fetch(context.Background(), srv.URL, nil) if err != nil { t.Fatal(err) } if !strings.Contains(d.Markdown, "static page content") { t.Fatalf("unexpected document: %.80q", d.Markdown) } } // A plain fetch presents the same identity the browser does. func TestStaticFetchUserAgentMatchesBrowser(t *testing.T) { var got string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = r.Header.Get("User-Agent") fmt.Fprint(w, `
`+strings.Repeat("page text here ", 20)+`
`) })) defer srv.Close() n := newNetwork(true) defer n.transport.CloseIdleConnections() f := &Fetcher{net: n, maxBytes: 1 << 20} if _, err := f.fetch(context.Background(), srv.URL, nil); err != nil { t.Fatal(err) } if !strings.Contains(got, "Chrome/") || strings.Contains(got, "Headless") { t.Fatalf("plain fetch announced an unusual identity: %q", got) } } // An oversized declared length is rejected before the body is read. func TestOversizedContentLengthRejectedEarly(t *testing.T) { body := strings.Repeat("x", 8192) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", fmt.Sprint(len(body))) w.Write([]byte(body)) })) defer srv.Close() n := newNetwork(true) defer n.transport.CloseIdleConnections() f := &Fetcher{net: n, maxBytes: 1024} if _, err := f.fetchStatic(context.Background(), srv.URL, nil); err == nil { t.Fatal("oversized response was accepted") } } // The proxy pool rotates, and an empty pool selects a direct connection. func TestProxyPoolRotation(t *testing.T) { n := newNetwork(true) if n.pickProxy() != nil { t.Fatal("empty pool did not select a direct connection") } a, _ := url.Parse("http://proxy-a.test:8080") b, _ := url.Parse("http://proxy-b.test:8080") pooled := &Network{proxies: []*url.URL{a, b}} seen := []string{} for i := 0; i < 4; i++ { seen = append(seen, pooled.pickProxy().Host) } want := []string{"proxy-a.test:8080", "proxy-b.test:8080", "proxy-a.test:8080", "proxy-b.test:8080"} for i := range want { if seen[i] != want[i] { t.Fatalf("rotation order = %v, want %v", seen, want) } } } // A configured pool must actually carry the traffic: this runs a real CONNECT // proxy and asserts both that the origin was reached and that it saw the proxy. func TestProxyPoolRoutesStaticFetchThroughProxy(t *testing.T) { origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `
`+strings.Repeat("proxied page content ", 20)+`
`) })) defer origin.Close() tunnels := atomic.Int32{} proxy := startConnectProxy(t, &tunnels) defer proxy.Close() proxyURL, _ := url.Parse(proxy.URL) n := newNetworkWithPool(true, []*url.URL{proxyURL}) defer n.closeIdle() f := &Fetcher{net: n, maxBytes: 1 << 20} d, err := f.fetchStatic(context.Background(), origin.URL, n.pickProxy()) if err != nil { t.Fatalf("proxied fetch failed: %v", err) } if !strings.Contains(d.Markdown, "proxied page content") { t.Fatalf("unexpected document: %.80q", d.Markdown) } if tunnels.Load() == 0 { t.Fatal("the request did not go through the configured proxy") } } // A proxy that refuses the tunnel surfaces as a fetch failure, not a hang. func TestProxyRefusalIsReported(t *testing.T) { refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusForbidden) })) defer refusing.Close() proxyURL, _ := url.Parse(refusing.URL) n := newNetworkWithPool(true, []*url.URL{proxyURL}) defer n.closeIdle() f := &Fetcher{net: n, maxBytes: 1 << 20} if _, err := f.fetchStatic(context.Background(), "https://example.test/", n.pickProxy()); err == nil { t.Fatal("a refusing proxy was treated as a successful fetch") } } // TestLiveFetchRendersBeforeFetching proves the render path is primary: the // page's text exists only after its script runs, and the plain fetch cannot see // it. func TestLiveFetchRendersBeforeFetching(t *testing.T) { if os.Getenv("LIVE_BROWSER") != "1" { t.Skip("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome") } if os.Getenv("BROWSER_PATH") == "" { t.Fatal("BROWSER_PATH is required") } page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") fmt.Fprint(w, `
loading
`) })) defer page.Close() path, err := filepath.Abs(os.Getenv("BROWSER_PATH")) if err != nil { t.Fatal(err) } n := newNetwork(true) defer n.transport.CloseIdleConnections() b := &Browser{path: path, network: n, slots: make(chan struct{}, 1), stealth: true, headless: true, waitMS: 1200 * time.Millisecond} f := &Fetcher{net: n, browser: b, maxBytes: 5 << 20} ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() d, err := f.fetch(ctx, page.URL, nil) if err != nil { t.Fatal(err) } if !strings.Contains(d.Markdown, "rendered sentence number 29") { t.Fatalf("script-generated content was not captured: %.200q", d.Markdown) } } // startConnectProxy runs a minimal forward proxy: CONNECT tunnels are // established directly, and absolute-URI requests are forwarded. That mirrors // what a real proxy offers and covers both transport paths. func startConnectProxy(t *testing.T, tunnels *atomic.Int32) *httptest.Server { t.Helper() forwarder := &http.Transport{} return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodConnect { tunnels.Add(1) out := r.Clone(r.Context()) out.RequestURI = "" res, err := forwarder.RoundTrip(out) if err != nil { w.WriteHeader(http.StatusBadGateway) return } defer res.Body.Close() for key, values := range res.Header { for _, v := range values { w.Header().Add(key, v) } } w.WriteHeader(res.StatusCode) io.Copy(w, res.Body) return } upstream, err := net.DialTimeout("tcp", r.Host, 10*time.Second) if err != nil { w.WriteHeader(http.StatusBadGateway) return } defer upstream.Close() hijacker, ok := w.(http.Hijacker) if !ok { w.WriteHeader(http.StatusInternalServerError) return } client, buffered, err := hijacker.Hijack() if err != nil { return } defer client.Close() tunnels.Add(1) fmt.Fprint(buffered, "HTTP/1.1 200 Connection Established\r\n\r\n") buffered.Flush() done := make(chan struct{}, 2) go func() { io.Copy(upstream, buffered); done <- struct{}{} }() go func() { io.Copy(client, upstream); done <- struct{}{} }() <-done })) }