package app
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
)
// The render path reports a thin page with a distinct tag, because that tag is
// the only thing that triggers the plain-fetch retry. A navigation failure or an
// access challenge must not be retried: those are real diagnoses.
func TestOnlyIncompleteRenderTriggersStaticRetry(t *testing.T) {
incomplete := verifyContent("too short", "https://example.test/")
var api *APIError
if !isIncompleteRender(incomplete) {
t.Fatalf("thin content was not reported as incomplete: %v", incomplete)
}
if !errors.As(incomplete, &api) || api.Status != 502 || api.Tag != incompleteTag {
t.Fatalf("incomplete render has the wrong contract: %+v", api)
}
rich := strings.Repeat("real page content ", 20)
if err := verifyContent(rich, "https://example.test/"); err != nil {
t.Fatalf("content above the floor was rejected: %v", err)
}
for _, terminal := range []error{
failure(403, "CRAWL_CHALLENGE", "browser access challenge did not resolve"),
failure(502, "CRAWL_HTTP_ERROR", "browser page returned http 502"),
failure(504, "REQUEST_TIMEOUT", "navigation did not complete"),
fmt.Errorf("browser navigation failed: connection reset"),
} {
if isIncompleteRender(terminal) {
t.Errorf("terminal error would be retried as a thin render: %v", terminal)
}
}
}
// The floor counts non-whitespace characters, so a page padded with layout
// whitespace cannot pass as content.
func TestVerifyContentCountsNonWhitespaceOnly(t *testing.T) {
if err := verifyContent(strings.Repeat("\n \t", 500), "https://example.test/"); !isIncompleteRender(err) {
t.Fatalf("whitespace-only page passed the floor: %v", err)
}
// Exactly at the floor is content; one below is not.
if err := verifyContent(strings.Repeat("x", minMeaningfulChars), "https://example.test/"); err != nil {
t.Fatalf("content at the floor was rejected: %v", err)
}
if err := verifyContent(strings.Repeat("x", minMeaningfulChars-1), "https://example.test/"); !isIncompleteRender(err) {
t.Fatalf("content one below the floor was accepted: %v", err)
}
}
// A .pdf URL never goes to the browser: a render cannot add text to a document
// that has no DOM.
func TestPDFURLsSkipTheBrowser(t *testing.T) {
pdf := fixturePDF(t, "report.pdf")
hits := atomic.Int32{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Type", "application/pdf")
w.Write(pdf)
}))
defer srv.Close()
n := newNetwork(true)
defer n.transport.CloseIdleConnections()
// A browser path that would fail loudly if it were used.
f := &Fetcher{net: n, browser: &Browser{path: "/nonexistent/chrome"}, maxBytes: 1 << 20}
d, err := f.fetch(context.Background(), srv.URL+"/report.pdf", nil)
if err != nil {
t.Fatalf("pdf fetch failed: %v", err)
}
if hits.Load() == 0 {
t.Fatal("pdf was not fetched over http")
}
if !strings.Contains(d.Markdown, "evidence") {
t.Fatalf("unexpected pdf text: %.80q", d.Markdown)
}
}
// The fallback ordering still holds when no browser is configured: every fetch
// takes the plain path.
func TestNoBrowserUsesStaticOnly(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `
`+strings.Repeat("static page content ", 20)+``)
}))
defer srv.Close()
n := newNetwork(true)
defer n.transport.CloseIdleConnections()
f := &Fetcher{net: n, maxBytes: 1 << 20}
if f.canRender() {
t.Fatal("fetcher without a browser claims it can render")
}
d, err := f.fetch(context.Background(), srv.URL, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(d.Markdown, "static page content") {
t.Fatalf("unexpected document: %.80q", d.Markdown)
}
}
// A plain fetch presents the same identity the browser does.
func TestStaticFetchUserAgentMatchesBrowser(t *testing.T) {
var got string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Get("User-Agent")
fmt.Fprint(w, ``+strings.Repeat("page text here ", 20)+``)
}))
defer srv.Close()
n := newNetwork(true)
defer n.transport.CloseIdleConnections()
f := &Fetcher{net: n, maxBytes: 1 << 20}
if _, err := f.fetch(context.Background(), srv.URL, nil); err != nil {
t.Fatal(err)
}
if !strings.Contains(got, "Chrome/") || strings.Contains(got, "Headless") {
t.Fatalf("plain fetch announced an unusual identity: %q", got)
}
}
// An oversized declared length is rejected before the body is read.
func TestOversizedContentLengthRejectedEarly(t *testing.T) {
body := strings.Repeat("x", 8192)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", fmt.Sprint(len(body)))
w.Write([]byte(body))
}))
defer srv.Close()
n := newNetwork(true)
defer n.transport.CloseIdleConnections()
f := &Fetcher{net: n, maxBytes: 1024}
if _, err := f.fetchStatic(context.Background(), srv.URL, nil); err == nil {
t.Fatal("oversized response was accepted")
}
}
// The proxy pool rotates, and an empty pool selects a direct connection.
func TestProxyPoolRotation(t *testing.T) {
n := newNetwork(true)
if n.pickProxy() != nil {
t.Fatal("empty pool did not select a direct connection")
}
a, _ := url.Parse("http://proxy-a.test:8080")
b, _ := url.Parse("http://proxy-b.test:8080")
pooled := &Network{proxies: []*url.URL{a, b}}
seen := []string{}
for i := 0; i < 4; i++ {
seen = append(seen, pooled.pickProxy().Host)
}
want := []string{"proxy-a.test:8080", "proxy-b.test:8080", "proxy-a.test:8080", "proxy-b.test:8080"}
for i := range want {
if seen[i] != want[i] {
t.Fatalf("rotation order = %v, want %v", seen, want)
}
}
}
// A configured pool must actually carry the traffic: this runs a real CONNECT
// proxy and asserts both that the origin was reached and that it saw the proxy.
func TestProxyPoolRoutesStaticFetchThroughProxy(t *testing.T) {
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, ``+strings.Repeat("proxied page content ", 20)+``)
}))
defer origin.Close()
tunnels := atomic.Int32{}
proxy := startConnectProxy(t, &tunnels)
defer proxy.Close()
proxyURL, _ := url.Parse(proxy.URL)
n := newNetworkWithPool(true, []*url.URL{proxyURL})
defer n.closeIdle()
f := &Fetcher{net: n, maxBytes: 1 << 20}
d, err := f.fetchStatic(context.Background(), origin.URL, n.pickProxy())
if err != nil {
t.Fatalf("proxied fetch failed: %v", err)
}
if !strings.Contains(d.Markdown, "proxied page content") {
t.Fatalf("unexpected document: %.80q", d.Markdown)
}
if tunnels.Load() == 0 {
t.Fatal("the request did not go through the configured proxy")
}
}
// A proxy that refuses the tunnel surfaces as a fetch failure, not a hang.
func TestProxyRefusalIsReported(t *testing.T) {
refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
}))
defer refusing.Close()
proxyURL, _ := url.Parse(refusing.URL)
n := newNetworkWithPool(true, []*url.URL{proxyURL})
defer n.closeIdle()
f := &Fetcher{net: n, maxBytes: 1 << 20}
if _, err := f.fetchStatic(context.Background(), "https://example.test/", n.pickProxy()); err == nil {
t.Fatal("a refusing proxy was treated as a successful fetch")
}
}
// TestLiveFetchRendersBeforeFetching proves the render path is primary: the
// page's text exists only after its script runs, and the plain fetch cannot see
// it.
func TestLiveFetchRendersBeforeFetching(t *testing.T) {
if os.Getenv("LIVE_BROWSER") != "1" {
t.Skip("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome")
}
if os.Getenv("BROWSER_PATH") == "" {
t.Fatal("BROWSER_PATH is required")
}
page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
fmt.Fprint(w, `loading`)
}))
defer page.Close()
path, err := filepath.Abs(os.Getenv("BROWSER_PATH"))
if err != nil {
t.Fatal(err)
}
n := newNetwork(true)
defer n.transport.CloseIdleConnections()
b := &Browser{path: path, network: n, slots: make(chan struct{}, 1), stealth: true, headless: true, waitMS: 1200 * time.Millisecond}
f := &Fetcher{net: n, browser: b, maxBytes: 5 << 20}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
d, err := f.fetch(ctx, page.URL, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(d.Markdown, "rendered sentence number 29") {
t.Fatalf("script-generated content was not captured: %.200q", d.Markdown)
}
}
// startConnectProxy runs a minimal forward proxy: CONNECT tunnels are
// established directly, and absolute-URI requests are forwarded. That mirrors
// what a real proxy offers and covers both transport paths.
func startConnectProxy(t *testing.T, tunnels *atomic.Int32) *httptest.Server {
t.Helper()
forwarder := &http.Transport{}
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodConnect {
tunnels.Add(1)
out := r.Clone(r.Context())
out.RequestURI = ""
res, err := forwarder.RoundTrip(out)
if err != nil {
w.WriteHeader(http.StatusBadGateway)
return
}
defer res.Body.Close()
for key, values := range res.Header {
for _, v := range values {
w.Header().Add(key, v)
}
}
w.WriteHeader(res.StatusCode)
io.Copy(w, res.Body)
return
}
upstream, err := net.DialTimeout("tcp", r.Host, 10*time.Second)
if err != nil {
w.WriteHeader(http.StatusBadGateway)
return
}
defer upstream.Close()
hijacker, ok := w.(http.Hijacker)
if !ok {
w.WriteHeader(http.StatusInternalServerError)
return
}
client, buffered, err := hijacker.Hijack()
if err != nil {
return
}
defer client.Close()
tunnels.Add(1)
fmt.Fprint(buffered, "HTTP/1.1 200 Connection Established\r\n\r\n")
buffered.Flush()
done := make(chan struct{}, 2)
go func() { io.Copy(upstream, buffered); done <- struct{}{} }()
go func() { io.Copy(client, upstream); done <- struct{}{} }()
<-done
}))
}