okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"errors" 6"fmt" 7"io" 8"net" 9"net/http" 10"net/http/httptest" 11"net/url" 12"os" 13"path/filepath" 14"strings" 15"sync/atomic" 16"testing" 17"time" 18) 19 20// The render path reports a thin page with a distinct tag, because that tag is 21// the only thing that triggers the plain-fetch retry. A navigation failure or an 22// access challenge must not be retried: those are real diagnoses. 23func TestOnlyIncompleteRenderTriggersStaticRetry (t * testing.T ) { 24incomplete := verifyContent ("too short" ,"https://example.test/" ) 25var api * APIError 26if ! isIncompleteRender (incomplete ) { 27t .Fatalf ("thin content was not reported as incomplete: %v" ,incomplete ) 28 } 29if ! errors .As (incomplete ,& api )|| api .Status != 502 || api .Tag != incompleteTag { 30t .Fatalf ("incomplete render has the wrong contract: %+v" ,api ) 31 } 32 33rich := strings .Repeat ("real page content " ,20 ) 34if err := verifyContent (rich ,"https://example.test/" );err != nil { 35t .Fatalf ("content above the floor was rejected: %v" ,err ) 36 } 37 38for _ ,terminal := range []error { 39failure (403 ,"CRAWL_CHALLENGE" ,"browser access challenge did not resolve" ), 40failure (502 ,"CRAWL_HTTP_ERROR" ,"browser page returned http 502" ), 41failure (504 ,"REQUEST_TIMEOUT" ,"navigation did not complete" ), 42fmt .Errorf ("browser navigation failed: connection reset" ), 43 } { 44if isIncompleteRender (terminal ) { 45t .Errorf ("terminal error would be retried as a thin render: %v" ,terminal ) 46 } 47 } 48} 49 50// The floor counts non-whitespace characters, so a page padded with layout 51// whitespace cannot pass as content. 52func TestVerifyContentCountsNonWhitespaceOnly (t * testing.T ) { 53if err := verifyContent (strings .Repeat ("\n \t" ,500 ),"https://example.test/" );! isIncompleteRender (err ) { 54t .Fatalf ("whitespace-only page passed the floor: %v" ,err ) 55 } 56// Exactly at the floor is content; one below is not. 57if err := verifyContent (strings .Repeat ("x" ,minMeaningfulChars ),"https://example.test/" );err != nil { 58t .Fatalf ("content at the floor was rejected: %v" ,err ) 59 } 60if err := verifyContent (strings .Repeat ("x" ,minMeaningfulChars - 1 ),"https://example.test/" );! isIncompleteRender (err ) { 61t .Fatalf ("content one below the floor was accepted: %v" ,err ) 62 } 63} 64 65// A .pdf URL never goes to the browser: a render cannot add text to a document 66// that has no DOM. 67func TestPDFURLsSkipTheBrowser (t * testing.T ) { 68:= fixturePDF (t ,"report.pdf" ) 69hits := atomic.Int32 {} 70srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 71hits .Add (1 ) 72w .Header ().Set ("Content-Type" ,"application/pdf" ) 73w .Write (defer srv .Close () 76 77n := newNetwork (true ) 78defer n .transport .CloseIdleConnections () 79// A browser path that would fail loudly if it were used. 80f := & Fetcher {net :n ,browser :& Browser {path :"/nonexistent/chrome" },maxBytes :1 << 20 } 81d ,err := f .fetch (context .Background (),srv .URL + "/report.pdf" ,nil ) 82if err != nil { 83t .Fatalf ("pdf fetch failed: %v" ,err ) 84 } 85if hits .Load ()== 0 { 86t .Fatal ("pdf was not fetched over http" ) 87 } 88if ! strings .Contains (d .Markdown ,"evidence" ) { 89t .Fatalf ("unexpected pdf text: %.80q" ,d .Markdown ) 90 } 91} 92 93// The fallback ordering still holds when no browser is configured: every fetch 94// takes the plain path. 95func TestNoBrowserUsesStaticOnly (t * testing.T ) { 96srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 97fmt .Fprint (w ,`<html><body><main>` + strings .Repeat ("static page content " ,20 )+ `</main></body></html>` ) 98 })) 99defer srv .Close () 100 101n := newNetwork (true ) 102defer n .transport .CloseIdleConnections () 103f := & Fetcher {net :n ,maxBytes :1 << 20 } 104if f .canRender () { 105t .Fatal ("fetcher without a browser claims it can render" ) 106 } 107d ,err := f .fetch (context .Background (),srv .URL ,nil ) 108if err != nil { 109t .Fatal (err ) 110 } 111if ! strings .Contains (d .Markdown ,"static page content" ) { 112t .Fatalf ("unexpected document: %.80q" ,d .Markdown ) 113 } 114} 115 116// A plain fetch presents the same identity the browser does. 117func TestStaticFetchUserAgentMatchesBrowser (t * testing.T ) { 118var got string 119srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 120got = r .Header .Get ("User-Agent" ) 121fmt .Fprint (w ,`<html><body><main>` + strings .Repeat ("page text here " ,20 )+ `</main></body></html>` ) 122 })) 123defer srv .Close () 124 125n := newNetwork (true ) 126defer n .transport .CloseIdleConnections () 127f := & Fetcher {net :n ,maxBytes :1 << 20 } 128if _ ,err := f .fetch (context .Background (),srv .URL ,nil );err != nil { 129t .Fatal (err ) 130 } 131if ! strings .Contains (got ,"Chrome/" )|| strings .Contains (got ,"Headless" ) { 132t .Fatalf ("plain fetch announced an unusual identity: %q" ,got ) 133 } 134} 135 136// An oversized declared length is rejected before the body is read. 137func TestOversizedContentLengthRejectedEarly (t * testing.T ) { 138body := strings .Repeat ("x" ,8192 ) 139srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 140w .Header ().Set ("Content-Length" ,fmt .Sprint (len (body ))) 141w .Write ([]byte (body )) 142 })) 143defer srv .Close () 144 145n := newNetwork (true ) 146defer n .transport .CloseIdleConnections () 147f := & Fetcher {net :n ,maxBytes :1024 } 148if _ ,err := f .fetchStatic (context .Background (),srv .URL ,nil );err == nil { 149t .Fatal ("oversized response was accepted" ) 150 } 151} 152 153// The proxy pool rotates, and an empty pool selects a direct connection. 154func TestProxyPoolRotation (t * testing.T ) { 155n := newNetwork (true ) 156if n .pickProxy ()!= nil { 157t .Fatal ("empty pool did not select a direct connection" ) 158 } 159 160a ,_ := url .Parse ("http://proxy-a.test:8080" ) 161b ,_ := url .Parse ("http://proxy-b.test:8080" ) 162pooled := & Network {proxies : []* url.URL {a ,b }} 163seen := []string {} 164for i := 0 ;i < 4 ;i ++ { 165seen = append (seen ,pooled .pickProxy ().Host ) 166 } 167want := []string {"proxy-a.test:8080" ,"proxy-b.test:8080" ,"proxy-a.test:8080" ,"proxy-b.test:8080" } 168for i := range want { 169if seen [i ]!= want [i ] { 170t .Fatalf ("rotation order = %v, want %v" ,seen ,want ) 171 } 172 } 173} 174 175// A configured pool must actually carry the traffic: this runs a real CONNECT 176// proxy and asserts both that the origin was reached and that it saw the proxy. 177func TestProxyPoolRoutesStaticFetchThroughProxy (t * testing.T ) { 178origin := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 179fmt .Fprint (w ,`<html><body><main>` + strings .Repeat ("proxied page content " ,20 )+ `</main></body></html>` ) 180 })) 181defer origin .Close () 182 183tunnels := atomic.Int32 {} 184proxy := startConnectProxy (t ,& tunnels ) 185defer proxy .Close () 186 187proxyURL ,_ := url .Parse (proxy .URL ) 188n := newNetworkWithPool (true , []* url.URL {proxyURL }) 189defer n .closeIdle () 190 191f := & Fetcher {net :n ,maxBytes :1 << 20 } 192d ,err := f .fetchStatic (context .Background (),origin .URL ,n .pickProxy ()) 193if err != nil { 194t .Fatalf ("proxied fetch failed: %v" ,err ) 195 } 196if ! strings .Contains (d .Markdown ,"proxied page content" ) { 197t .Fatalf ("unexpected document: %.80q" ,d .Markdown ) 198 } 199if tunnels .Load ()== 0 { 200t .Fatal ("the request did not go through the configured proxy" ) 201 } 202} 203 204// A proxy that refuses the tunnel surfaces as a fetch failure, not a hang. 205func TestProxyRefusalIsReported (t * testing.T ) { 206refusing := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 207w .WriteHeader (http .StatusForbidden ) 208 })) 209defer refusing .Close () 210 211proxyURL ,_ := url .Parse (refusing .URL ) 212n := newNetworkWithPool (true , []* url.URL {proxyURL }) 213defer n .closeIdle () 214 215f := & Fetcher {net :n ,maxBytes :1 << 20 } 216if _ ,err := f .fetchStatic (context .Background (),"https://example.test/" ,n .pickProxy ());err == nil { 217t .Fatal ("a refusing proxy was treated as a successful fetch" ) 218 } 219} 220 221// TestLiveFetchRendersBeforeFetching proves the render path is primary: the 222// page's text exists only after its script runs, and the plain fetch cannot see 223// it. 224func TestLiveFetchRendersBeforeFetching (t * testing.T ) { 225if os .Getenv ("LIVE_BROWSER" )!= "1" { 226t .Skip ("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome" ) 227 } 228if os .Getenv ("BROWSER_PATH" )== "" { 229t .Fatal ("BROWSER_PATH is required" ) 230 } 231page := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 232w .Header ().Set ("Content-Type" ,"text/html" ) 233fmt .Fprint (w ,`<html><body><main>loading</main><script> 234const parts = []; 235for (let i = 0; i < 30; i++) parts.push('rendered sentence number ' + i + '.'); 236document.querySelector('main').textContent = parts.join(' '); 237</script></body></html>` ) 238 })) 239defer page .Close () 240 241path ,err := filepath .Abs (os .Getenv ("BROWSER_PATH" )) 242if err != nil { 243t .Fatal (err ) 244 } 245n := newNetwork (true ) 246defer n .transport .CloseIdleConnections () 247b := & Browser {path :path ,network :n ,slots :make (chan struct {},1 ),stealth :true ,headless :true ,waitMS :1200 * time .Millisecond } 248f := & Fetcher {net :n ,browser :b ,maxBytes :5 << 20 } 249 250ctx ,cancel := context .WithTimeout (context .Background (),90 * time .Second ) 251defer cancel () 252d ,err := f .fetch (ctx ,page .URL ,nil ) 253if err != nil { 254t .Fatal (err ) 255 } 256if ! strings .Contains (d .Markdown ,"rendered sentence number 29" ) { 257t .Fatalf ("script-generated content was not captured: %.200q" ,d .Markdown ) 258 } 259} 260 261// startConnectProxy runs a minimal forward proxy: CONNECT tunnels are 262// established directly, and absolute-URI requests are forwarded. That mirrors 263// what a real proxy offers and covers both transport paths. 264func startConnectProxy (t * testing.T ,tunnels * atomic.Int32 )* httptest.Server { 265t .Helper () 266forwarder := & http.Transport {} 267return httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 268if r .Method != http .MethodConnect { 269tunnels .Add (1 ) 270out := r .Clone (r .Context ()) 271out .RequestURI = "" 272res ,err := forwarder .RoundTrip (out ) 273if err != nil { 274w .WriteHeader (http .StatusBadGateway ) 275return 276 } 277defer res .Body .Close () 278for key ,values := range res .Header { 279for _ ,v := range values { 280w .Header ().Add (key ,v ) 281 } 282 } 283w .WriteHeader (res .StatusCode ) 284io .Copy (w ,res .Body ) 285return 286 } 287upstream ,err := net .DialTimeout ("tcp" ,r .Host ,10 * time .Second ) 288if err != nil { 289w .WriteHeader (http .StatusBadGateway ) 290return 291 } 292defer upstream .Close () 293hijacker ,ok := w .(http.Hijacker ) 294if ! ok { 295w .WriteHeader (http .StatusInternalServerError ) 296return 297 } 298client ,buffered ,err := hijacker .Hijack () 299if err != nil { 300return 301 } 302defer client .Close () 303tunnels .Add (1 ) 304fmt .Fprint (buffered ,"HTTP/1.1 200 Connection Established\r\n\r\n" ) 305buffered .Flush () 306 307done := make (chan struct {},2 ) 308go func () {io .Copy (upstream ,buffered );done <- struct {}{} }() 309go func () {io .Copy (client ,upstream );done <- struct {}{} }() 310<- done 311 })) 312}