okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"errors" 6"fmt" 7"net/http" 8"net/http/httptest" 9"os" 10"path/filepath" 11"strings" 12"sync/atomic" 13"testing" 14"time" 15) 16 17func TestBrowserWaitKeepsPhaseAndCallerDeadlinesSeparate (t * testing.T ) { 18ctx ,cancel := context .WithTimeout (context .Background (),time .Second ) 19defer cancel () 20err := browserWait (ctx ,5 * time .Millisecond ,time .Millisecond ,func (ctx context.Context ) (bool ,error ) { 21<- ctx .Done () 22return false ,ctx .Err () 23 }) 24if err != nil || ctx .Err ()!= nil { 25t .Fatalf ("phase timeout killed parent: %v, %v" ,err ,ctx .Err ()) 26 } 27cancel () 28if err := browserWait (ctx ,time .Second ,time .Millisecond ,func (ctx context.Context ) (bool ,error ) { 29return true ,nil 30 });! errors .Is (err ,context .Canceled ) { 31t .Fatalf ("ignored caller cancellation: %v" ,err ) 32 } 33want := errors .New ("browser disconnected" ) 34if err := browserWait (context .Background (),time .Second ,time .Millisecond ,func (context.Context ) (bool ,error ) { 35return false ,want 36 });! errors .Is (err ,want ) { 37t .Fatalf ("hid browser error: %v" ,err ) 38 } 39} 40 41func TestBrowserQueueCancellation (t * testing.T ) { 42n := newNetwork (true ) 43defer n .closeIdle () 44b := & Browser {path :"not launched" ,network :n ,slots :make (chan struct {},1 )} 45b .slots <- struct {}{} 46ctx ,cancel := context .WithTimeout (context .Background (),5 * time .Millisecond ) 47defer cancel () 48_ ,_ ,err := b .render (ctx ,"https://example.com" ,false ,"" ,"" ) 49if ! errors .Is (err ,context .DeadlineExceeded )|| len (b .slots )!= 1 { 50t .Fatalf ("queue cancellation changed occupied slot: %v" ,err ) 51 } 52} 53 54func TestKagiMissingResultsDiagnosis (t * testing.T ) { 55for _ ,tc := range []struct {html ,final ,tag string }{ 56 {`<title>sign in to kagi</title>` ,"https://kagi.com/signin" ,"SEARCH_SESSION_REJECTED" }, 57 {`<form><input type="password"></form>` ,"https://kagi.com/search" ,"SEARCH_SESSION_REJECTED" }, 58 {`<p>no results; <a href="/signin">sign in</a></p>` ,"https://kagi.com/search" ,"SEARCH_UPSTREAM_ERROR" }, 59 {`<p>unrecognized search markup</p>` ,"https://kagi.com/search" ,"SEARCH_UPSTREAM_ERROR" }, 60 } { 61var api * APIError 62if err := kagiPageError (tc .html ,tc .final );! errors .As (err ,& api )|| api .Tag != tc .tag { 63t .Errorf ("%s: %v" ,tc .html ,err ) 64 } 65 } 66} 67 68func TestChallengeContentIsNotEvidence (t * testing.T ) { 69for _ ,html := range []string { 70`<title>Just a moment...</title>` ,`<title>Attention Required</title>` , 71`<div id="challenge-form">checking your browser</div>` , 72`<div id="cf-browser-verification">checking your browser</div>` , 73`<script src="/cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1?cf-chl-token=1"></script>` , 74 } { 75_ ,err := htmlDocument ("https://example.com" ,"https://example.com" , []byte (html )) 76var api * APIError 77if ! errors .As (err ,& api )|| api .Tag != "CRAWL_CHALLENGE" { 78t .Errorf ("challenge accepted as evidence: %s, %v" ,html ,err ) 79 } 80 } 81if isAccessChallenge ("cloudflare documentation" ,"<p>how access challenges work</p>" ) { 82t .Fatal ("ordinary content classified as a challenge" ) 83 } 84} 85 86// These cases exercise real Chrome and the guarded proxy, without external 87// requests or model calls. Keep opt-in consistent with TestLiveBrowser. 88func TestLiveBrowserRecovery (t * testing.T ) { 89if os .Getenv ("LIVE_BROWSER" )!= "1" { 90t .Skip ("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome" ) 91 } 92if os .Getenv ("BROWSER_PATH" )== "" { 93t .Fatal ("BROWSER_PATH is required" ) 94 } 95profiles := t .TempDir () 96t .Setenv ("TMPDIR" ,profiles ) 97var blockedHits ,scriptHits atomic.Int32 98page := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 99w .Header ().Set ("Content-Type" ,"text/html" ) 100switch r .URL .Path { 101case "/never" : 102<- r .Context ().Done () 103case "/hung" ,"/empty" : 104body := "" 105if r .URL .Path == "/hung" { 106body = "usable evidence despite an unfinished third-party iframe" 107 } 108fmt .Fprintf (w ,`<body><main>%s</main><iframe src="/never"></iframe></body>` ,body ) 109case "/stream" : 110fmt .Fprint (w ,`<body><main></main><script>setTimeout(() => document.querySelector('main').innerHTML = '<div class="search-result"><a class="__sri_title_link" href="https://example.com">streamed result</a></div>', 700)</script></body>` ) 111case "/missing" : 112fmt .Fprint (w ,`<title>sign in to kagi</title><body><form><input type="password"></form></body>` ) 113case "/hydrate" : 114fmt .Fprint (w ,`<body><main>loading</main><script>for (let i=1;i<=4;i++) setTimeout(() => document.querySelector('main').textContent = 'hydration stage ' + i + '.'.repeat(i), i*250)</script></body>` ) 115case "/challenge" ,"/resolved" : 116w .WriteHeader (http .StatusForbidden ) 117fmt .Fprint (w ,`<title>just a moment</title><body><div id="challenge-form">checking browser</div>` ) 118if r .URL .Path == "/resolved" { 119fmt .Fprint (w ,`<script>setTimeout(() => { document.title='source'; document.body.innerHTML='<main>resolved source evidence</main>'; }, 1200)</script>` ) 120 } 121case "/resources" : 122fmt .Fprint (w ,`<body><main>rendering</main><img src="/tracker.avif"><script src="/necessary.js"></script></body>` ) 123case "/tracker.avif" : 124blockedHits .Add (1 ) 125case "/necessary.js" : 126scriptHits .Add (1 ) 127w .Header ().Set ("Content-Type" ,"text/javascript" ) 128fmt .Fprint (w ,`document.querySelector('main').textContent='necessary javascript ran'` ) 129case "/http-error" : 130w .WriteHeader (http .StatusBadGateway ) 131fmt .Fprint (w ,`<body>upstream unavailable; this is not source evidence</body>` ) 132default : 133fmt .Fprint (w ,`<body><main>ordinary source evidence</main></body>` ) 134 } 135 })) 136defer page .Close () 137n := newNetwork (true ) 138defer n .transport .CloseIdleConnections () 139b := & Browser {path :os .Getenv ("BROWSER_PATH" ),network :n ,slots :make (chan struct {},2 )} 140for _ ,tc := range []struct { 141path ,want ,tag string 142kagi bool 143max time.Duration 144 }{ 145 {"/hung" ,"<main>usable evidence despite an unfinished third-party iframe</main>" ,"" ,false ,20 * time .Second }, 146 {"/empty" ,"" ,"REQUEST_TIMEOUT" ,false ,20 * time .Second }, 147 {"/stream" ,`class="search-result"` ,"" ,true ,6 * time .Second }, 148 {"/missing" ,`type="password"` ,"" ,true ,15 * time .Second }, 149 {"/hydrate" ,"<main>hydration stage 4....</main>" ,"" ,false ,6 * time .Second }, 150 {"/resolved" ,"<main>resolved source evidence</main>" ,"" ,false ,8 * time .Second }, 151 {"/challenge" ,"" ,"CRAWL_CHALLENGE" ,false ,18 * time .Second }, 152 {"/resources" ,"<main>necessary javascript ran</main>" ,"" ,false ,6 * time .Second }, 153 {"/http-error" ,"" ,"CRAWL_HTTP_ERROR" ,false ,6 * time .Second }, 154 } { 155t .Run (tc .path ,func (t * testing.T ) { 156ctx ,cancel := context .WithTimeout (context .Background (),tc .max + 5 * time .Second ) 157defer cancel () 158start := time .Now () 159html ,final ,err := b .render (ctx ,page .URL + tc .path ,tc .kagi ,"" ,"" ) 160elapsed := time .Since (start ) 161if tc .tag != "" { 162var api * APIError 163if ! errors .As (err ,& api )|| api .Tag != tc .tag || html != "" { 164t .Fatalf ("want %s without content, got %q, %v" ,tc .tag ,html ,err ) 165 } 166 }else if err != nil || ! strings .Contains (html ,tc .want )|| final != page .URL + tc .path { 167t .Fatalf ("render mismatch: url=%s error=%v html=%s" ,final ,err ,html ) 168 } 169if elapsed > tc .max { 170t .Errorf ("phase wait was not bounded: %s" ,elapsed ) 171 } 172if len (b .slots )!= 0 { 173t .Fatal ("browser slot leaked" ) 174 } 175left ,_ := filepath .Glob (filepath .Join (profiles ,"chromedp-runner*" )) 176if len (left )!= 0 { 177t .Fatalf ("temporary browser profiles leaked: %v" ,left ) 178 } 179t .Logf ("elapsed=%s" ,elapsed .Round (time .Millisecond )) 180 }) 181 } 182if blockedHits .Load ()!= 0 || scriptHits .Load ()!= 1 { 183t .Errorf ("resource blocking: image=%d necessary script=%d" ,blockedHits .Load (),scriptHits .Load ()) 184 } 185t .Run ("caller cancellation" ,func (t * testing.T ) { 186ctx ,cancel := context .WithTimeout (context .Background (),2 * time .Second ) 187defer cancel () 188_ ,_ ,err := b .render (ctx ,page .URL + "/hung" ,false ,"" ,"" ) 189if ! errors .Is (err ,context .DeadlineExceeded )|| len (b .slots )!= 0 { 190t .Fatalf ("caller cancellation did not terminate browser: %v" ,err ) 191 } 192 }) 193t .Run ("navigation failure" ,func (t * testing.T ) { 194dead := httptest .NewServer (http .NotFoundHandler ()) 195dead .Close () 196// HTTPS forces a failed CONNECT/navigation, rather than the proxy's 197// ordinary HTTP 502 document (covered separately above). 198_ ,_ ,err := b .render (context .Background (),strings .Replace (dead .URL ,"http:" ,"https:" ,1 ),false ,"" ,"" ) 199if err == nil || errors .Is (err ,context .DeadlineExceeded ) { 200t .Fatalf ("navigation failure was hidden: %v" ,err ) 201 } 202 }) 203}