okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"fmt" 7"io" 8"net/http" 9"net/http/httptest" 10"strconv" 11"strings" 12"sync/atomic" 13"testing" 14"time" 15) 16 17func TestJevBatchesKeepItemsIsolatedAndMapAnswers (t * testing.T ) { 18for _ ,tc := range []struct { 19name ,text string 20items ,calls int 21 }{ 22 {"empty" ,"" ,0 ,0 }, 23 {"whole_page" ,"small" ,24 ,1 }, 24 {"question_limit" ,"small" ,61 ,3 }, 25 {"byte_limit" ,strings .Repeat ("界<" ,2400 ),3 ,3 }, 26 } { 27t .Run (tc .name ,func (t * testing.T ) { 28var calls atomic.Int32 29provider := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 30calls .Add (1 ) 31body ,_ := io .ReadAll (r .Body ) 32var req struct { 33State map [string ]string `json:"state"` 34Questions map [string ]struct { 35Instructions struct { 36Question string `json:"question"` 37Passage map [string ]string `json:"passage"` 38 }`json:"instructions"` 39 }`json:"questions"` 40 } 41if err := json .Unmarshal (body ,& req );err != nil { 42t .Error (err ) 43 } 44if len (body )> maxJevRequestBytes || len (req .Questions )> 96 || len (req .State )!= 1 || req .State ["query" ]!= "limits" { 45t .Error ("batch exceeded budget or leaked items into shared state" ) 46 } 47answers := map [string ]Decision {} 48for id ,q := range req .Questions { 49index ,_ := strconv .Atoi (strings .SplitN (id ,"_" ,2 )[0 ]) 50if q .Instructions .Passage ["heading" ]!= strconv .Itoa (index )|| q .Instructions .Passage ["text" ]!= tc .text || q .Instructions .Question == "" { 51t .Error ("question lost its isolated passage" ,id ) 52 } 53answers [id ]= Decision {Type :"noul" ,Noul :float64 (index )/ 100 } 54 } 55writeJSON (w ,200 ,JevResponse {Model :"test" ,Answers :answers }) 56 })) 57defer provider .Close () 58j := & Jev {URL :provider .URL ,Key :"key" ,Model :"test" ,HTTP :provider .Client (),slots :make (chan struct {},2 )} 59items := make ([]map [string ]string ,tc .items ) 60for i := range items { 61items [i ]= map [string ]string {"heading" :strconv .Itoa (i ),"text" :tc .text } 62 } 63answers ,err := j .evaluateItems (context .Background (),"passage" ,"limits" ,"passage" ,items ,passageQuestions ) 64if err != nil || len (answers )!= tc .items || int (calls .Load ())!= tc .calls { 65t .Fatal (err ,len (answers ),calls .Load ()) 66 } 67for i ,a := range answers { 68for name := range passageQuestions { 69if a [name ].Noul != float64 (i )/ 100 { 70t .Fatal ("misrouted answer" ,i ,name ,a [name ]) 71 } 72 } 73 } 74 }) 75 } 76} 77 78func TestJevBatchRejectsOversizedItemsBeforeCalling (t * testing.T ) { 79j := & Jev {Model :"test" } 80items := []map [string ]string {{"text" :"small" }, {"text" :strings .Repeat ("x" ,maxJevRequestBytes )}} 81if _ ,err := j .evaluateItems (context .Background (),"passage" ,"limits" ,"passage" ,items ,passageQuestions );err == nil { 82t .Fatal ("oversized item accepted" ) 83 } 84} 85 86func TestJevBatchesRespectGlobalConcurrencyAndCancellation (t * testing.T ) { 87started ,release := make (chan struct {},3 ),make (chan struct {}) 88var active ,peak atomic.Int32 89provider := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 90n := active .Add (1 ) 91defer active .Add (- 1 ) 92for old := peak .Load ();n > old ;old = peak .Load () { 93if peak .CompareAndSwap (old ,n ) { 94break 95 } 96 } 97started <- struct {}{} 98select { 99case <- release : 100case <- r .Context ().Done (): 101return 102 } 103var req struct { 104Questions map [string ]Question `json:"questions"` 105 } 106_ = json .NewDecoder (r .Body ).Decode (& req ) 107answers := map [string ]Decision {} 108for id := range req .Questions { 109answers [id ]= Decision {Type :"noul" ,Noul :.5 } 110 } 111writeJSON (w ,200 ,JevResponse {Model :"test" ,Answers :answers }) 112 })) 113defer provider .Close () 114j := & Jev {URL :provider .URL ,Key :"key" ,Model :"test" ,HTTP :provider .Client (),slots :make (chan struct {},2 )} 115ctx ,cancel := context .WithTimeout (context .Background (),5 * time .Second ) 116defer cancel () 117done := make (chan error ,1 ) 118items := make ([]map [string ]string ,60 ) 119go func () { 120_ ,err := j .evaluateItems (ctx ,"passage" ,"limits" ,"passage" ,items ,passageQuestions ) 121done <- err 122 }() 123for range 2 { 124select { 125case <- started : 126case <- ctx .Done (): 127t .Fatal ("batches did not run concurrently" ) 128 } 129 } 130close (release ) 131if err := <- done ;err != nil || peak .Load ()!= 2 { 132t .Fatal (err ,peak .Load ()) 133 } 134canceled ,stop := context .WithCancel (context .Background ()) 135stop () 136if _ ,err := j .evaluateItems (canceled ,"passage" ,"limits" ,"passage" ,items ,passageQuestions );err == nil { 137t .Fatal ("canceled batch succeeded" ) 138 } 139} 140 141func TestDeadlineReturnsCompletedSourcesAndTimeoutStatuses (t * testing.T ) { 142s ,page := fixtureService (t ) 143s .cfg .Concurrency = 1 144s .cfg .RequestTimeout = 100 * time .Millisecond 145slow := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) {<- r .Context ().Done () })) 146defer slow .Close () 147status ,payload := callHTTP (t ,s ,"/contents" ,fmt .Sprintf (`{"urls":[%q,%q,%q]}` ,page .URL ,slow .URL ,slow .URL + "/queued" )) 148var out ResultsResponse 149_ = json .Unmarshal ([]byte (jsonText (payload )),& out ) 150if status != 200 || len (out .Results )!= 1 || len (out .Statuses )!= 3 { 151t .Fatal (status ,out ) 152 } 153for i ,item := range out .Statuses { 154if item .ID == "" || i == 0 && item .Status != "success" || i > 0 && (item .Error == nil || item .Error .Tag != "REQUEST_TIMEOUT" || item .Error .HTTPStatusCode != 504 ) { 155t .Fatal ("bad per-url status" ,item ) 156 } 157 } 158var outcome ,detail string 159if err := s .log .db .QueryRow ("select outcome,detail from requests where request_id=? and endpoint='contents'" ,out .RequestID ).Scan (& outcome ,& detail );err != nil || outcome != "partial" || ! strings .Contains (detail ,"REQUEST_TIMEOUT" ) { 160t .Fatal ("partial timeout lost from logs" ,err ,outcome ,detail ) 161 } 162status ,_ = callHTTP (t ,s ,"/contents" ,fmt .Sprintf (`{"urls":[%q]}` ,slow .URL )) 163if status != 504 { 164t .Fatal ("empty timed-out request must still fail" ,status ) 165 } 166}