okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
3009bc8
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"net/http" 7"net/http/httptest" 8"os" 9"strings" 10"sync/atomic" 11"testing" 12"time" 13 14"github.com/joho/godotenv" 15) 16 17func frozenPages (t * testing.T ) []Document { 18t .Helper () 19b ,err := os .ReadFile ("testdata/evidence-pages.json" ) 20if err != nil { 21t .Fatal (err ) 22 } 23var pages []Document 24if err := json .Unmarshal (b ,& pages );err != nil { 25t .Fatal (err ) 26 } 27return pages 28} 29 30func TestShortlistRetainsFalsePremiseCorrection (t * testing.T ) { 31d := frozenPages (t )[0 ] 32q := "How does SQLite WAL allow multiple writers to write simultaneously?" 33selected := shortlist (passages (d .Markdown ),q ,24 ) 34if ! strings .Contains (evidenceText (selected ),"only be one writer at a time" ) { 35t .Fatal ("the lexical shortlist discarded the explicit correction before jev could judge it" ) 36 } 37if len (selected )> 24 { 38t .Fatal ("candidate cap exceeded" ) 39 } 40} 41 42func TestOversizedBlocksDoNotConsumeCandidateSlots (t * testing.T ) { 43s ,_ := fixtureService (t ) 44d := Document {URL :"https://example.com" ,FinalURL :"https://example.com" ,Markdown :strings .Repeat ("request limits " ,100 )+ "\n\nrequests may contain up to 8 mib." } 45pack ,err := s .jev .selectEvidence (context .Background (),d ,"request limits" ,1 ,200 ) 46if err != nil { 47t .Fatal (err ) 48 } 49if len (pack .Passages )!= 1 || ! strings .Contains (pack .Passages [0 ].Text ,"8 mib" ) { 50t .Fatalf ("lost eligible passage: %+v" ,pack ) 51 } 52} 53 54func TestEmptySelectionDescribesCoverageNotSourceAbsence (t * testing.T ) { 55s ,_ := fixtureService (t ) 56reply ,err := s .llm .preview (context .Background (),Document {},EvidencePack {},"none" ) 57if err != nil || reply != "no usable evidence was selected from the inspected passages for this query." { 58t .Fatalf ("overstated empty selection: %q, %v" ,reply ,err ) 59 } 60var n int 61if err := s .log .db .QueryRow ("select count(*) from completions" ).Scan (& n );err != nil || n != 0 { 62t .Fatalf ("fallback must not become a training completion: %d, %v" ,n ,err ) 63 } 64} 65 66func TestSummaryRetriesAreBoundedAndPreserveRawTrainingReplies (t * testing.T ) { 67for _ ,alwaysInvalid := range []bool {false ,true } { 68name := "corrected" 69if alwaysInvalid { 70name = "still_invalid" 71 } 72t .Run (name ,func (t * testing.T ) { 73s ,_ := fixtureService (t ) 74var calls atomic.Int32 75provider := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 76var body CompletionRequest 77if err := json .NewDecoder (r .Body ).Decode (& body );err != nil { 78t .Error (err ) 79 } 80n := calls .Add (1 ) 81if n > 1 && ! strings .Contains (body .Messages [0 ].Content ,"Format correction" ) { 82t .Error ("missing format-only retry instruction" ) 83 } 84reply := `{"summary":"use errors.Is for equivalence."}` 85if n > 1 && ! alwaysInvalid { 86reply = "use errors.Is for equivalence." 87 } 88writeJSON (w ,200 ,map [string ]any {"model" :"test" ,"choices" : []any {map [string ]any {"message" :Message {"assistant" ,reply },"finish_reason" :"stop" }}}) 89 })) 90defer provider .Close () 91s .llm .URL = provider .URL 92reply ,err := s .llm .complete (context .Background (),"preview_summary" ,previewSystem ,"original evidence" ,160 ) 93if calls .Load ()!= 2 || (err != nil )!= alwaysInvalid { 94t .Fatalf ("calls=%d error=%v" ,calls .Load (),err ) 95 } 96if ! alwaysInvalid && usableSummary (reply )!= nil { 97t .Fatal ("returned invalid prose" ) 98 } 99var raw ,request string 100var failed bool 101if err := s .log .db .QueryRow ("select response_text,request_json,error is not null from completions order by id limit 1" ).Scan (& raw ,& request ,& failed );err != nil { 102t .Fatal (err ) 103 } 104if raw != `{"summary":"use errors.Is for equivalence."}` || ! failed { 105t .Fatalf ("rejected raw reply lost or counted as valid: %q %v" ,raw ,failed ) 106 } 107var body map [string ]json.RawMessage 108if err := json .Unmarshal ([]byte (request ),& body );err != nil || len (body )!= 5 || string (body ["chat_template_kwargs" ])!= `{"enable_thinking":false}` { 109t .Fatal ("training request shape changed" ,request ) 110 } 111var successful int 112if err := s .log .db .QueryRow ("select count(*) from completions where error is null" ).Scan (& successful );err != nil { 113t .Fatal (err ) 114 } 115wantSuccessful := 1 116if alwaysInvalid { 117wantSuccessful = 0 118 } 119if successful != wantSuccessful { 120t .Fatalf ("wrong successful row count %d" ,successful ) 121 } 122 }) 123 } 124} 125 126func TestLiveEvidenceRegression (t * testing.T ) { 127if os .Getenv ("LIVE_EVIDENCE_REGRESSION" )!= "1" { 128t .Skip ("opt in to real jev and summarizer calls on frozen source pages" ) 129 } 130_ = godotenv .Load ("../../.env" ) 131cfg ,err := readConfig () 132if err != nil { 133t .Fatal (err ) 134 } 135s ,_ := fixtureService (t ) 136s .jev .URL ,s .jev .Key ,s .jev .Model = cfg .JevURL ,cfg .JevKey ,cfg .JevModel 137s .llm .URL ,s .llm .Key ,s .llm .Model = cfg .LLMURL ,cfg .LLMKey ,cfg .LLMModel 138pages := frozenPages (t ) 139var report []map [string ]any 140for _ ,tc := range []struct { 141name ,query string 142page int 143want []string 144 }{ 145 {"counterevidence" ,"How does SQLite WAL allow multiple writers to write simultaneously?" ,0 , []string {"one writer" }}, 146 {"partial_answer" ,"Go errors Is As wrapped errors difference percent w" ,1 , []string {"%w" ,"Is" ,"As" }}, 147 } { 148t .Run (tc .name ,func (t * testing.T ) { 149ctx ,cancel := context .WithTimeout (context .Background (),90 * time .Second ) 150defer cancel () 151ctx = context .WithValue (ctx ,contextKey {},callContext {ID :newID (),Caller :"frozen-regression" }) 152started := time .Now () 153p ,err := s .jev .selectEvidence (ctx ,pages [tc .page ],tc .query ,24 ,5500 ) 154if err != nil { 155t .Fatal (err ) 156 } 157label ,err := s .jev .answerability (ctx ,tc .query ,p ) 158if err != nil { 159t .Fatal (err ) 160 } 161summary ,err := s .llm .preview (ctx ,pages [tc .page ],p ,label ) 162entry := map [string ]any {"case" :tc .name ,"pack" :p ,"label" :label ,"summary" :summary ,"seconds" :time .Since (started ).Seconds ()} 163if err != nil { 164entry ["error" ]= err .Error () 165 } 166report = append (report ,entry ) 167if path := os .Getenv ("EVIDENCE_REPORT_PATH" );path != "" { 168b ,_ := json .MarshalIndent (report ,"" ," " ) 169if err := os .WriteFile (path ,b ,0600 );err != nil { 170t .Fatal (err ) 171 } 172 } 173if err != nil { 174t .Fatal (err ) 175 } 176for _ ,want := range tc .want { 177if ! strings .Contains (evidenceText (p .Passages ),want ) { 178t .Errorf ("selected evidence missed %q" ,want ) 179 } 180if ! strings .Contains (summary ,want ) { 181t .Errorf ("summary missed %q: %s" ,want ,summary ) 182 } 183 } 184if err := usableSummary (summary );err != nil { 185t .Error (err ) 186 } 187t .Logf ("%d passages, %d chars, %s: %s" ,len (p .Passages ),p .UsedChars ,label ,summary ) 188 }) 189 } 190}