okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"math" 7"os" 8"testing" 9"time" 10 11"github.com/joho/godotenv" 12) 13 14// Replay exported decision rows against the same queries, passages and rubrics. 15// Opt-in: no production database writes, and no fetching changed live pages. 16func TestLiveJevBatchReplay (t * testing.T ) { 17path := os .Getenv ("JEV_REPLAY_PATH" ) 18if path == "" { 19t .Skip ("set JEV_REPLAY_PATH to a frozen pre-batching decision export" ) 20 } 21data ,err := os .ReadFile (path ) 22if err != nil { 23t .Fatal (err ) 24 } 25var rows []struct { 26Task string `json:"task"` 27Request struct { 28State struct { 29Query string `json:"query"` 30Passage map [string ]string `json:"passage"` 31Candidate map [string ]string `json:"candidate"` 32 }`json:"state"` 33Questions map [string ]Question `json:"questions"` 34 }`json:"request"` 35Response JevResponse `json:"response"` 36 } 37if err := json .Unmarshal (data ,& rows );err != nil || len (rows )== 0 { 38t .Fatal ("invalid replay export" ,err ) 39 } 40_ = godotenv .Load ("../../.env" ) 41cfg ,err := readConfig () 42if err != nil { 43t .Fatal (err ) 44 } 45s ,_ := fixtureService (t ) 46s .jev .URL ,s .jev .Key ,s .jev .Model = cfg .JevURL ,cfg .JevKey ,cfg .JevModel 47s .jev .slots = make (chan struct {},cfg .JevConcurrency ) 48groups := map [string ][]int {} 49order := []string {} 50oldTokens := 0 51for i ,r := range rows { 52key := r .Task + "\x00" + r .Request .State .Query + "\x00" + r .Request .State .Passage ["url" ] 53if _ ,ok := groups [key ];! ok { 54order = append (order ,key ) 55 } 56groups [key ]= append (groups [key ],i ) 57oldTokens += r .Response .Usage .InputTokens 58 } 59ctx ,cancel := context .WithTimeout (context .Background (),90 * time .Second ) 60defer cancel () 61started := time .Now () 62decisions ,oldKept ,retained ,newKept := 0 ,0 ,0 ,0 63lost := []map [string ]any {} 64var absoluteDifference float64 65keep := func (a map [string ]Decision )bool { 66return a ["injection" ].Noul < .85 && (a ["contradiction" ].Noul >= .65 || a ["relevant" ].Noul >= .35 && a ["evidence" ].Noul >= .5 ) 67 } 68for _ ,key := range order { 69indices := groups [key ] 70first := rows [indices [0 ]] 71field := "passage" 72if first .Task == "rerank" { 73field = "candidate" 74 } 75items := make ([]map [string ]string ,len (indices )) 76for i ,index := range indices { 77if field == "passage" { 78items [i ]= rows [index ].Request .State .Passage 79 }else { 80items [i ]= rows [index ].Request .State .Candidate 81 } 82 } 83answers ,err := s .jev .evaluateItems (ctx ,first .Task ,first .Request .State .Query ,field ,items ,first .Request .Questions ) 84if err != nil { 85var raw string 86_ = s .log .db .QueryRow ("select coalesce(response_json,'') from decisions where error is not null order by id desc limit 1" ).Scan (& raw ) 87t .Fatal (err ,first .Task ,len (items ),takeRunes (raw ,1600 )) 88 } 89for i ,a := range answers { 90old := rows [indices [i ]].Response .Answers 91for name ,v := range a { 92decisions ++ 93absoluteDifference += math .Abs (v .Noul - old [name ].Noul ) 94 } 95if field == "passage" { 96if keep (a ) { 97newKept ++ 98 } 99if keep (old ) { 100oldKept ++ 101if keep (a ) { 102retained ++ 103 }else { 104lost = append (lost ,map [string ]any {"item" :items [i ],"old" :old ,"new" :a }) 105 } 106 } 107 } 108 } 109 } 110var calls ,newTokens int 111if err := s .log .db .QueryRow ("select count(*),coalesce(sum(json_extract(response_json,'$.usage.input_tokens')),0) from decisions" ).Scan (& calls ,& newTokens );err != nil { 112t .Fatal (err ) 113 } 114report := map [string ]any {"old_calls" :len (rows ),"new_calls" :calls ,"decisions" :decisions ,"seconds" :time .Since (started ).Seconds (),"mean_absolute_probability_change" :absoluteDifference / float64 (decisions ),"old_eligible" :oldKept ,"old_eligible_retained" :retained ,"new_eligible" :newKept ,"old_input_tokens" :oldTokens ,"new_input_tokens" :newTokens } 115t .Log (jsonText (report )) 116if path := os .Getenv ("JEV_REPLAY_REPORT_PATH" );path != "" { 117report ["lost" ]= lost 118if err := os .WriteFile (path , []byte (jsonText (report )+ "\n" ),0600 );err != nil { 119t .Fatal (err ) 120 } 121 } 122if calls >= len (rows ) { 123t .Fatal ("batching did not reduce calls" ) 124 } 125// Historical model votes are diagnostics, not ground-truth labels. The live 126// evidence regression separately asserts required facts and counterevidence. 127}