okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"fmt" 7"sort" 8) 9 10// Question-local data keeps unrelated candidates out of each judgment's context. 11// Jev evaluates the questions independently in parallel; IDs only route answers. 12// Bound both question count and encoded bytes without dropping/truncating items. 13func (j * Jev )evaluateItems (ctx context.Context ,task ,query ,field string ,items []map [string ]string ,questions map [string ]Question ) ([]map [string ]Decision ,error ) { 14const maxQuestions = 96 15state := map [string ]string {"query" :query } 16empty ,err := j .requestBody (state ,map [string ]Question {}) 17if err != nil { 18return nil ,err 19 } 20names := make ([]string ,0 ,len (questions )) 21for name := range questions { 22names = append (names ,name ) 23 } 24sort .Strings (names ) 25batches := []map [string ]Question {} 26current := map [string ]Question {} 27bytes := len (empty ) 28for i ,item := range items { 29for _ ,name := range names { 30q := questions [name ] 31q .Instructions = map [string ]any {"question" :q .Instructions ,field :item } 32id := fmt .Sprintf ("%d_%s" ,i ,name ) 33encoded ,err := json .Marshal (map [string ]Question {id :q }) 34if err != nil { 35return nil ,err 36 } 37size := len (encoded )- 2 // exclude the enclosing braces 38if len (empty )+ size > maxJevRequestBytes { 39return nil ,fmt .Errorf ("jev item exceeds the application state budget" ) 40 } 41if len (current )> 0 && (bytes + 1 + size > maxJevRequestBytes || len (current )== maxQuestions ) { 42batches = append (batches ,current ) 43current = map [string ]Question {} 44bytes = len (empty ) 45 } 46if len (current )> 0 { 47bytes ++ // comma between questions 48 } 49current [id ]= q 50bytes += size 51 } 52 } 53if len (current )> 0 { 54batches = append (batches ,current ) 55 } 56replies := make ([]JevResponse ,len (batches )) 57err = parallel (ctx ,len (batches ),cap (j .slots ),func (ctx context.Context ,i int )error { 58var err error 59replies [i ],err = j .evaluate (ctx ,task ,state ,batches [i ]) 60return err 61 }) 62if err != nil { 63return nil ,err 64 } 65flat := map [string ]Decision {} 66for _ ,r := range replies { 67for id ,a := range r .Answers { 68flat [id ]= a 69 } 70 } 71answers := make ([]map [string ]Decision ,len (items )) 72for i := range items { 73answers [i ]= make (map [string ]Decision ,len (questions )) 74for _ ,name := range names { 75answers [i ][name ]= flat [fmt .Sprintf ("%d_%s" ,i ,name )] 76 } 77 } 78return answers ,nil 79}