okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
8af1784
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"os" 7"strings" 8"time" 9) 10 11// decisionCost totals the decision-model calls of each request: tokens the 12// model reported, or about len/4 of the request body where it reported none 13// (the local model), so jevUsd is what the same calls would cost on Jev. 14// The filter keeps the scan to the requests asked about (decisions is indexed by 15// request_id and created_at). 16func decisionCost (filter string )string { 17return `SELECT request_id, count(*) AS calls, 18sum(coalesce(nullif(json_extract(response_json,'$.usage.input_tokens'),0), length(request_json)/4)) AS inputTokens, 19coalesce(sum(json_extract(response_json,'$.usage.output_tokens')),0) AS outputTokens, 20sum(coalesce(json_extract(response_json,'$.usage.input_tokens'),0) = 0) AS estimatedCalls, 21group_concat(DISTINCT model) AS models 22FROM decisions WHERE error IS NULL AND ` + filter + ` GROUP BY request_id` 23} 24 25// jevUsd prices decision input tokens at JEV_PRICE_PER_MILLION; Jev does not bill output. 26func (s * Service )jevUsd (row map [string ]any ) { 27tokens ,_ := row ["inputTokens" ].(int64 ) 28row ["jevUsd" ]= float64 (tokens )* s .cfg .JevPricePerMillion / 1e6 29} 30 31// attachCosts adds each row's decision calls, tokens and Jev price. 32func (s * Service )attachCosts (ctx context.Context ,rows []map [string ]any )error { 33if len (rows )== 0 { 34return nil 35 } 36ids := make ([]any ,len (rows )) 37for i ,row := range rows { 38ids [i ]= row ["requestId" ] 39 } 40costs ,err := s .log .adminRows (ctx ,decisionCost ("request_id IN (?" + strings .Repeat (",?" ,len (ids )- 1 )+ ")" ),ids ... ) 41if err != nil { 42return err 43 } 44byID := map [any ]map [string ]any {} 45for _ ,c := range costs { 46byID [c ["request_id" ]]= c 47 } 48for _ ,row := range rows { 49c := byID [row ["requestId" ]] 50if c == nil { 51c = map [string ]any {"calls" :int64 (0 ),"inputTokens" :int64 (0 )} 52 } 53row ["decisionCalls" ],row ["inputTokens" ],row ["decisionModels" ]= c ["calls" ],c ["inputTokens" ],c ["models" ] 54s .jevUsd (row ) 55 } 56return nil 57} 58 59type adminRequestPage struct { 60Rows []map [string ]any `json:"rows"` 61NextBefore int64 `json:"nextBefore,omitempty"` 62} 63 64type adminRequestTrace struct { 65Requests []map [string ]any `json:"requests"` 66Completions []map [string ]any `json:"completions"` 67Decisions []map [string ]any `json:"decisions"` 68Evidence []map [string ]any `json:"evidence"` 69} 70 71func (s * Service )adminOverview (ctx context.Context ) (any ,error ) { 72since := time .Now ().Add (- 24 * time .Hour ).Unix () 73requests ,err := s .log .adminRows (ctx ,`SELECT count(*) AS total, coalesce(sum(outcome != 'ok'),0) AS errors 74FROM requests WHERE created_at >= ? AND endpoint IN ('search','contents','answer')` ,since ) 75if err != nil { 76return nil ,err 77 } 78models ,err := s .log .adminRows (ctx ,`SELECT count(*) AS calls, coalesce(sum(error IS NOT NULL),0) AS errors, 79coalesce(avg(latency_ms),0) AS meanLatencyMs FROM ( 80SELECT error,latency_ms FROM completions WHERE created_at >= ? UNION ALL 81SELECT error,latency_ms FROM decisions WHERE created_at >= ?)` ,since ,since ) 82if err != nil { 83return nil ,err 84 } 85tokens ,err := s .log .adminRows (ctx ,`SELECT coalesce(sum(prompt_tokens),0) AS input, coalesce(sum(completion_tokens),0) AS output, 86count(*) AS calls, coalesce(sum(prompt_tokens IS NOT NULL AND completion_tokens IS NOT NULL),0) AS reportedCalls 87FROM completions WHERE created_at >= ?` ,since ) 88if err != nil { 89return nil ,err 90 } 91decisions ,err := s .log .adminRows (ctx ,`SELECT count(DISTINCT r.request_id) AS requests, coalesce(sum(d.calls),0) AS calls, 92coalesce(sum(d.inputTokens),0) AS inputTokens, coalesce(sum(d.outputTokens),0) AS outputTokens, 93coalesce(sum(d.estimatedCalls),0) AS estimatedCalls 94FROM (SELECT DISTINCT request_id FROM requests WHERE created_at >= ? AND endpoint IN ('search','contents','answer')) r 95LEFT JOIN (` + decisionCost ("created_at >= ?" )+ `) d USING (request_id)` ,since ,since ) 96if err != nil { 97return nil ,err 98 } 99s .jevUsd (decisions [0 ]) 100var dbBytes int64 101for _ ,suffix := range []string {"" ,"-wal" ,"-shm" } { 102if info ,err := os .Stat (s .cfg .DBPath + suffix );err == nil { 103dbBytes += info .Size () 104 } 105 } 106return map [string ]any {"since" :since ,"requests" :requests [0 ],"models" :models [0 ],"tokens" :tokens [0 ],"decisions" :decisions [0 ],"databaseBytes" :dbBytes , 107"config" :map [string ]any { 108"jevModel" :s .cfg .JevModel ,"summarizerModel" :s .cfg .LLMModel ,"jevPricePerMillion" :s .cfg .JevPricePerMillion , 109"jevConfigured" :s .cfg .JevKey != "" ,"summarizerConfigured" :s .cfg .LLMKey != "" , 110"searchConfigured" :s .cfg .KagiSession != "" || s .cfg .KagiToken != "" , 111"browserConfigured" :s .cfg .BrowserPath != "" ,"passageCandidates" :s .cfg .Passages , 112"evidenceCharacters" :s .cfg .EvidenceChars ,"requestTimeoutSeconds" :s .cfg .RequestTimeout .Seconds (), 113 }},nil 114} 115 116func (s * Service )adminRequests (ctx context.Context ,before int64 ,endpoint string ,errorsOnly bool ) (adminRequestPage ,error ) { 117query := `SELECT id,created_at AS createdAt,request_id AS requestId,endpoint,caller,detail,outcome 118FROM requests WHERE endpoint IN ('search','contents','answer')` 119args := []any {} 120if before > 0 { 121query += " AND id < ?" 122args = append (args ,before ) 123 } 124if endpoint != "" { 125query += " AND endpoint = ?" 126args = append (args ,endpoint ) 127 } 128if errorsOnly { 129query += " AND outcome != 'ok'" 130 } 131rows ,err := s .log .adminRows (ctx ,query + " ORDER BY id DESC LIMIT 31" ,args ... ) 132if err != nil { 133return adminRequestPage {},err 134 } 135if err := s .attachCosts (ctx ,rows );err != nil { 136return adminRequestPage {},err 137 } 138page := adminRequestPage {Rows :rows } 139if len (rows )> 30 { 140page .Rows = rows [:30 ] 141page .NextBefore = rows [29 ]["id" ].(int64 ) 142 } 143return page ,err 144} 145 146func (l * Log )adminTrace (ctx context.Context ,id string ) (adminRequestTrace ,error ) { 147var trace adminRequestTrace 148for _ ,part := range []struct { 149target * []map [string ]any 150query string 151 }{ 152 {& trace .Requests ,`SELECT id,created_at AS createdAt,endpoint,caller,detail,outcome FROM requests WHERE request_id = ? ORDER BY id LIMIT 100` }, 153 {& trace .Completions ,`SELECT id,created_at AS createdAt,task,model,request_json AS request,response_text AS response,error,prompt_tokens AS inputTokens,completion_tokens AS outputTokens,latency_ms AS latencyMs FROM completions WHERE request_id = ? ORDER BY id LIMIT 200` }, 154 {& trace .Decisions ,`SELECT id,created_at AS createdAt,task,model,request_json AS request,response_json AS response,error,latency_ms AS latencyMs FROM decisions WHERE request_id = ? ORDER BY id LIMIT 200` }, 155 {& trace .Evidence ,`SELECT id,source_id AS sourceId,policy_version AS policy,packet_json AS packet FROM evidence WHERE request_id = ? ORDER BY id LIMIT 100` }, 156 } { 157rows ,err := l .adminRows (ctx ,part .query ,id ) 158if err != nil { 159return trace ,err 160 } 161* part .target = rows 162 } 163return trace ,nil 164} 165 166// SQL projections are the admin read model; this does not modify training rows. 167// Parse valid JSON cells for inspection, retaining malformed legacy cells as text. 168func (l * Log )adminRows (ctx context.Context ,query string ,args ... any ) ([]map [string ]any ,error ) { 169rows ,err := l .db .QueryContext (ctx ,query ,args ... ) 170if err != nil { 171return nil ,err 172 } 173defer rows .Close () 174columns ,err := rows .Columns () 175if err != nil { 176return nil ,err 177 } 178result := []map [string ]any {} 179for rows .Next () { 180values ,pointers := make ([]any ,len (columns )),make ([]any ,len (columns )) 181for i := range values { 182pointers [i ]= & values [i ] 183 } 184if err := rows .Scan (pointers ... );err != nil { 185return nil ,err 186 } 187row := map [string ]any {} 188for i ,name := range columns { 189value := values [i ] 190if b ,ok := value .([]byte );ok { 191value = string (b ) 192 } 193if text ,ok := value .(string );ok && (name == "detail" || name == "request" || name == "packet" ) { 194var decoded any 195if json .Unmarshal ([]byte (text ),& decoded )== nil { 196value = decoded 197 } 198 } 199row [name ]= value 200 } 201result = append (result ,row ) 202 } 203return result ,rows .Err () 204}