okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
84fcc80
main
1package app 2 3import ( 4"context" 5"database/sql" 6"encoding/json" 7"log/slog" 8"net/url" 9"os" 10"path/filepath" 11"time" 12 13 _"github.com/mattn/go-sqlite3" 14) 15 16// These two tables deliberately retain the existing training export contract. 17const logSchema = ` 18CREATE TABLE IF NOT EXISTS completions ( 19id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, task TEXT NOT NULL, 20caller TEXT, model TEXT NOT NULL, request_json TEXT NOT NULL, response_text TEXT, 21error TEXT, prompt_tokens INTEGER, completion_tokens INTEGER, latency_ms INTEGER NOT NULL, 22request_id TEXT 23); 24CREATE INDEX IF NOT EXISTS idx_completions_task ON completions(task); 25CREATE INDEX IF NOT EXISTS idx_completions_created_at ON completions(created_at); 26CREATE TABLE IF NOT EXISTS requests ( 27id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL, 28endpoint TEXT NOT NULL, caller TEXT, detail TEXT, outcome TEXT NOT NULL 29); 30CREATE INDEX IF NOT EXISTS idx_requests_created_at ON requests(created_at); 31CREATE INDEX IF NOT EXISTS idx_requests_request_id ON requests(request_id); 32CREATE TABLE IF NOT EXISTS decisions ( 33id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL, 34task TEXT NOT NULL, model TEXT NOT NULL, request_json TEXT NOT NULL, response_json TEXT, 35error TEXT, latency_ms INTEGER NOT NULL 36); 37CREATE TABLE IF NOT EXISTS evidence ( 38id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL, 39source_id TEXT NOT NULL, policy_version TEXT NOT NULL, packet_json TEXT NOT NULL 40);` 41 42// Log is the training log. Calls from the public, unauthenticated endpoint are 43// written only when the caller opts in, and then to public instead. 44type Log struct { 45db * sql.DB 46public * Log 47} 48 49type callContext struct { 50ID ,Caller string 51// Public marks an unauthenticated call; Contribute marks its opt-in to logging. 52Public ,Contribute bool 53} 54type contextKey struct {} 55 56func callInfo (ctx context.Context )callContext { 57c ,_ := ctx .Value (contextKey {}).(callContext ) 58return c 59} 60 61func openLog (path string ) (* Log ,error ) { 62path ,err := filepath .Abs (path ) 63if err != nil { 64return nil ,err 65 } 66if err := os .MkdirAll (filepath .Dir (path ),0700 );err != nil { 67return nil ,err 68 } 69u := url.URL {Scheme :"file" ,Path :path } 70q := u .Query () 71q .Set ("_busy_timeout" ,"5000" ) 72q .Set ("_journal_mode" ,"WAL" ) 73u .RawQuery = q .Encode () 74db ,err := sql .Open ("sqlite3" ,u .String ()) 75if err != nil { 76return nil ,err 77 } 78db .SetMaxOpenConns (1 ) 79if _ ,err = db .Exec (logSchema );err != nil { 80db .Close () 81return nil ,err 82 } 83rows ,err := db .Query ("PRAGMA table_info(completions)" ) 84if err != nil { 85db .Close () 86return nil ,err 87 } 88hasID := false 89for rows .Next () { 90var n ,notNull ,pk int 91var name ,kind string 92var def any 93if err = rows .Scan (& n ,& name ,& kind ,& notNull ,& def ,& pk );err != nil { 94rows .Close () 95db .Close () 96return nil ,err 97 } 98hasID = hasID || name == "request_id" 99 } 100err = rows .Err () 101rows .Close () 102if err != nil { 103db .Close () 104return nil ,err 105 } 106if ! hasID { 107if _ ,err = db .Exec ("ALTER TABLE completions ADD COLUMN request_id TEXT" );err != nil { 108db .Close () 109return nil ,err 110 } 111 } 112if _ ,err = db .Exec (`CREATE INDEX IF NOT EXISTS idx_completions_request_id ON completions(request_id); 113CREATE INDEX IF NOT EXISTS idx_decisions_request_id ON decisions(request_id); 114CREATE INDEX IF NOT EXISTS idx_decisions_created_at ON decisions(created_at); 115CREATE INDEX IF NOT EXISTS idx_evidence_request_id ON evidence(request_id);` );err != nil { 116db .Close () 117return nil ,err 118 } 119return & Log {db :db },nil 120} 121 122// target is where a call's rows go: the main log for authenticated callers, the 123// public log for public callers who opted in, and nowhere for the rest. 124func (l * Log )target (ctx context.Context )* Log { 125if l == nil { 126return nil 127 } 128c := callInfo (ctx ) 129switch { 130case ! c .Public : 131return l 132case c .Contribute : 133return l .public 134default : 135return nil 136 } 137} 138 139func (l * Log )write (query string ,args ... any ) { 140if l == nil { 141return 142 } 143ctx ,cancel := context .WithTimeout (context .Background (),5 * time .Second ) 144defer cancel () 145if _ ,err := l .db .ExecContext (ctx ,query ,args ... );err != nil { 146slog .Error ("training log write failed" ,"error" ,err ) 147 } 148} 149func nullable (s string )any { 150if s == "" { 151return nil 152 } 153return s 154} 155func errorText (err error )any { 156if err == nil { 157return nil 158 } 159return err .Error () 160} 161func jsonText (v any )string { 162b ,err := json .Marshal (v ) 163if err != nil { 164panic (err ) 165 } 166return string (b ) 167} 168 169func (l * Log )completion (ctx context.Context ,task ,model string ,request []byte ,reply string ,err error ,input ,output * int ,latency time.Duration ) { 170c := callInfo (ctx ) 171l .target (ctx ).write (`INSERT INTO completions(created_at,task,request_id,caller,model,request_json,response_text,error,prompt_tokens,completion_tokens,latency_ms) VALUES(?,?,?,?,?,?,?,?,?,?,?)` ,time .Now ().Unix (),task ,c .ID ,nullable (c .Caller ),model ,string (request ),nullable (reply ),errorText (err ),input ,output ,latency .Milliseconds ()) 172} 173func (l * Log )decision (ctx context.Context ,task ,model string ,request ,response []byte ,err error ,latency time.Duration ) { 174l .target (ctx ).write (`INSERT INTO decisions(created_at,request_id,task,model,request_json,response_json,error,latency_ms) VALUES(?,?,?,?,?,?,?,?)` ,time .Now ().Unix (),callInfo (ctx ).ID ,task ,model ,string (request ),nullable (string (response )),errorText (err ),latency .Milliseconds ()) 175} 176func (l * Log )evidence (ctx context.Context ,source string ,p EvidencePack ) { 177l .target (ctx ).write (`INSERT INTO evidence(created_at,request_id,source_id,policy_version,packet_json) VALUES(?,?,?,?,?)` ,time .Now ().Unix (),callInfo (ctx ).ID ,source ,"jev-extractive-v2" ,jsonText (p )) 178} 179func (l * Log )request (ctx context.Context ,endpoint string ,detail any ,outcome string ) { 180c := callInfo (ctx ) 181l .target (ctx ).write (`INSERT INTO requests(created_at,request_id,endpoint,caller,detail,outcome) VALUES(?,?,?,?,?,?)` ,time .Now ().Unix (),c .ID ,endpoint ,nullable (c .Caller ),jsonText (detail ),outcome ) 182}