package app import ( "context" "database/sql" "encoding/json" "log/slog" "net/url" "os" "path/filepath" "time" _ "github.com/mattn/go-sqlite3" ) // These two tables deliberately retain the existing training export contract. const logSchema = ` CREATE TABLE IF NOT EXISTS completions ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, task TEXT NOT NULL, caller TEXT, model TEXT NOT NULL, request_json TEXT NOT NULL, response_text TEXT, error TEXT, prompt_tokens INTEGER, completion_tokens INTEGER, latency_ms INTEGER NOT NULL, request_id TEXT ); CREATE INDEX IF NOT EXISTS idx_completions_task ON completions(task); CREATE INDEX IF NOT EXISTS idx_completions_created_at ON completions(created_at); CREATE TABLE IF NOT EXISTS requests ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL, endpoint TEXT NOT NULL, caller TEXT, detail TEXT, outcome TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_requests_created_at ON requests(created_at); CREATE INDEX IF NOT EXISTS idx_requests_request_id ON requests(request_id); CREATE TABLE IF NOT EXISTS decisions ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL, task TEXT NOT NULL, model TEXT NOT NULL, request_json TEXT NOT NULL, response_json TEXT, error TEXT, latency_ms INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS evidence ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL, source_id TEXT NOT NULL, policy_version TEXT NOT NULL, packet_json TEXT NOT NULL );` // Log is the training log. Calls from the public, unauthenticated endpoint are // written only when the caller opts in, and then to public instead. type Log struct { db *sql.DB public *Log } type callContext struct { ID, Caller string // Public marks an unauthenticated call; Contribute marks its opt-in to logging. Public, Contribute bool } type contextKey struct{} func callInfo(ctx context.Context) callContext { c, _ := ctx.Value(contextKey{}).(callContext) return c } func openLog(path string) (*Log, error) { path, err := filepath.Abs(path) if err != nil { return nil, err } if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return nil, err } u := url.URL{Scheme: "file", Path: path} q := u.Query() q.Set("_busy_timeout", "5000") q.Set("_journal_mode", "WAL") u.RawQuery = q.Encode() db, err := sql.Open("sqlite3", u.String()) if err != nil { return nil, err } db.SetMaxOpenConns(1) if _, err = db.Exec(logSchema); err != nil { db.Close() return nil, err } rows, err := db.Query("PRAGMA table_info(completions)") if err != nil { db.Close() return nil, err } hasID := false for rows.Next() { var n, notNull, pk int var name, kind string var def any if err = rows.Scan(&n, &name, &kind, ¬Null, &def, &pk); err != nil { rows.Close() db.Close() return nil, err } hasID = hasID || name == "request_id" } err = rows.Err() rows.Close() if err != nil { db.Close() return nil, err } if !hasID { if _, err = db.Exec("ALTER TABLE completions ADD COLUMN request_id TEXT"); err != nil { db.Close() return nil, err } } if _, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_completions_request_id ON completions(request_id); CREATE INDEX IF NOT EXISTS idx_decisions_request_id ON decisions(request_id); CREATE INDEX IF NOT EXISTS idx_decisions_created_at ON decisions(created_at); CREATE INDEX IF NOT EXISTS idx_evidence_request_id ON evidence(request_id);`); err != nil { db.Close() return nil, err } return &Log{db: db}, nil } // target is where a call's rows go: the main log for authenticated callers, the // public log for public callers who opted in, and nowhere for the rest. func (l *Log) target(ctx context.Context) *Log { if l == nil { return nil } c := callInfo(ctx) switch { case !c.Public: return l case c.Contribute: return l.public default: return nil } } func (l *Log) write(query string, args ...any) { if l == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if _, err := l.db.ExecContext(ctx, query, args...); err != nil { slog.Error("training log write failed", "error", err) } } func nullable(s string) any { if s == "" { return nil } return s } func errorText(err error) any { if err == nil { return nil } return err.Error() } func jsonText(v any) string { b, err := json.Marshal(v) if err != nil { panic(err) } return string(b) } func (l *Log) completion(ctx context.Context, task, model string, request []byte, reply string, err error, input, output *int, latency time.Duration) { c := callInfo(ctx) l.target(ctx).write(`INSERT INTO completions(created_at,task,request_id,caller,model,request_json,response_text,error,prompt_tokens,completion_tokens,latency_ms) VALUES(?,?,?,?,?,?,?,?,?,?,?)`, time.Now().Unix(), task, c.ID, nullable(c.Caller), model, string(request), nullable(reply), errorText(err), input, output, latency.Milliseconds()) } func (l *Log) decision(ctx context.Context, task, model string, request, response []byte, err error, latency time.Duration) { l.target(ctx).write(`INSERT INTO decisions(created_at,request_id,task,model,request_json,response_json,error,latency_ms) VALUES(?,?,?,?,?,?,?,?)`, time.Now().Unix(), callInfo(ctx).ID, task, model, string(request), nullable(string(response)), errorText(err), latency.Milliseconds()) } func (l *Log) evidence(ctx context.Context, source string, p EvidencePack) { l.target(ctx).write(`INSERT INTO evidence(created_at,request_id,source_id,policy_version,packet_json) VALUES(?,?,?,?,?)`, time.Now().Unix(), callInfo(ctx).ID, source, "jev-extractive-v2", jsonText(p)) } func (l *Log) request(ctx context.Context, endpoint string, detail any, outcome string) { c := callInfo(ctx) l.target(ctx).write(`INSERT INTO requests(created_at,request_id,endpoint,caller,detail,outcome) VALUES(?,?,?,?,?,?)`, time.Now().Unix(), c.ID, endpoint, nullable(c.Caller), jsonText(detail), outcome) }