okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petpublic access: serve token-less callers with rate limits and opt-in logging84fcc80

main
5.9 KiB182 linesraw
1package app
2
3import (
4	"context"
5	"database/sql"
6	"encoding/json"
7	"log/slog"
8	"net/url"
9	"os"
10	"path/filepath"
11	"time"
12
13	_ "github.com/mattn/go-sqlite3"
14)
15
16// These two tables deliberately retain the existing training export contract.
17const logSchema = `
18CREATE TABLE IF NOT EXISTS completions (
19 id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, task TEXT NOT NULL,
20 caller TEXT, model TEXT NOT NULL, request_json TEXT NOT NULL, response_text TEXT,
21 error TEXT, prompt_tokens INTEGER, completion_tokens INTEGER, latency_ms INTEGER NOT NULL,
22 request_id TEXT
23);
24CREATE INDEX IF NOT EXISTS idx_completions_task ON completions(task);
25CREATE INDEX IF NOT EXISTS idx_completions_created_at ON completions(created_at);
26CREATE TABLE IF NOT EXISTS requests (
27 id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL,
28 endpoint TEXT NOT NULL, caller TEXT, detail TEXT, outcome TEXT NOT NULL
29);
30CREATE INDEX IF NOT EXISTS idx_requests_created_at ON requests(created_at);
31CREATE INDEX IF NOT EXISTS idx_requests_request_id ON requests(request_id);
32CREATE TABLE IF NOT EXISTS decisions (
33 id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL,
34 task TEXT NOT NULL, model TEXT NOT NULL, request_json TEXT NOT NULL, response_json TEXT,
35 error TEXT, latency_ms INTEGER NOT NULL
36);
37CREATE TABLE IF NOT EXISTS evidence (
38 id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, request_id TEXT NOT NULL,
39 source_id TEXT NOT NULL, policy_version TEXT NOT NULL, packet_json TEXT NOT NULL
40);`
41
42// Log is the training log. Calls from the public, unauthenticated endpoint are
43// written only when the caller opts in, and then to public instead.
44type Log struct {
45	db     *sql.DB
46	public *Log
47}
48
49type callContext struct {
50	ID, Caller string
51	// Public marks an unauthenticated call; Contribute marks its opt-in to logging.
52	Public, Contribute bool
53}
54type contextKey struct{}
55
56func callInfo(ctx context.Context) callContext {
57	c, _ := ctx.Value(contextKey{}).(callContext)
58	return c
59}
60
61func openLog(path string) (*Log, error) {
62	path, err := filepath.Abs(path)
63	if err != nil {
64		return nil, err
65	}
66	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
67		return nil, err
68	}
69	u := url.URL{Scheme: "file", Path: path}
70	q := u.Query()
71	q.Set("_busy_timeout", "5000")
72	q.Set("_journal_mode", "WAL")
73	u.RawQuery = q.Encode()
74	db, err := sql.Open("sqlite3", u.String())
75	if err != nil {
76		return nil, err
77	}
78	db.SetMaxOpenConns(1)
79	if _, err = db.Exec(logSchema); err != nil {
80		db.Close()
81		return nil, err
82	}
83	rows, err := db.Query("PRAGMA table_info(completions)")
84	if err != nil {
85		db.Close()
86		return nil, err
87	}
88	hasID := false
89	for rows.Next() {
90		var n, notNull, pk int
91		var name, kind string
92		var def any
93		if err = rows.Scan(&n, &name, &kind, &notNull, &def, &pk); err != nil {
94			rows.Close()
95			db.Close()
96			return nil, err
97		}
98		hasID = hasID || name == "request_id"
99	}
100	err = rows.Err()
101	rows.Close()
102	if err != nil {
103		db.Close()
104		return nil, err
105	}
106	if !hasID {
107		if _, err = db.Exec("ALTER TABLE completions ADD COLUMN request_id TEXT"); err != nil {
108			db.Close()
109			return nil, err
110		}
111	}
112	if _, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_completions_request_id ON completions(request_id);
113	CREATE INDEX IF NOT EXISTS idx_decisions_request_id ON decisions(request_id);
114	CREATE INDEX IF NOT EXISTS idx_decisions_created_at ON decisions(created_at);
115	CREATE INDEX IF NOT EXISTS idx_evidence_request_id ON evidence(request_id);`); err != nil {
116		db.Close()
117		return nil, err
118	}
119	return &Log{db: db}, nil
120}
121
122// target is where a call's rows go: the main log for authenticated callers, the
123// public log for public callers who opted in, and nowhere for the rest.
124func (l *Log) target(ctx context.Context) *Log {
125	if l == nil {
126		return nil
127	}
128	c := callInfo(ctx)
129	switch {
130	case !c.Public:
131		return l
132	case c.Contribute:
133		return l.public
134	default:
135		return nil
136	}
137}
138
139func (l *Log) write(query string, args ...any) {
140	if l == nil {
141		return
142	}
143	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
144	defer cancel()
145	if _, err := l.db.ExecContext(ctx, query, args...); err != nil {
146		slog.Error("training log write failed", "error", err)
147	}
148}
149func nullable(s string) any {
150	if s == "" {
151		return nil
152	}
153	return s
154}
155func errorText(err error) any {
156	if err == nil {
157		return nil
158	}
159	return err.Error()
160}
161func jsonText(v any) string {
162	b, err := json.Marshal(v)
163	if err != nil {
164		panic(err)
165	}
166	return string(b)
167}
168
169func (l *Log) completion(ctx context.Context, task, model string, request []byte, reply string, err error, input, output *int, latency time.Duration) {
170	c := callInfo(ctx)
171	l.target(ctx).write(`INSERT INTO completions(created_at,task,request_id,caller,model,request_json,response_text,error,prompt_tokens,completion_tokens,latency_ms) VALUES(?,?,?,?,?,?,?,?,?,?,?)`, time.Now().Unix(), task, c.ID, nullable(c.Caller), model, string(request), nullable(reply), errorText(err), input, output, latency.Milliseconds())
172}
173func (l *Log) decision(ctx context.Context, task, model string, request, response []byte, err error, latency time.Duration) {
174	l.target(ctx).write(`INSERT INTO decisions(created_at,request_id,task,model,request_json,response_json,error,latency_ms) VALUES(?,?,?,?,?,?,?,?)`, time.Now().Unix(), callInfo(ctx).ID, task, model, string(request), nullable(string(response)), errorText(err), latency.Milliseconds())
175}
176func (l *Log) evidence(ctx context.Context, source string, p EvidencePack) {
177	l.target(ctx).write(`INSERT INTO evidence(created_at,request_id,source_id,policy_version,packet_json) VALUES(?,?,?,?,?)`, time.Now().Unix(), callInfo(ctx).ID, source, "jev-extractive-v2", jsonText(p))
178}
179func (l *Log) request(ctx context.Context, endpoint string, detail any, outcome string) {
180	c := callInfo(ctx)
181	l.target(ctx).write(`INSERT INTO requests(created_at,request_id,endpoint,caller,detail,outcome) VALUES(?,?,?,?,?,?)`, time.Now().Unix(), c.ID, endpoint, nullable(c.Caller), jsonText(detail), outcome)
182}