okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
4.1 KiB127 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"math"
7	"os"
8	"testing"
9	"time"
10
11	"github.com/joho/godotenv"
12)
13
14// Replay exported decision rows against the same queries, passages and rubrics.
15// Opt-in: no production database writes, and no fetching changed live pages.
16func TestLiveJevBatchReplay(t *testing.T) {
17	path := os.Getenv("JEV_REPLAY_PATH")
18	if path == "" {
19		t.Skip("set JEV_REPLAY_PATH to a frozen pre-batching decision export")
20	}
21	data, err := os.ReadFile(path)
22	if err != nil {
23		t.Fatal(err)
24	}
25	var rows []struct {
26		Task    string `json:"task"`
27		Request struct {
28			State struct {
29				Query     string            `json:"query"`
30				Passage   map[string]string `json:"passage"`
31				Candidate map[string]string `json:"candidate"`
32			} `json:"state"`
33			Questions map[string]Question `json:"questions"`
34		} `json:"request"`
35		Response JevResponse `json:"response"`
36	}
37	if err := json.Unmarshal(data, &rows); err != nil || len(rows) == 0 {
38		t.Fatal("invalid replay export", err)
39	}
40	_ = godotenv.Load("../../.env")
41	cfg, err := readConfig()
42	if err != nil {
43		t.Fatal(err)
44	}
45	s, _ := fixtureService(t)
46	s.jev.URL, s.jev.Key, s.jev.Model = cfg.JevURL, cfg.JevKey, cfg.JevModel
47	s.jev.slots = make(chan struct{}, cfg.JevConcurrency)
48	groups := map[string][]int{}
49	order := []string{}
50	oldTokens := 0
51	for i, r := range rows {
52		key := r.Task + "\x00" + r.Request.State.Query + "\x00" + r.Request.State.Passage["url"]
53		if _, ok := groups[key]; !ok {
54			order = append(order, key)
55		}
56		groups[key] = append(groups[key], i)
57		oldTokens += r.Response.Usage.InputTokens
58	}
59	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
60	defer cancel()
61	started := time.Now()
62	decisions, oldKept, retained, newKept := 0, 0, 0, 0
63	lost := []map[string]any{}
64	var absoluteDifference float64
65	keep := func(a map[string]Decision) bool {
66		return a["injection"].Noul < .85 && (a["contradiction"].Noul >= .65 || a["relevant"].Noul >= .35 && a["evidence"].Noul >= .5)
67	}
68	for _, key := range order {
69		indices := groups[key]
70		first := rows[indices[0]]
71		field := "passage"
72		if first.Task == "rerank" {
73			field = "candidate"
74		}
75		items := make([]map[string]string, len(indices))
76		for i, index := range indices {
77			if field == "passage" {
78				items[i] = rows[index].Request.State.Passage
79			} else {
80				items[i] = rows[index].Request.State.Candidate
81			}
82		}
83		answers, err := s.jev.evaluateItems(ctx, first.Task, first.Request.State.Query, field, items, first.Request.Questions)
84		if err != nil {
85			var raw string
86			_ = s.log.db.QueryRow("select coalesce(response_json,'') from decisions where error is not null order by id desc limit 1").Scan(&raw)
87			t.Fatal(err, first.Task, len(items), takeRunes(raw, 1600))
88		}
89		for i, a := range answers {
90			old := rows[indices[i]].Response.Answers
91			for name, v := range a {
92				decisions++
93				absoluteDifference += math.Abs(v.Noul - old[name].Noul)
94			}
95			if field == "passage" {
96				if keep(a) {
97					newKept++
98				}
99				if keep(old) {
100					oldKept++
101					if keep(a) {
102						retained++
103					} else {
104						lost = append(lost, map[string]any{"item": items[i], "old": old, "new": a})
105					}
106				}
107			}
108		}
109	}
110	var calls, newTokens int
111	if err := s.log.db.QueryRow("select count(*),coalesce(sum(json_extract(response_json,'$.usage.input_tokens')),0) from decisions").Scan(&calls, &newTokens); err != nil {
112		t.Fatal(err)
113	}
114	report := map[string]any{"old_calls": len(rows), "new_calls": calls, "decisions": decisions, "seconds": time.Since(started).Seconds(), "mean_absolute_probability_change": absoluteDifference / float64(decisions), "old_eligible": oldKept, "old_eligible_retained": retained, "new_eligible": newKept, "old_input_tokens": oldTokens, "new_input_tokens": newTokens}
115	t.Log(jsonText(report))
116	if path := os.Getenv("JEV_REPLAY_REPORT_PATH"); path != "" {
117		report["lost"] = lost
118		if err := os.WriteFile(path, []byte(jsonText(report)+"\n"), 0600); err != nil {
119			t.Fatal(err)
120		}
121	}
122	if calls >= len(rows) {
123		t.Fatal("batching did not reduce calls")
124	}
125	// Historical model votes are diagnostics, not ground-truth labels. The live
126	// evidence regression separately asserts required facts and counterevidence.
127}