package app import ( "context" "encoding/json" "math" "os" "testing" "time" "github.com/joho/godotenv" ) // Replay exported decision rows against the same queries, passages and rubrics. // Opt-in: no production database writes, and no fetching changed live pages. func TestLiveJevBatchReplay(t *testing.T) { path := os.Getenv("JEV_REPLAY_PATH") if path == "" { t.Skip("set JEV_REPLAY_PATH to a frozen pre-batching decision export") } data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var rows []struct { Task string `json:"task"` Request struct { State struct { Query string `json:"query"` Passage map[string]string `json:"passage"` Candidate map[string]string `json:"candidate"` } `json:"state"` Questions map[string]Question `json:"questions"` } `json:"request"` Response JevResponse `json:"response"` } if err := json.Unmarshal(data, &rows); err != nil || len(rows) == 0 { t.Fatal("invalid replay export", err) } _ = godotenv.Load("../../.env") cfg, err := readConfig() if err != nil { t.Fatal(err) } s, _ := fixtureService(t) s.jev.URL, s.jev.Key, s.jev.Model = cfg.JevURL, cfg.JevKey, cfg.JevModel s.jev.slots = make(chan struct{}, cfg.JevConcurrency) groups := map[string][]int{} order := []string{} oldTokens := 0 for i, r := range rows { key := r.Task + "\x00" + r.Request.State.Query + "\x00" + r.Request.State.Passage["url"] if _, ok := groups[key]; !ok { order = append(order, key) } groups[key] = append(groups[key], i) oldTokens += r.Response.Usage.InputTokens } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() started := time.Now() decisions, oldKept, retained, newKept := 0, 0, 0, 0 lost := []map[string]any{} var absoluteDifference float64 keep := func(a map[string]Decision) bool { return a["injection"].Noul < .85 && (a["contradiction"].Noul >= .65 || a["relevant"].Noul >= .35 && a["evidence"].Noul >= .5) } for _, key := range order { indices := groups[key] first := rows[indices[0]] field := "passage" if first.Task == "rerank" { field = "candidate" } items := make([]map[string]string, len(indices)) for i, index := range indices { if field == "passage" { items[i] = rows[index].Request.State.Passage } else { items[i] = rows[index].Request.State.Candidate } } answers, err := s.jev.evaluateItems(ctx, first.Task, first.Request.State.Query, field, items, first.Request.Questions) if err != nil { var raw string _ = s.log.db.QueryRow("select coalesce(response_json,'') from decisions where error is not null order by id desc limit 1").Scan(&raw) t.Fatal(err, first.Task, len(items), takeRunes(raw, 1600)) } for i, a := range answers { old := rows[indices[i]].Response.Answers for name, v := range a { decisions++ absoluteDifference += math.Abs(v.Noul - old[name].Noul) } if field == "passage" { if keep(a) { newKept++ } if keep(old) { oldKept++ if keep(a) { retained++ } else { lost = append(lost, map[string]any{"item": items[i], "old": old, "new": a}) } } } } } var calls, newTokens int if err := s.log.db.QueryRow("select count(*),coalesce(sum(json_extract(response_json,'$.usage.input_tokens')),0) from decisions").Scan(&calls, &newTokens); err != nil { t.Fatal(err) } report := map[string]any{"old_calls": len(rows), "new_calls": calls, "decisions": decisions, "seconds": time.Since(started).Seconds(), "mean_absolute_probability_change": absoluteDifference / float64(decisions), "old_eligible": oldKept, "old_eligible_retained": retained, "new_eligible": newKept, "old_input_tokens": oldTokens, "new_input_tokens": newTokens} t.Log(jsonText(report)) if path := os.Getenv("JEV_REPLAY_REPORT_PATH"); path != "" { report["lost"] = lost if err := os.WriteFile(path, []byte(jsonText(report)+"\n"), 0600); err != nil { t.Fatal(err) } } if calls >= len(rows) { t.Fatal("batching did not reduce calls") } // Historical model votes are diagnostics, not ground-truth labels. The live // evidence regression separately asserts required facts and counterevidence. }