okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petsummaries: thinking off for previews, capped at 256 tokens for answers3009bc8

main
6.4 KiB190 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"net/http"
7	"net/http/httptest"
8	"os"
9	"strings"
10	"sync/atomic"
11	"testing"
12	"time"
13
14	"github.com/joho/godotenv"
15)
16
17func frozenPages(t *testing.T) []Document {
18	t.Helper()
19	b, err := os.ReadFile("testdata/evidence-pages.json")
20	if err != nil {
21		t.Fatal(err)
22	}
23	var pages []Document
24	if err := json.Unmarshal(b, &pages); err != nil {
25		t.Fatal(err)
26	}
27	return pages
28}
29
30func TestShortlistRetainsFalsePremiseCorrection(t *testing.T) {
31	d := frozenPages(t)[0]
32	q := "How does SQLite WAL allow multiple writers to write simultaneously?"
33	selected := shortlist(passages(d.Markdown), q, 24)
34	if !strings.Contains(evidenceText(selected), "only be one writer at a time") {
35		t.Fatal("the lexical shortlist discarded the explicit correction before jev could judge it")
36	}
37	if len(selected) > 24 {
38		t.Fatal("candidate cap exceeded")
39	}
40}
41
42func TestOversizedBlocksDoNotConsumeCandidateSlots(t *testing.T) {
43	s, _ := fixtureService(t)
44	d := Document{URL: "https://example.com", FinalURL: "https://example.com", Markdown: strings.Repeat("request limits ", 100) + "\n\nrequests may contain up to 8 mib."}
45	pack, err := s.jev.selectEvidence(context.Background(), d, "request limits", 1, 200)
46	if err != nil {
47		t.Fatal(err)
48	}
49	if len(pack.Passages) != 1 || !strings.Contains(pack.Passages[0].Text, "8 mib") {
50		t.Fatalf("lost eligible passage: %+v", pack)
51	}
52}
53
54func TestEmptySelectionDescribesCoverageNotSourceAbsence(t *testing.T) {
55	s, _ := fixtureService(t)
56	reply, err := s.llm.preview(context.Background(), Document{}, EvidencePack{}, "none")
57	if err != nil || reply != "no usable evidence was selected from the inspected passages for this query." {
58		t.Fatalf("overstated empty selection: %q, %v", reply, err)
59	}
60	var n int
61	if err := s.log.db.QueryRow("select count(*) from completions").Scan(&n); err != nil || n != 0 {
62		t.Fatalf("fallback must not become a training completion: %d, %v", n, err)
63	}
64}
65
66func TestSummaryRetriesAreBoundedAndPreserveRawTrainingReplies(t *testing.T) {
67	for _, alwaysInvalid := range []bool{false, true} {
68		name := "corrected"
69		if alwaysInvalid {
70			name = "still_invalid"
71		}
72		t.Run(name, func(t *testing.T) {
73			s, _ := fixtureService(t)
74			var calls atomic.Int32
75			provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
76				var body CompletionRequest
77				if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
78					t.Error(err)
79				}
80				n := calls.Add(1)
81				if n > 1 && !strings.Contains(body.Messages[0].Content, "Format correction") {
82					t.Error("missing format-only retry instruction")
83				}
84				reply := `{"summary":"use errors.Is for equivalence."}`
85				if n > 1 && !alwaysInvalid {
86					reply = "use errors.Is for equivalence."
87				}
88				writeJSON(w, 200, map[string]any{"model": "test", "choices": []any{map[string]any{"message": Message{"assistant", reply}, "finish_reason": "stop"}}})
89			}))
90			defer provider.Close()
91			s.llm.URL = provider.URL
92			reply, err := s.llm.complete(context.Background(), "preview_summary", previewSystem, "original evidence", 160)
93			if calls.Load() != 2 || (err != nil) != alwaysInvalid {
94				t.Fatalf("calls=%d error=%v", calls.Load(), err)
95			}
96			if !alwaysInvalid && usableSummary(reply) != nil {
97				t.Fatal("returned invalid prose")
98			}
99			var raw, request string
100			var failed bool
101			if err := s.log.db.QueryRow("select response_text,request_json,error is not null from completions order by id limit 1").Scan(&raw, &request, &failed); err != nil {
102				t.Fatal(err)
103			}
104			if raw != `{"summary":"use errors.Is for equivalence."}` || !failed {
105				t.Fatalf("rejected raw reply lost or counted as valid: %q %v", raw, failed)
106			}
107			var body map[string]json.RawMessage
108			if err := json.Unmarshal([]byte(request), &body); err != nil || len(body) != 5 || string(body["chat_template_kwargs"]) != `{"enable_thinking":false}` {
109				t.Fatal("training request shape changed", request)
110			}
111			var successful int
112			if err := s.log.db.QueryRow("select count(*) from completions where error is null").Scan(&successful); err != nil {
113				t.Fatal(err)
114			}
115			wantSuccessful := 1
116			if alwaysInvalid {
117				wantSuccessful = 0
118			}
119			if successful != wantSuccessful {
120				t.Fatalf("wrong successful row count %d", successful)
121			}
122		})
123	}
124}
125
126func TestLiveEvidenceRegression(t *testing.T) {
127	if os.Getenv("LIVE_EVIDENCE_REGRESSION") != "1" {
128		t.Skip("opt in to real jev and summarizer calls on frozen source pages")
129	}
130	_ = godotenv.Load("../../.env")
131	cfg, err := readConfig()
132	if err != nil {
133		t.Fatal(err)
134	}
135	s, _ := fixtureService(t)
136	s.jev.URL, s.jev.Key, s.jev.Model = cfg.JevURL, cfg.JevKey, cfg.JevModel
137	s.llm.URL, s.llm.Key, s.llm.Model = cfg.LLMURL, cfg.LLMKey, cfg.LLMModel
138	pages := frozenPages(t)
139	var report []map[string]any
140	for _, tc := range []struct {
141		name, query string
142		page        int
143		want        []string
144	}{
145		{"counterevidence", "How does SQLite WAL allow multiple writers to write simultaneously?", 0, []string{"one writer"}},
146		{"partial_answer", "Go errors Is As wrapped errors difference percent w", 1, []string{"%w", "Is", "As"}},
147	} {
148		t.Run(tc.name, func(t *testing.T) {
149			ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
150			defer cancel()
151			ctx = context.WithValue(ctx, contextKey{}, callContext{ID: newID(), Caller: "frozen-regression"})
152			started := time.Now()
153			p, err := s.jev.selectEvidence(ctx, pages[tc.page], tc.query, 24, 5500)
154			if err != nil {
155				t.Fatal(err)
156			}
157			label, err := s.jev.answerability(ctx, tc.query, p)
158			if err != nil {
159				t.Fatal(err)
160			}
161			summary, err := s.llm.preview(ctx, pages[tc.page], p, label)
162			entry := map[string]any{"case": tc.name, "pack": p, "label": label, "summary": summary, "seconds": time.Since(started).Seconds()}
163			if err != nil {
164				entry["error"] = err.Error()
165			}
166			report = append(report, entry)
167			if path := os.Getenv("EVIDENCE_REPORT_PATH"); path != "" {
168				b, _ := json.MarshalIndent(report, "", "  ")
169				if err := os.WriteFile(path, b, 0600); err != nil {
170					t.Fatal(err)
171				}
172			}
173			if err != nil {
174				t.Fatal(err)
175			}
176			for _, want := range tc.want {
177				if !strings.Contains(evidenceText(p.Passages), want) {
178					t.Errorf("selected evidence missed %q", want)
179				}
180				if !strings.Contains(summary, want) {
181					t.Errorf("summary missed %q: %s", want, summary)
182				}
183			}
184			if err := usableSummary(summary); err != nil {
185				t.Error(err)
186			}
187			t.Logf("%d passages, %d chars, %s: %s", len(p.Passages), p.UsedChars, label, summary)
188		})
189	}
190}