package app import ( "context" "encoding/json" "net/http" "net/http/httptest" "os" "strings" "sync/atomic" "testing" "time" "github.com/joho/godotenv" ) func frozenPages(t *testing.T) []Document { t.Helper() b, err := os.ReadFile("testdata/evidence-pages.json") if err != nil { t.Fatal(err) } var pages []Document if err := json.Unmarshal(b, &pages); err != nil { t.Fatal(err) } return pages } func TestShortlistRetainsFalsePremiseCorrection(t *testing.T) { d := frozenPages(t)[0] q := "How does SQLite WAL allow multiple writers to write simultaneously?" selected := shortlist(passages(d.Markdown), q, 24) if !strings.Contains(evidenceText(selected), "only be one writer at a time") { t.Fatal("the lexical shortlist discarded the explicit correction before jev could judge it") } if len(selected) > 24 { t.Fatal("candidate cap exceeded") } } func TestOversizedBlocksDoNotConsumeCandidateSlots(t *testing.T) { s, _ := fixtureService(t) d := Document{URL: "https://example.com", FinalURL: "https://example.com", Markdown: strings.Repeat("request limits ", 100) + "\n\nrequests may contain up to 8 mib."} pack, err := s.jev.selectEvidence(context.Background(), d, "request limits", 1, 200) if err != nil { t.Fatal(err) } if len(pack.Passages) != 1 || !strings.Contains(pack.Passages[0].Text, "8 mib") { t.Fatalf("lost eligible passage: %+v", pack) } } func TestEmptySelectionDescribesCoverageNotSourceAbsence(t *testing.T) { s, _ := fixtureService(t) reply, err := s.llm.preview(context.Background(), Document{}, EvidencePack{}, "none") if err != nil || reply != "no usable evidence was selected from the inspected passages for this query." { t.Fatalf("overstated empty selection: %q, %v", reply, err) } var n int if err := s.log.db.QueryRow("select count(*) from completions").Scan(&n); err != nil || n != 0 { t.Fatalf("fallback must not become a training completion: %d, %v", n, err) } } func TestSummaryRetriesAreBoundedAndPreserveRawTrainingReplies(t *testing.T) { for _, alwaysInvalid := range []bool{false, true} { name := "corrected" if alwaysInvalid { name = "still_invalid" } t.Run(name, func(t *testing.T) { s, _ := fixtureService(t) var calls atomic.Int32 provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var body CompletionRequest if err := json.NewDecoder(r.Body).Decode(&body); err != nil { t.Error(err) } n := calls.Add(1) if n > 1 && !strings.Contains(body.Messages[0].Content, "Format correction") { t.Error("missing format-only retry instruction") } reply := `{"summary":"use errors.Is for equivalence."}` if n > 1 && !alwaysInvalid { reply = "use errors.Is for equivalence." } writeJSON(w, 200, map[string]any{"model": "test", "choices": []any{map[string]any{"message": Message{"assistant", reply}, "finish_reason": "stop"}}}) })) defer provider.Close() s.llm.URL = provider.URL reply, err := s.llm.complete(context.Background(), "preview_summary", previewSystem, "original evidence", 160) if calls.Load() != 2 || (err != nil) != alwaysInvalid { t.Fatalf("calls=%d error=%v", calls.Load(), err) } if !alwaysInvalid && usableSummary(reply) != nil { t.Fatal("returned invalid prose") } var raw, request string var failed bool if err := s.log.db.QueryRow("select response_text,request_json,error is not null from completions order by id limit 1").Scan(&raw, &request, &failed); err != nil { t.Fatal(err) } if raw != `{"summary":"use errors.Is for equivalence."}` || !failed { t.Fatalf("rejected raw reply lost or counted as valid: %q %v", raw, failed) } var body map[string]json.RawMessage if err := json.Unmarshal([]byte(request), &body); err != nil || len(body) != 5 || string(body["chat_template_kwargs"]) != `{"enable_thinking":false}` { t.Fatal("training request shape changed", request) } var successful int if err := s.log.db.QueryRow("select count(*) from completions where error is null").Scan(&successful); err != nil { t.Fatal(err) } wantSuccessful := 1 if alwaysInvalid { wantSuccessful = 0 } if successful != wantSuccessful { t.Fatalf("wrong successful row count %d", successful) } }) } } func TestLiveEvidenceRegression(t *testing.T) { if os.Getenv("LIVE_EVIDENCE_REGRESSION") != "1" { t.Skip("opt in to real jev and summarizer calls on frozen source pages") } _ = godotenv.Load("../../.env") cfg, err := readConfig() if err != nil { t.Fatal(err) } s, _ := fixtureService(t) s.jev.URL, s.jev.Key, s.jev.Model = cfg.JevURL, cfg.JevKey, cfg.JevModel s.llm.URL, s.llm.Key, s.llm.Model = cfg.LLMURL, cfg.LLMKey, cfg.LLMModel pages := frozenPages(t) var report []map[string]any for _, tc := range []struct { name, query string page int want []string }{ {"counterevidence", "How does SQLite WAL allow multiple writers to write simultaneously?", 0, []string{"one writer"}}, {"partial_answer", "Go errors Is As wrapped errors difference percent w", 1, []string{"%w", "Is", "As"}}, } { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() ctx = context.WithValue(ctx, contextKey{}, callContext{ID: newID(), Caller: "frozen-regression"}) started := time.Now() p, err := s.jev.selectEvidence(ctx, pages[tc.page], tc.query, 24, 5500) if err != nil { t.Fatal(err) } label, err := s.jev.answerability(ctx, tc.query, p) if err != nil { t.Fatal(err) } summary, err := s.llm.preview(ctx, pages[tc.page], p, label) entry := map[string]any{"case": tc.name, "pack": p, "label": label, "summary": summary, "seconds": time.Since(started).Seconds()} if err != nil { entry["error"] = err.Error() } report = append(report, entry) if path := os.Getenv("EVIDENCE_REPORT_PATH"); path != "" { b, _ := json.MarshalIndent(report, "", " ") if err := os.WriteFile(path, b, 0600); err != nil { t.Fatal(err) } } if err != nil { t.Fatal(err) } for _, want := range tc.want { if !strings.Contains(evidenceText(p.Passages), want) { t.Errorf("selected evidence missed %q", want) } if !strings.Contains(summary, want) { t.Errorf("summary missed %q: %s", want, summary) } } if err := usableSummary(summary); err != nil { t.Error(err) } t.Logf("%d passages, %d chars, %s: %s", len(p.Passages), p.UsedChars, label, summary) }) } }