okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petdecisions: ask a local model first and keep only its confident answers70a1e29

main
2.7 KiB80 linesraw
1package app
2
3import (
4	"encoding/json"
5	"net/http"
6	"net/http/httptest"
7	"path/filepath"
8	"sync"
9	"testing"
10)
11
12// decisionServer answers every noul question with the probability given for
13// its id (default 0.5) and records the question ids it was asked.
14func decisionServer(t *testing.T, scores map[string]float64, status int) (*httptest.Server, *[][]string) {
15	t.Helper()
16	var mu sync.Mutex
17	var asked [][]string
18	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
19		var req struct {
20			Questions map[string]Question `json:"questions"`
21		}
22		_ = json.NewDecoder(r.Body).Decode(&req)
23		ids := []string{}
24		answers := map[string]any{}
25		for id := range req.Questions {
26			ids = append(ids, id)
27			p, ok := scores[id]
28			if !ok {
29				p = 0.5
30			}
31			answers[id] = map[string]any{"type": "noul", "noul": p}
32		}
33		mu.Lock()
34		asked = append(asked, ids)
35		mu.Unlock()
36		if status != 200 {
37			w.WriteHeader(status)
38			return
39		}
40		writeJSON(w, 200, map[string]any{"model": "m", "answers": answers})
41	}))
42	t.Cleanup(srv.Close)
43	return srv, &asked
44}
45
46func TestLocalTierKeepsConfidentAnswersAndForwardsTheRest(t *testing.T) {
47	log, err := openLog(filepath.Join(t.TempDir(), "log.db"))
48	if err != nil {
49		t.Fatal(err)
50	}
51	t.Cleanup(func() { log.db.Close() })
52	local, _ := decisionServer(t, map[string]float64{"0_relevant": 0.02, "0_evidence": 0.97, "1_relevant": 0.5}, 200)
53	remote, asked := decisionServer(t, map[string]float64{"1_relevant": 0.8, "0_injection": 0.01}, 200)
54	mk := func(url string) *Jev {
55		return &Jev{URL: url, Key: "k", Model: "m", HTTP: http.DefaultClient, Log: log, slots: make(chan struct{}, 2)}
56	}
57	j := mk(remote.URL)
58	j.Local = &LocalTier{Jev: mk(local.URL), Names: map[string]bool{"relevant": true, "evidence": true}, Low: 0.1, High: 0.9}
59	qs := map[string]Question{"0_relevant": noul("r"), "0_evidence": noul("e"), "1_relevant": noul("r"), "0_injection": noul("i")}
60	out, err := j.evaluate(t.Context(), "passage", map[string]string{"query": "q"}, qs)
61	if err != nil {
62		t.Fatal(err)
63	}
64	if len(*asked) != 1 || len((*asked)[0]) != 2 {
65		t.Fatalf("remote should get only the unsure and non-local questions, got %v", *asked)
66	}
67	want := map[string]float64{"0_relevant": 0.02, "0_evidence": 0.97, "1_relevant": 0.8, "0_injection": 0.01}
68	for id, p := range want {
69		if out.Answers[id].Noul != p {
70			t.Fatalf("%s: got %v want %v", id, out.Answers[id].Noul, p)
71		}
72	}
73
74	broken, _ := decisionServer(t, nil, 500)
75	j.Local.Jev = mk(broken.URL)
76	*asked = nil
77	if _, err := j.evaluate(t.Context(), "passage", map[string]string{"query": "q"}, qs); err != nil || len(*asked) != 1 || len((*asked)[0]) != 4 {
78		t.Fatalf("a failing local model must send everything remote: %v %v", err, *asked)
79	}
80}