okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
5.7 KiB166 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"fmt"
7	"io"
8	"net/http"
9	"net/http/httptest"
10	"strconv"
11	"strings"
12	"sync/atomic"
13	"testing"
14	"time"
15)
16
17func TestJevBatchesKeepItemsIsolatedAndMapAnswers(t *testing.T) {
18	for _, tc := range []struct {
19		name, text   string
20		items, calls int
21	}{
22		{"empty", "", 0, 0},
23		{"whole_page", "small", 24, 1},
24		{"question_limit", "small", 61, 3},
25		{"byte_limit", strings.Repeat("界<", 2400), 3, 3},
26	} {
27		t.Run(tc.name, func(t *testing.T) {
28			var calls atomic.Int32
29			provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
30				calls.Add(1)
31				body, _ := io.ReadAll(r.Body)
32				var req struct {
33					State     map[string]string `json:"state"`
34					Questions map[string]struct {
35						Instructions struct {
36							Question string            `json:"question"`
37							Passage  map[string]string `json:"passage"`
38						} `json:"instructions"`
39					} `json:"questions"`
40				}
41				if err := json.Unmarshal(body, &req); err != nil {
42					t.Error(err)
43				}
44				if len(body) > maxJevRequestBytes || len(req.Questions) > 96 || len(req.State) != 1 || req.State["query"] != "limits" {
45					t.Error("batch exceeded budget or leaked items into shared state")
46				}
47				answers := map[string]Decision{}
48				for id, q := range req.Questions {
49					index, _ := strconv.Atoi(strings.SplitN(id, "_", 2)[0])
50					if q.Instructions.Passage["heading"] != strconv.Itoa(index) || q.Instructions.Passage["text"] != tc.text || q.Instructions.Question == "" {
51						t.Error("question lost its isolated passage", id)
52					}
53					answers[id] = Decision{Type: "noul", Noul: float64(index) / 100}
54				}
55				writeJSON(w, 200, JevResponse{Model: "test", Answers: answers})
56			}))
57			defer provider.Close()
58			j := &Jev{URL: provider.URL, Key: "key", Model: "test", HTTP: provider.Client(), slots: make(chan struct{}, 2)}
59			items := make([]map[string]string, tc.items)
60			for i := range items {
61				items[i] = map[string]string{"heading": strconv.Itoa(i), "text": tc.text}
62			}
63			answers, err := j.evaluateItems(context.Background(), "passage", "limits", "passage", items, passageQuestions)
64			if err != nil || len(answers) != tc.items || int(calls.Load()) != tc.calls {
65				t.Fatal(err, len(answers), calls.Load())
66			}
67			for i, a := range answers {
68				for name := range passageQuestions {
69					if a[name].Noul != float64(i)/100 {
70						t.Fatal("misrouted answer", i, name, a[name])
71					}
72				}
73			}
74		})
75	}
76}
77
78func TestJevBatchRejectsOversizedItemsBeforeCalling(t *testing.T) {
79	j := &Jev{Model: "test"}
80	items := []map[string]string{{"text": "small"}, {"text": strings.Repeat("x", maxJevRequestBytes)}}
81	if _, err := j.evaluateItems(context.Background(), "passage", "limits", "passage", items, passageQuestions); err == nil {
82		t.Fatal("oversized item accepted")
83	}
84}
85
86func TestJevBatchesRespectGlobalConcurrencyAndCancellation(t *testing.T) {
87	started, release := make(chan struct{}, 3), make(chan struct{})
88	var active, peak atomic.Int32
89	provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
90		n := active.Add(1)
91		defer active.Add(-1)
92		for old := peak.Load(); n > old; old = peak.Load() {
93			if peak.CompareAndSwap(old, n) {
94				break
95			}
96		}
97		started <- struct{}{}
98		select {
99		case <-release:
100		case <-r.Context().Done():
101			return
102		}
103		var req struct {
104			Questions map[string]Question `json:"questions"`
105		}
106		_ = json.NewDecoder(r.Body).Decode(&req)
107		answers := map[string]Decision{}
108		for id := range req.Questions {
109			answers[id] = Decision{Type: "noul", Noul: .5}
110		}
111		writeJSON(w, 200, JevResponse{Model: "test", Answers: answers})
112	}))
113	defer provider.Close()
114	j := &Jev{URL: provider.URL, Key: "key", Model: "test", HTTP: provider.Client(), slots: make(chan struct{}, 2)}
115	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
116	defer cancel()
117	done := make(chan error, 1)
118	items := make([]map[string]string, 60)
119	go func() {
120		_, err := j.evaluateItems(ctx, "passage", "limits", "passage", items, passageQuestions)
121		done <- err
122	}()
123	for range 2 {
124		select {
125		case <-started:
126		case <-ctx.Done():
127			t.Fatal("batches did not run concurrently")
128		}
129	}
130	close(release)
131	if err := <-done; err != nil || peak.Load() != 2 {
132		t.Fatal(err, peak.Load())
133	}
134	canceled, stop := context.WithCancel(context.Background())
135	stop()
136	if _, err := j.evaluateItems(canceled, "passage", "limits", "passage", items, passageQuestions); err == nil {
137		t.Fatal("canceled batch succeeded")
138	}
139}
140
141func TestDeadlineReturnsCompletedSourcesAndTimeoutStatuses(t *testing.T) {
142	s, page := fixtureService(t)
143	s.cfg.Concurrency = 1
144	s.cfg.RequestTimeout = 100 * time.Millisecond
145	slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() }))
146	defer slow.Close()
147	status, payload := callHTTP(t, s, "/contents", fmt.Sprintf(`{"urls":[%q,%q,%q]}`, page.URL, slow.URL, slow.URL+"/queued"))
148	var out ResultsResponse
149	_ = json.Unmarshal([]byte(jsonText(payload)), &out)
150	if status != 200 || len(out.Results) != 1 || len(out.Statuses) != 3 {
151		t.Fatal(status, out)
152	}
153	for i, item := range out.Statuses {
154		if item.ID == "" || i == 0 && item.Status != "success" || i > 0 && (item.Error == nil || item.Error.Tag != "REQUEST_TIMEOUT" || item.Error.HTTPStatusCode != 504) {
155			t.Fatal("bad per-url status", item)
156		}
157	}
158	var outcome, detail string
159	if err := s.log.db.QueryRow("select outcome,detail from requests where request_id=? and endpoint='contents'", out.RequestID).Scan(&outcome, &detail); err != nil || outcome != "partial" || !strings.Contains(detail, "REQUEST_TIMEOUT") {
160		t.Fatal("partial timeout lost from logs", err, outcome, detail)
161	}
162	status, _ = callHTTP(t, s, "/contents", fmt.Sprintf(`{"urls":[%q]}`, slow.URL))
163	if status != 504 {
164		t.Fatal("empty timed-out request must still fail", status)
165	}
166}