okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
2.3 KiB79 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"fmt"
7	"sort"
8)
9
10// Question-local data keeps unrelated candidates out of each judgment's context.
11// Jev evaluates the questions independently in parallel; IDs only route answers.
12// Bound both question count and encoded bytes without dropping/truncating items.
13func (j *Jev) evaluateItems(ctx context.Context, task, query, field string, items []map[string]string, questions map[string]Question) ([]map[string]Decision, error) {
14	const maxQuestions = 96
15	state := map[string]string{"query": query}
16	empty, err := j.requestBody(state, map[string]Question{})
17	if err != nil {
18		return nil, err
19	}
20	names := make([]string, 0, len(questions))
21	for name := range questions {
22		names = append(names, name)
23	}
24	sort.Strings(names)
25	batches := []map[string]Question{}
26	current := map[string]Question{}
27	bytes := len(empty)
28	for i, item := range items {
29		for _, name := range names {
30			q := questions[name]
31			q.Instructions = map[string]any{"question": q.Instructions, field: item}
32			id := fmt.Sprintf("%d_%s", i, name)
33			encoded, err := json.Marshal(map[string]Question{id: q})
34			if err != nil {
35				return nil, err
36			}
37			size := len(encoded) - 2 // exclude the enclosing braces
38			if len(empty)+size > maxJevRequestBytes {
39				return nil, fmt.Errorf("jev item exceeds the application state budget")
40			}
41			if len(current) > 0 && (bytes+1+size > maxJevRequestBytes || len(current) == maxQuestions) {
42				batches = append(batches, current)
43				current = map[string]Question{}
44				bytes = len(empty)
45			}
46			if len(current) > 0 {
47				bytes++ // comma between questions
48			}
49			current[id] = q
50			bytes += size
51		}
52	}
53	if len(current) > 0 {
54		batches = append(batches, current)
55	}
56	replies := make([]JevResponse, len(batches))
57	err = parallel(ctx, len(batches), cap(j.slots), func(ctx context.Context, i int) error {
58		var err error
59		replies[i], err = j.evaluate(ctx, task, state, batches[i])
60		return err
61	})
62	if err != nil {
63		return nil, err
64	}
65	flat := map[string]Decision{}
66	for _, r := range replies {
67		for id, a := range r.Answers {
68			flat[id] = a
69		}
70	}
71	answers := make([]map[string]Decision, len(items))
72	for i := range items {
73		answers[i] = make(map[string]Decision, len(questions))
74		for _, name := range names {
75			answers[i][name] = flat[fmt.Sprintf("%d_%s", i, name)]
76		}
77	}
78	return answers, nil
79}