okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petanswer: read only sources the reranker found useful, and give it Searx snippetsf89560d

main
5.5 KiB200 linesraw
1package app
2
3import (
4	"bytes"
5	"encoding/json"
6	"fmt"
7	"io"
8	"strings"
9)
10
11// A content feature is absent, disabled, enabled, or an options object.
12type Feature struct {
13	Set, Enabled  bool
14	Query         string
15	MaxCharacters int
16}
17
18func (f *Feature) UnmarshalJSON(b []byte) error {
19	f.Set = true
20	switch string(bytes.TrimSpace(b)) {
21	case "true":
22		f.Enabled = true
23		return nil
24	case "false", "null":
25		return nil
26	}
27	var opts struct {
28		Query         string `json:"query"`
29		MaxCharacters int    `json:"maxCharacters"`
30	}
31	if err := strictJSON(b, &opts); err != nil {
32		return err
33	}
34	if opts.MaxCharacters < 0 || opts.MaxCharacters > 10000 || len(opts.Query) > 4000 {
35		return fmt.Errorf("invalid content query or maxCharacters (1–10000)")
36	}
37	*f = Feature{Set: true, Enabled: true, Query: opts.Query, MaxCharacters: opts.MaxCharacters}
38	return nil
39}
40
41type ContentsOptions struct {
42	Text             Feature `json:"text"`
43	Highlights       Feature `json:"highlights"`
44	Summary          Feature `json:"summary"`
45	MaxAgeHours      *int    `json:"maxAgeHours"`
46	LivecrawlTimeout int     `json:"livecrawlTimeout"`
47}
48
49func (o *ContentsOptions) defaults(contents bool) error {
50	if contents && !o.Text.Set && !o.Highlights.Set && !o.Summary.Set {
51		o.Text = Feature{Set: true, Enabled: true}
52	}
53	if o.MaxAgeHours != nil && (*o.MaxAgeHours < -1 || *o.MaxAgeHours > 720) {
54		return fmt.Errorf("maxAgeHours must be -1 through 720")
55	}
56	if o.LivecrawlTimeout < 0 || o.LivecrawlTimeout > 90000 {
57		return fmt.Errorf("livecrawlTimeout must be 1–90000 milliseconds")
58	}
59	if o.Summary.MaxCharacters != 0 {
60		return fmt.Errorf("summary accepts query, not maxCharacters")
61	}
62	if o.Text.Query != "" {
63		return fmt.Errorf("text accepts maxCharacters, not query")
64	}
65	return nil
66}
67
68type SearchRequest struct {
69	Query          string           `json:"query"`
70	Type           string           `json:"type"`
71	NumResults     int              `json:"numResults"`
72	IncludeDomains []string         `json:"includeDomains"`
73	ExcludeDomains []string         `json:"excludeDomains"`
74	Contents       *ContentsOptions `json:"contents"`
75	// minScore drops candidates the reranker scored lower, keeping at least the
76	// best one; answer() uses it so that off-topic results never become sources.
77	minScore float64
78}
79
80func (q *SearchRequest) validate() error {
81	q.Query = strings.TrimSpace(q.Query)
82	if q.Query == "" || len(q.Query) > 4000 {
83		return fmt.Errorf("query must contain 1–4000 bytes")
84	}
85	if q.Type != "" && q.Type != "auto" {
86		return fmt.Errorf("only type=auto is supported; this is kagi + jev, not exa's index")
87	}
88	if q.NumResults == 0 {
89		q.NumResults = 10
90	}
91	if q.NumResults < 1 || q.NumResults > 20 {
92		return fmt.Errorf("numResults must be 1–20")
93	}
94	for _, domains := range [][]string{q.IncludeDomains, q.ExcludeDomains} {
95		if len(domains) > 20 {
96			return fmt.Errorf("at most 20 domain filters")
97		}
98		for _, d := range domains {
99			if _, err := domainFilter(d); err != nil {
100				return err
101			}
102		}
103	}
104	if q.Contents != nil {
105		return q.Contents.defaults(false)
106	}
107	return nil
108}
109
110type ContentsRequest struct {
111	IDs  []string `json:"ids"`
112	URLs []string `json:"urls"`
113	ContentsOptions
114}
115
116func (q *ContentsRequest) validate() ([]string, error) {
117	if (len(q.IDs) == 0) == (len(q.URLs) == 0) {
118		return nil, fmt.Errorf("provide exactly one of ids or urls")
119	}
120	urls := q.URLs
121	if len(q.IDs) > 0 {
122		urls = q.IDs
123	}
124	if len(urls) > 20 {
125		return nil, fmt.Errorf("at most 20 urls per request")
126	}
127	for _, raw := range urls {
128		if _, err := parseWebURL(raw); err != nil {
129			return nil, err
130		}
131	}
132	return urls, q.ContentsOptions.defaults(true)
133}
134
135type AnswerRequest struct {
136	Query  string `json:"query"`
137	Text   bool   `json:"text"`
138	Stream bool   `json:"stream"`
139}
140
141type Result struct {
142	ID              string     `json:"id"`
143	URL             string     `json:"url"`
144	Title           string     `json:"title"`
145	PublishedDate   string     `json:"publishedDate,omitempty"`
146	Author          string     `json:"author,omitempty"`
147	Text            *string    `json:"text,omitempty"`
148	Highlights      *[]string  `json:"highlights,omitempty"`
149	HighlightScores *[]float64 `json:"highlightScores,omitempty"`
150	Summary         *string    `json:"summary,omitempty"`
151	// Internal evidence is never confused with generated text or put on Exa's wire.
152	pack    EvidencePack
153	score   float64
154	snippet string
155}
156
157type ItemError struct {
158	Tag            string `json:"tag"`
159	HTTPStatusCode int    `json:"httpStatusCode,omitempty"`
160}
161type ItemStatus struct {
162	ID     string     `json:"id"`
163	Status string     `json:"status"`
164	Source string     `json:"source,omitempty"`
165	Error  *ItemError `json:"error,omitempty"`
166}
167type ResultsResponse struct {
168	RequestID string       `json:"requestId"`
169	Results   []Result     `json:"results"`
170	Statuses  []ItemStatus `json:"statuses,omitempty"`
171}
172type AnswerResponse struct {
173	RequestID string   `json:"requestId"`
174	Answer    string   `json:"answer"`
175	Citations []Result `json:"citations"`
176}
177type APIError struct {
178	Status    int    `json:"-"`
179	RequestID string `json:"requestId"`
180	Message   string `json:"error"`
181	Tag       string `json:"tag"`
182}
183
184func (e *APIError) Error() string { return e.Message }
185func failure(status int, tag, message string) error {
186	return &APIError{Status: status, Tag: tag, Message: message}
187}
188
189func strictJSON(b []byte, dst any) error {
190	d := json.NewDecoder(bytes.NewReader(b))
191	d.DisallowUnknownFields()
192	if err := d.Decode(dst); err != nil {
193		return err
194	}
195	var extra any
196	if err := d.Decode(&extra); err != io.EOF {
197		return fmt.Errorf("expected one json value")
198	}
199	return nil
200}