okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petsearx: ask the general, it and science categories75c10dd

main
6.1 KiB211 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"fmt"
7	"io"
8	"net/http"
9
10	"net/url"
11	"sort"
12	"strings"
13
14	"github.com/PuerkitoBio/goquery"
15)
16
17type Searcher interface {
18	Search(context.Context, string, int) ([]Result, error)
19}
20type Kagi struct {
21	browser        *Browser
22	Session, Token string
23	maxBytes       int
24}
25
26// Search renders the Kagi results page in the browser.
27//
28// Kagi streams its organic result cards in after the initial load and only
29// serves them to a rendered session, so the browser is the only path that sees
30// the results. A plain fetch is not attempted first: it cannot produce them,
31// and the render already carries the session cookies.
32func (k *Kagi) Search(ctx context.Context, query string, limit int) ([]Result, error) {
33	if k.Session == "" || k.Token == "" {
34		return nil, failure(503, "SEARCH_NOT_CONFIGURED", "KAGI_SESSION and KAGI_SEARCH_TOKEN are required")
35	}
36	u := "https://kagi.com/search?q=" + url.QueryEscape(query)
37	rendered, final, err := k.browser.render(ctx, u, true, k.Session, k.Token)
38	if err != nil {
39		return nil, err
40	}
41	if len(rendered) > k.maxBytes {
42		return nil, fmt.Errorf("kagi response exceeds size limit")
43	}
44	hits := parseKagi(rendered, limit)
45	if len(hits) == 0 {
46		return nil, kagiPageError(rendered, final)
47	}
48	return hits, nil
49}
50
51
52// Searx is the self-hosted searxng meta search on valefar. It speaks the
53// searx JSON API (server.formats must include "json"), no browser needed.
54type Searx struct {
55	BaseURL  string
56	HTTP     HTTPClient
57	maxBytes int
58}
59
60func (s *Searx) Search(ctx context.Context, query string, limit int) ([]Result, error) {
61	if s.BaseURL == "" {
62		return nil, failure(503, "SEARCH_NOT_CONFIGURED", "SEARX_URL is required")
63	}
64	// Technical and research questions are often answered by it and science
65	// engines (Stack Exchange, MDN, arXiv), which searx leaves out of "general".
66	u := strings.TrimRight(s.BaseURL, "/") + "/search?q=" + url.QueryEscape(query) + "&format=json&language=en&categories=general,it,science"
67	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
68	if err != nil {
69		return nil, err
70	}
71	req.Header.Set("User-Agent", "web-extract/1.0 (family appliance)")
72	resp, err := s.HTTP.Do(req)
73	if err != nil {
74		return nil, err
75	}
76	defer resp.Body.Close()
77	body, err := io.ReadAll(io.LimitReader(resp.Body, int64(s.maxBytes)+1))
78	if err != nil {
79		return nil, err
80	}
81	if len(body) > s.maxBytes {
82		return nil, fmt.Errorf("searx response exceeds size limit")
83	}
84	if resp.StatusCode != http.StatusOK {
85		return nil, failure(502, "SEARCH_UPSTREAM_ERROR", fmt.Sprintf("searx returned status %d", resp.StatusCode))
86	}
87	var parsed struct {
88		Results []struct {
89			URL     string `json:"url"`
90			Title   string `json:"title"`
91			Content string `json:"content"`
92		} `json:"results"`
93	}
94	if err := json.Unmarshal(body, &parsed); err != nil {
95		return nil, failure(502, "SEARCH_UPSTREAM_ERROR", "searx returned unparseable json")
96	}
97	var out []Result
98	seen := map[string]bool{}
99	for _, r := range parsed.Results {
100		if len(out) >= limit {
101			break
102		}
103		u, err := parseWebURL(r.URL)
104		if err != nil {
105			continue
106		}
107		u.Fragment = ""
108		id := u.String()
109		if seen[id] {
110			continue
111		}
112		seen[id] = true
113		// The snippet is what the reranker judges the candidate by, as with Kagi.
114		text := r.Content
115		out = append(out, Result{ID: id, URL: r.URL, Title: r.Title, Text: &text, snippet: strings.Join(strings.Fields(r.Content), " ")})
116	}
117	if len(out) == 0 {
118		return nil, failure(502, "SEARCH_UPSTREAM_ERROR", "searx returned no recognizable results")
119	}
120	return out, nil
121}
122
123type HTTPClient interface {
124	Do(*http.Request) (*http.Response, error)
125}
126
127func kagiPageError(html, final string) error {
128	d, _ := goquery.NewDocumentFromReader(strings.NewReader(html))
129	u, _ := url.Parse(final)
130	title := strings.ToLower(d.Find("title").Text())
131	login := d.Find(`input[type="password"]`).Length() > 0 || strings.Contains(title, "sign in") || strings.Contains(title, "log in") || strings.Contains(title, "login")
132	if u != nil {
133		login = login || u.Path == "/signin" || u.Path == "/login"
134	}
135	if login {
136		return failure(502, "SEARCH_SESSION_REJECTED", "the kagi session was not accepted")
137	}
138	return failure(502, "SEARCH_UPSTREAM_ERROR", "the rendered kagi page contained no recognizable results")
139}
140
141func parseKagi(html string, limit int) []Result {
142	d, err := goquery.NewDocumentFromReader(strings.NewReader(html))
143	if err != nil {
144		return nil
145	}
146	out := []Result{}
147	seen := map[string]bool{}
148	d.Find(".search-result, .__srgi").Each(func(_ int, s *goquery.Selection) {
149		link := s.Find(".__sri_title_link, .__srgi-title a, .__sri-title a").First()
150		href, _ := link.Attr("href")
151		u, err := parseWebURL(href)
152		title := strings.Join(strings.Fields(link.Text()), " ")
153		if err != nil || title == "" || len(out) >= limit {
154			return
155		}
156		u.Fragment = ""
157		id := u.String()
158		if seen[id] {
159			return
160		}
161		seen[id] = true
162		out = append(out, Result{ID: id, URL: href, Title: title, snippet: strings.Join(strings.Fields(s.Find(".__sri-desc").First().Text()), " ")})
163	})
164	return out
165}
166
167func domainFilter(raw string) (*url.URL, error) {
168	if !strings.Contains(raw, "://") {
169		raw = "https://" + raw
170	}
171	u, err := parseWebURL(raw)
172	if err != nil || u.Port() != "" || u.RawQuery != "" || u.Fragment != "" {
173		return nil, fmt.Errorf("invalid domain filter")
174	}
175	return u, nil
176}
177func domainMatches(raw, filter string) bool {
178	u, err := url.Parse(raw)
179	if err != nil {
180		return false
181	}
182	f, err := domainFilter(filter)
183	if err != nil {
184		return false
185	}
186	host := strings.ToLower(u.Hostname())
187	want := strings.ToLower(f.Hostname())
188	path := strings.TrimSuffix(f.Path, "/")
189	return (host == want || strings.HasSuffix(host, "."+want)) && (path == "" || u.Path == path || strings.HasPrefix(u.Path, path+"/"))
190}
191func filtered(hits []Result, include, exclude []string) []Result {
192	out := []Result{}
193	for _, h := range hits {
194		ok := len(include) == 0
195		for _, f := range include {
196			ok = ok || domainMatches(h.URL, f)
197		}
198		for _, f := range exclude {
199			if domainMatches(h.URL, f) {
200				ok = false
201			}
202		}
203		if ok {
204			out = append(out, h)
205		}
206	}
207	return out
208}
209func rankResults(hits []Result) {
210	sort.SliceStable(hits, func(i, j int) bool { return hits[i].score > hits[j].score })
211}