package app import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "sort" "strings" "github.com/PuerkitoBio/goquery" ) type Searcher interface { Search(context.Context, string, int) ([]Result, error) } type Kagi struct { browser *Browser Session, Token string maxBytes int } // Search renders the Kagi results page in the browser. // // Kagi streams its organic result cards in after the initial load and only // serves them to a rendered session, so the browser is the only path that sees // the results. A plain fetch is not attempted first: it cannot produce them, // and the render already carries the session cookies. func (k *Kagi) Search(ctx context.Context, query string, limit int) ([]Result, error) { if k.Session == "" || k.Token == "" { return nil, failure(503, "SEARCH_NOT_CONFIGURED", "KAGI_SESSION and KAGI_SEARCH_TOKEN are required") } u := "https://kagi.com/search?q=" + url.QueryEscape(query) rendered, final, err := k.browser.render(ctx, u, true, k.Session, k.Token) if err != nil { return nil, err } if len(rendered) > k.maxBytes { return nil, fmt.Errorf("kagi response exceeds size limit") } hits := parseKagi(rendered, limit) if len(hits) == 0 { return nil, kagiPageError(rendered, final) } return hits, nil } // Searx is the self-hosted searxng meta search on valefar. It speaks the // searx JSON API (server.formats must include "json"), no browser needed. type Searx struct { BaseURL string HTTP HTTPClient maxBytes int } func (s *Searx) Search(ctx context.Context, query string, limit int) ([]Result, error) { if s.BaseURL == "" { return nil, failure(503, "SEARCH_NOT_CONFIGURED", "SEARX_URL is required") } // Technical and research questions are often answered by it and science // engines (Stack Exchange, MDN, arXiv), which searx leaves out of "general". u := strings.TrimRight(s.BaseURL, "/") + "/search?q=" + url.QueryEscape(query) + "&format=json&language=en&categories=general,it,science" req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "web-extract/1.0 (family appliance)") resp, err := s.HTTP.Do(req) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(io.LimitReader(resp.Body, int64(s.maxBytes)+1)) if err != nil { return nil, err } if len(body) > s.maxBytes { return nil, fmt.Errorf("searx response exceeds size limit") } if resp.StatusCode != http.StatusOK { return nil, failure(502, "SEARCH_UPSTREAM_ERROR", fmt.Sprintf("searx returned status %d", resp.StatusCode)) } var parsed struct { Results []struct { URL string `json:"url"` Title string `json:"title"` Content string `json:"content"` } `json:"results"` } if err := json.Unmarshal(body, &parsed); err != nil { return nil, failure(502, "SEARCH_UPSTREAM_ERROR", "searx returned unparseable json") } var out []Result seen := map[string]bool{} for _, r := range parsed.Results { if len(out) >= limit { break } u, err := parseWebURL(r.URL) if err != nil { continue } u.Fragment = "" id := u.String() if seen[id] { continue } seen[id] = true // The snippet is what the reranker judges the candidate by, as with Kagi. text := r.Content out = append(out, Result{ID: id, URL: r.URL, Title: r.Title, Text: &text, snippet: strings.Join(strings.Fields(r.Content), " ")}) } if len(out) == 0 { return nil, failure(502, "SEARCH_UPSTREAM_ERROR", "searx returned no recognizable results") } return out, nil } type HTTPClient interface { Do(*http.Request) (*http.Response, error) } func kagiPageError(html, final string) error { d, _ := goquery.NewDocumentFromReader(strings.NewReader(html)) u, _ := url.Parse(final) title := strings.ToLower(d.Find("title").Text()) login := d.Find(`input[type="password"]`).Length() > 0 || strings.Contains(title, "sign in") || strings.Contains(title, "log in") || strings.Contains(title, "login") if u != nil { login = login || u.Path == "/signin" || u.Path == "/login" } if login { return failure(502, "SEARCH_SESSION_REJECTED", "the kagi session was not accepted") } return failure(502, "SEARCH_UPSTREAM_ERROR", "the rendered kagi page contained no recognizable results") } func parseKagi(html string, limit int) []Result { d, err := goquery.NewDocumentFromReader(strings.NewReader(html)) if err != nil { return nil } out := []Result{} seen := map[string]bool{} d.Find(".search-result, .__srgi").Each(func(_ int, s *goquery.Selection) { link := s.Find(".__sri_title_link, .__srgi-title a, .__sri-title a").First() href, _ := link.Attr("href") u, err := parseWebURL(href) title := strings.Join(strings.Fields(link.Text()), " ") if err != nil || title == "" || len(out) >= limit { return } u.Fragment = "" id := u.String() if seen[id] { return } seen[id] = true out = append(out, Result{ID: id, URL: href, Title: title, snippet: strings.Join(strings.Fields(s.Find(".__sri-desc").First().Text()), " ")}) }) return out } func domainFilter(raw string) (*url.URL, error) { if !strings.Contains(raw, "://") { raw = "https://" + raw } u, err := parseWebURL(raw) if err != nil || u.Port() != "" || u.RawQuery != "" || u.Fragment != "" { return nil, fmt.Errorf("invalid domain filter") } return u, nil } func domainMatches(raw, filter string) bool { u, err := url.Parse(raw) if err != nil { return false } f, err := domainFilter(filter) if err != nil { return false } host := strings.ToLower(u.Hostname()) want := strings.ToLower(f.Hostname()) path := strings.TrimSuffix(f.Path, "/") return (host == want || strings.HasSuffix(host, "."+want)) && (path == "" || u.Path == path || strings.HasPrefix(u.Path, path+"/")) } func filtered(hits []Result, include, exclude []string) []Result { out := []Result{} for _, h := range hits { ok := len(include) == 0 for _, f := range include { ok = ok || domainMatches(h.URL, f) } for _, f := range exclude { if domainMatches(h.URL, f) { ok = false } } if ok { out = append(out, h) } } return out } func rankResults(hits []Result) { sort.SliceStable(hits, func(i, j int) bool { return hits[i].score > hits[j].score }) }