okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
75c10dd
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"fmt" 7"io" 8"net/http" 9 10"net/url" 11"sort" 12"strings" 13 14"github.com/PuerkitoBio/goquery" 15) 16 17type Searcher interface { 18Search (context.Context ,string ,int ) ([]Result ,error ) 19} 20type Kagi struct { 21browser * Browser 22Session ,Token string 23maxBytes int 24} 25 26// Search renders the Kagi results page in the browser. 27// 28// Kagi streams its organic result cards in after the initial load and only 29// serves them to a rendered session, so the browser is the only path that sees 30// the results. A plain fetch is not attempted first: it cannot produce them, 31// and the render already carries the session cookies. 32func (k * Kagi )Search (ctx context.Context ,query string ,limit int ) ([]Result ,error ) { 33if k .Session == "" || k .Token == "" { 34return nil ,failure (503 ,"SEARCH_NOT_CONFIGURED" ,"KAGI_SESSION and KAGI_SEARCH_TOKEN are required" ) 35 } 36u := "https://kagi.com/search?q=" + url .QueryEscape (query ) 37rendered ,final ,err := k .browser .render (ctx ,u ,true ,k .Session ,k .Token ) 38if err != nil { 39return nil ,err 40 } 41if len (rendered )> k .maxBytes { 42return nil ,fmt .Errorf ("kagi response exceeds size limit" ) 43 } 44hits := parseKagi (rendered ,limit ) 45if len (hits )== 0 { 46return nil ,kagiPageError (rendered ,final ) 47 } 48return hits ,nil 49} 50 51 52// Searx is the self-hosted searxng meta search on valefar. It speaks the 53// searx JSON API (server.formats must include "json"), no browser needed. 54type Searx struct { 55BaseURL string 56HTTP HTTPClient 57maxBytes int 58} 59 60func (s * Searx )Search (ctx context.Context ,query string ,limit int ) ([]Result ,error ) { 61if s .BaseURL == "" { 62return nil ,failure (503 ,"SEARCH_NOT_CONFIGURED" ,"SEARX_URL is required" ) 63 } 64// Technical and research questions are often answered by it and science 65// engines (Stack Exchange, MDN, arXiv), which searx leaves out of "general". 66u := strings .TrimRight (s .BaseURL ,"/" )+ "/search?q=" + url .QueryEscape (query )+ "&format=json&language=en&categories=general,it,science" 67req ,err := http .NewRequestWithContext (ctx ,http .MethodGet ,u ,nil ) 68if err != nil { 69return nil ,err 70 } 71req .Header .Set ("User-Agent" ,"web-extract/1.0 (family appliance)" ) 72resp ,err := s .HTTP .Do (req ) 73if err != nil { 74return nil ,err 75 } 76defer resp .Body .Close () 77body ,err := io .ReadAll (io .LimitReader (resp .Body ,int64 (s .maxBytes )+ 1 )) 78if err != nil { 79return nil ,err 80 } 81if len (body )> s .maxBytes { 82return nil ,fmt .Errorf ("searx response exceeds size limit" ) 83 } 84if resp .StatusCode != http .StatusOK { 85return nil ,failure (502 ,"SEARCH_UPSTREAM_ERROR" ,fmt .Sprintf ("searx returned status %d" ,resp .StatusCode )) 86 } 87var parsed struct { 88Results []struct { 89URL string `json:"url"` 90Title string `json:"title"` 91Content string `json:"content"` 92 }`json:"results"` 93 } 94if err := json .Unmarshal (body ,& parsed );err != nil { 95return nil ,failure (502 ,"SEARCH_UPSTREAM_ERROR" ,"searx returned unparseable json" ) 96 } 97var out []Result 98seen := map [string ]bool {} 99for _ ,r := range parsed .Results { 100if len (out )>= limit { 101break 102 } 103u ,err := parseWebURL (r .URL ) 104if err != nil { 105continue 106 } 107u .Fragment = "" 108id := u .String () 109if seen [id ] { 110continue 111 } 112seen [id ]= true 113// The snippet is what the reranker judges the candidate by, as with Kagi. 114text := r .Content 115out = append (out ,Result {ID :id ,URL :r .URL ,Title :r .Title ,Text :& text ,snippet :strings .Join (strings .Fields (r .Content )," " )}) 116 } 117if len (out )== 0 { 118return nil ,failure (502 ,"SEARCH_UPSTREAM_ERROR" ,"searx returned no recognizable results" ) 119 } 120return out ,nil 121} 122 123type HTTPClient interface { 124Do (* http.Request ) (* http.Response ,error ) 125} 126 127func kagiPageError (html ,final string )error { 128d ,_ := goquery .NewDocumentFromReader (strings .NewReader (html )) 129u ,_ := url .Parse (final ) 130title := strings .ToLower (d .Find ("title" ).Text ()) 131login := d .Find (`input[type="password"]` ).Length ()> 0 || strings .Contains (title ,"sign in" )|| strings .Contains (title ,"log in" )|| strings .Contains (title ,"login" ) 132if u != nil { 133login = login || u .Path == "/signin" || u .Path == "/login" 134 } 135if login { 136return failure (502 ,"SEARCH_SESSION_REJECTED" ,"the kagi session was not accepted" ) 137 } 138return failure (502 ,"SEARCH_UPSTREAM_ERROR" ,"the rendered kagi page contained no recognizable results" ) 139} 140 141func parseKagi (html string ,limit int ) []Result { 142d ,err := goquery .NewDocumentFromReader (strings .NewReader (html )) 143if err != nil { 144return nil 145 } 146out := []Result {} 147seen := map [string ]bool {} 148d .Find (".search-result, .__srgi" ).Each (func (_ int ,s * goquery.Selection ) { 149link := s .Find (".__sri_title_link, .__srgi-title a, .__sri-title a" ).First () 150href ,_ := link .Attr ("href" ) 151u ,err := parseWebURL (href ) 152title := strings .Join (strings .Fields (link .Text ())," " ) 153if err != nil || title == "" || len (out )>= limit { 154return 155 } 156u .Fragment = "" 157id := u .String () 158if seen [id ] { 159return 160 } 161seen [id ]= true 162out = append (out ,Result {ID :id ,URL :href ,Title :title ,snippet :strings .Join (strings .Fields (s .Find (".__sri-desc" ).First ().Text ())," " )}) 163 }) 164return out 165} 166 167func domainFilter (raw string ) (* url.URL ,error ) { 168if ! strings .Contains (raw ,"://" ) { 169raw = "https://" + raw 170 } 171u ,err := parseWebURL (raw ) 172if err != nil || u .Port ()!= "" || u .RawQuery != "" || u .Fragment != "" { 173return nil ,fmt .Errorf ("invalid domain filter" ) 174 } 175return u ,nil 176} 177func domainMatches (raw ,filter string )bool { 178u ,err := url .Parse (raw ) 179if err != nil { 180return false 181 } 182f ,err := domainFilter (filter ) 183if err != nil { 184return false 185 } 186host := strings .ToLower (u .Hostname ()) 187want := strings .ToLower (f .Hostname ()) 188path := strings .TrimSuffix (f .Path ,"/" ) 189return (host == want || strings .HasSuffix (host ,"." + want ))&& (path == "" || u .Path == path || strings .HasPrefix (u .Path ,path + "/" )) 190} 191func filtered (hits []Result ,include ,exclude []string ) []Result { 192out := []Result {} 193for _ ,h := range hits { 194ok := len (include )== 0 195for _ ,f := range include { 196ok = ok || domainMatches (h .URL ,f ) 197 } 198for _ ,f := range exclude { 199if domainMatches (h .URL ,f ) { 200ok = false 201 } 202 } 203if ok { 204out = append (out ,h ) 205 } 206 } 207return out 208} 209func rankResults (hits []Result ) { 210sort .SliceStable (hits ,func (i ,j int )bool {return hits [i ].score > hits [j ].score }) 211}