okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
f89560d
main
1package app 2 3import ( 4"bytes" 5"encoding/json" 6"fmt" 7"io" 8"strings" 9) 10 11// A content feature is absent, disabled, enabled, or an options object. 12type Feature struct { 13Set ,Enabled bool 14Query string 15MaxCharacters int 16} 17 18func (f * Feature )UnmarshalJSON (b []byte )error { 19f .Set = true 20switch string (bytes .TrimSpace (b )) { 21case "true" : 22f .Enabled = true 23return nil 24case "false" ,"null" : 25return nil 26 } 27var opts struct { 28Query string `json:"query"` 29MaxCharacters int `json:"maxCharacters"` 30 } 31if err := strictJSON (b ,& opts );err != nil { 32return err 33 } 34if opts .MaxCharacters < 0 || opts .MaxCharacters > 10000 || len (opts .Query )> 4000 { 35return fmt .Errorf ("invalid content query or maxCharacters (1–10000)" ) 36 } 37* f = Feature {Set :true ,Enabled :true ,Query :opts .Query ,MaxCharacters :opts .MaxCharacters } 38return nil 39} 40 41type ContentsOptions struct { 42Text Feature `json:"text"` 43Highlights Feature `json:"highlights"` 44Summary Feature `json:"summary"` 45MaxAgeHours * int `json:"maxAgeHours"` 46LivecrawlTimeout int `json:"livecrawlTimeout"` 47} 48 49func (o * ContentsOptions )defaults (contents bool )error { 50if contents && ! o .Text .Set && ! o .Highlights .Set && ! o .Summary .Set { 51o .Text = Feature {Set :true ,Enabled :true } 52 } 53if o .MaxAgeHours != nil && (* o .MaxAgeHours < - 1 || * o .MaxAgeHours > 720 ) { 54return fmt .Errorf ("maxAgeHours must be -1 through 720" ) 55 } 56if o .LivecrawlTimeout < 0 || o .LivecrawlTimeout > 90000 { 57return fmt .Errorf ("livecrawlTimeout must be 1–90000 milliseconds" ) 58 } 59if o .Summary .MaxCharacters != 0 { 60return fmt .Errorf ("summary accepts query, not maxCharacters" ) 61 } 62if o .Text .Query != "" { 63return fmt .Errorf ("text accepts maxCharacters, not query" ) 64 } 65return nil 66} 67 68type SearchRequest struct { 69Query string `json:"query"` 70Type string `json:"type"` 71NumResults int `json:"numResults"` 72IncludeDomains []string `json:"includeDomains"` 73ExcludeDomains []string `json:"excludeDomains"` 74Contents * ContentsOptions `json:"contents"` 75// minScore drops candidates the reranker scored lower, keeping at least the 76// best one; answer() uses it so that off-topic results never become sources. 77minScore float64 78} 79 80func (q * SearchRequest )validate ()error { 81q .Query = strings .TrimSpace (q .Query ) 82if q .Query == "" || len (q .Query )> 4000 { 83return fmt .Errorf ("query must contain 1–4000 bytes" ) 84 } 85if q .Type != "" && q .Type != "auto" { 86return fmt .Errorf ("only type=auto is supported; this is kagi + jev, not exa's index" ) 87 } 88if q .NumResults == 0 { 89q .NumResults = 10 90 } 91if q .NumResults < 1 || q .NumResults > 20 { 92return fmt .Errorf ("numResults must be 1–20" ) 93 } 94for _ ,domains := range [][]string {q .IncludeDomains ,q .ExcludeDomains } { 95if len (domains )> 20 { 96return fmt .Errorf ("at most 20 domain filters" ) 97 } 98for _ ,d := range domains { 99if _ ,err := domainFilter (d );err != nil { 100return err 101 } 102 } 103 } 104if q .Contents != nil { 105return q .Contents .defaults (false ) 106 } 107return nil 108} 109 110type ContentsRequest struct { 111IDs []string `json:"ids"` 112URLs []string `json:"urls"` 113ContentsOptions 114} 115 116func (q * ContentsRequest )validate () ([]string ,error ) { 117if (len (q .IDs )== 0 )== (len (q .URLs )== 0 ) { 118return nil ,fmt .Errorf ("provide exactly one of ids or urls" ) 119 } 120urls := q .URLs 121if len (q .IDs )> 0 { 122urls = q .IDs 123 } 124if len (urls )> 20 { 125return nil ,fmt .Errorf ("at most 20 urls per request" ) 126 } 127for _ ,raw := range urls { 128if _ ,err := parseWebURL (raw );err != nil { 129return nil ,err 130 } 131 } 132return urls ,q .ContentsOptions .defaults (true ) 133} 134 135type AnswerRequest struct { 136Query string `json:"query"` 137Text bool `json:"text"` 138Stream bool `json:"stream"` 139} 140 141type Result struct { 142ID string `json:"id"` 143URL string `json:"url"` 144Title string `json:"title"` 145PublishedDate string `json:"publishedDate,omitempty"` 146Author string `json:"author,omitempty"` 147Text * string `json:"text,omitempty"` 148Highlights * []string `json:"highlights,omitempty"` 149HighlightScores * []float64 `json:"highlightScores,omitempty"` 150Summary * string `json:"summary,omitempty"` 151// Internal evidence is never confused with generated text or put on Exa's wire. 152pack EvidencePack 153score float64 154snippet string 155} 156 157type ItemError struct { 158Tag string `json:"tag"` 159HTTPStatusCode int `json:"httpStatusCode,omitempty"` 160} 161type ItemStatus struct { 162ID string `json:"id"` 163Status string `json:"status"` 164Source string `json:"source,omitempty"` 165Error * ItemError `json:"error,omitempty"` 166} 167type ResultsResponse struct { 168RequestID string `json:"requestId"` 169Results []Result `json:"results"` 170Statuses []ItemStatus `json:"statuses,omitempty"` 171} 172type AnswerResponse struct { 173RequestID string `json:"requestId"` 174Answer string `json:"answer"` 175Citations []Result `json:"citations"` 176} 177type APIError struct { 178Status int `json:"-"` 179RequestID string `json:"requestId"` 180Message string `json:"error"` 181Tag string `json:"tag"` 182} 183 184func (e * APIError )Error ()string {return e .Message } 185func failure (status int ,tag ,message string )error { 186return & APIError {Status :status ,Tag :tag ,Message :message } 187} 188 189func strictJSON (b []byte ,dst any )error { 190d := json .NewDecoder (bytes .NewReader (b )) 191d .DisallowUnknownFields () 192if err := d .Decode (dst );err != nil { 193return err 194 } 195var extra any 196if err := d .Decode (& extra );err != io .EOF { 197return fmt .Errorf ("expected one json value" ) 198 } 199return nil 200}