okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
24e631b
main
1package app 2 3import ( 4"context" 5"errors" 6"fmt" 7"strings" 8"unicode/utf8" 9) 10 11type Service struct { 12cfg Config 13fetch * Fetcher 14searcher Searcher 15jev * Jev 16llm * LLM 17log * Log 18auth * AuthStore 19// publicJev and gate serve the public endpoint; both nil unless it is enabled. 20publicJev * Jev 21gate * publicGate 22} 23 24func (s * Service )search (ctx context.Context ,q SearchRequest ) (ResultsResponse ,error ) { 25out := ResultsResponse {RequestID :callInfo (ctx ).ID ,Results : []Result {}} 26if err := q .validate ();err != nil { 27return out ,failure (400 ,"INVALID_REQUEST" ,err .Error ()) 28 } 29if s .decider (ctx ).Key == "" { 30return out ,failure (503 ,"JEV_NOT_CONFIGURED" ,"TYPESAFE_API_KEY is not configured" ) 31 } 32hits ,err := s .searcher .Search (ctx ,q .Query ,max (q .NumResults ,s .cfg .Candidates )) 33if err != nil { 34return out ,err 35 } 36hits = filtered (hits ,q .IncludeDomains ,q .ExcludeDomains ) 37if err = s .decider (ctx ).rank (ctx ,q .Query ,hits );err != nil { 38return out ,err 39 } 40rankResults (hits ) 41if q .minScore > 0 { 42keep := 1 43for keep < len (hits )&& hits [keep ].score >= q .minScore { 44keep ++ 45 } 46hits = hits [:min (keep ,len (hits ))] 47 } 48hits = hits [:min (q .NumResults ,len (hits ))] 49if q .Contents == nil { 50out .Results = hits 51return out ,nil 52 } 53results ,statuses ,err := s .enrich (ctx ,hits ,* q .Contents ,q .Query ) 54out .Results = results 55out .Statuses = statuses 56rankResults (out .Results ) 57return out ,err 58} 59 60func (s * Service )contents (ctx context.Context ,q ContentsRequest ) (ResultsResponse ,error ) { 61out := ResultsResponse {RequestID :callInfo (ctx ).ID ,Results : []Result {}} 62urls ,err := q .validate () 63if err != nil { 64return out ,failure (400 ,"INVALID_REQUEST" ,err .Error ()) 65 } 66hits := make ([]Result ,len (urls )) 67for i ,u := range urls { 68hits [i ]= Result {ID :u ,URL :u } 69 } 70out .Results ,out .Statuses ,err = s .enrich (ctx ,hits ,q .ContentsOptions ,"" ) 71return out ,err 72} 73 74func (s * Service )enrich (ctx context.Context ,hits []Result ,o ContentsOptions ,query string ) ([]Result , []ItemStatus ,error ) { 75values := make ([]Result ,len (hits )) 76statuses := make ([]ItemStatus ,len (hits )) 77success := make ([]bool ,len (hits )) 78for i ,hit := range hits { 79statuses [i ]= ItemStatus {ID :hit .ID ,Status :"error" ,Error :& ItemError {Tag :"REQUEST_TIMEOUT" ,HTTPStatusCode :504 }} 80 } 81err := parallel (ctx ,len (hits ),s .cfg .Concurrency ,func (ctx context.Context ,i int )error { 82result ,source ,err := s .enrichOne (ctx ,hits [i ],o ,query ) 83statuses [i ]= ItemStatus {ID :hits [i ].ID ,Status :"success" ,Source :source } 84if err != nil { 85tag := "FETCH_DOCUMENT_ERROR" 86code := 502 87var a * APIError 88if errors .As (err ,& a ) { 89tag = a .Tag 90code = a .Status 91 }else if errors .Is (err ,context .DeadlineExceeded )|| errors .Is (err ,context .Canceled ) { 92tag ,code = "REQUEST_TIMEOUT" ,504 93 } 94statuses [i ]= ItemStatus {ID :hits [i ].ID ,Status :"error" ,Error :& ItemError {Tag :tag ,HTTPStatusCode :code }} 95 }else { 96values [i ]= result 97success [i ]= true 98 } 99return nil // a failed page must not cancel independent source retrieval 100 }) 101out := []Result {} 102for i ,r := range values { 103if success [i ] { 104out = append (out ,r ) 105 } 106 } 107// A slow tail must not discard already completed sources at the deadline. 108if len (out )> 0 && errors .Is (err ,context .DeadlineExceeded ) { 109err = nil 110 } 111return out ,statuses ,err 112} 113 114func (s * Service )enrichOne (ctx context.Context ,r Result ,o ContentsOptions ,query string ) (Result ,string ,error ) { 115d ,source ,err := s .fetch .load (ctx ,r .URL ,o ) 116if err != nil { 117return r ,source ,err 118 } 119r .URL = d .FinalURL 120if d .Title != "" { 121r .Title = d .Title 122 } 123r .Author = d .Author 124r .PublishedDate = d .Published 125if o .Text .Enabled { 126text := takeRunes (d .Markdown ,o .Text .MaxCharacters ) 127r .Text = & text 128 } 129// One evidence selection serves both features when their query is identical. 130packs := map [string ]EvidencePack {} 131selectFor := func (q string ) (EvidencePack ,error ) { 132if q == "" { 133q = "summarize the main content, important facts, and material caveats of this document" 134 } 135if p ,ok := packs [q ];ok { 136return p ,nil 137 } 138p ,err := s .decider (ctx ).selectEvidence (ctx ,d ,q ,s .cfg .Passages ,s .cfg .EvidenceChars ) 139if err == nil { 140packs [q ]= p 141 } 142return p ,err 143 } 144if o .Highlights .Enabled { 145q := o .Highlights .Query 146if q == "" { 147q = query 148 } 149p ,err := selectFor (q ) 150if err != nil { 151return r ,source ,err 152 } 153r .pack = p 154r .score = bestScore (p ) 155texts := []string {} 156scores := []float64 {} 157remaining := o .Highlights .MaxCharacters 158if remaining == 0 { 159remaining = s .cfg .EvidenceChars 160 } 161for _ ,passage := range p .Passages { 162n := utf8 .RuneCountInString (passage .Text ) 163if n > remaining { 164continue 165 } 166texts = append (texts ,passage .Text ) 167scores = append (scores ,passage .Judgments .Evidence ) 168remaining -= n 169 } 170r .Highlights = & texts 171r .HighlightScores = & scores 172 } 173if o .Summary .Enabled { 174q := o .Summary .Query 175if q == "" { 176q = query 177 } 178p ,err := selectFor (q ) 179if err != nil { 180return r ,source ,err 181 } 182r .pack = p 183r .score = bestScore (p ) 184label ,err := s .decider (ctx ).answerability (ctx ,p .Query ,p ) 185if err != nil { 186return r ,source ,err 187 } 188summary ,err := s .llm .preview (ctx ,d ,p ,label ) 189if err != nil { 190return r ,source ,err 191 } 192r .Summary = & summary 193 } 194return r ,source ,nil 195} 196func bestScore (p EvidencePack )float64 { 197score := 0.0 198for _ ,v := range p .Passages { 199score = max (score ,v .Judgments .Evidence ,v .Judgments .Contradiction ) 200 } 201return score 202} 203 204// answerSourceFloor is the lowest rerank score a search result needs to be read 205// as an answer source. Off-topic results score about 0.02. 206const answerSourceFloor = 0.1 207 208func (s * Service )answer (ctx context.Context ,q AnswerRequest ) (AnswerResponse ,error ) { 209out := AnswerResponse {RequestID :callInfo (ctx ).ID ,Citations : []Result {}} 210if q .Stream { 211return out ,failure (400 ,"FEATURE_DISABLED" ,"streaming is not implemented" ) 212 } 213query := SearchRequest {Query :q .Query ,NumResults :5 ,minScore :answerSourceFloor ,Contents :& ContentsOptions {Highlights :Feature {Enabled :true ,Set :true }}} 214r ,err := s .search (ctx ,query ) 215if err != nil { 216return out ,err 217 } 218var evidence strings.Builder 219remaining := s .cfg .EvidenceChars 220combined := EvidencePack {Query :q .Query ,Passages : []Passage {},BudgetChars :remaining ,Model :s .decider (ctx ).Model } 221for _ ,result := range r .Results { 222// Sources are numbered as they appear in citations, so an answer's [n] 223// markers point at citations[n-1]. 224n := len (out .Citations )+ 1 225accepted := []Passage {} 226header := fmt .Sprintf ("[%d] %s\n%s\n" ,n ,result .Title ,result .URL ) 227for _ ,p := range result .pack .Passages { 228p .ID = fmt .Sprintf ("source-%d:%s" ,n ,p .ID ) 229cost := utf8 .RuneCountInString (evidenceText ([]Passage {p }))+ 2 230if len (accepted )== 0 { 231cost += utf8 .RuneCountInString (header ) 232 } 233if cost > remaining { 234continue 235 } 236remaining -= cost 237accepted = append (accepted ,p ) 238 } 239if len (accepted )== 0 { 240continue 241 } 242evidence .WriteString (header ) 243evidence .WriteString (evidenceText (accepted )) 244evidence .WriteString ("\n\n" ) 245combined .Passages = append (combined .Passages ,accepted ... ) 246citation := Result {ID :result .ID ,URL :result .URL ,Title :result .Title ,Author :result .Author ,PublishedDate :result .PublishedDate } 247if q .Text { 248v := evidenceText (accepted ) 249citation .Text = & v 250 } 251out .Citations = append (out .Citations ,citation ) 252 } 253if len (out .Citations )== 0 { 254return out ,failure (422 ,"NO_CONTENT_FOUND" ,"no usable source evidence was retained" ) 255 } 256combined .UsedChars = utf8 .RuneCountInString (evidence .String ()) 257s .log .evidence (ctx ,"answer" ,combined ) 258label ,err := s .decider (ctx ).answerability (ctx ,q .Query ,combined ) 259if err != nil { 260return out ,err 261 } 262assessment := map [string ]string {"direct" :"answered" ,"partial" :"partial" ,"background" :"unanswered" ,"none" :"unanswered" }[label ] 263out .Answer ,err = s .llm .digest (ctx ,q .Query ,evidence .String (),assessment ) 264return out ,err 265}