okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
24e631b
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"net/http" 7"net/http/httptest" 8"strings" 9"testing" 10) 11 12func TestAnswerReadsOnlySourcesTheRerankerFoundUseful (t * testing.T ) { 13s ,page := fixtureService (t ) 14status ,body := callHTTP (t ,s ,"/answer" ,`{"query":"request limits"}` ) 15if status != 200 { 16t .Fatal (status ,string (body ["error" ])) 17 } 18var citations []Result 19_ = json .Unmarshal (body ["citations" ],& citations ) 20if len (citations )!= 1 || citations [0 ].URL != page .URL { 21t .Fatalf ("a candidate scored 0.02 became a source: %+v" ,citations ) 22 } 23var digest string 24_ = s .log .db .QueryRow ("select request_json from completions where task='synthesis_digest'" ).Scan (& digest ) 25if ! strings .Contains (digest ,`\n[1] ` )|| strings .Contains (digest ,`\n[2] ` )|| strings .Contains (digest ,"[source 1]" ) { 26t .Fatal ("sources are not numbered as their citations" ) 27 } 28var passages string 29_ = s .log .db .QueryRow ("select group_concat(request_json) from decisions where task='passage'" ).Scan (& passages ) 30if strings .Contains (passages ,page .URL + "/bad" ) { 31t .Fatal ("a low-scored candidate was fetched and judged" ) 32 } 33} 34 35func TestAnswerKeepsTheBestSourceWhenAllScoreLow (t * testing.T ) { 36s ,page := fixtureService (t ) 37s .searcher = fixedSearch {{ID :page .URL + "/bad" ,URL :page .URL + "/bad" ,Title :"bad candidate" }} 38if status ,body := callHTTP (t ,s ,"/answer" ,`{"query":"request limits"}` );status != 200 { 39t .Fatal ("the floor removed every source:" ,status ,string (body ["error" ])) 40 } 41} 42 43func TestSearxCandidatesCarryTheirSnippetAndID (t * testing.T ) { 44srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 45if got := r .URL .Query ().Get ("categories" );got != "general,it,science" { 46t .Errorf ("searx categories %q" ,got ) 47 } 48writeJSON (w ,200 ,map [string ]any {"results" : []map [string ]string { 49 {"url" :"https://docs.example.com/limits#top" ,"title" :"limits" ,"content" :" requests may contain\n up to 8 mib " }, 50 {"url" :"https://docs.example.com/limits" ,"title" :"limits again" ,"content" :"duplicate" }, 51 {"url" :"not a url" ,"title" :"junk" ,"content" :"" }, 52 }}) 53 })) 54t .Cleanup (srv .Close ) 55hits ,err := (& Searx {BaseURL :srv .URL ,HTTP :srv .Client (),maxBytes :1 << 20 }).Search (context .Background (),"limits" ,10 ) 56if err != nil { 57t .Fatal (err ) 58 } 59if len (hits )!= 1 || hits [0 ].ID != "https://docs.example.com/limits" || hits [0 ].snippet != "requests may contain up to 8 mib" { 60t .Fatalf ("unexpected searx candidates: %+v" ,hits ) 61 } 62}