okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
b4741f0
main
1package app 2 3import ( 4"context" 5"crypto/rand" 6"crypto/subtle" 7"encoding/hex" 8"encoding/json" 9"errors" 10"fmt" 11"log/slog" 12"net/http" 13"strconv" 14"strings" 15) 16 17func (s * Service )apiHandler () http.Handler { 18return http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 19id := newID () 20w .Header ().Set ("x-request-id" ,id ) 21if r .Method == "GET" && r .URL .Path == "/healthz" { 22writeJSON (w ,200 ,map [string ]any {"ok" :true ,"jevConfigured" :s .jev .Key != "" ,"summarizerConfigured" :s .llm .Key != "" }) 23return 24 } 25token := r .Header .Get ("x-api-key" ) 26if token == "" { 27parts := strings .Fields (r .Header .Get ("Authorization" )) 28if len (parts )== 2 && strings .EqualFold (parts [0 ],"bearer" ) { 29token = parts [1 ] 30 } 31 } 32admin := s .cfg .Token != "" && subtle .ConstantTimeCompare ([]byte (token ), []byte (s .cfg .Token ))== 1 33caller := "" 34if ! admin { 35var err error 36caller ,err = s .auth .caller (token ) 37if err != nil { 38slog .Error ("issued-token store unavailable" ,"error" ,err ) 39writeJSON (w ,503 ,APIError {RequestID :id ,Tag :"AUTH_STORE_UNAVAILABLE" ,Message :"authentication store is unavailable" }) 40return 41 } 42 } 43public := token == "" && s .gate != nil 44if public && r .Method == "GET" && (r .URL .Path == "/" || r .URL .Path == "/docs" ) { 45w .Header ().Set ("Content-Type" ,"text/html; charset=utf-8" ) 46if r .URL .Path == "/docs" { 47_ ,_ = w .Write (publicDocs ()) 48 }else { 49_ ,_ = w .Write ([]byte (publicNotice )) 50 } 51return 52 } 53if ! admin && caller == "" && ! public { 54w .Header ().Set ("WWW-Authenticate" ,"Bearer" ) 55writeJSON (w ,401 ,APIError {RequestID :id ,Tag :"INVALID_API_KEY" ,Message :"invalid api key" }) 56return 57 } 58if r .Method != "POST" { 59w .Header ().Set ("Allow" ,"POST" ) 60writeJSON (w ,405 ,APIError {RequestID :id ,Tag :"METHOD_NOT_ALLOWED" ,Message :"use post" }) 61return 62 } 63ctx ,cancel := context .WithTimeout (r .Context (),s .cfg .RequestTimeout ) 64defer cancel () 65if admin { 66caller = takeRunes (r .Header .Get ("x-caller" ),200 ) 67if caller == "" { 68caller = takeRunes (r .Header .Get ("x-agent-name" ),200 ) 69 } 70if caller == "" { 71caller = "admin" 72 } 73 } 74call := callContext {ID :id ,Caller :caller } 75path := r .URL .Path 76if public { 77call = callContext {ID :id ,Caller :"public" ,Public :true ,Contribute :contributes (r )} 78if path == "/mcp/contribute" { 79path = "/mcp" 80 } 81 } 82ctx = context .WithValue (ctx ,contextKey {},call ) 83body ,err := readBounded (r .Body ,64 << 10 ) 84if err != nil { 85s .respond (w ,ctx ,nil ,failure (413 ,"INVALID_REQUEST_BODY" ,"request body exceeds 64 kib" )) 86return 87 } 88if public { 89if ok ,wait := s .gate .admit (clientAddr (r ,s .cfg .Public .TrustedProxies ),runsTool (path ,body ));! ok { 90w .Header ().Set ("Retry-After" ,strconv .Itoa (wait )) 91s .respond (w ,ctx ,nil ,failure (429 ,"RATE_LIMITED" ,"too many requests; see Retry-After" )) 92return 93 } 94 } 95if path == "/mcp" { 96if result := s .mcp (ctx ,body );result != nil { 97writeJSON (w ,200 ,result ) 98 }else { 99w .WriteHeader (http .StatusAccepted ) 100 } 101return 102 } 103var out any 104endpoint := strings .TrimPrefix (path ,"/" ) 105switch path { 106case "/search" : 107var q SearchRequest 108if err = decodeRequest (body ,& q );err == nil { 109out ,err = s .search (ctx ,q ) 110 } 111case "/contents" : 112var q ContentsRequest 113if err = decodeRequest (body ,& q );err == nil { 114out ,err = s .contents (ctx ,q ) 115 } 116case "/answer" : 117var q AnswerRequest 118if err = decodeRequest (body ,& q );err == nil { 119out ,err = s .answer (ctx ,q ) 120 } 121case "/v1/extract" ,"/v1/preview" ,"/v1/summarize" : 122endpoint = "contents" 123out ,err = s .pageAlias (ctx ,strings .TrimPrefix (path ,"/v1/" ),body ) 124default : 125err = failure (404 ,"NOT_FOUND" ,"unknown endpoint" ) 126 } 127if err != nil { 128var syntax * json.SyntaxError 129var typeErr * json.UnmarshalTypeError 130if errors .As (err ,& syntax )|| errors .As (err ,& typeErr )|| strings .HasPrefix (err .Error (),"json:" )|| err .Error ()== "expected one json value" { 131err = failure (400 ,"INVALID_REQUEST_BODY" ,err .Error ()) 132 } 133 } 134s .recordAPI (ctx ,endpoint ,body ,out ,err ) 135s .respond (w ,ctx ,out ,err ) 136 }) 137} 138 139func (s * Service )recordAPI (ctx context.Context ,endpoint string ,body []byte ,out any ,err error ) { 140detail := map [string ]any {} 141outcome := "ok" 142var fields map [string ]json.RawMessage 143_ = json .Unmarshal (body ,& fields ) 144if q := fields ["query" ];q != nil { 145var v string 146_ = json .Unmarshal (q ,& v ) 147detail ["query" ]= v 148 } 149if result ,ok := out .(ResultsResponse );ok { 150urls := []string {} 151for _ ,r := range result .Results { 152urls = append (urls ,r .ID ) 153 } 154if endpoint == "search" { 155detail ["result_urls" ]= urls 156 } 157if endpoint == "contents" { 158detail ["urls" ]= urls 159 } 160if len (result .Statuses )> 0 { 161detail ["statuses" ]= result .Statuses 162for _ ,status := range result .Statuses { 163if status .Status == "error" { 164outcome = "partial" 165 } 166 } 167 } 168 } 169if err != nil { 170outcome = apiError (ctx ,err ).Tag 171 } 172s .log .request (ctx ,endpoint ,detail ,outcome ) 173// Preserve the old exporter’s search→extract follow-through labels for 174// explicit /contents reads without inventing model completions. 175if endpoint == "contents" { 176if result ,ok := out .(ResultsResponse );ok { 177for _ ,r := range result .Results { 178s .log .request (ctx ,"extract" ,map [string ]any {"url" :r .ID },"ok" ) 179 } 180 } 181 } 182} 183func apiError (ctx context.Context ,err error )APIError { 184var a * APIError 185if errors .As (err ,& a ) { 186copy := * a 187copy .RequestID = callInfo (ctx ).ID 188return copy 189 } 190status ,tag := 502 ,"UPSTREAM_ERROR" 191if errors .Is (err ,context .DeadlineExceeded )|| errors .Is (err ,context .Canceled ) { 192status = 504 193tag = "REQUEST_TIMEOUT" 194 } 195return APIError {Status :status ,RequestID :callInfo (ctx ).ID ,Tag :tag ,Message :err .Error ()} 196} 197func (s * Service )respond (w http.ResponseWriter ,ctx context.Context ,out any ,err error ) { 198if err != nil { 199a := apiError (ctx ,err ) 200writeJSON (w ,a .Status ,a ) 201return 202 } 203writeJSON (w ,200 ,out ) 204} 205func writeJSON (w http.ResponseWriter ,status int ,v any ) { 206w .Header ().Set ("Content-Type" ,"application/json" ) 207w .WriteHeader (status ) 208_ = json .NewEncoder (w ).Encode (v ) 209} 210func newID ()string { 211var b [16 ]byte 212if _ ,err := rand .Read (b [:]);err != nil { 213panic (err ) 214 } 215return hex .EncodeToString (b [:]) 216} 217 218func decodeRequest (b []byte ,v any )error { 219if err := strictJSON (b ,v );err != nil { 220return failure (400 ,"INVALID_REQUEST_BODY" ,err .Error ()) 221 } 222return nil 223} 224 225// MCP transport metadata is not a tool argument. Keep the service's strict 226// validation for every actual argument, including unknown nested fields. 227func stripMCPMeta (arguments json.RawMessage ) (json.RawMessage ,error ) { 228var fields map [string ]json.RawMessage 229if err := json .Unmarshal (arguments ,& fields );err != nil { 230return nil ,err 231 } 232if _ ,ok := fields ["_meta" ];! ok { 233return arguments ,nil 234 } 235delete (fields ,"_meta" ) 236return json .Marshal (fields ) 237} 238 239var toolsSchema = json .RawMessage (`[ 240{"name":"web_search","description":"Search the web (SearXNG), rerank with a decision model, and optionally return text, highlights or grounded summaries. Exa-shaped input and output.","inputSchema":{"type":"object","required":["query"],"additionalProperties":false,"properties":{"query":{"type":"string"},"type":{"type":"string","enum":["auto"]},"numResults":{"type":"integer","minimum":1,"maximum":20},"includeDomains":{"type":"array","items":{"type":"string"}},"excludeDomains":{"type":"array","items":{"type":"string"}},"contents":{"type":"object","properties":{"text":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum returned text characters (1–10000); 0 or omitted means uncapped."}}}]},"highlights":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"},"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum total highlight characters (1–10000); 0 or omitted uses the configured evidence budget."}}}]},"summary":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"}}}]},"maxAgeHours":{"type":"integer","minimum":-1,"maximum":720},"livecrawlTimeout":{"type":"integer","minimum":1,"maximum":90000}},"additionalProperties":false}}}}, 241{"name":"web_contents","description":"Read up to 20 known urls. Full text by default; request verbatim highlights picked by the decision model or a query-focused summary explicitly.","inputSchema":{"type":"object","additionalProperties":false,"properties":{"urls":{"type":"array","minItems":1,"maxItems":20,"items":{"type":"string"}},"ids":{"type":"array","minItems":1,"maxItems":20,"items":{"type":"string"}},"text":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum returned text characters (1–10000); 0 or omitted means uncapped."}}}]},"highlights":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"},"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum total highlight characters (1–10000); 0 or omitted uses the configured evidence budget."}}}]},"summary":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"}}}]},"maxAgeHours":{"type":"integer","minimum":-1,"maximum":720},"livecrawlTimeout":{"type":"integer","minimum":1,"maximum":90000}},"oneOf":[{"required":["urls"]},{"required":["ids"]}]}}, 242{"name":"web_answer","description":"Search and return one grounded plain-prose answer plus sources, using the original evidence the decision model picked rather than summaries of summaries.","inputSchema":{"type":"object","required":["query"],"additionalProperties":false,"properties":{"query":{"type":"string"},"text":{"type":"boolean"},"stream":{"type":"boolean","enum":[false]}}}}, 243{"name":"web_extract","description":"Single-url alias for web_contents with text enabled, without model calls. Returns the new requestId/results/statuses shape, not legacy markdown fields.","inputSchema":{"type":"object","required":["url"],"additionalProperties":false,"properties":{"url":{"type":"string"},"max_chars":{"type":"integer","minimum":0,"maximum":10000}}}}, 244{"name":"web_preview","description":"Single-url alias for web_contents with a query-focused, evidence-grounded summary. Returns the new requestId/results/statuses shape. Inspect per-url statuses.","inputSchema":{"type":"object","required":["url","query"],"additionalProperties":false,"properties":{"url":{"type":"string"},"query":{"type":"string","minLength":1,"maxLength":4000},"include_markdown":{"type":"boolean"},"max_chars":{"type":"integer","minimum":0,"maximum":10000}}}}, 245{"name":"web_summarize","description":"Single-url alias for web_contents with an evidence-grounded summary. Optional query (or instructions as the fallback query) focuses the summary. Returns the new requestId/results/statuses shape.","inputSchema":{"type":"object","required":["url"],"additionalProperties":false,"properties":{"url":{"type":"string"},"query":{"type":"string","maxLength":4000},"instructions":{"type":"string","maxLength":4000},"include_markdown":{"type":"boolean"},"max_chars":{"type":"integer","minimum":0,"maximum":10000}}}} 246]` ) 247 248func (s * Service )mcp (ctx context.Context ,body []byte )any { 249var req struct { 250JSONRPC string `json:"jsonrpc"` 251ID json.RawMessage `json:"id"` 252Method string `json:"method"` 253Params json.RawMessage `json:"params"` 254 } 255if err := json .Unmarshal (body ,& req );err != nil || req .JSONRPC != "2.0" { 256return map [string ]any {"jsonrpc" :"2.0" ,"id" :nil ,"error" :map [string ]any {"code" :- 32600 ,"message" :"invalid json-rpc request" }} 257 } 258if len (req .ID )== 0 { 259return nil 260 } 261reply := func (result any )any {return map [string ]any {"jsonrpc" :"2.0" ,"id" :req .ID ,"result" :result } } 262switch req .Method { 263case "initialize" : 264return reply (map [string ]any {"protocolVersion" :"2025-03-26" ,"capabilities" :map [string ]any {"tools" :map [string ]bool {"listChanged" :false }},"serverInfo" :map [string ]string {"name" :"web-extract" ,"version" :"2.0.0" }}) 265case "ping" : 266return reply (map [string ]any {}) 267case "tools/list" : 268return reply (map [string ]any {"tools" :toolsSchema }) 269case "tools/call" : 270var p struct { 271Name string `json:"name"` 272Arguments json.RawMessage `json:"arguments"` 273Meta json.RawMessage `json:"_meta"` 274 } 275err := strictJSON (req .Params ,& p ) 276var out any 277endpoint := "" 278if err == nil { 279p .Arguments ,err = stripMCPMeta (p .Arguments ) 280 } 281if err == nil { 282switch p .Name { 283case "web_search" : 284endpoint = "search" 285var q SearchRequest 286if err = strictJSON (p .Arguments ,& q );err == nil { 287out ,err = s .search (ctx ,q ) 288 } 289case "web_contents" : 290endpoint = "contents" 291var q ContentsRequest 292if err = strictJSON (p .Arguments ,& q );err == nil { 293out ,err = s .contents (ctx ,q ) 294 } 295case "web_answer" : 296endpoint = "answer" 297var q AnswerRequest 298if err = strictJSON (p .Arguments ,& q );err == nil { 299out ,err = s .answer (ctx ,q ) 300 } 301case "web_extract" ,"web_preview" ,"web_summarize" : 302endpoint = "contents" 303out ,err = s .pageAlias (ctx ,strings .TrimPrefix (p .Name ,"web_" ),p .Arguments ) 304default : 305err = fmt .Errorf ("unknown tool" ) 306 } 307 } 308s .recordAPI (ctx ,endpoint ,p .Arguments ,out ,err ) 309if err != nil { 310a := apiError (ctx ,err ) 311return reply (map [string ]any {"isError" :true ,"content" : []any {map [string ]string {"type" :"text" ,"text" :jsonText (a )}},"structuredContent" :a }) 312 }else { 313return reply (map [string ]any {"isError" :false ,"content" : []any {map [string ]string {"type" :"text" ,"text" :jsonText (out )}},"structuredContent" :out }) 314 } 315default : 316return map [string ]any {"jsonrpc" :"2.0" ,"id" :req .ID ,"error" :map [string ]any {"code" :- 32601 ,"message" :"method not found" }} 317 } 318}