okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petpublic access: explain the models on the notice and add an API pageb4741f0

main
13.5 KiB318 linesraw
1package app
2
3import (
4	"context"
5	"crypto/rand"
6	"crypto/subtle"
7	"encoding/hex"
8	"encoding/json"
9	"errors"
10	"fmt"
11	"log/slog"
12	"net/http"
13	"strconv"
14	"strings"
15)
16
17func (s *Service) apiHandler() http.Handler {
18	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
19		id := newID()
20		w.Header().Set("x-request-id", id)
21		if r.Method == "GET" && r.URL.Path == "/healthz" {
22			writeJSON(w, 200, map[string]any{"ok": true, "jevConfigured": s.jev.Key != "", "summarizerConfigured": s.llm.Key != ""})
23			return
24		}
25		token := r.Header.Get("x-api-key")
26		if token == "" {
27			parts := strings.Fields(r.Header.Get("Authorization"))
28			if len(parts) == 2 && strings.EqualFold(parts[0], "bearer") {
29				token = parts[1]
30			}
31		}
32		admin := s.cfg.Token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(s.cfg.Token)) == 1
33		caller := ""
34		if !admin {
35			var err error
36			caller, err = s.auth.caller(token)
37			if err != nil {
38				slog.Error("issued-token store unavailable", "error", err)
39				writeJSON(w, 503, APIError{RequestID: id, Tag: "AUTH_STORE_UNAVAILABLE", Message: "authentication store is unavailable"})
40				return
41			}
42		}
43		public := token == "" && s.gate != nil
44		if public && r.Method == "GET" && (r.URL.Path == "/" || r.URL.Path == "/docs") {
45			w.Header().Set("Content-Type", "text/html; charset=utf-8")
46			if r.URL.Path == "/docs" {
47				_, _ = w.Write(publicDocs())
48			} else {
49				_, _ = w.Write([]byte(publicNotice))
50			}
51			return
52		}
53		if !admin && caller == "" && !public {
54			w.Header().Set("WWW-Authenticate", "Bearer")
55			writeJSON(w, 401, APIError{RequestID: id, Tag: "INVALID_API_KEY", Message: "invalid api key"})
56			return
57		}
58		if r.Method != "POST" {
59			w.Header().Set("Allow", "POST")
60			writeJSON(w, 405, APIError{RequestID: id, Tag: "METHOD_NOT_ALLOWED", Message: "use post"})
61			return
62		}
63		ctx, cancel := context.WithTimeout(r.Context(), s.cfg.RequestTimeout)
64		defer cancel()
65		if admin {
66			caller = takeRunes(r.Header.Get("x-caller"), 200)
67			if caller == "" {
68				caller = takeRunes(r.Header.Get("x-agent-name"), 200)
69			}
70			if caller == "" {
71				caller = "admin"
72			}
73		}
74		call := callContext{ID: id, Caller: caller}
75		path := r.URL.Path
76		if public {
77			call = callContext{ID: id, Caller: "public", Public: true, Contribute: contributes(r)}
78			if path == "/mcp/contribute" {
79				path = "/mcp"
80			}
81		}
82		ctx = context.WithValue(ctx, contextKey{}, call)
83		body, err := readBounded(r.Body, 64<<10)
84		if err != nil {
85			s.respond(w, ctx, nil, failure(413, "INVALID_REQUEST_BODY", "request body exceeds 64 kib"))
86			return
87		}
88		if public {
89			if ok, wait := s.gate.admit(clientAddr(r, s.cfg.Public.TrustedProxies), runsTool(path, body)); !ok {
90				w.Header().Set("Retry-After", strconv.Itoa(wait))
91				s.respond(w, ctx, nil, failure(429, "RATE_LIMITED", "too many requests; see Retry-After"))
92				return
93			}
94		}
95		if path == "/mcp" {
96			if result := s.mcp(ctx, body); result != nil {
97				writeJSON(w, 200, result)
98			} else {
99				w.WriteHeader(http.StatusAccepted)
100			}
101			return
102		}
103		var out any
104		endpoint := strings.TrimPrefix(path, "/")
105		switch path {
106		case "/search":
107			var q SearchRequest
108			if err = decodeRequest(body, &q); err == nil {
109				out, err = s.search(ctx, q)
110			}
111		case "/contents":
112			var q ContentsRequest
113			if err = decodeRequest(body, &q); err == nil {
114				out, err = s.contents(ctx, q)
115			}
116		case "/answer":
117			var q AnswerRequest
118			if err = decodeRequest(body, &q); err == nil {
119				out, err = s.answer(ctx, q)
120			}
121		case "/v1/extract", "/v1/preview", "/v1/summarize":
122			endpoint = "contents"
123			out, err = s.pageAlias(ctx, strings.TrimPrefix(path, "/v1/"), body)
124		default:
125			err = failure(404, "NOT_FOUND", "unknown endpoint")
126		}
127		if err != nil {
128			var syntax *json.SyntaxError
129			var typeErr *json.UnmarshalTypeError
130			if errors.As(err, &syntax) || errors.As(err, &typeErr) || strings.HasPrefix(err.Error(), "json:") || err.Error() == "expected one json value" {
131				err = failure(400, "INVALID_REQUEST_BODY", err.Error())
132			}
133		}
134		s.recordAPI(ctx, endpoint, body, out, err)
135		s.respond(w, ctx, out, err)
136	})
137}
138
139func (s *Service) recordAPI(ctx context.Context, endpoint string, body []byte, out any, err error) {
140	detail := map[string]any{}
141	outcome := "ok"
142	var fields map[string]json.RawMessage
143	_ = json.Unmarshal(body, &fields)
144	if q := fields["query"]; q != nil {
145		var v string
146		_ = json.Unmarshal(q, &v)
147		detail["query"] = v
148	}
149	if result, ok := out.(ResultsResponse); ok {
150		urls := []string{}
151		for _, r := range result.Results {
152			urls = append(urls, r.ID)
153		}
154		if endpoint == "search" {
155			detail["result_urls"] = urls
156		}
157		if endpoint == "contents" {
158			detail["urls"] = urls
159		}
160		if len(result.Statuses) > 0 {
161			detail["statuses"] = result.Statuses
162			for _, status := range result.Statuses {
163				if status.Status == "error" {
164					outcome = "partial"
165				}
166			}
167		}
168	}
169	if err != nil {
170		outcome = apiError(ctx, err).Tag
171	}
172	s.log.request(ctx, endpoint, detail, outcome)
173	// Preserve the old exporter’s search→extract follow-through labels for
174	// explicit /contents reads without inventing model completions.
175	if endpoint == "contents" {
176		if result, ok := out.(ResultsResponse); ok {
177			for _, r := range result.Results {
178				s.log.request(ctx, "extract", map[string]any{"url": r.ID}, "ok")
179			}
180		}
181	}
182}
183func apiError(ctx context.Context, err error) APIError {
184	var a *APIError
185	if errors.As(err, &a) {
186		copy := *a
187		copy.RequestID = callInfo(ctx).ID
188		return copy
189	}
190	status, tag := 502, "UPSTREAM_ERROR"
191	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
192		status = 504
193		tag = "REQUEST_TIMEOUT"
194	}
195	return APIError{Status: status, RequestID: callInfo(ctx).ID, Tag: tag, Message: err.Error()}
196}
197func (s *Service) respond(w http.ResponseWriter, ctx context.Context, out any, err error) {
198	if err != nil {
199		a := apiError(ctx, err)
200		writeJSON(w, a.Status, a)
201		return
202	}
203	writeJSON(w, 200, out)
204}
205func writeJSON(w http.ResponseWriter, status int, v any) {
206	w.Header().Set("Content-Type", "application/json")
207	w.WriteHeader(status)
208	_ = json.NewEncoder(w).Encode(v)
209}
210func newID() string {
211	var b [16]byte
212	if _, err := rand.Read(b[:]); err != nil {
213		panic(err)
214	}
215	return hex.EncodeToString(b[:])
216}
217
218func decodeRequest(b []byte, v any) error {
219	if err := strictJSON(b, v); err != nil {
220		return failure(400, "INVALID_REQUEST_BODY", err.Error())
221	}
222	return nil
223}
224
225// MCP transport metadata is not a tool argument. Keep the service's strict
226// validation for every actual argument, including unknown nested fields.
227func stripMCPMeta(arguments json.RawMessage) (json.RawMessage, error) {
228	var fields map[string]json.RawMessage
229	if err := json.Unmarshal(arguments, &fields); err != nil {
230		return nil, err
231	}
232	if _, ok := fields["_meta"]; !ok {
233		return arguments, nil
234	}
235	delete(fields, "_meta")
236	return json.Marshal(fields)
237}
238
239var toolsSchema = json.RawMessage(`[
240 {"name":"web_search","description":"Search the web (SearXNG), rerank with a decision model, and optionally return text, highlights or grounded summaries. Exa-shaped input and output.","inputSchema":{"type":"object","required":["query"],"additionalProperties":false,"properties":{"query":{"type":"string"},"type":{"type":"string","enum":["auto"]},"numResults":{"type":"integer","minimum":1,"maximum":20},"includeDomains":{"type":"array","items":{"type":"string"}},"excludeDomains":{"type":"array","items":{"type":"string"}},"contents":{"type":"object","properties":{"text":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum returned text characters (1–10000); 0 or omitted means uncapped."}}}]},"highlights":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"},"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum total highlight characters (1–10000); 0 or omitted uses the configured evidence budget."}}}]},"summary":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"}}}]},"maxAgeHours":{"type":"integer","minimum":-1,"maximum":720},"livecrawlTimeout":{"type":"integer","minimum":1,"maximum":90000}},"additionalProperties":false}}}},
241 {"name":"web_contents","description":"Read up to 20 known urls. Full text by default; request verbatim highlights picked by the decision model or a query-focused summary explicitly.","inputSchema":{"type":"object","additionalProperties":false,"properties":{"urls":{"type":"array","minItems":1,"maxItems":20,"items":{"type":"string"}},"ids":{"type":"array","minItems":1,"maxItems":20,"items":{"type":"string"}},"text":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum returned text characters (1–10000); 0 or omitted means uncapped."}}}]},"highlights":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"},"maxCharacters":{"type":"integer","minimum":0,"maximum":10000,"description":"Maximum total highlight characters (1–10000); 0 or omitted uses the configured evidence budget."}}}]},"summary":{"oneOf":[{"type":"boolean"},{"type":"object","properties":{"query":{"type":"string"}}}]},"maxAgeHours":{"type":"integer","minimum":-1,"maximum":720},"livecrawlTimeout":{"type":"integer","minimum":1,"maximum":90000}},"oneOf":[{"required":["urls"]},{"required":["ids"]}]}},
242 {"name":"web_answer","description":"Search and return one grounded plain-prose answer plus sources, using the original evidence the decision model picked rather than summaries of summaries.","inputSchema":{"type":"object","required":["query"],"additionalProperties":false,"properties":{"query":{"type":"string"},"text":{"type":"boolean"},"stream":{"type":"boolean","enum":[false]}}}},
243 {"name":"web_extract","description":"Single-url alias for web_contents with text enabled, without model calls. Returns the new requestId/results/statuses shape, not legacy markdown fields.","inputSchema":{"type":"object","required":["url"],"additionalProperties":false,"properties":{"url":{"type":"string"},"max_chars":{"type":"integer","minimum":0,"maximum":10000}}}},
244 {"name":"web_preview","description":"Single-url alias for web_contents with a query-focused, evidence-grounded summary. Returns the new requestId/results/statuses shape. Inspect per-url statuses.","inputSchema":{"type":"object","required":["url","query"],"additionalProperties":false,"properties":{"url":{"type":"string"},"query":{"type":"string","minLength":1,"maxLength":4000},"include_markdown":{"type":"boolean"},"max_chars":{"type":"integer","minimum":0,"maximum":10000}}}},
245 {"name":"web_summarize","description":"Single-url alias for web_contents with an evidence-grounded summary. Optional query (or instructions as the fallback query) focuses the summary. Returns the new requestId/results/statuses shape.","inputSchema":{"type":"object","required":["url"],"additionalProperties":false,"properties":{"url":{"type":"string"},"query":{"type":"string","maxLength":4000},"instructions":{"type":"string","maxLength":4000},"include_markdown":{"type":"boolean"},"max_chars":{"type":"integer","minimum":0,"maximum":10000}}}}
246]`)
247
248func (s *Service) mcp(ctx context.Context, body []byte) any {
249	var req struct {
250		JSONRPC string          `json:"jsonrpc"`
251		ID      json.RawMessage `json:"id"`
252		Method  string          `json:"method"`
253		Params  json.RawMessage `json:"params"`
254	}
255	if err := json.Unmarshal(body, &req); err != nil || req.JSONRPC != "2.0" {
256		return map[string]any{"jsonrpc": "2.0", "id": nil, "error": map[string]any{"code": -32600, "message": "invalid json-rpc request"}}
257	}
258	if len(req.ID) == 0 {
259		return nil
260	}
261	reply := func(result any) any { return map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result} }
262	switch req.Method {
263	case "initialize":
264		return reply(map[string]any{"protocolVersion": "2025-03-26", "capabilities": map[string]any{"tools": map[string]bool{"listChanged": false}}, "serverInfo": map[string]string{"name": "web-extract", "version": "2.0.0"}})
265	case "ping":
266		return reply(map[string]any{})
267	case "tools/list":
268		return reply(map[string]any{"tools": toolsSchema})
269	case "tools/call":
270		var p struct {
271			Name      string          `json:"name"`
272			Arguments json.RawMessage `json:"arguments"`
273			Meta      json.RawMessage `json:"_meta"`
274		}
275		err := strictJSON(req.Params, &p)
276		var out any
277		endpoint := ""
278		if err == nil {
279			p.Arguments, err = stripMCPMeta(p.Arguments)
280		}
281		if err == nil {
282			switch p.Name {
283			case "web_search":
284				endpoint = "search"
285				var q SearchRequest
286				if err = strictJSON(p.Arguments, &q); err == nil {
287					out, err = s.search(ctx, q)
288				}
289			case "web_contents":
290				endpoint = "contents"
291				var q ContentsRequest
292				if err = strictJSON(p.Arguments, &q); err == nil {
293					out, err = s.contents(ctx, q)
294				}
295			case "web_answer":
296				endpoint = "answer"
297				var q AnswerRequest
298				if err = strictJSON(p.Arguments, &q); err == nil {
299					out, err = s.answer(ctx, q)
300				}
301			case "web_extract", "web_preview", "web_summarize":
302				endpoint = "contents"
303				out, err = s.pageAlias(ctx, strings.TrimPrefix(p.Name, "web_"), p.Arguments)
304			default:
305				err = fmt.Errorf("unknown tool")
306			}
307		}
308		s.recordAPI(ctx, endpoint, p.Arguments, out, err)
309		if err != nil {
310			a := apiError(ctx, err)
311			return reply(map[string]any{"isError": true, "content": []any{map[string]string{"type": "text", "text": jsonText(a)}}, "structuredContent": a})
312		} else {
313			return reply(map[string]any{"isError": false, "content": []any{map[string]string{"type": "text", "text": jsonText(out)}}, "structuredContent": out})
314		}
315	default:
316		return map[string]any{"jsonrpc": "2.0", "id": req.ID, "error": map[string]any{"code": -32601, "message": "method not found"}}
317	}
318}