okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petpublic access: readable code on the notice in dark mode3bd2407

main
8.9 KiB236 linesraw
1package app
2
3import (
4	"context"
5	"encoding/json"
6	"fmt"
7	"html"
8	"net"
9	"net/http"
10	"net/netip"
11	"slices"
12	"strconv"
13	"strings"
14	"sync"
15	"time"
16)
17
18// publicGate limits the public endpoint: a fixed one-minute window per client
19// address and, when daily is above zero, a daily count of tool calls across all
20// public clients.
21type publicGate struct {
22	mu         sync.Mutex
23	perMinute  int
24	daily      int
25	minute     int64
26	perClient  map[netip.Addr]int
27	day        string
28	callsToday int
29	now        func() time.Time
30}
31
32func newPublicGate(perMinute, daily int) *publicGate {
33	return &publicGate{perMinute: perMinute, daily: daily, perClient: map[netip.Addr]int{}, now: time.Now}
34}
35
36// admit counts one request from client. work marks a request that runs a tool;
37// only those count toward the daily limit. It returns the seconds to wait when
38// the request is refused.
39func (g *publicGate) admit(client netip.Addr, work bool) (ok bool, retryAfter int) {
40	g.mu.Lock()
41	defer g.mu.Unlock()
42	now := g.now().UTC()
43	if minute := now.Unix() / 60; minute != g.minute {
44		g.minute, g.perClient = minute, map[netip.Addr]int{}
45	}
46	if day := now.Format(time.DateOnly); day != g.day {
47		g.day, g.callsToday = day, 0
48	}
49	// An IPv6 host usually holds a whole /64, so count the /64.
50	if client.Is6() {
51		client = netip.PrefixFrom(client, 64).Masked().Addr()
52	}
53	if g.perClient[client] >= g.perMinute {
54		return false, 60 - int(now.Unix()%60)
55	}
56	if work && g.daily > 0 && g.callsToday >= g.daily {
57		midnight := now.Truncate(24 * time.Hour).Add(24 * time.Hour)
58		return false, int(midnight.Sub(now).Seconds()) + 1
59	}
60	g.perClient[client]++
61	if work {
62		g.callsToday++
63	}
64	return true, 0
65}
66
67// clientAddr is the caller's address. A trusted proxy's X-Forwarded-For is
68// believed, using its last entry, which that proxy appended itself.
69func clientAddr(r *http.Request, trusted []netip.Prefix) netip.Addr {
70	host, _, err := net.SplitHostPort(r.RemoteAddr)
71	if err != nil {
72		host = r.RemoteAddr
73	}
74	peer, err := netip.ParseAddr(host)
75	if err != nil {
76		return netip.Addr{}
77	}
78	peer = peer.Unmap()
79	for _, p := range trusted {
80		if !p.Contains(peer) {
81			continue
82		}
83		forwarded := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
84		if addr, err := netip.ParseAddr(strings.TrimSpace(forwarded[len(forwarded)-1])); err == nil {
85			return addr.Unmap()
86		}
87		break
88	}
89	return peer
90}
91
92// contributes reports whether a public caller opted in to having its calls
93// logged: the X-Contribute header, or the /mcp/contribute endpoint.
94func contributes(r *http.Request) bool {
95	if r.URL.Path == "/mcp/contribute" {
96		return true
97	}
98	v, err := strconv.ParseBool(strings.TrimSpace(r.Header.Get("X-Contribute")))
99	return err == nil && v
100}
101
102// runsTool reports whether a request does work: any REST call, or an MCP
103// tools/call. MCP handshakes and listings are free.
104func runsTool(path string, body []byte) bool {
105	if path != "/mcp" {
106		return true
107	}
108	var msg struct {
109		Method string `json:"method"`
110	}
111	if json.Unmarshal(body, &msg) != nil {
112		return true
113	}
114	return msg.Method == "tools/call"
115}
116
117// decider is the decision model for a call: the public one for public callers
118// when it is configured.
119func (s *Service) decider(ctx context.Context) *Jev {
120	if callInfo(ctx).Public && s.publicJev != nil {
121		return s.publicJev
122	}
123	return s.jev
124}
125
126// publicStyle is shared by the notice and the API page.
127const publicStyle = `<meta charset="utf-8">
128<meta name="viewport" content="width=device-width, initial-scale=1">
129<style>
130:root{--fg:#222;--bg:#fbfaf8;--code:#ebe9e5;--line:#ddd;--link:#2456a6}
131@media (prefers-color-scheme:dark){:root{--fg:#ddd;--bg:#18171a;--code:#2c2b30;--line:#333;--link:#9cc4ff}}
132body{font:16px/1.6 system-ui,sans-serif;max-width:44rem;margin:3rem auto;padding:0 1.25rem;color:var(--fg);background:var(--bg)}
133a{color:var(--link)}
134code{color:inherit;background:var(--code);padding:.1em .3em;border-radius:4px;font-size:.92em}
135pre{overflow-x:auto}
136h1{font-size:1.4rem}
137table{border-collapse:collapse;width:100%;font-size:.95em}
138td,th{border-bottom:1px solid var(--line);padding:.35rem .5rem;text-align:left;vertical-align:top}
139</style>`
140
141const publicNotice = `<!doctype html>
142<html lang="en">
143<title>search.nkp.pet</title>
144` + publicStyle + `
145<h1>search.nkp.pet</h1>
146<p>A web search and page reading tool for AI agents, shared with friends. It searches the web,
147reads pages, keeps the parts that answer your question, and can summarize them. No account needed.</p>
148<h2>Use it</h2>
149<p>Add an MCP server over streamable HTTP:</p>
150<pre><code>https://search.nkp.pet/mcp</code></pre>
151<p>Tools: <code>web_search</code>, <code>web_contents</code>, <code>web_answer</code> and a few single-page shortcuts.
152Each address can make 20 requests a minute. <a href="/docs">All tools and HTTP endpoints</a>.</p>
153<h2>How it works</h2>
154<p>Search results come from a self-hosted SearXNG. Pages are read in a real browser. Deciding which results
155and passages are relevant is done by Cloudflare-hosted <a href="https://developers.cloudflare.com/workers-ai/models/clef-flash/">Clef-Flash</a>;
156it will move to a self-hosted Jev-like model at some point. Summaries and answers are written by
157Bonsai 2 27B, a 1-bit model run on a home server, not fine-tuned yet. Expect it to be slower and rougher than a big hosted model.</p>
158<h2>Your data</h2>
159<p><strong>By default nothing is kept</strong>: your queries, the pages read and the answers are not stored.</p>
160<p>If you want to help, you can opt in. Opted-in calls are saved (the query, the page excerpts and the
161model's answers) and used for one thing: fine-tuning the model that writes the summaries, so they get better.
162Send the header <code>X-Contribute: 1</code>, or use <code>https://search.nkp.pet/mcp/contribute</code> instead.</p>
163<p>This is run by one person. The data is never sold or shared, and nobody reads through it; it only goes
164into training.</p>
165<p>Please don't send anything private through it either way: it fetches pages and talks to model providers on your behalf.</p>
166</html>
167`
168
169// publicDocs renders the API page from the MCP tool schemas, so it lists what
170// the server actually accepts.
171var publicDocs = sync.OnceValue(func() []byte {
172	var tools []struct {
173		Name        string `json:"name"`
174		Description string `json:"description"`
175		InputSchema struct {
176			Required   []string                   `json:"required"`
177			Properties map[string]json.RawMessage `json:"properties"`
178		} `json:"inputSchema"`
179	}
180	if err := json.Unmarshal(toolsSchema, &tools); err != nil {
181		panic(err)
182	}
183	var b strings.Builder
184	b.WriteString("<!doctype html>\n<html lang=\"en\">\n<title>search.nkp.pet API</title>\n" + publicStyle + `
185<h1>search.nkp.pet API</h1>
186<p><a href="/">Back</a>. No authentication. 20 requests a minute per address; past that you get 429 with <code>Retry-After</code>.
187Nothing is stored unless you opt in with <code>X-Contribute: 1</code> (or <code>/mcp/contribute</code>).</p>
188<h2>Endpoints</h2>
189<table>
190<tr><th>endpoint</th><th></th></tr>
191<tr><td><code>POST /mcp</code></td><td>MCP over streamable HTTP (JSON-RPC): <code>initialize</code>, <code>tools/list</code>, <code>tools/call</code>.</td></tr>
192<tr><td><code>POST /mcp/contribute</code></td><td>The same, with this call's data kept for training.</td></tr>
193<tr><td><code>POST /search</code></td><td>The <code>web_search</code> tool as plain JSON: send its arguments as the body.</td></tr>
194<tr><td><code>POST /contents</code></td><td>The <code>web_contents</code> tool as plain JSON.</td></tr>
195<tr><td><code>POST /answer</code></td><td>The <code>web_answer</code> tool as plain JSON.</td></tr>
196<tr><td><code>POST /v1/extract</code>, <code>/v1/preview</code>, <code>/v1/summarize</code></td><td>The single-page tools as plain JSON.</td></tr>
197<tr><td><code>GET /healthz</code></td><td>Liveness.</td></tr>
198</table>
199<p>Example:</p>
200<pre><code>curl https://search.nkp.pet/search -H 'content-type: application/json' \
201  -d '{"query":"what is the atproto firehose","numResults":3,"contents":{"highlights":true}}'</code></pre>
202<h2>MCP tools</h2>
203`)
204	for _, tool := range tools {
205		fmt.Fprintf(&b, "<h3><code>%s</code></h3>\n<p>%s</p>\n<table>\n<tr><th>argument</th><th>type</th><th></th></tr>\n",
206			html.EscapeString(tool.Name), html.EscapeString(tool.Description))
207		names := make([]string, 0, len(tool.InputSchema.Properties))
208		for name := range tool.InputSchema.Properties {
209			names = append(names, name)
210		}
211		slices.Sort(names)
212		for _, name := range names {
213			var p struct {
214				Type        any    `json:"type"`
215				Description string `json:"description"`
216				OneOf       []any  `json:"oneOf"`
217			}
218			_ = json.Unmarshal(tool.InputSchema.Properties[name], &p)
219			kind := fmt.Sprint(p.Type)
220			if p.Type == nil {
221				kind = "object"
222				if p.OneOf != nil {
223					kind = "boolean or object"
224				}
225			}
226			if slices.Contains(tool.InputSchema.Required, name) {
227				kind += ", required"
228			}
229			fmt.Fprintf(&b, "<tr><td><code>%s</code></td><td>%s</td><td>%s</td></tr>\n",
230				html.EscapeString(name), html.EscapeString(kind), html.EscapeString(p.Description))
231		}
232		b.WriteString("</table>\n")
233	}
234	b.WriteString("</html>\n")
235	return []byte(b.String())
236})