okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
3bd2407
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"fmt" 7"html" 8"net" 9"net/http" 10"net/netip" 11"slices" 12"strconv" 13"strings" 14"sync" 15"time" 16) 17 18// publicGate limits the public endpoint: a fixed one-minute window per client 19// address and, when daily is above zero, a daily count of tool calls across all 20// public clients. 21type publicGate struct { 22mu sync.Mutex 23perMinute int 24daily int 25minute int64 26perClient map [netip.Addr ]int 27day string 28callsToday int 29now func () time.Time 30} 31 32func newPublicGate (perMinute ,daily int )* publicGate { 33return & publicGate {perMinute :perMinute ,daily :daily ,perClient :map [netip.Addr ]int {},now :time .Now } 34} 35 36// admit counts one request from client. work marks a request that runs a tool; 37// only those count toward the daily limit. It returns the seconds to wait when 38// the request is refused. 39func (g * publicGate )admit (client netip.Addr ,work bool ) (ok bool ,retryAfter int ) { 40g .mu .Lock () 41defer g .mu .Unlock () 42now := g .now ().UTC () 43if minute := now .Unix ()/ 60 ;minute != g .minute { 44g .minute ,g .perClient = minute ,map [netip.Addr ]int {} 45 } 46if day := now .Format (time .DateOnly );day != g .day { 47g .day ,g .callsToday = day ,0 48 } 49// An IPv6 host usually holds a whole /64, so count the /64. 50if client .Is6 () { 51client = netip .PrefixFrom (client ,64 ).Masked ().Addr () 52 } 53if g .perClient [client ]>= g .perMinute { 54return false ,60 - int (now .Unix ()% 60 ) 55 } 56if work && g .daily > 0 && g .callsToday >= g .daily { 57midnight := now .Truncate (24 * time .Hour ).Add (24 * time .Hour ) 58return false ,int (midnight .Sub (now ).Seconds ())+ 1 59 } 60g .perClient [client ]++ 61if work { 62g .callsToday ++ 63 } 64return true ,0 65} 66 67// clientAddr is the caller's address. A trusted proxy's X-Forwarded-For is 68// believed, using its last entry, which that proxy appended itself. 69func clientAddr (r * http.Request ,trusted []netip.Prefix ) netip.Addr { 70host ,_ ,err := net .SplitHostPort (r .RemoteAddr ) 71if err != nil { 72host = r .RemoteAddr 73 } 74peer ,err := netip .ParseAddr (host ) 75if err != nil { 76return netip.Addr {} 77 } 78peer = peer .Unmap () 79for _ ,p := range trusted { 80if ! p .Contains (peer ) { 81continue 82 } 83forwarded := strings .Split (r .Header .Get ("X-Forwarded-For" ),"," ) 84if addr ,err := netip .ParseAddr (strings .TrimSpace (forwarded [len (forwarded )- 1 ]));err == nil { 85return addr .Unmap () 86 } 87break 88 } 89return peer 90} 91 92// contributes reports whether a public caller opted in to having its calls 93// logged: the X-Contribute header, or the /mcp/contribute endpoint. 94func contributes (r * http.Request )bool { 95if r .URL .Path == "/mcp/contribute" { 96return true 97 } 98v ,err := strconv .ParseBool (strings .TrimSpace (r .Header .Get ("X-Contribute" ))) 99return err == nil && v 100} 101 102// runsTool reports whether a request does work: any REST call, or an MCP 103// tools/call. MCP handshakes and listings are free. 104func runsTool (path string ,body []byte )bool { 105if path != "/mcp" { 106return true 107 } 108var msg struct { 109Method string `json:"method"` 110 } 111if json .Unmarshal (body ,& msg )!= nil { 112return true 113 } 114return msg .Method == "tools/call" 115} 116 117// decider is the decision model for a call: the public one for public callers 118// when it is configured. 119func (s * Service )decider (ctx context.Context )* Jev { 120if callInfo (ctx ).Public && s .publicJev != nil { 121return s .publicJev 122 } 123return s .jev 124} 125 126// publicStyle is shared by the notice and the API page. 127const publicStyle = `<meta charset="utf-8"> 128<meta name="viewport" content="width=device-width, initial-scale=1"> 129<style> 130:root{--fg:#222;--bg:#fbfaf8;--code:#ebe9e5;--line:#ddd;--link:#2456a6} 131@media (prefers-color-scheme:dark){:root{--fg:#ddd;--bg:#18171a;--code:#2c2b30;--line:#333;--link:#9cc4ff}} 132body{font:16px/1.6 system-ui,sans-serif;max-width:44rem;margin:3rem auto;padding:0 1.25rem;color:var(--fg);background:var(--bg)} 133a{color:var(--link)} 134code{color:inherit;background:var(--code);padding:.1em .3em;border-radius:4px;font-size:.92em} 135pre{overflow-x:auto} 136h1{font-size:1.4rem} 137table{border-collapse:collapse;width:100%;font-size:.95em} 138td,th{border-bottom:1px solid var(--line);padding:.35rem .5rem;text-align:left;vertical-align:top} 139</style>` 140 141const publicNotice = `<!doctype html> 142<html lang="en"> 143<title>search.nkp.pet</title> 144` + publicStyle + ` 145<h1>search.nkp.pet</h1> 146<p>A web search and page reading tool for AI agents, shared with friends. It searches the web, 147reads pages, keeps the parts that answer your question, and can summarize them. No account needed.</p> 148<h2>Use it</h2> 149<p>Add an MCP server over streamable HTTP:</p> 150<pre><code>https://search.nkp.pet/mcp</code></pre> 151<p>Tools: <code>web_search</code>, <code>web_contents</code>, <code>web_answer</code> and a few single-page shortcuts. 152Each address can make 20 requests a minute. <a href="/docs">All tools and HTTP endpoints</a>.</p> 153<h2>How it works</h2> 154<p>Search results come from a self-hosted SearXNG. Pages are read in a real browser. Deciding which results 155and passages are relevant is done by Cloudflare-hosted <a href="https://developers.cloudflare.com/workers-ai/models/clef-flash/">Clef-Flash</a>; 156it will move to a self-hosted Jev-like model at some point. Summaries and answers are written by 157Bonsai 2 27B, a 1-bit model run on a home server, not fine-tuned yet. Expect it to be slower and rougher than a big hosted model.</p> 158<h2>Your data</h2> 159<p><strong>By default nothing is kept</strong>: your queries, the pages read and the answers are not stored.</p> 160<p>If you want to help, you can opt in. Opted-in calls are saved (the query, the page excerpts and the 161model's answers) and used for one thing: fine-tuning the model that writes the summaries, so they get better. 162Send the header <code>X-Contribute: 1</code>, or use <code>https://search.nkp.pet/mcp/contribute</code> instead.</p> 163<p>This is run by one person. The data is never sold or shared, and nobody reads through it; it only goes 164into training.</p> 165<p>Please don't send anything private through it either way: it fetches pages and talks to model providers on your behalf.</p> 166</html> 167` 168 169// publicDocs renders the API page from the MCP tool schemas, so it lists what 170// the server actually accepts. 171var publicDocs = sync .OnceValue (func () []byte { 172var tools []struct { 173Name string `json:"name"` 174Description string `json:"description"` 175InputSchema struct { 176Required []string `json:"required"` 177Properties map [string ]json.RawMessage `json:"properties"` 178 }`json:"inputSchema"` 179 } 180if err := json .Unmarshal (toolsSchema ,& tools );err != nil { 181panic (err ) 182 } 183var b strings.Builder 184b .WriteString ("<!doctype html>\n<html lang=\"en\">\n<title>search.nkp.pet API</title>\n" + publicStyle + ` 185<h1>search.nkp.pet API</h1> 186<p><a href="/">Back</a>. No authentication. 20 requests a minute per address; past that you get 429 with <code>Retry-After</code>. 187Nothing is stored unless you opt in with <code>X-Contribute: 1</code> (or <code>/mcp/contribute</code>).</p> 188<h2>Endpoints</h2> 189<table> 190<tr><th>endpoint</th><th></th></tr> 191<tr><td><code>POST /mcp</code></td><td>MCP over streamable HTTP (JSON-RPC): <code>initialize</code>, <code>tools/list</code>, <code>tools/call</code>.</td></tr> 192<tr><td><code>POST /mcp/contribute</code></td><td>The same, with this call's data kept for training.</td></tr> 193<tr><td><code>POST /search</code></td><td>The <code>web_search</code> tool as plain JSON: send its arguments as the body.</td></tr> 194<tr><td><code>POST /contents</code></td><td>The <code>web_contents</code> tool as plain JSON.</td></tr> 195<tr><td><code>POST /answer</code></td><td>The <code>web_answer</code> tool as plain JSON.</td></tr> 196<tr><td><code>POST /v1/extract</code>, <code>/v1/preview</code>, <code>/v1/summarize</code></td><td>The single-page tools as plain JSON.</td></tr> 197<tr><td><code>GET /healthz</code></td><td>Liveness.</td></tr> 198</table> 199<p>Example:</p> 200<pre><code>curl https://search.nkp.pet/search -H 'content-type: application/json' \ 201-d '{"query":"what is the atproto firehose","numResults":3,"contents":{"highlights":true}}'</code></pre> 202<h2>MCP tools</h2> 203` ) 204for _ ,tool := range tools { 205fmt .Fprintf (& b ,"<h3><code>%s</code></h3>\n<p>%s</p>\n<table>\n<tr><th>argument</th><th>type</th><th></th></tr>\n" , 206html .EscapeString (tool .Name ),html .EscapeString (tool .Description )) 207names := make ([]string ,0 ,len (tool .InputSchema .Properties )) 208for name := range tool .InputSchema .Properties { 209names = append (names ,name ) 210 } 211slices .Sort (names ) 212for _ ,name := range names { 213var p struct { 214Type any `json:"type"` 215Description string `json:"description"` 216OneOf []any `json:"oneOf"` 217 } 218_ = json .Unmarshal (tool .InputSchema .Properties [name ],& p ) 219kind := fmt .Sprint (p .Type ) 220if p .Type == nil { 221kind = "object" 222if p .OneOf != nil { 223kind = "boolean or object" 224 } 225 } 226if slices .Contains (tool .InputSchema .Required ,name ) { 227kind += ", required" 228 } 229fmt .Fprintf (& b ,"<tr><td><code>%s</code></td><td>%s</td><td>%s</td></tr>\n" , 230html .EscapeString (name ),html .EscapeString (kind ),html .EscapeString (p .Description )) 231 } 232b .WriteString ("</table>\n" ) 233 } 234b .WriteString ("</html>\n" ) 235return []byte (b .String ()) 236})