okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a5f5bc
main
1package app 2 3import ( 4"net/http" 5"net/http/httptest" 6"net/http/httputil" 7"net/netip" 8"net/url" 9"path/filepath" 10"strconv" 11"strings" 12"sync/atomic" 13"testing" 14"time" 15) 16 17// publicFixture is fixtureService with the public endpoint enabled. Its public 18// decision model forwards to the fixture model and counts the calls it gets. 19func publicFixture (t * testing.T ,perMinute ,daily int ) (* Service ,* atomic.Int32 ) { 20t .Helper () 21s ,_ := fixtureService (t ) 22public ,err := openLog (filepath .Join (t .TempDir (),"public.db" )) 23if err != nil { 24t .Fatal (err ) 25 } 26t .Cleanup (func () {public .db .Close () }) 27s .log .public = public 28s .gate = newPublicGate (perMinute ,daily ) 29target ,_ := url .Parse (s .jev .URL ) 30var calls atomic.Int32 31proxy := httputil .NewSingleHostReverseProxy (target ) 32clef := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 33calls .Add (1 ) 34proxy .ServeHTTP (w ,r ) 35 })) 36t .Cleanup (clef .Close ) 37s .publicJev = & Jev {URL :clef .URL ,Key :"model-secret" ,Model :"clef-test" ,HTTP :clef .Client (),Log :s .log ,slots :make (chan struct {},4 )} 38return s ,& calls 39} 40 41func rows (t * testing.T ,l * Log ,table string )int { 42t .Helper () 43var n int 44if err := l .db .QueryRow ("select count(*) from " + table ).Scan (& n );err != nil { 45t .Fatal (err ) 46 } 47return n 48} 49 50const searchCall = `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"web_search","arguments":{"query":"request size","numResults":1}}}` 51 52func TestPublicCallsAreServedWithoutATokenAndLoggedOnlyOnOptIn (t * testing.T ) { 53s ,clefCalls := publicFixture (t ,100 ,100 ) 54h := s .handler () 55if w := adminCall (h ,"POST" ,"/mcp" ,searchCall ,nil ,nil );w .Code != 200 { 56t .Fatalf ("public call: %d %s" ,w .Code ,w .Body .String ()) 57 } 58if clefCalls .Load ()== 0 { 59t .Fatal ("public call did not use the public decision model" ) 60 } 61for _ ,table := range []string {"requests" ,"decisions" } { 62if rows (t ,s .log ,table )!= 0 || rows (t ,s .log .public ,table )!= 0 { 63t .Fatalf ("a call without opt-in was logged in %s" ,table ) 64 } 65 } 66if w := adminCall (h ,"POST" ,"/mcp" ,searchCall ,nil ,map [string ]string {"X-Contribute" :"1" });w .Code != 200 { 67t .Fatal (w .Code ) 68 } 69if w := adminCall (h ,"POST" ,"/mcp/contribute" ,searchCall ,nil ,nil );w .Code != 200 { 70t .Fatal (w .Code ) 71 } 72if rows (t ,s .log .public ,"requests" )!= 2 || rows (t ,s .log .public ,"decisions" )== 0 { 73t .Fatal ("opted-in calls were not logged to the public log" ) 74 } 75if rows (t ,s .log ,"requests" )!= 0 { 76t .Fatal ("public calls reached the main log" ) 77 } 78before := clefCalls .Load () 79if w := adminCall (h ,"POST" ,"/mcp" ,searchCall ,nil ,map [string ]string {"Authorization" :"Bearer test-token" });w .Code != 200 { 80t .Fatal (w .Code ) 81 } 82if clefCalls .Load ()!= before || rows (t ,s .log ,"requests" )!= 1 { 83t .Fatal ("an authenticated call changed model or skipped the main log" ) 84 } 85if w := adminCall (h ,"POST" ,"/mcp" ,searchCall ,nil ,map [string ]string {"Authorization" :"Bearer wrong" });w .Code != 401 { 86t .Fatal ("a wrong token fell through to public access" ) 87 } 88if w := adminCall (h ,"GET" ,"/" ,"" ,nil ,nil );w .Code != 200 || w .Header ().Get ("Content-Type" )!= "text/html; charset=utf-8" { 89t .Fatal ("no notice page" ) 90 } 91w := adminCall (h ,"GET" ,"/docs" ,"" ,nil ,nil ) 92for _ ,want := range []string {"<code>web_search</code>" ,"<code>numResults</code>" ,"integer" ,"<code>query</code></td><td>string, required" ,"POST /mcp/contribute" } { 93if w .Code != 200 || ! strings .Contains (w .Body .String (),want ) { 94t .Fatalf ("api page lacks %q" ,want ) 95 } 96 } 97} 98 99func TestPublicAccessIsOffUnlessEnabled (t * testing.T ) { 100s ,_ := fixtureService (t ) 101if w := adminCall (s .handler (),"POST" ,"/mcp" ,searchCall ,nil ,nil );w .Code != 401 { 102t .Fatal (w .Code ) 103 } 104if w := adminCall (s .handler (),"GET" ,"/" ,"" ,nil ,nil );w .Code != 401 { 105t .Fatal (w .Code ) 106 } 107} 108 109func TestPublicRateLimits (t * testing.T ) { 110s ,_ := publicFixture (t ,3 ,2 ) 111h := s .handler () 112list := `{"jsonrpc":"2.0","id":1,"method":"tools/list"}` 113// Two tool calls use up the day; a listing still works until the minute's limit. 114for i ,body := range []string {searchCall ,searchCall ,list } { 115if w := adminCall (h ,"POST" ,"/mcp" ,body ,nil ,nil );w .Code != 200 { 116t .Fatalf ("call %d: %d" ,i ,w .Code ) 117 } 118 } 119w := adminCall (h ,"POST" ,"/mcp" ,list ,nil ,nil ) 120if w .Code != 429 { 121t .Fatal ("per-minute limit not applied" ,w .Code ) 122 } 123if wait ,err := strconv .Atoi (w .Header ().Get ("Retry-After" ));err != nil || wait < 1 || wait > 60 { 124t .Fatal ("bad Retry-After" ,w .Header ().Get ("Retry-After" )) 125 } 126 127g := newPublicGate (3 ,2 ) 128now := time .Date (2026 ,10 ,17 ,23 ,59 ,30 ,0 ,time .UTC ) 129g .now = func () time.Time {return now } 130a ,b := netip .MustParseAddr ("192.0.2.1" ),netip .MustParseAddr ("192.0.2.2" ) 131for range 2 { 132if ok ,_ := g .admit (a ,true );! ok { 133t .Fatal ("refused within limits" ) 134 } 135 } 136if ok ,wait := g .admit (b ,true );ok || wait != 31 { 137t .Fatal ("daily limit not shared across clients" ,ok ,wait ) 138 } 139if ok ,_ := g .admit (b ,false );! ok { 140t .Fatal ("a listing counted against the daily limit" ) 141 } 142now = now .Add (time .Minute ) 143if ok ,_ := g .admit (b ,true );! ok { 144t .Fatal ("limits did not reset at midnight UTC" ) 145 } 146v6 := newPublicGate (2 ,0 ) 147for _ ,addr := range []string {"2001:db8:1:2::a" ,"2001:db8:1:2::b" } { 148if ok ,_ := v6 .admit (netip .MustParseAddr (addr ),false );! ok { 149t .Fatal ("refused within limits" ) 150 } 151 } 152if ok ,_ := v6 .admit (netip .MustParseAddr ("2001:db8:1:2:ffff::1" ),false );ok { 153t .Fatal ("addresses in one IPv6 /64 were counted separately" ) 154 } 155if ok ,_ := v6 .admit (netip .MustParseAddr ("2001:db8:1:3::1" ),false );! ok { 156t .Fatal ("another /64 shared the limit" ) 157 } 158unlimited := newPublicGate (1000 ,0 ) 159for range 600 { 160if ok ,_ := unlimited .admit (a ,true );! ok { 161t .Fatal ("a zero daily limit capped calls" ) 162 } 163 } 164} 165 166func TestClientAddrTrustsOnlyConfiguredProxies (t * testing.T ) { 167proxy := []netip.Prefix {netip .MustParsePrefix ("100.64.0.9/32" )} 168r := httptest .NewRequest ("POST" ,"/mcp" ,nil ) 169r .Header .Set ("X-Forwarded-For" ,"203.0.113.5, 198.51.100.7" ) 170r .RemoteAddr = "100.64.0.9:41000" 171if got := clientAddr (r ,proxy );got != netip .MustParseAddr ("198.51.100.7" ) { 172t .Fatal ("trusted proxy's appended address not used" ,got ) 173 } 174r .RemoteAddr = "192.0.2.9:41000" 175if got := clientAddr (r ,proxy );got != netip .MustParseAddr ("192.0.2.9" ) { 176t .Fatal ("an untrusted peer chose its own address" ,got ) 177 } 178} 179 180func TestPublicDecisionModelFallsBackWhenItFails (t * testing.T ) { 181s ,clefCalls := publicFixture (t ,100 ,0 ) 182broken := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 183http .Error (w ,`{"error":"out of daily allocation"}` ,http .StatusTooManyRequests ) 184 })) 185t .Cleanup (broken .Close ) 186fallback := s .publicJev 187s .publicJev = & Jev {URL :broken .URL ,Key :"model-secret" ,Model :"clef-broken" ,HTTP :broken .Client (),Log :s .log ,slots :make (chan struct {},4 ),Fallback :fallback } 188w := adminCall (s .handler (),"POST" ,"/mcp/contribute" ,searchCall ,nil ,nil ) 189if w .Code != 200 || strings .Contains (w .Body .String (),`"isError":true` ) { 190t .Fatalf ("search failed instead of falling back: %d %s" ,w .Code ,w .Body .String ()) 191 } 192if clefCalls .Load ()== 0 { 193t .Fatal ("the fallback model was not called" ) 194 } 195var models string 196if err := s .log .public .db .QueryRow ("select group_concat(distinct model) from decisions" ).Scan (& models );err != nil || ! strings .Contains (models ,"jev-test" ) { 197t .Fatalf ("decisions do not name the model that answered: %q %v" ,models ,err ) 198 } 199}