okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
8af1784
main
1package app 2 3import ( 4"context" 5"encoding/json" 6"fmt" 7"net/http" 8"net/http/httptest" 9"os" 10"os/signal" 11"strings" 12"syscall" 13"testing" 14"time" 15) 16 17func adminCall (handler http.Handler ,method ,path ,body string ,cookie * http.Cookie ,headers map [string ]string )* httptest.ResponseRecorder { 18r := httptest .NewRequest (method ,path ,strings .NewReader (body )) 19r .Header .Set ("Content-Type" ,"application/json" ) 20if cookie != nil { 21r .AddCookie (cookie ) 22 } 23for key ,value := range headers { 24r .Header .Set (key ,value ) 25 } 26w := httptest .NewRecorder () 27handler .ServeHTTP (w ,r ) 28return w 29} 30 31func adminLogin (t * testing.T ,handler http.Handler ,password string )* http.Cookie { 32t .Helper () 33w := adminCall (handler ,"POST" ,"/admin/api/login" ,jsonText (map [string ]string {"password" :password }),nil ,nil ) 34if w .Code != 200 { 35t .Fatalf ("login: %d %s" ,w .Code ,w .Body .String ()) 36 } 37cookies := w .Result ().Cookies () 38if len (cookies )!= 1 || cookies [0 ].Name != adminCookie { 39t .Fatal ("missing session cookie" ) 40 } 41return cookies [0 ] 42} 43 44func TestAdminAuthenticationBoundary (t * testing.T ) { 45s ,_ := fixtureService (t ) 46for _ ,path := range []string {"/admin" ,"/admin/" ,"/admin/api/overview" } { 47if w := adminCall (s .handler (),"GET" ,path ,"" ,nil ,nil );w .Code != 404 { 48t .Fatalf ("disabled dashboard: %s %d" ,path ,w .Code ) 49 } 50 } 51s .cfg .AdminPassword ,s .cfg .AdminCookieSecure = "test-admin-password" ,true 52h := s .handler () 53for _ ,path := range []string {"/admin/api/session" ,"/admin/api/overview" ,"/admin/api/requests" ,"/admin/api/requests/secret" } { 54w := adminCall (h ,"GET" ,path ,"" ,nil ,map [string ]string {"x-api-key" :s .cfg .Token }) 55if w .Code != 401 { 56t .Fatalf ("api key bypass: %s %d" ,path ,w .Code ) 57 } 58if w .Header ().Get ("Cache-Control" )!= "no-store" { 59t .Fatal ("sensitive response may be cached" ) 60 } 61 } 62if w := adminCall (h ,"POST" ,"/admin/api/login" ,`{"password":"wrong-password"}` ,nil ,nil );w .Code != 401 { 63t .Fatal (w .Code ) 64 } 65cookie := adminLogin (t ,h ,s .cfg .AdminPassword ) 66if ! cookie .HttpOnly || ! cookie .Secure || cookie .SameSite != http .SameSiteStrictMode || cookie .Path != "/admin" || cookie .MaxAge != 43200 { 67t .Fatalf ("unsafe cookie: %+v" ,cookie ) 68 } 69if w := adminCall (h ,"GET" ,"/admin/api/session" ,"" ,cookie ,nil );w .Code != 200 { 70t .Fatal (w .Code ) 71 } 72if w := adminCall (s .handler (),"GET" ,"/admin/api/session" ,"" ,cookie ,nil );w .Code != 401 { 73t .Fatal ("a new server handler accepted an old process-local session" ) 74 } 75if w := adminCall (h ,"POST" ,"/search" ,`{"query":"limits"}` ,cookie ,nil );w .Code != 401 { 76t .Fatal ("admin session bypassed public api auth" ) 77 } 78if w := adminCall (h ,"POST" ,"/admin/api/mcp" ,`{}` ,cookie ,nil );w .Code != 404 { 79t .Fatal ("unexpected mcp bridge" ) 80 } 81for _ ,headers := range []map [string ]string {{"Origin" :"https://evil.example" }, {"Sec-Fetch-Site" :"cross-site" }} { 82w := adminCall (h ,"POST" ,"/admin/api/logout" ,`{}` ,cookie ,headers ) 83if w .Code != 403 { 84t .Fatalf ("csrf accepted: %d" ,w .Code ) 85 } 86w = adminCall (h ,"POST" ,"/admin/api/login" ,`{"password":"test-admin-password"}` ,nil ,headers ) 87if w .Code != 403 { 88t .Fatal ("login csrf accepted" ) 89 } 90 } 91w := adminCall (h ,"POST" ,"/admin/api/logout" ,`{}` ,cookie ,map [string ]string {"Content-Type" :"text/plain" }) 92if w .Code != 415 { 93t .Fatal ("simple cross-site content type accepted" ) 94 } 95w = adminCall (h ,"POST" ,"/admin/api/logout" ,`{}` ,cookie ,nil ) 96if w .Code != 200 || w .Result ().Cookies ()[0 ].MaxAge != - 1 { 97t .Fatal ("logout failed" ) 98 } 99if w := adminCall (h ,"GET" ,"/admin/api/overview" ,"" ,cookie ,nil );w .Code != 401 { 100t .Fatal ("revoked cookie still authorized" ) 101 } 102if w := adminCall (h ,"GET" ,"/admin/api/session" ,"" ,& http.Cookie {Name :adminCookie ,Value :"forged" },nil );w .Code != 401 { 103t .Fatal ("forged cookie accepted" ) 104 } 105page := adminCall (h ,"GET" ,"/admin/" ,"" ,nil ,nil ) 106if page .Code != 200 || ! strings .Contains (page .Body .String (),"/admin/assets/" ) { 107t .Fatal ("embedded ui missing" ) 108 } 109if ! strings .Contains (page .Header ().Get ("Content-Security-Policy" ),"frame-ancestors 'none'" ) { 110t .Fatal ("missing csp" ) 111 } 112if w := adminCall (h ,"GET" ,"/admin/.env" ,"" ,nil ,nil );w .Code != 404 { 113t .Fatal ("static handler exposed non-assets" ) 114 } 115} 116 117func TestAdminLoginRateLimit (t * testing.T ) { 118s ,_ := fixtureService (t ) 119s .cfg .AdminPassword = "test-admin-password" 120h := s .handler () 121for i := range 11 { 122w := adminCall (h ,"POST" ,"/admin/api/login" ,`{"password":"incorrect"}` ,nil ,nil ) 123want := 401 124if i == 10 { 125want = 429 126 } 127if w .Code != want { 128t .Fatalf ("attempt %d: %d" ,i ,w .Code ) 129 } 130 } 131if w := adminCall (h ,"POST" ,"/admin/api/login" ,`{"password":"test-admin-password"}` ,nil ,nil );w .Code != 429 { 132t .Fatal ("rate limit bypass" ) 133 } 134} 135 136func TestAdminPlaygroundLogsAndReadModels (t * testing.T ) { 137s ,page := fixtureService (t ) 138s .cfg .AdminPassword = "test-admin-password" 139s .cfg .JevKey ,s .cfg .LLMKey = "secret-jev-key" ,"secret-llm-key" 140h := s .handler () 141cookie := adminLogin (t ,h ,s .cfg .AdminPassword ) 142body := fmt .Sprintf (`{"urls":[%q,%q],"highlights":{"query":"request size"},"summary":{"query":"request size"}}` ,page .URL ,page .URL + "/missing" ) 143w := adminCall (h ,"POST" ,"/admin/api/contents" ,body ,cookie ,nil ) 144if w .Code != 200 { 145t .Fatalf ("contents: %s" ,w .Body .String ()) 146 } 147var response ResultsResponse 148if err := json .Unmarshal (w .Body .Bytes (),& response );err != nil { 149t .Fatal (err ) 150 } 151if len (response .Results )!= 1 || len (response .Statuses )!= 2 || response .Statuses [1 ].Error == nil { 152t .Fatal ("lost partial failure" ) 153 } 154trace ,err := s .log .adminTrace (context .Background (),response .RequestID ) 155if err != nil { 156t .Fatal (err ) 157 } 158if len (trace .Requests )!= 2 || len (trace .Completions )!= 1 || len (trace .Decisions )== 0 || len (trace .Evidence )!= 1 { 159t .Fatalf ("incomplete trace: %+v" ,trace ) 160 } 161if trace .Requests [0 ]["caller" ]!= "admin-dashboard" { 162t .Fatal ("missing dashboard caller" ) 163 } 164if trace .Completions [0 ]["response" ]!= * response .Results [0 ].Summary { 165t .Fatal ("training reply changed" ) 166 } 167w = adminCall (h ,"POST" ,"/admin/api/search" ,`{"query":""}` ,cookie ,nil ) 168if w .Code != 400 { 169t .Fatal ("public request validation not reused" ) 170 } 171w = adminCall (h ,"GET" ,"/admin/api/overview" ,"" ,cookie ,nil ) 172if w .Code != 200 { 173t .Fatal (w .Body .String ()) 174 } 175for _ ,secret := range []string {s .cfg .AdminPassword ,s .cfg .Token ,s .cfg .JevKey ,s .cfg .LLMKey } { 176if strings .Contains (w .Body .String (),secret ) { 177t .Fatal ("credentials in overview" ) 178 } 179 } 180var stats struct { 181Requests struct {Total ,Errors int } 182Models struct {Calls int } 183Tokens struct {Input ,Output int } 184 } 185if err := json .Unmarshal (w .Body .Bytes (),& stats );err != nil { 186t .Fatal (err ) 187 } 188if stats .Requests .Total != 2 || stats .Requests .Errors != 2 || stats .Models .Calls != len (trace .Decisions )+ 1 || stats .Tokens .Input != 250 || stats .Tokens .Output != 20 { 189t .Fatalf ("wrong totals: %+v" ,stats ) 190 } 191for _ ,path := range []string {"/admin/api/requests?before=oops" ,"/admin/api/requests?endpoint=extract" } { 192if w := adminCall (h ,"GET" ,path ,"" ,cookie ,nil );w .Code != 400 { 193t .Fatal ("invalid filter accepted" ) 194 } 195 } 196if w := adminCall (h ,"GET" ,"/admin/api/requests/missing" ,"" ,cookie ,nil );w .Code != 404 { 197t .Fatal ("unknown trace should be missing" ) 198 } 199} 200 201// Opt-in browser fixture: real HTTP/pipeline/SQLite, local mocked providers, no paid calls. 202func TestAdminBrowserFixture (t * testing.T ) { 203path := os .Getenv ("ADMIN_BROWSER_FIXTURE" ) 204if path == "" { 205t .Skip ("set ADMIN_BROWSER_FIXTURE to start the browser-test server" ) 206 } 207s ,page := fixtureService (t ) 208s .cfg .AdminPassword = "browser-fixture-password" 209s .cfg .JevKey ,s .cfg .JevModel ,s .cfg .LLMKey ,s .cfg .LLMModel = "fixture" ,"jev-test" ,"fixture" ,"configured-model" 210h := s .handler () 211cookie := adminLogin (t ,h ,s .cfg .AdminPassword ) 212for i := range 32 { 213adminCall (h ,"POST" ,"/admin/api/search" ,fmt .Sprintf (`{"query":"fixture request limits %d","numResults":1}` ,i ),cookie ,nil ) 214 } 215adminCall (h ,"POST" ,"/admin/api/search" ,`{"query":""}` ,cookie ,nil ) 216server := httptest .NewServer (h ) 217defer server .Close () 218if err := os .WriteFile (path , []byte (jsonText (map [string ]string {"url" :server .URL ,"sourceURL" :page .URL ,"password" :s .cfg .AdminPassword })),0600 );err != nil { 219t .Fatal (err ) 220 } 221ctx ,stop := signal .NotifyContext (context .Background (),os .Interrupt ,syscall .SIGTERM ) 222defer stop () 223<- ctx .Done () 224} 225 226func TestAdminPaginationAndHistoricalRows (t * testing.T ) { 227s ,_ := fixtureService (t ) 228for i := range 65 { 229ctx := context .WithValue (context .Background (),contextKey {},callContext {ID :fmt .Sprint (i ),Caller :"test" }) 230s .log .request (ctx ,"search" ,map [string ]string {"query" :"query" },"ok" ) 231 } 232ctx := context .Background () 233first ,err := s .adminRequests (ctx ,0 ,"" ,false ) 234if err != nil || len (first .Rows )!= 30 || first .NextBefore != 36 { 235t .Fatalf ("first: %+v %v" ,first ,err ) 236 } 237second ,err := s .adminRequests (ctx ,first .NextBefore ,"search" ,false ) 238if err != nil || len (second .Rows )!= 30 || second .Rows [0 ]["id" ]!= int64 (35 ) { 239t .Fatal ("pagination skipped or duplicated rows" ) 240 } 241third ,err := s .adminRequests (ctx ,second .NextBefore ,"" ,false ) 242if err != nil || len (third .Rows )!= 5 || third .NextBefore != 0 { 243t .Fatal ("last page invalid" ) 244 } 245empty ,err := s .adminRequests (ctx ,0 ,"" ,true ) 246if err != nil || empty .Rows == nil || len (empty .Rows )!= 0 { 247t .Fatal ("empty page should be an array" ) 248 } 249s .log .write (`INSERT INTO requests(created_at,request_id,endpoint,detail,outcome) VALUES(?,?,?,?,?)` ,time .Now ().Unix (),"legacy" ,"search" ,"not-json" ,"ok" ) 250trace ,err := s .log .adminTrace (ctx ,"legacy" ) 251if err != nil || trace .Requests [0 ]["detail" ]!= "not-json" || trace .Decisions == nil { 252t .Fatal ("legacy rows not preserved" ) 253 } 254} 255 256func TestAdminConfig (t * testing.T ) { 257t .Setenv ("API_AUTH_TOKEN" ,"local-token" ) 258t .Setenv ("ADMIN_PASSWORD" ,"short" ) 259if _ ,err := readConfig ();err == nil { 260t .Fatal ("weak password accepted" ) 261 } 262t .Setenv ("ADMIN_PASSWORD" ,"test-admin-password" ) 263t .Setenv ("ADMIN_COOKIE_SECURE" ,"true" ) 264c ,err := readConfig () 265if err != nil || ! c .AdminCookieSecure || c .AdminPassword != "test-admin-password" { 266t .Fatal ("admin config missing" ) 267 } 268}