okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
20f80af
main
1package app 2 3import ( 4"context" 5"flag" 6"log/slog" 7"net/http" 8"os" 9"os/signal" 10"syscall" 11"time" 12 13"github.com/joho/godotenv" 14) 15 16// Run starts the configured HTTP or stdio service. 17func Run ()error { 18stdio := flag .Bool ("stdio" ,false ,"serve mcp over stdin/stdout instead of http" ) 19flag .Parse () 20_ = godotenv .Load () 21cfg ,err := readConfig () 22if err != nil { 23return err 24 } 25auth ,err := loadAuthStore (cfg .AuthStorePath ) 26if err != nil { 27return err 28 } 29log ,err := openLog (cfg .DBPath ) 30if err != nil { 31return err 32 } 33defer log .db .Close () 34network := newNetworkWithPool (cfg .AllowPrivate ,cfg .ProxyPool ) 35defer network .transport .CloseIdleConnections () 36browser := newBrowser (cfg ,network ) 37modelHTTP := & http.Client {Timeout :60 * time .Second ,CheckRedirect :func (* http.Request , []* http.Request )error {return http .ErrUseLastResponse }} 38s := & Service {cfg :cfg ,log :log ,auth :auth , 39fetch :& Fetcher {net :network ,browser :browser ,maxBytes :cfg .MaxPageBytes ,cache :& DocumentCache {docs :map [string ]Document {},max :cfg .CacheBytes }}, 40searcher :newSearcher (cfg ,browser ,network ,modelHTTP ), 41jev :& Jev {URL :cfg .JevURL ,Key :cfg .JevKey ,Model :cfg .JevModel ,HTTP :modelHTTP ,Log :log ,slots :make (chan struct {},cfg .JevConcurrency )}, 42llm :& LLM {URL :cfg .LLMURL ,Key :cfg .LLMKey ,Model :cfg .LLMModel ,Temperature :cfg .LLMTemperature ,HTTP :modelHTTP ,Log :log ,slots :make (chan struct {},cfg .Concurrency )}, 43 } 44if cfg .Public .Enabled { 45public ,err := openLog (cfg .Public .LogPath ) 46if err != nil { 47return err 48 } 49defer public .db .Close () 50log .public = public 51s .gate = newPublicGate (cfg .Public .RatePerMinute ,cfg .Public .DailyLimit ) 52if cfg .Public .JevURL != "" { 53s .publicJev = & Jev {URL :cfg .Public .JevURL ,Key :cfg .Public .JevKey ,Model :cfg .Public .JevModel ,HTTP :modelHTTP ,Log :log ,slots :make (chan struct {},cfg .JevConcurrency )} 54if cfg .Public .FallbackURL != "" { 55s .publicJev .Fallback = & Jev {URL :cfg .Public .FallbackURL ,Key :cfg .Public .FallbackKey ,Model :cfg .Public .FallbackModel ,HTTP :modelHTTP ,Log :log ,slots :make (chan struct {},cfg .JevConcurrency )} 56 } 57 } 58 } 59if cfg .Local .URL != "" { 60names := map [string ]bool {} 61for _ ,n := range cfg .Local .Questions { 62names [n ]= true 63 } 64tier := & LocalTier {Names :names ,Low :cfg .Local .Low ,High :cfg .Local .High , 65Jev :& Jev {URL :cfg .Local .URL ,Key :"local" ,Model :cfg .Local .Model ,HTTP :modelHTTP ,Log :log ,slots :make (chan struct {},cfg .JevConcurrency )}} 66if cfg .Local .Scope == "all" { 67s .jev .Local = tier 68 } 69if s .publicJev != nil { 70s .publicJev .Local = tier 71 } 72 } 73ctx ,stop := signal .NotifyContext (context .Background (),os .Interrupt ,syscall .SIGTERM ) 74defer stop () 75if * stdio { 76return s .serveStdio (ctx ,os .Stdin ,os .Stdout ) 77 } 78server := & http.Server {Addr :cfg .Addr ,Handler :s .handler (),ReadHeaderTimeout :5 * time .Second ,ReadTimeout :10 * time .Second ,WriteTimeout :cfg .RequestTimeout + 5 * time .Second ,IdleTimeout :30 * time .Second ,MaxHeaderBytes :16384 } 79go func () { 80<- ctx .Done () 81shutdown ,cancel := context .WithTimeout (context .Background (),10 * time .Second ) 82defer cancel () 83_ = server .Shutdown (shutdown ) 84 }() 85slog .Info ("web-extract listening" ,"address" ,cfg .Addr ,"jev_configured" ,cfg .JevKey != "" ,"summarizer_configured" ,cfg .LLMKey != "" ) 86err = server .ListenAndServe () 87if err == http .ErrServerClosed { 88return nil 89 } 90return err 91} 92 93// newSearcher picks the search backend: searxng when SEARX_URL is set, 94// rendered kagi otherwise. 95func newSearcher (cfg Config ,browser * Browser ,network * Network ,modelHTTP * http.Client )Searcher { 96if cfg .SearxURL != "" { 97return & Searx {BaseURL :cfg .SearxURL ,HTTP :modelHTTP ,maxBytes :cfg .MaxPageBytes } 98 } 99return & Kagi {browser :browser ,Session :cfg .KagiSession ,Token :cfg .KagiToken ,maxBytes :cfg .MaxPageBytes } 100}