okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
20f80af
main
1package app 2 3import ( 4"fmt" 5"net/netip" 6"net/url" 7"os" 8"strconv" 9"strings" 10"time" 11) 12 13type Config struct { 14AdminPassword string 15AdminCookieSecure bool 16AuthStorePath string 17Addr ,Token ,DBPath ,BrowserPath ,KagiSession ,KagiToken ,SearxURL string 18JevURL ,JevKey ,JevModel ,LLMURL ,LLMKey ,LLMModel string 19Concurrency ,Candidates ,Passages ,EvidenceChars ,MaxPageBytes ,CacheBytes int 20JevConcurrency int 21RequestTimeout time.Duration 22RenderWaitMS time.Duration 23ProxyPool []* url.URL 24AllowPrivate bool 25Stealth bool 26Headless bool 27Public PublicConfig 28Local LocalConfig 29// JevPricePerMillion prices decision input tokens on the admin dashboard. 30JevPricePerMillion float64 31// LLMTemperature is the summarizer sampling temperature (LLM_TEMPERATURE, 0-2). 32LLMTemperature float64 33} 34 35// LocalConfig is a local decision model asked before the remote one; see LocalTier. 36type LocalConfig struct { 37URL ,Model string 38Questions []string 39Low ,High float64 40// Scope is "public" (public calls only) or "all". 41Scope string 42} 43 44// PublicConfig opens the API to callers without a token, for the public 45// instance: rate limited, logged only on opt-in, and optionally with its own 46// decision model. 47type PublicConfig struct { 48Enabled bool 49RatePerMinute int 50DailyLimit int 51LogPath string 52JevURL ,JevKey ,JevModel string 53// Fallback decision model for public calls, used when the public one fails. 54FallbackURL ,FallbackKey ,FallbackModel string 55// TrustedProxies may set X-Forwarded-For; the client address is read from it. 56TrustedProxies []netip.Prefix 57} 58 59func readConfig () (Config ,error ) { 60c := Config { 61AuthStorePath :os .Getenv ("AUTH_STORE_PATH" ), 62AdminPassword :os .Getenv ("ADMIN_PASSWORD" ), 63Addr :env ("BIND_ADDR" ,"127.0.0.1:8788" ),Token :os .Getenv ("API_AUTH_TOKEN" ), 64DBPath :env ("COMPLETIONS_DB_PATH" ,"data/completions.db" ),BrowserPath :os .Getenv ("BROWSER_PATH" ), 65KagiSession :os .Getenv ("KAGI_SESSION" ),KagiToken :os .Getenv ("KAGI_SEARCH_TOKEN" ),SearxURL :os .Getenv ("SEARX_URL" ), 66JevURL :env ("TYPESAFE_BASE_URL" ,"https://api.typesafe.ai/v1" ),JevKey :os .Getenv ("TYPESAFE_API_KEY" ),JevModel :env ("TYPESAFE_MODEL" ,"jev-1.13.0" ), 67LLMURL :env ("LLM_BASE_URL" ,"https://api.deepseek.com" ),LLMKey :os .Getenv ("LLM_API_KEY" ),LLMModel :env ("LLM_MODEL" ,"deepseek-v4-flash" ), 68Concurrency :4 ,Candidates :20 ,Passages :24 ,EvidenceChars :5500 ,MaxPageBytes :5 << 20 ,CacheBytes :32 << 20 , 69JevConcurrency :8 ,JevPricePerMillion :0.042 ,LLMTemperature :1.0 , 70RequestTimeout :90 * time .Second , 71RenderWaitMS :1200 * time .Millisecond , 72Stealth :true , 73Headless :true , 74 } 75if c .Token == "" || strings .ContainsAny (c .Token ," \t\r\n" ) { 76return c ,fmt .Errorf ("API_AUTH_TOKEN must be set without whitespace" ) 77 } 78if c .AdminPassword != "" && len (c .AdminPassword )< 12 { 79return c ,fmt .Errorf ("ADMIN_PASSWORD must be at least 12 bytes, or empty to disable the dashboard" ) 80 } 81if value := os .Getenv ("ADMIN_COOKIE_SECURE" );value != "" { 82var err error 83c .AdminCookieSecure ,err = strconv .ParseBool (value ) 84if err != nil { 85return c ,fmt .Errorf ("ADMIN_COOKIE_SECURE must be a boolean" ) 86 } 87 } 88for name ,target := range map [string ]* int {"CONCURRENCY" :& c .Concurrency ,"JEV_CONCURRENCY" :& c .JevConcurrency ,"SEARCH_CANDIDATES" :& c .Candidates ,"PASSAGE_CANDIDATES" :& c .Passages ,"EVIDENCE_CHARS" :& c .EvidenceChars ,"MAX_PAGE_BYTES" :& c .MaxPageBytes ,"CACHE_BYTES" :& c .CacheBytes } { 89if s := os .Getenv (name );s != "" { 90v ,err := strconv .Atoi (s ) 91if err != nil || v < 1 { 92return c ,fmt .Errorf ("%s must be a positive integer" ,name ) 93 } 94* target = v 95 } 96 } 97if c .Concurrency > 16 || c .JevConcurrency > 16 || c .Candidates > 40 || c .Passages > 64 || c .EvidenceChars > 20000 || c .MaxPageBytes > 25 << 20 || c .CacheBytes > 256 << 20 { 98return c ,fmt .Errorf ("configured resource limits exceed supported bounds" ) 99 } 100if s := os .Getenv ("JEV_PRICE_PER_MILLION" );s != "" { 101v ,err := strconv .ParseFloat (s ,64 ) 102if err != nil || v < 0 { 103return c ,fmt .Errorf ("JEV_PRICE_PER_MILLION must be a non-negative number" ) 104 } 105c .JevPricePerMillion = v 106 } 107if s := os .Getenv ("LLM_TEMPERATURE" );s != "" { 108v ,err := strconv .ParseFloat (s ,64 ) 109if err != nil || v < 0 || v > 2 { 110return c ,fmt .Errorf ("LLM_TEMPERATURE must be a number from 0 to 2" ) 111 } 112c .LLMTemperature = v 113 } 114if s := os .Getenv ("RENDER_WAIT_MS" );s != "" { 115v ,err := strconv .Atoi (s ) 116if err != nil || v < 0 || v > 10000 { 117return c ,fmt .Errorf ("RENDER_WAIT_MS must be 0–10000 milliseconds" ) 118 } 119c .RenderWaitMS = time .Duration (v )* time .Millisecond 120 } 121for name ,target := range map [string ]* bool {"STEALTH" :& c .Stealth ,"BROWSER_HEADLESS" :& c .Headless } { 122if s := os .Getenv (name );s != "" { 123v ,err := strconv .ParseBool (s ) 124if err != nil { 125return c ,fmt .Errorf ("%s must be a boolean" ,name ) 126 } 127* target = v 128 } 129 } 130if s := os .Getenv ("PROXY_POOL" );s != "" { 131for _ ,entry := range strings .Split (s ,"," ) { 132entry = strings .TrimSpace (entry ) 133if entry == "" { 134continue 135 } 136u ,err := url .Parse (entry ) 137if err != nil || u .Host == "" || (u .Scheme != "https" && u .Scheme != "http" ) { 138return c ,fmt .Errorf ("PROXY_POOL entries must be absolute http(s) proxy urls" ) 139 } 140c .ProxyPool = append (c .ProxyPool ,u ) 141 } 142 } 143if s := os .Getenv ("ALLOW_PRIVATE_NETWORK" );s != "" { 144b ,err := strconv .ParseBool (s ) 145if err != nil { 146return c ,err 147 } 148c .AllowPrivate = b 149 } 150public ,err := readPublicConfig () 151if err != nil { 152return c ,err 153 } 154c .Public = public 155if c .Local ,err = readLocalConfig ();err != nil { 156return c ,err 157 } 158for _ ,endpoint := range []string {c .JevURL ,c .LLMURL ,c .Public .JevURL ,c .Public .FallbackURL ,c .Local .URL } { 159if endpoint == "" { 160continue 161 } 162u ,err := url .Parse (endpoint ) 163if err != nil || u .Host == "" || (u .Scheme != "https" && u .Scheme != "http" )|| u .User != nil { 164return c ,fmt .Errorf ("invalid model base url" ) 165 } 166 } 167return c ,nil 168} 169 170func env (name ,fallback string )string { 171if s := os .Getenv (name );s != "" { 172return s 173 } 174return fallback 175} 176 177func readPublicConfig () (PublicConfig ,error ) { 178p := PublicConfig { 179RatePerMinute :20 , 180LogPath :env ("PUBLIC_LOG_PATH" ,"data/public.db" ), 181JevURL :os .Getenv ("PUBLIC_JEV_URL" ),JevKey :os .Getenv ("PUBLIC_JEV_KEY" ),JevModel :env ("PUBLIC_JEV_MODEL" ,"clef-flash" ), 182FallbackURL :os .Getenv ("PUBLIC_JEV_FALLBACK_URL" ),FallbackKey :os .Getenv ("PUBLIC_JEV_FALLBACK_KEY" ), 183FallbackModel :env ("PUBLIC_JEV_FALLBACK_MODEL" ,"jev-1.13-free" ), 184 } 185if s := os .Getenv ("PUBLIC_ACCESS" );s != "" { 186b ,err := strconv .ParseBool (s ) 187if err != nil { 188return p ,fmt .Errorf ("PUBLIC_ACCESS must be a boolean" ) 189 } 190p .Enabled = b 191 } 192if s := os .Getenv ("PUBLIC_RATE_PER_MINUTE" );s != "" { 193v ,err := strconv .Atoi (s ) 194if err != nil || v < 1 { 195return p ,fmt .Errorf ("PUBLIC_RATE_PER_MINUTE must be a positive integer" ) 196 } 197p .RatePerMinute = v 198 } 199// 0, the default, means no daily limit. 200if s := os .Getenv ("PUBLIC_DAILY_LIMIT" );s != "" { 201v ,err := strconv .Atoi (s ) 202if err != nil || v < 0 { 203return p ,fmt .Errorf ("PUBLIC_DAILY_LIMIT must be a non-negative integer" ) 204 } 205p .DailyLimit = v 206 } 207if (p .JevURL == "" )!= (p .JevKey == "" )|| (p .FallbackURL == "" )!= (p .FallbackKey == "" ) { 208return p ,fmt .Errorf ("PUBLIC_JEV_URL/KEY and PUBLIC_JEV_FALLBACK_URL/KEY each go together" ) 209 } 210if p .FallbackURL != "" && p .JevURL == "" { 211return p ,fmt .Errorf ("PUBLIC_JEV_FALLBACK_URL needs PUBLIC_JEV_URL" ) 212 } 213for _ ,entry := range strings .Split (os .Getenv ("TRUSTED_PROXIES" ),"," ) { 214entry = strings .TrimSpace (entry ) 215if entry == "" { 216continue 217 } 218prefix ,err := netip .ParsePrefix (entry ) 219if err != nil { 220addr ,e := netip .ParseAddr (entry ) 221if e != nil { 222return p ,fmt .Errorf ("TRUSTED_PROXIES entries must be addresses or prefixes" ) 223 } 224prefix = netip .PrefixFrom (addr ,addr .BitLen ()) 225 } 226p .TrustedProxies = append (p .TrustedProxies ,prefix ) 227 } 228return p ,nil 229} 230 231func readLocalConfig () (LocalConfig ,error ) { 232l := LocalConfig { 233URL :os .Getenv ("LOCAL_DECISION_URL" ),Model :env ("LOCAL_DECISION_MODEL" ,"local-decision" ), 234Low :0.1 ,High :0.9 ,Scope :env ("LOCAL_DECISION_SCOPE" ,"public" ), 235 } 236for _ ,name := range strings .Split (env ("LOCAL_DECISION_QUESTIONS" ,"evidence,useful" ),"," ) { 237if name = strings .TrimSpace (name );name != "" { 238l .Questions = append (l .Questions ,name ) 239 } 240 } 241for name ,target := range map [string ]* float64 {"LOCAL_DECISION_LOW" :& l .Low ,"LOCAL_DECISION_HIGH" :& l .High } { 242if s := os .Getenv (name );s != "" { 243v ,err := strconv .ParseFloat (s ,64 ) 244if err != nil || v < 0 || v > 1 { 245return l ,fmt .Errorf ("%s must be between 0 and 1" ,name ) 246 } 247* target = v 248 } 249 } 250if l .Low >= l .High { 251return l ,fmt .Errorf ("LOCAL_DECISION_LOW must be below LOCAL_DECISION_HIGH" ) 252 } 253if l .Scope != "public" && l .Scope != "all" { 254return l ,fmt .Errorf ("LOCAL_DECISION_SCOPE must be public or all" ) 255 } 256return l ,nil 257}