okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"bufio" 5"context" 6"encoding/base64" 7"fmt" 8"io" 9"net" 10"net/http" 11"net/netip" 12"net/url" 13"strings" 14"sync" 15"sync/atomic" 16"time" 17) 18 19func parseWebURL (raw string ) (* url.URL ,error ) { 20u ,err := url .Parse (raw ) 21if err != nil || len (raw )> 2048 || u == nil || u .Hostname ()== "" || u .User != nil || (u .Scheme != "http" && u .Scheme != "https" ) { 22return nil ,fmt .Errorf ("only absolute http(s) urls without embedded credentials are supported" ) 23 } 24return u ,nil 25} 26 27func publicIP (a netip.Addr )bool { 28a = a .Unmap () 29if ! a .IsGlobalUnicast ()|| a .IsPrivate ()|| a .IsLoopback ()|| a .IsLinkLocalUnicast () { 30return false 31 } 32for _ ,raw := range []string {"0.0.0.0/8" ,"100.64.0.0/10" ,"192.0.0.0/24" ,"192.0.2.0/24" ,"198.18.0.0/15" ,"198.51.100.0/24" ,"203.0.113.0/24" ,"240.0.0.0/4" ,"2001:db8::/32" } { 33if netip .MustParsePrefix (raw ).Contains (a ) { 34return false 35 } 36 } 37return true 38} 39 40type Network struct { 41AllowPrivate bool 42proxies []* url.URL 43proxyCounter atomic.Uint64 44transport * http.Transport 45client * http.Client 46mu sync.Mutex 47byProxy map [string ]* http.Client 48} 49 50func newNetwork (allowPrivate bool )* Network {return newNetworkWithPool (allowPrivate ,nil ) } 51 52// newNetworkWithPool builds the guarded client. A proxy pool routes egress 53// through upstream proxies in rotation; an empty pool dials directly. 54func newNetworkWithPool (allowPrivate bool ,proxies []* url.URL )* Network { 55n := & Network {AllowPrivate :allowPrivate ,proxies :proxies ,byProxy :map [string ]* http.Client {}} 56n .transport = n .newTransport (nil ) 57n .client = n .newClient (n .transport ) 58return n 59} 60 61// newTransport builds a transport whose connections are guarded, and which 62// routes through proxy when one is selected. 63func (n * Network )newTransport (proxy * url.URL )* http.Transport { 64t := & http.Transport {DialContext :n .dial ,MaxIdleConns :32 ,MaxIdleConnsPerHost :4 ,IdleConnTimeout :30 * time .Second ,TLSHandshakeTimeout :10 * time .Second ,ResponseHeaderTimeout :20 * time .Second ,ForceAttemptHTTP2 :true } 65if proxy != nil { 66t .Proxy = http .ProxyURL (proxy ) 67 } 68return t 69} 70 71func (n * Network )newClient (t * http.Transport )* http.Client { 72return & http.Client {Transport :t ,Timeout :35 * time .Second ,CheckRedirect :func (r * http.Request ,via []* http.Request )error { 73if len (via )>= 10 { 74return fmt .Errorf ("too many redirects" ) 75 } 76_ ,err := parseWebURL (r .URL .String ()) 77return err 78 }} 79} 80 81// clientFor returns a client routed through proxy, caching one per proxy so 82// connections are reused. A nil proxy selects the direct, guarded client. 83// 84// The proxy is fixed for the whole fetch: redirects and retries must not move 85// to a different egress address mid-session. 86func (n * Network )clientFor (proxy * url.URL )* http.Client { 87if proxy == nil { 88return n .client 89 } 90key := proxy .String () 91n .mu .Lock () 92defer n .mu .Unlock () 93if c ,ok := n .byProxy [key ];ok { 94return c 95 } 96c := n .newClient (n .newTransport (proxy )) 97n .byProxy [key ]= c 98return c 99} 100 101// closeIdle releases pooled connections, including those held for proxies. 102func (n * Network )closeIdle () { 103n .transport .CloseIdleConnections () 104n .mu .Lock () 105defer n .mu .Unlock () 106for _ ,c := range n .byProxy { 107if t ,ok := c .Transport .(* http.Transport );ok { 108t .CloseIdleConnections () 109 } 110 } 111} 112 113// pickProxy rotates through the configured pool, returning nil for a direct 114// connection. One call per fetch keeps an egress address stable for that fetch. 115func (n * Network )pickProxy ()* url.URL { 116if len (n .proxies )== 0 { 117return nil 118 } 119index := n .proxyCounter .Add (1 )- 1 120return n .proxies [index % uint64 (len (n .proxies ))] 121} 122 123// guard is the local HTTP proxy Chrome is pointed at. Every connection the 124// browser makes is dialed through it, so redirects and subresources cannot 125// escape the policy. It optionally tunnels through an upstream proxy. 126type guard struct { 127net * Network 128upstream * url.URL 129} 130 131// dialTarget opens a connection to address, through the render's upstream proxy 132// when one was selected and directly through the guarded dialer otherwise. 133// 134// The proxy is reached through the same guarded dialer, so a proxy address on a 135// private or reserved network is refused unless private targets are allowed. 136func (g * guard )dialTarget (ctx context.Context ,address string ) (net.Conn ,error ) { 137if g .upstream == nil { 138return g .net .dial (ctx ,"tcp" ,address ) 139 } 140return g .net .dialViaProxy (ctx ,g .upstream ,address ) 141} 142 143// dialViaProxy establishes a CONNECT tunnel to address through an HTTP proxy. 144// The proxy resolves the destination, so the destination address cannot be 145// range-checked here; only the proxy itself is. 146func (n * Network )dialViaProxy (ctx context.Context ,proxy * url.URL ,address string ) (net.Conn ,error ) { 147port := proxy .Port () 148if port == "" { 149if proxy .Scheme == "https" { 150port = "443" 151 }else { 152port = "80" 153 } 154 } 155conn ,err := n .dial (ctx ,"tcp" ,net .JoinHostPort (proxy .Hostname (),port )) 156if err != nil { 157return nil ,fmt .Errorf ("proxy connect failed: %w" ,err ) 158 } 159req := & http.Request { 160Method :http .MethodConnect , 161URL :& url.URL {Opaque :address }, 162Host :address , 163Header : http.Header {}, 164 } 165if user := proxy .User ;user != nil { 166password ,_ := user .Password () 167credentials := base64 .StdEncoding .EncodeToString ([]byte (user .Username ()+ ":" + password )) 168req .Header .Set ("Proxy-Authorization" ,"Basic " + credentials ) 169 } 170if err := req .Write (conn );err != nil { 171conn .Close () 172return nil ,fmt .Errorf ("proxy request failed: %w" ,err ) 173 } 174reader := bufio .NewReader (conn ) 175res ,err := http .ReadResponse (reader ,req ) 176if err != nil { 177conn .Close () 178return nil ,fmt .Errorf ("proxy response failed: %w" ,err ) 179 } 180res .Body .Close () 181if res .StatusCode != http .StatusOK { 182conn .Close () 183return nil ,fmt .Errorf ("proxy refused the tunnel: %s" ,res .Status ) 184 } 185// The proxy may have pipelined tunnel bytes into the same read, so they must 186// be drained before the raw connection is used. 187if reader .Buffered ()> 0 { 188return & bufferedConn {Conn :conn ,reader :reader },nil 189 } 190return conn ,nil 191} 192 193// bufferedConn returns bytes already buffered by a proxy handshake before 194// reading from the underlying connection. 195type bufferedConn struct { 196 net.Conn 197reader * bufio.Reader 198} 199 200func (c * bufferedConn )Read (p []byte ) (int ,error ) {return c .reader .Read (p ) } 201func (n * Network )dial (ctx context.Context ,network ,address string ) (net.Conn ,error ) { 202host ,port ,err := net .SplitHostPort (address ) 203if err != nil { 204return nil ,err 205 } 206ips ,err := net .DefaultResolver .LookupNetIP (ctx ,"ip" ,host ) 207if err != nil { 208return nil ,fmt .Errorf ("resolve target: %w" ,err ) 209 } 210if len (ips )== 0 { 211return nil ,fmt .Errorf ("target has no addresses" ) 212 } 213for _ ,ip := range ips { 214if ! n .AllowPrivate && ! publicIP (ip ) { 215return nil ,fmt .Errorf ("private or reserved network target is blocked" ) 216 } 217 } 218var last error 219for _ ,ip := range ips { 220c ,err := (& net.Dialer {Timeout :10 * time .Second ,KeepAlive :30 * time .Second }).DialContext (ctx ,network ,net .JoinHostPort (ip .String (),port )) 221if err == nil { 222return c ,nil 223 } 224last = err 225 } 226return nil ,last 227} 228 229// Chrome sends every connection to this handler, including redirects and 230// subresources. CONNECT pins the checked address, avoiding a separate 231// check-then-browser-DNS lookup that would permit DNS rebinding. 232func (g * guard )ServeHTTP (w http.ResponseWriter ,r * http.Request ) { 233if r .Method == http .MethodConnect { 234up ,err := g .dialTarget (r .Context (),r .Host ) 235if err != nil { 236http .Error (w ,"blocked destination" ,http .StatusForbidden ) 237return 238 } 239down ,buf ,err := w .(http.Hijacker ).Hijack () 240if err != nil { 241up .Close () 242return 243 } 244_ ,err = buf .WriteString ("HTTP/1.1 200 Connection Established\r\n\r\n" ) 245if err == nil { 246err = buf .Flush () 247 } 248if err != nil { 249up .Close () 250down .Close () 251return 252 } 253up .SetDeadline (time .Now ().Add (95 * time .Second )) 254down .SetDeadline (time .Now ().Add (95 * time .Second )) 255go func () { 256defer up .Close () 257defer down .Close () 258go func () {io .Copy (up ,buf );up .Close () }() 259io .Copy (down ,up ) 260 }() 261return 262 } 263if _ ,err := parseWebURL (r .URL .String ());err != nil { 264http .Error (w ,"invalid destination" ,400 ) 265return 266 } 267r = r .Clone (r .Context ()) 268r .RequestURI = "" 269r .Header .Del ("Proxy-Authorization" ) 270r .Header .Del ("Proxy-Connection" ) 271transport ,ok := g .net .clientFor (g .upstream ).Transport .(* http.Transport ) 272if ! ok { 273http .Error (w ,"destination unavailable" ,502 ) 274return 275 } 276res ,err := transport .RoundTrip (r ) 277if err != nil { 278http .Error (w ,"destination unavailable" ,502 ) 279return 280 } 281defer res .Body .Close () 282for key ,values := range res .Header { 283if strings .EqualFold (key ,"Connection" ) { 284continue 285 } 286for _ ,v := range values { 287w .Header ().Add (key ,v ) 288 } 289 } 290w .WriteHeader (res .StatusCode ) 291io .Copy (w ,res .Body ) 292} 293 294func readBounded (r io.Reader ,limit int ) ([]byte ,error ) { 295b ,err := io .ReadAll (io .LimitReader (r ,int64 (limit )+ 1 )) 296if err != nil { 297return nil ,err 298 } 299if len (b )> limit { 300return nil ,fmt .Errorf ("response exceeds %d bytes" ,limit ) 301 } 302return b ,nil 303}