okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
8.8 KiB303 linesraw
1package app
2
3import (
4	"bufio"
5	"context"
6	"encoding/base64"
7	"fmt"
8	"io"
9	"net"
10	"net/http"
11	"net/netip"
12	"net/url"
13	"strings"
14	"sync"
15	"sync/atomic"
16	"time"
17)
18
19func parseWebURL(raw string) (*url.URL, error) {
20	u, err := url.Parse(raw)
21	if err != nil || len(raw) > 2048 || u == nil || u.Hostname() == "" || u.User != nil || (u.Scheme != "http" && u.Scheme != "https") {
22		return nil, fmt.Errorf("only absolute http(s) urls without embedded credentials are supported")
23	}
24	return u, nil
25}
26
27func publicIP(a netip.Addr) bool {
28	a = a.Unmap()
29	if !a.IsGlobalUnicast() || a.IsPrivate() || a.IsLoopback() || a.IsLinkLocalUnicast() {
30		return false
31	}
32	for _, raw := range []string{"0.0.0.0/8", "100.64.0.0/10", "192.0.0.0/24", "192.0.2.0/24", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "240.0.0.0/4", "2001:db8::/32"} {
33		if netip.MustParsePrefix(raw).Contains(a) {
34			return false
35		}
36	}
37	return true
38}
39
40type Network struct {
41	AllowPrivate bool
42	proxies      []*url.URL
43	proxyCounter atomic.Uint64
44	transport    *http.Transport
45	client       *http.Client
46	mu           sync.Mutex
47	byProxy      map[string]*http.Client
48}
49
50func newNetwork(allowPrivate bool) *Network { return newNetworkWithPool(allowPrivate, nil) }
51
52// newNetworkWithPool builds the guarded client. A proxy pool routes egress
53// through upstream proxies in rotation; an empty pool dials directly.
54func newNetworkWithPool(allowPrivate bool, proxies []*url.URL) *Network {
55	n := &Network{AllowPrivate: allowPrivate, proxies: proxies, byProxy: map[string]*http.Client{}}
56	n.transport = n.newTransport(nil)
57	n.client = n.newClient(n.transport)
58	return n
59}
60
61// newTransport builds a transport whose connections are guarded, and which
62// routes through proxy when one is selected.
63func (n *Network) newTransport(proxy *url.URL) *http.Transport {
64	t := &http.Transport{DialContext: n.dial, MaxIdleConns: 32, MaxIdleConnsPerHost: 4, IdleConnTimeout: 30 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ResponseHeaderTimeout: 20 * time.Second, ForceAttemptHTTP2: true}
65	if proxy != nil {
66		t.Proxy = http.ProxyURL(proxy)
67	}
68	return t
69}
70
71func (n *Network) newClient(t *http.Transport) *http.Client {
72	return &http.Client{Transport: t, Timeout: 35 * time.Second, CheckRedirect: func(r *http.Request, via []*http.Request) error {
73		if len(via) >= 10 {
74			return fmt.Errorf("too many redirects")
75		}
76		_, err := parseWebURL(r.URL.String())
77		return err
78	}}
79}
80
81// clientFor returns a client routed through proxy, caching one per proxy so
82// connections are reused. A nil proxy selects the direct, guarded client.
83//
84// The proxy is fixed for the whole fetch: redirects and retries must not move
85// to a different egress address mid-session.
86func (n *Network) clientFor(proxy *url.URL) *http.Client {
87	if proxy == nil {
88		return n.client
89	}
90	key := proxy.String()
91	n.mu.Lock()
92	defer n.mu.Unlock()
93	if c, ok := n.byProxy[key]; ok {
94		return c
95	}
96	c := n.newClient(n.newTransport(proxy))
97	n.byProxy[key] = c
98	return c
99}
100
101// closeIdle releases pooled connections, including those held for proxies.
102func (n *Network) closeIdle() {
103	n.transport.CloseIdleConnections()
104	n.mu.Lock()
105	defer n.mu.Unlock()
106	for _, c := range n.byProxy {
107		if t, ok := c.Transport.(*http.Transport); ok {
108			t.CloseIdleConnections()
109		}
110	}
111}
112
113// pickProxy rotates through the configured pool, returning nil for a direct
114// connection. One call per fetch keeps an egress address stable for that fetch.
115func (n *Network) pickProxy() *url.URL {
116	if len(n.proxies) == 0 {
117		return nil
118	}
119	index := n.proxyCounter.Add(1) - 1
120	return n.proxies[index%uint64(len(n.proxies))]
121}
122
123// guard is the local HTTP proxy Chrome is pointed at. Every connection the
124// browser makes is dialed through it, so redirects and subresources cannot
125// escape the policy. It optionally tunnels through an upstream proxy.
126type guard struct {
127	net      *Network
128	upstream *url.URL
129}
130
131// dialTarget opens a connection to address, through the render's upstream proxy
132// when one was selected and directly through the guarded dialer otherwise.
133//
134// The proxy is reached through the same guarded dialer, so a proxy address on a
135// private or reserved network is refused unless private targets are allowed.
136func (g *guard) dialTarget(ctx context.Context, address string) (net.Conn, error) {
137	if g.upstream == nil {
138		return g.net.dial(ctx, "tcp", address)
139	}
140	return g.net.dialViaProxy(ctx, g.upstream, address)
141}
142
143// dialViaProxy establishes a CONNECT tunnel to address through an HTTP proxy.
144// The proxy resolves the destination, so the destination address cannot be
145// range-checked here; only the proxy itself is.
146func (n *Network) dialViaProxy(ctx context.Context, proxy *url.URL, address string) (net.Conn, error) {
147	port := proxy.Port()
148	if port == "" {
149		if proxy.Scheme == "https" {
150			port = "443"
151		} else {
152			port = "80"
153		}
154	}
155	conn, err := n.dial(ctx, "tcp", net.JoinHostPort(proxy.Hostname(), port))
156	if err != nil {
157		return nil, fmt.Errorf("proxy connect failed: %w", err)
158	}
159	req := &http.Request{
160		Method: http.MethodConnect,
161		URL:    &url.URL{Opaque: address},
162		Host:   address,
163		Header: http.Header{},
164	}
165	if user := proxy.User; user != nil {
166		password, _ := user.Password()
167		credentials := base64.StdEncoding.EncodeToString([]byte(user.Username() + ":" + password))
168		req.Header.Set("Proxy-Authorization", "Basic "+credentials)
169	}
170	if err := req.Write(conn); err != nil {
171		conn.Close()
172		return nil, fmt.Errorf("proxy request failed: %w", err)
173	}
174	reader := bufio.NewReader(conn)
175	res, err := http.ReadResponse(reader, req)
176	if err != nil {
177		conn.Close()
178		return nil, fmt.Errorf("proxy response failed: %w", err)
179	}
180	res.Body.Close()
181	if res.StatusCode != http.StatusOK {
182		conn.Close()
183		return nil, fmt.Errorf("proxy refused the tunnel: %s", res.Status)
184	}
185	// The proxy may have pipelined tunnel bytes into the same read, so they must
186	// be drained before the raw connection is used.
187	if reader.Buffered() > 0 {
188		return &bufferedConn{Conn: conn, reader: reader}, nil
189	}
190	return conn, nil
191}
192
193// bufferedConn returns bytes already buffered by a proxy handshake before
194// reading from the underlying connection.
195type bufferedConn struct {
196	net.Conn
197	reader *bufio.Reader
198}
199
200func (c *bufferedConn) Read(p []byte) (int, error) { return c.reader.Read(p) }
201func (n *Network) dial(ctx context.Context, network, address string) (net.Conn, error) {
202	host, port, err := net.SplitHostPort(address)
203	if err != nil {
204		return nil, err
205	}
206	ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
207	if err != nil {
208		return nil, fmt.Errorf("resolve target: %w", err)
209	}
210	if len(ips) == 0 {
211		return nil, fmt.Errorf("target has no addresses")
212	}
213	for _, ip := range ips {
214		if !n.AllowPrivate && !publicIP(ip) {
215			return nil, fmt.Errorf("private or reserved network target is blocked")
216		}
217	}
218	var last error
219	for _, ip := range ips {
220		c, err := (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
221		if err == nil {
222			return c, nil
223		}
224		last = err
225	}
226	return nil, last
227}
228
229// Chrome sends every connection to this handler, including redirects and
230// subresources. CONNECT pins the checked address, avoiding a separate
231// check-then-browser-DNS lookup that would permit DNS rebinding.
232func (g *guard) ServeHTTP(w http.ResponseWriter, r *http.Request) {
233	if r.Method == http.MethodConnect {
234		up, err := g.dialTarget(r.Context(), r.Host)
235		if err != nil {
236			http.Error(w, "blocked destination", http.StatusForbidden)
237			return
238		}
239		down, buf, err := w.(http.Hijacker).Hijack()
240		if err != nil {
241			up.Close()
242			return
243		}
244		_, err = buf.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
245		if err == nil {
246			err = buf.Flush()
247		}
248		if err != nil {
249			up.Close()
250			down.Close()
251			return
252		}
253		up.SetDeadline(time.Now().Add(95 * time.Second))
254		down.SetDeadline(time.Now().Add(95 * time.Second))
255		go func() {
256			defer up.Close()
257			defer down.Close()
258			go func() { io.Copy(up, buf); up.Close() }()
259			io.Copy(down, up)
260		}()
261		return
262	}
263	if _, err := parseWebURL(r.URL.String()); err != nil {
264		http.Error(w, "invalid destination", 400)
265		return
266	}
267	r = r.Clone(r.Context())
268	r.RequestURI = ""
269	r.Header.Del("Proxy-Authorization")
270	r.Header.Del("Proxy-Connection")
271	transport, ok := g.net.clientFor(g.upstream).Transport.(*http.Transport)
272	if !ok {
273		http.Error(w, "destination unavailable", 502)
274		return
275	}
276	res, err := transport.RoundTrip(r)
277	if err != nil {
278		http.Error(w, "destination unavailable", 502)
279		return
280	}
281	defer res.Body.Close()
282	for key, values := range res.Header {
283		if strings.EqualFold(key, "Connection") {
284			continue
285		}
286		for _, v := range values {
287			w.Header().Add(key, v)
288		}
289	}
290	w.WriteHeader(res.StatusCode)
291	io.Copy(w, res.Body)
292}
293
294func readBounded(r io.Reader, limit int) ([]byte, error) {
295	b, err := io.ReadAll(io.LimitReader(r, int64(limit)+1))
296	if err != nil {
297		return nil, err
298	}
299	if len(b) > limit {
300		return nil, fmt.Errorf("response exceeds %d bytes", limit)
301	}
302	return b, nil
303}