okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
8.3 KiB203 linesraw
1package app
2
3import (
4	"context"
5	"errors"
6	"fmt"
7	"net/http"
8	"net/http/httptest"
9	"os"
10	"path/filepath"
11	"strings"
12	"sync/atomic"
13	"testing"
14	"time"
15)
16
17func TestBrowserWaitKeepsPhaseAndCallerDeadlinesSeparate(t *testing.T) {
18	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
19	defer cancel()
20	err := browserWait(ctx, 5*time.Millisecond, time.Millisecond, func(ctx context.Context) (bool, error) {
21		<-ctx.Done()
22		return false, ctx.Err()
23	})
24	if err != nil || ctx.Err() != nil {
25		t.Fatalf("phase timeout killed parent: %v, %v", err, ctx.Err())
26	}
27	cancel()
28	if err := browserWait(ctx, time.Second, time.Millisecond, func(ctx context.Context) (bool, error) {
29		return true, nil
30	}); !errors.Is(err, context.Canceled) {
31		t.Fatalf("ignored caller cancellation: %v", err)
32	}
33	want := errors.New("browser disconnected")
34	if err := browserWait(context.Background(), time.Second, time.Millisecond, func(context.Context) (bool, error) {
35		return false, want
36	}); !errors.Is(err, want) {
37		t.Fatalf("hid browser error: %v", err)
38	}
39}
40
41func TestBrowserQueueCancellation(t *testing.T) {
42	n := newNetwork(true)
43	defer n.closeIdle()
44	b := &Browser{path: "not launched", network: n, slots: make(chan struct{}, 1)}
45	b.slots <- struct{}{}
46	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond)
47	defer cancel()
48	_, _, err := b.render(ctx, "https://example.com", false, "", "")
49	if !errors.Is(err, context.DeadlineExceeded) || len(b.slots) != 1 {
50		t.Fatalf("queue cancellation changed occupied slot: %v", err)
51	}
52}
53
54func TestKagiMissingResultsDiagnosis(t *testing.T) {
55	for _, tc := range []struct{ html, final, tag string }{
56		{`<title>sign in to kagi</title>`, "https://kagi.com/signin", "SEARCH_SESSION_REJECTED"},
57		{`<form><input type="password"></form>`, "https://kagi.com/search", "SEARCH_SESSION_REJECTED"},
58		{`<p>no results; <a href="/signin">sign in</a></p>`, "https://kagi.com/search", "SEARCH_UPSTREAM_ERROR"},
59		{`<p>unrecognized search markup</p>`, "https://kagi.com/search", "SEARCH_UPSTREAM_ERROR"},
60	} {
61		var api *APIError
62		if err := kagiPageError(tc.html, tc.final); !errors.As(err, &api) || api.Tag != tc.tag {
63			t.Errorf("%s: %v", tc.html, err)
64		}
65	}
66}
67
68func TestChallengeContentIsNotEvidence(t *testing.T) {
69	for _, html := range []string{
70		`<title>Just a moment...</title>`, `<title>Attention Required</title>`,
71		`<div id="challenge-form">checking your browser</div>`,
72		`<div id="cf-browser-verification">checking your browser</div>`,
73		`<script src="/cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1?cf-chl-token=1"></script>`,
74	} {
75		_, err := htmlDocument("https://example.com", "https://example.com", []byte(html))
76		var api *APIError
77		if !errors.As(err, &api) || api.Tag != "CRAWL_CHALLENGE" {
78			t.Errorf("challenge accepted as evidence: %s, %v", html, err)
79		}
80	}
81	if isAccessChallenge("cloudflare documentation", "<p>how access challenges work</p>") {
82		t.Fatal("ordinary content classified as a challenge")
83	}
84}
85
86// These cases exercise real Chrome and the guarded proxy, without external
87// requests or model calls. Keep opt-in consistent with TestLiveBrowser.
88func TestLiveBrowserRecovery(t *testing.T) {
89	if os.Getenv("LIVE_BROWSER") != "1" {
90		t.Skip("set LIVE_BROWSER=1 and BROWSER_PATH to exercise chrome")
91	}
92	if os.Getenv("BROWSER_PATH") == "" {
93		t.Fatal("BROWSER_PATH is required")
94	}
95	profiles := t.TempDir()
96	t.Setenv("TMPDIR", profiles)
97	var blockedHits, scriptHits atomic.Int32
98	page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
99		w.Header().Set("Content-Type", "text/html")
100		switch r.URL.Path {
101		case "/never":
102			<-r.Context().Done()
103		case "/hung", "/empty":
104			body := ""
105			if r.URL.Path == "/hung" {
106				body = "usable evidence despite an unfinished third-party iframe"
107			}
108			fmt.Fprintf(w, `<body><main>%s</main><iframe src="/never"></iframe></body>`, body)
109		case "/stream":
110			fmt.Fprint(w, `<body><main></main><script>setTimeout(() => document.querySelector('main').innerHTML = '<div class="search-result"><a class="__sri_title_link" href="https://example.com">streamed result</a></div>', 700)</script></body>`)
111		case "/missing":
112			fmt.Fprint(w, `<title>sign in to kagi</title><body><form><input type="password"></form></body>`)
113		case "/hydrate":
114			fmt.Fprint(w, `<body><main>loading</main><script>for (let i=1;i<=4;i++) setTimeout(() => document.querySelector('main').textContent = 'hydration stage ' + i + '.'.repeat(i), i*250)</script></body>`)
115		case "/challenge", "/resolved":
116			w.WriteHeader(http.StatusForbidden)
117			fmt.Fprint(w, `<title>just a moment</title><body><div id="challenge-form">checking browser</div>`)
118			if r.URL.Path == "/resolved" {
119				fmt.Fprint(w, `<script>setTimeout(() => { document.title='source'; document.body.innerHTML='<main>resolved source evidence</main>'; }, 1200)</script>`)
120			}
121		case "/resources":
122			fmt.Fprint(w, `<body><main>rendering</main><img src="/tracker.avif"><script src="/necessary.js"></script></body>`)
123		case "/tracker.avif":
124			blockedHits.Add(1)
125		case "/necessary.js":
126			scriptHits.Add(1)
127			w.Header().Set("Content-Type", "text/javascript")
128			fmt.Fprint(w, `document.querySelector('main').textContent='necessary javascript ran'`)
129		case "/http-error":
130			w.WriteHeader(http.StatusBadGateway)
131			fmt.Fprint(w, `<body>upstream unavailable; this is not source evidence</body>`)
132		default:
133			fmt.Fprint(w, `<body><main>ordinary source evidence</main></body>`)
134		}
135	}))
136	defer page.Close()
137	n := newNetwork(true)
138	defer n.transport.CloseIdleConnections()
139	b := &Browser{path: os.Getenv("BROWSER_PATH"), network: n, slots: make(chan struct{}, 2)}
140	for _, tc := range []struct {
141		path, want, tag string
142		kagi            bool
143		max             time.Duration
144	}{
145		{"/hung", "<main>usable evidence despite an unfinished third-party iframe</main>", "", false, 20 * time.Second},
146		{"/empty", "", "REQUEST_TIMEOUT", false, 20 * time.Second},
147		{"/stream", `class="search-result"`, "", true, 6 * time.Second},
148		{"/missing", `type="password"`, "", true, 15 * time.Second},
149		{"/hydrate", "<main>hydration stage 4....</main>", "", false, 6 * time.Second},
150		{"/resolved", "<main>resolved source evidence</main>", "", false, 8 * time.Second},
151		{"/challenge", "", "CRAWL_CHALLENGE", false, 18 * time.Second},
152		{"/resources", "<main>necessary javascript ran</main>", "", false, 6 * time.Second},
153		{"/http-error", "", "CRAWL_HTTP_ERROR", false, 6 * time.Second},
154	} {
155		t.Run(tc.path, func(t *testing.T) {
156			ctx, cancel := context.WithTimeout(context.Background(), tc.max+5*time.Second)
157			defer cancel()
158			start := time.Now()
159			html, final, err := b.render(ctx, page.URL+tc.path, tc.kagi, "", "")
160			elapsed := time.Since(start)
161			if tc.tag != "" {
162				var api *APIError
163				if !errors.As(err, &api) || api.Tag != tc.tag || html != "" {
164					t.Fatalf("want %s without content, got %q, %v", tc.tag, html, err)
165				}
166			} else if err != nil || !strings.Contains(html, tc.want) || final != page.URL+tc.path {
167				t.Fatalf("render mismatch: url=%s error=%v html=%s", final, err, html)
168			}
169			if elapsed > tc.max {
170				t.Errorf("phase wait was not bounded: %s", elapsed)
171			}
172			if len(b.slots) != 0 {
173				t.Fatal("browser slot leaked")
174			}
175			left, _ := filepath.Glob(filepath.Join(profiles, "chromedp-runner*"))
176			if len(left) != 0 {
177				t.Fatalf("temporary browser profiles leaked: %v", left)
178			}
179			t.Logf("elapsed=%s", elapsed.Round(time.Millisecond))
180		})
181	}
182	if blockedHits.Load() != 0 || scriptHits.Load() != 1 {
183		t.Errorf("resource blocking: image=%d necessary script=%d", blockedHits.Load(), scriptHits.Load())
184	}
185	t.Run("caller cancellation", func(t *testing.T) {
186		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
187		defer cancel()
188		_, _, err := b.render(ctx, page.URL+"/hung", false, "", "")
189		if !errors.Is(err, context.DeadlineExceeded) || len(b.slots) != 0 {
190			t.Fatalf("caller cancellation did not terminate browser: %v", err)
191		}
192	})
193	t.Run("navigation failure", func(t *testing.T) {
194		dead := httptest.NewServer(http.NotFoundHandler())
195		dead.Close()
196		// HTTPS forces a failed CONNECT/navigation, rather than the proxy's
197		// ordinary HTTP 502 document (covered separately above).
198		_, _, err := b.render(context.Background(), strings.Replace(dead.URL, "http:", "https:", 1), false, "", "")
199		if err == nil || errors.Is(err, context.DeadlineExceeded) {
200			t.Fatalf("navigation failure was hidden: %v", err)
201		}
202	})
203}