okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
11.2 KiB356 linesraw
1package app
2
3import (
4	"bytes"
5	"context"
6	"crypto/sha256"
7	"errors"
8	"fmt"
9	"net/http"
10	"net/url"
11	"path/filepath"
12	"strings"
13	"sync"
14	"time"
15	"unicode/utf8"
16
17	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
18	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
19	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
20	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
21	"github.com/PuerkitoBio/goquery"
22)
23
24// minMeaningfulChars is the reference implementation's floor for document text
25// worth returning: 80 non-whitespace characters. Below it a page is a failed
26// extraction rather than content, and the browser path retries as a plain
27// fetch.
28const minMeaningfulChars = 80
29
30// maxPDFBytes is the separate input cap for PDFs. PDFs legitimately run much
31// larger than HTML pages (figures, embedded fonts), so applying the page cap to
32// them rejects ordinary documents.
33const maxPDFBytes = 25 << 20
34
35type Document struct {
36	URL, FinalURL, Title, Author, Published, Markdown, Hash string
37	Fetched                                                 time.Time
38}
39type DocumentCache struct {
40	mu         sync.Mutex
41	docs       map[string]Document
42	bytes, max int
43}
44
45func (c *DocumentCache) get(key string, age time.Duration) (Document, bool) {
46	c.mu.Lock()
47	defer c.mu.Unlock()
48	d, ok := c.docs[key]
49	return d, ok && (age < 0 || time.Since(d.Fetched) <= age)
50}
51func (c *DocumentCache) put(key string, d Document) {
52	c.mu.Lock()
53	defer c.mu.Unlock()
54	if len(d.Markdown) > c.max {
55		return
56	}
57	if old, ok := c.docs[key]; ok {
58		c.bytes -= len(old.Markdown)
59	}
60	for c.bytes+len(d.Markdown) > c.max {
61		var oldest string
62		var at time.Time
63		for k, v := range c.docs {
64			if k == key {
65				continue
66			}
67			if oldest == "" || v.Fetched.Before(at) {
68				oldest = k
69				at = v.Fetched
70			}
71		}
72		if oldest == "" {
73			break
74		}
75		c.bytes -= len(c.docs[oldest].Markdown)
76		delete(c.docs, oldest)
77	}
78	c.docs[key] = d
79	c.bytes += len(d.Markdown)
80}
81
82type Fetcher struct {
83	net      *Network
84	browser  *Browser
85	cache    *DocumentCache
86	maxBytes int
87}
88
89func (f *Fetcher) load(ctx context.Context, raw string, o ContentsOptions) (Document, string, error) {
90	if _, err := parseWebURL(raw); err != nil {
91		return Document{}, "", err
92	}
93	age := 5 * time.Minute
94	if o.MaxAgeHours != nil {
95		age = time.Duration(*o.MaxAgeHours) * time.Hour
96	}
97	if age != 0 {
98		if d, ok := f.cache.get(raw, age); ok {
99			return d, "cached", nil
100		}
101	}
102	if o.MaxAgeHours != nil && *o.MaxAgeHours == -1 {
103		return Document{}, "", failure(404, "CRAWL_CACHE_MISS", "document is not cached")
104	}
105	timeout := 30 * time.Second
106	if o.LivecrawlTimeout > 0 {
107		timeout = time.Duration(o.LivecrawlTimeout) * time.Millisecond
108	}
109	ctx, cancel := context.WithTimeout(ctx, timeout)
110	defer cancel()
111	// One proxy per load keeps the egress address stable across the render, its
112	// subresources, redirects, and the plain-fetch retry.
113	proxy := f.net.pickProxy()
114	d, err := f.fetch(ctx, raw, proxy)
115	if err == nil {
116		f.cache.put(raw, d)
117	}
118	return d, "crawled", err
119}
120
121// fetch retrieves a document by rendering it in the browser first, falling back
122// to a plain fetch only when the render produced too little text to be the real
123// page content.
124//
125// Rendering is the primary path because that is what reaches JavaScript-built
126// pages, access challenges, and bot walls. The plain fetch is the retry for the
127// documents a browser renders badly: PDFs served without a .pdf suffix, and
128// pages that serve their text only to non-browser clients.
129func (f *Fetcher) fetch(ctx context.Context, raw string, proxy *url.URL) (Document, error) {
130	// A PDF has no DOM, so a render cannot add text to it; the reference sends
131	// .pdf URLs straight to the plain fetch as well.
132	if !f.canRender() || isPDFURL(raw) {
133		return f.fetchStatic(ctx, raw, proxy)
134	}
135	d, err := f.fetchRendered(ctx, raw, proxy)
136	if err == nil {
137		return d, nil
138	}
139	if !isIncompleteRender(err) {
140		return Document{}, err
141	}
142	// Only a thin render is retried. A navigation failure, an HTTP error, or an
143	// unresolved challenge is a real diagnosis and is reported as-is.
144	return f.fetchStatic(ctx, raw, proxy)
145}
146
147// canRender reports whether a browser is configured to render with.
148func (f *Fetcher) canRender() bool {
149	return f.browser != nil && f.browser.path != ""
150}
151
152// incompleteTag marks a render that produced too little text to be the real
153// page content. It is the trigger for the plain-fetch retry.
154const incompleteTag = "INCOMPLETE_RENDER"
155
156func isIncompleteRender(err error) bool {
157	var api *APIError
158	return errors.As(err, &api) && api.Tag == incompleteTag
159}
160
161// fetchRendered renders raw in the browser and converts the resulting DOM. It
162// enforces the content floor, so a render that produces only a shell is
163// reported as incomplete rather than returned as content.
164func (f *Fetcher) fetchRendered(ctx context.Context, raw string, proxy *url.URL) (Document, error) {
165	page, err := f.browser.renderDocument(ctx, raw, proxy)
166	if err != nil {
167		return Document{}, err
168	}
169	if len(page.HTML) > f.maxBytes {
170		return Document{}, fmt.Errorf("rendered page exceeds size limit")
171	}
172	d, err := htmlDocument(raw, page.URL, []byte(page.HTML))
173	if err != nil {
174		return Document{}, err
175	}
176	if err := verifyContent(d.Markdown, d.FinalURL); err != nil {
177		return Document{}, err
178	}
179	return f.finish(d)
180}
181
182// meaningfulChars counts non-whitespace characters, so boilerplate whitespace
183// cannot make an empty page look full.
184func meaningfulChars(markdown string) int {
185	return utf8.RuneCountInString(strings.Join(strings.Fields(markdown), ""))
186}
187
188// verifyContent applies the content floor. A document with too little readable
189// text is a failed extraction, not content, and is reported with a distinct tag
190// so the caller can retry through the other fetch path.
191func verifyContent(markdown, finalURL string) error {
192	if n := meaningfulChars(markdown); n < minMeaningfulChars {
193		return failure(502, incompleteTag, fmt.Sprintf("%s produced only %d meaningful characters (minimum %d)", finalURL, n, minMeaningfulChars))
194	}
195	return nil
196}
197
198func (f *Fetcher) fetchStatic(ctx context.Context, raw string, proxy *url.URL) (Document, error) {
199	req, err := http.NewRequestWithContext(ctx, "GET", raw, nil)
200	if err != nil {
201		return Document{}, err
202	}
203	req.Header.Set("User-Agent", f.userAgent())
204	res, err := f.net.clientFor(proxy).Do(req)
205	if err != nil {
206		return Document{}, fmt.Errorf("page fetch failed: %w", err)
207	}
208	defer res.Body.Close()
209	if res.StatusCode >= 400 {
210		return Document{}, failure(res.StatusCode, "CRAWL_HTTP_ERROR", fmt.Sprintf("page returned http %d", res.StatusCode))
211	}
212	// A PDF is recognized from the URL or declared type before the body is
213	// read, so the larger PDF cap can apply to the read itself.
214	pdfLike := isPDFURL(raw) || isPDFContentType(res.Header.Get("Content-Type"))
215	limit := f.maxBytes
216	if pdfLike {
217		limit = max(limit, maxPDFBytes)
218	}
219	// Reject an oversized body from its declared length before reading it,
220	// instead of reading up to the limit and failing after the transfer.
221	if res.ContentLength > int64(limit) {
222		return Document{}, fmt.Errorf("response exceeds %d bytes", limit)
223	}
224	b, err := readBounded(res.Body, limit)
225	if err != nil {
226		return Document{}, err
227	}
228	final := res.Request.URL.String()
229	var d Document
230	switch {
231	case pdfLike || looksLikePDF(b):
232		text, err := extractPDFText(b)
233		if err != nil {
234			return Document{}, err
235		}
236		d = Document{URL: raw, FinalURL: final, Title: filepath.Base(res.Request.URL.Path), Markdown: text}
237	case strings.HasPrefix(res.Header.Get("Content-Type"), "text/plain") || strings.HasPrefix(res.Header.Get("Content-Type"), "text/markdown"):
238		d = Document{URL: raw, FinalURL: final, Markdown: string(b)}
239	default:
240		d, err = htmlDocument(raw, final, b)
241		if err != nil {
242			return Document{}, err
243		}
244	}
245	if err := verifyContent(d.Markdown, final); err != nil {
246		return Document{}, err
247	}
248	return f.finish(d)
249}
250
251// userAgent is the identity a plain fetch presents. It matches the browser
252// identity so a site cannot distinguish the two paths, and because many sites
253// serve their text only to a browser user agent.
254func (f *Fetcher) userAgent() string {
255	if f.browser != nil {
256		return f.browser.userAgent()
257	}
258	return chromeUserAgent(probeChromeVersion(""))
259}
260
261// finish applies the size gate and stamps the document identity. The content
262// floor is applied by verifyContent before this point.
263func (f *Fetcher) finish(d Document) (Document, error) {
264	if len(d.Markdown) > f.maxBytes {
265		return Document{}, fmt.Errorf("converted document exceeds size limit")
266	}
267	d.Hash = fmt.Sprintf("%x", sha256.Sum256([]byte(d.Markdown)))
268	d.Fetched = time.Now()
269	return d, nil
270}
271
272// isPDFURL reports whether a URL path ends in .pdf, ignoring its query string
273// and fragment. Both are common on PDF links: "report.pdf?download=1" and
274// "report.pdf#page=2".
275func isPDFURL(raw string) bool {
276	path := raw
277	if i := strings.IndexAny(path, "?#"); i >= 0 {
278		path = path[:i]
279	}
280	return len(path) >= 4 && strings.EqualFold(path[len(path)-4:], ".pdf")
281}
282
283// isPDFContentType reports whether a declared content type is a PDF.
284func isPDFContentType(contentType string) bool {
285	return strings.Contains(strings.ToLower(contentType), "pdf")
286}
287
288// looksLikePDF reports whether bytes begin with a PDF header after any leading
289// whitespace or byte-order marks. Some servers prepend either.
290func looksLikePDF(b []byte) bool {
291	i := 0
292	for {
293		for i < len(b) && isASCIIWhitespace(b[i]) {
294			i++
295		}
296		if bytes.HasPrefix(b[i:], []byte("\xef\xbb\xbf")) {
297			i += 3
298			continue
299		}
300		break
301	}
302	return bytes.HasPrefix(b[i:], []byte("%PDF-"))
303}
304
305func isASCIIWhitespace(c byte) bool {
306	return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'
307}
308
309func htmlDocument(raw, final string, b []byte) (Document, error) {
310	h, err := goquery.NewDocumentFromReader(bytes.NewReader(b))
311	if err != nil {
312		return Document{}, err
313	}
314	d := Document{URL: raw, FinalURL: final, Title: strings.TrimSpace(h.Find("title").First().Text())}
315	d.Author, _ = h.Find(`meta[name="author"]`).Attr("content")
316	date, _ := h.Find(`meta[property="article:published_time"]`).Attr("content")
317	if t, err := time.Parse(time.RFC3339, date); err == nil {
318		d.Published = t.Format(time.RFC3339)
319	}
320	if isAccessChallenge(d.Title, string(b)) {
321		return d, failure(403, "CRAWL_CHALLENGE", "page is an access challenge, not source evidence")
322	}
323	h.Find("script,style,noscript,nav,footer,form,iframe,svg,[hidden],[aria-hidden=true]").Remove()
324	h.Find("img").Each(func(_ int, s *goquery.Selection) { alt, _ := s.Attr("alt"); s.ReplaceWithHtml(escapeHTML(alt)) })
325	content := h.Find("main").First()
326	if content.Length() == 0 {
327		content = h.Find("article").First()
328	}
329	if content.Length() == 0 {
330		content = h.Find("body").First()
331	}
332	markup, err := content.Html()
333	if err != nil {
334		return d, err
335	}
336	c := converter.NewConverter(converter.WithPlugins(base.NewBasePlugin(), commonmark.NewCommonmarkPlugin(), table.NewTablePlugin()))
337	d.Markdown, err = c.ConvertString(markup, converter.WithDomain(final))
338	d.Markdown = strings.TrimSpace(d.Markdown)
339	return d, err
340}
341
342func escapeHTML(s string) string {
343	return strings.NewReplacer("&", "&amp;", "<", "&lt;", ">", "&gt;", "\"", "&quot;").Replace(s)
344}
345
346type limitedBuffer struct {
347	bytes.Buffer
348	limit int
349}
350
351func (b *limitedBuffer) Write(p []byte) (int, error) {
352	if b.Len()+len(p) > b.limit {
353		return 0, fmt.Errorf("output exceeds limit")
354	}
355	return b.Buffer.Write(p)
356}