package app import ( "bytes" "context" "crypto/sha256" "errors" "fmt" "net/http" "net/url" "path/filepath" "strings" "sync" "time" "unicode/utf8" "github.com/JohannesKaufmann/html-to-markdown/v2/converter" "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table" "github.com/PuerkitoBio/goquery" ) // minMeaningfulChars is the reference implementation's floor for document text // worth returning: 80 non-whitespace characters. Below it a page is a failed // extraction rather than content, and the browser path retries as a plain // fetch. const minMeaningfulChars = 80 // maxPDFBytes is the separate input cap for PDFs. PDFs legitimately run much // larger than HTML pages (figures, embedded fonts), so applying the page cap to // them rejects ordinary documents. const maxPDFBytes = 25 << 20 type Document struct { URL, FinalURL, Title, Author, Published, Markdown, Hash string Fetched time.Time } type DocumentCache struct { mu sync.Mutex docs map[string]Document bytes, max int } func (c *DocumentCache) get(key string, age time.Duration) (Document, bool) { c.mu.Lock() defer c.mu.Unlock() d, ok := c.docs[key] return d, ok && (age < 0 || time.Since(d.Fetched) <= age) } func (c *DocumentCache) put(key string, d Document) { c.mu.Lock() defer c.mu.Unlock() if len(d.Markdown) > c.max { return } if old, ok := c.docs[key]; ok { c.bytes -= len(old.Markdown) } for c.bytes+len(d.Markdown) > c.max { var oldest string var at time.Time for k, v := range c.docs { if k == key { continue } if oldest == "" || v.Fetched.Before(at) { oldest = k at = v.Fetched } } if oldest == "" { break } c.bytes -= len(c.docs[oldest].Markdown) delete(c.docs, oldest) } c.docs[key] = d c.bytes += len(d.Markdown) } type Fetcher struct { net *Network browser *Browser cache *DocumentCache maxBytes int } func (f *Fetcher) load(ctx context.Context, raw string, o ContentsOptions) (Document, string, error) { if _, err := parseWebURL(raw); err != nil { return Document{}, "", err } age := 5 * time.Minute if o.MaxAgeHours != nil { age = time.Duration(*o.MaxAgeHours) * time.Hour } if age != 0 { if d, ok := f.cache.get(raw, age); ok { return d, "cached", nil } } if o.MaxAgeHours != nil && *o.MaxAgeHours == -1 { return Document{}, "", failure(404, "CRAWL_CACHE_MISS", "document is not cached") } timeout := 30 * time.Second if o.LivecrawlTimeout > 0 { timeout = time.Duration(o.LivecrawlTimeout) * time.Millisecond } ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() // One proxy per load keeps the egress address stable across the render, its // subresources, redirects, and the plain-fetch retry. proxy := f.net.pickProxy() d, err := f.fetch(ctx, raw, proxy) if err == nil { f.cache.put(raw, d) } return d, "crawled", err } // fetch retrieves a document by rendering it in the browser first, falling back // to a plain fetch only when the render produced too little text to be the real // page content. // // Rendering is the primary path because that is what reaches JavaScript-built // pages, access challenges, and bot walls. The plain fetch is the retry for the // documents a browser renders badly: PDFs served without a .pdf suffix, and // pages that serve their text only to non-browser clients. func (f *Fetcher) fetch(ctx context.Context, raw string, proxy *url.URL) (Document, error) { // A PDF has no DOM, so a render cannot add text to it; the reference sends // .pdf URLs straight to the plain fetch as well. if !f.canRender() || isPDFURL(raw) { return f.fetchStatic(ctx, raw, proxy) } d, err := f.fetchRendered(ctx, raw, proxy) if err == nil { return d, nil } if !isIncompleteRender(err) { return Document{}, err } // Only a thin render is retried. A navigation failure, an HTTP error, or an // unresolved challenge is a real diagnosis and is reported as-is. return f.fetchStatic(ctx, raw, proxy) } // canRender reports whether a browser is configured to render with. func (f *Fetcher) canRender() bool { return f.browser != nil && f.browser.path != "" } // incompleteTag marks a render that produced too little text to be the real // page content. It is the trigger for the plain-fetch retry. const incompleteTag = "INCOMPLETE_RENDER" func isIncompleteRender(err error) bool { var api *APIError return errors.As(err, &api) && api.Tag == incompleteTag } // fetchRendered renders raw in the browser and converts the resulting DOM. It // enforces the content floor, so a render that produces only a shell is // reported as incomplete rather than returned as content. func (f *Fetcher) fetchRendered(ctx context.Context, raw string, proxy *url.URL) (Document, error) { page, err := f.browser.renderDocument(ctx, raw, proxy) if err != nil { return Document{}, err } if len(page.HTML) > f.maxBytes { return Document{}, fmt.Errorf("rendered page exceeds size limit") } d, err := htmlDocument(raw, page.URL, []byte(page.HTML)) if err != nil { return Document{}, err } if err := verifyContent(d.Markdown, d.FinalURL); err != nil { return Document{}, err } return f.finish(d) } // meaningfulChars counts non-whitespace characters, so boilerplate whitespace // cannot make an empty page look full. func meaningfulChars(markdown string) int { return utf8.RuneCountInString(strings.Join(strings.Fields(markdown), "")) } // verifyContent applies the content floor. A document with too little readable // text is a failed extraction, not content, and is reported with a distinct tag // so the caller can retry through the other fetch path. func verifyContent(markdown, finalURL string) error { if n := meaningfulChars(markdown); n < minMeaningfulChars { return failure(502, incompleteTag, fmt.Sprintf("%s produced only %d meaningful characters (minimum %d)", finalURL, n, minMeaningfulChars)) } return nil } func (f *Fetcher) fetchStatic(ctx context.Context, raw string, proxy *url.URL) (Document, error) { req, err := http.NewRequestWithContext(ctx, "GET", raw, nil) if err != nil { return Document{}, err } req.Header.Set("User-Agent", f.userAgent()) res, err := f.net.clientFor(proxy).Do(req) if err != nil { return Document{}, fmt.Errorf("page fetch failed: %w", err) } defer res.Body.Close() if res.StatusCode >= 400 { return Document{}, failure(res.StatusCode, "CRAWL_HTTP_ERROR", fmt.Sprintf("page returned http %d", res.StatusCode)) } // A PDF is recognized from the URL or declared type before the body is // read, so the larger PDF cap can apply to the read itself. pdfLike := isPDFURL(raw) || isPDFContentType(res.Header.Get("Content-Type")) limit := f.maxBytes if pdfLike { limit = max(limit, maxPDFBytes) } // Reject an oversized body from its declared length before reading it, // instead of reading up to the limit and failing after the transfer. if res.ContentLength > int64(limit) { return Document{}, fmt.Errorf("response exceeds %d bytes", limit) } b, err := readBounded(res.Body, limit) if err != nil { return Document{}, err } final := res.Request.URL.String() var d Document switch { case pdfLike || looksLikePDF(b): text, err := extractPDFText(b) if err != nil { return Document{}, err } d = Document{URL: raw, FinalURL: final, Title: filepath.Base(res.Request.URL.Path), Markdown: text} case strings.HasPrefix(res.Header.Get("Content-Type"), "text/plain") || strings.HasPrefix(res.Header.Get("Content-Type"), "text/markdown"): d = Document{URL: raw, FinalURL: final, Markdown: string(b)} default: d, err = htmlDocument(raw, final, b) if err != nil { return Document{}, err } } if err := verifyContent(d.Markdown, final); err != nil { return Document{}, err } return f.finish(d) } // userAgent is the identity a plain fetch presents. It matches the browser // identity so a site cannot distinguish the two paths, and because many sites // serve their text only to a browser user agent. func (f *Fetcher) userAgent() string { if f.browser != nil { return f.browser.userAgent() } return chromeUserAgent(probeChromeVersion("")) } // finish applies the size gate and stamps the document identity. The content // floor is applied by verifyContent before this point. func (f *Fetcher) finish(d Document) (Document, error) { if len(d.Markdown) > f.maxBytes { return Document{}, fmt.Errorf("converted document exceeds size limit") } d.Hash = fmt.Sprintf("%x", sha256.Sum256([]byte(d.Markdown))) d.Fetched = time.Now() return d, nil } // isPDFURL reports whether a URL path ends in .pdf, ignoring its query string // and fragment. Both are common on PDF links: "report.pdf?download=1" and // "report.pdf#page=2". func isPDFURL(raw string) bool { path := raw if i := strings.IndexAny(path, "?#"); i >= 0 { path = path[:i] } return len(path) >= 4 && strings.EqualFold(path[len(path)-4:], ".pdf") } // isPDFContentType reports whether a declared content type is a PDF. func isPDFContentType(contentType string) bool { return strings.Contains(strings.ToLower(contentType), "pdf") } // looksLikePDF reports whether bytes begin with a PDF header after any leading // whitespace or byte-order marks. Some servers prepend either. func looksLikePDF(b []byte) bool { i := 0 for { for i < len(b) && isASCIIWhitespace(b[i]) { i++ } if bytes.HasPrefix(b[i:], []byte("\xef\xbb\xbf")) { i += 3 continue } break } return bytes.HasPrefix(b[i:], []byte("%PDF-")) } func isASCIIWhitespace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r' } func htmlDocument(raw, final string, b []byte) (Document, error) { h, err := goquery.NewDocumentFromReader(bytes.NewReader(b)) if err != nil { return Document{}, err } d := Document{URL: raw, FinalURL: final, Title: strings.TrimSpace(h.Find("title").First().Text())} d.Author, _ = h.Find(`meta[name="author"]`).Attr("content") date, _ := h.Find(`meta[property="article:published_time"]`).Attr("content") if t, err := time.Parse(time.RFC3339, date); err == nil { d.Published = t.Format(time.RFC3339) } if isAccessChallenge(d.Title, string(b)) { return d, failure(403, "CRAWL_CHALLENGE", "page is an access challenge, not source evidence") } h.Find("script,style,noscript,nav,footer,form,iframe,svg,[hidden],[aria-hidden=true]").Remove() h.Find("img").Each(func(_ int, s *goquery.Selection) { alt, _ := s.Attr("alt"); s.ReplaceWithHtml(escapeHTML(alt)) }) content := h.Find("main").First() if content.Length() == 0 { content = h.Find("article").First() } if content.Length() == 0 { content = h.Find("body").First() } markup, err := content.Html() if err != nil { return d, err } c := converter.NewConverter(converter.WithPlugins(base.NewBasePlugin(), commonmark.NewCommonmarkPlugin(), table.NewTablePlugin())) d.Markdown, err = c.ConvertString(markup, converter.WithDomain(final)) d.Markdown = strings.TrimSpace(d.Markdown) return d, err } func escapeHTML(s string) string { return strings.NewReplacer("&", "&", "<", "<", ">", ">", "\"", """).Replace(s) } type limitedBuffer struct { bytes.Buffer limit int } func (b *limitedBuffer) Write(p []byte) (int, error) { if b.Len()+len(p) > b.limit { return 0, fmt.Errorf("output exceeds limit") } return b.Buffer.Write(p) }