okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"bytes" 5"context" 6"crypto/sha256" 7"errors" 8"fmt" 9"net/http" 10"net/url" 11"path/filepath" 12"strings" 13"sync" 14"time" 15"unicode/utf8" 16 17"github.com/JohannesKaufmann/html-to-markdown/v2/converter" 18"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" 19"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" 20"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table" 21"github.com/PuerkitoBio/goquery" 22) 23 24// minMeaningfulChars is the reference implementation's floor for document text 25// worth returning: 80 non-whitespace characters. Below it a page is a failed 26// extraction rather than content, and the browser path retries as a plain 27// fetch. 28const minMeaningfulChars = 80 29 30// maxPDFBytes is the separate input cap for PDFs. PDFs legitimately run much 31// larger than HTML pages (figures, embedded fonts), so applying the page cap to 32// them rejects ordinary documents. 33const maxPDFBytes = 25 << 20 34 35type Document struct { 36URL ,FinalURL ,Title ,Author ,Published ,Markdown ,Hash string 37Fetched time.Time 38} 39type DocumentCache struct { 40mu sync.Mutex 41docs map [string ]Document 42bytes ,max int 43} 44 45func (c * DocumentCache )get (key string ,age time.Duration ) (Document ,bool ) { 46c .mu .Lock () 47defer c .mu .Unlock () 48d ,ok := c .docs [key ] 49return d ,ok && (age < 0 || time .Since (d .Fetched )<= age ) 50} 51func (c * DocumentCache )put (key string ,d Document ) { 52c .mu .Lock () 53defer c .mu .Unlock () 54if len (d .Markdown )> c .max { 55return 56 } 57if old ,ok := c .docs [key ];ok { 58c .bytes -= len (old .Markdown ) 59 } 60for c .bytes + len (d .Markdown )> c .max { 61var oldest string 62var at time.Time 63for k ,v := range c .docs { 64if k == key { 65continue 66 } 67if oldest == "" || v .Fetched .Before (at ) { 68oldest = k 69at = v .Fetched 70 } 71 } 72if oldest == "" { 73break 74 } 75c .bytes -= len (c .docs [oldest ].Markdown ) 76delete (c .docs ,oldest ) 77 } 78c .docs [key ]= d 79c .bytes += len (d .Markdown ) 80} 81 82type Fetcher struct { 83net * Network 84browser * Browser 85cache * DocumentCache 86maxBytes int 87} 88 89func (f * Fetcher )load (ctx context.Context ,raw string ,o ContentsOptions ) (Document ,string ,error ) { 90if _ ,err := parseWebURL (raw );err != nil { 91return Document {},"" ,err 92 } 93age := 5 * time .Minute 94if o .MaxAgeHours != nil { 95age = time .Duration (* o .MaxAgeHours )* time .Hour 96 } 97if age != 0 { 98if d ,ok := f .cache .get (raw ,age );ok { 99return d ,"cached" ,nil 100 } 101 } 102if o .MaxAgeHours != nil && * o .MaxAgeHours == - 1 { 103return Document {},"" ,failure (404 ,"CRAWL_CACHE_MISS" ,"document is not cached" ) 104 } 105timeout := 30 * time .Second 106if o .LivecrawlTimeout > 0 { 107timeout = time .Duration (o .LivecrawlTimeout )* time .Millisecond 108 } 109ctx ,cancel := context .WithTimeout (ctx ,timeout ) 110defer cancel () 111// One proxy per load keeps the egress address stable across the render, its 112// subresources, redirects, and the plain-fetch retry. 113proxy := f .net .pickProxy () 114d ,err := f .fetch (ctx ,raw ,proxy ) 115if err == nil { 116f .cache .put (raw ,d ) 117 } 118return d ,"crawled" ,err 119} 120 121// fetch retrieves a document by rendering it in the browser first, falling back 122// to a plain fetch only when the render produced too little text to be the real 123// page content. 124// 125// Rendering is the primary path because that is what reaches JavaScript-built 126// pages, access challenges, and bot walls. The plain fetch is the retry for the 127// documents a browser renders badly: PDFs served without a .pdf suffix, and 128// pages that serve their text only to non-browser clients. 129func (f * Fetcher )fetch (ctx context.Context ,raw string ,proxy * url.URL ) (Document ,error ) { 130// A PDF has no DOM, so a render cannot add text to it; the reference sends 131// .pdf URLs straight to the plain fetch as well. 132if ! f .canRender ()|| isPDFURL (raw ) { 133return f .fetchStatic (ctx ,raw ,proxy ) 134 } 135d ,err := f .fetchRendered (ctx ,raw ,proxy ) 136if err == nil { 137return d ,nil 138 } 139if ! isIncompleteRender (err ) { 140return Document {},err 141 } 142// Only a thin render is retried. A navigation failure, an HTTP error, or an 143// unresolved challenge is a real diagnosis and is reported as-is. 144return f .fetchStatic (ctx ,raw ,proxy ) 145} 146 147// canRender reports whether a browser is configured to render with. 148func (f * Fetcher )canRender ()bool { 149return f .browser != nil && f .browser .path != "" 150} 151 152// incompleteTag marks a render that produced too little text to be the real 153// page content. It is the trigger for the plain-fetch retry. 154const incompleteTag = "INCOMPLETE_RENDER" 155 156func isIncompleteRender (err error )bool { 157var api * APIError 158return errors .As (err ,& api )&& api .Tag == incompleteTag 159} 160 161// fetchRendered renders raw in the browser and converts the resulting DOM. It 162// enforces the content floor, so a render that produces only a shell is 163// reported as incomplete rather than returned as content. 164func (f * Fetcher )fetchRendered (ctx context.Context ,raw string ,proxy * url.URL ) (Document ,error ) { 165page ,err := f .browser .renderDocument (ctx ,raw ,proxy ) 166if err != nil { 167return Document {},err 168 } 169if len (page .HTML )> f .maxBytes { 170return Document {},fmt .Errorf ("rendered page exceeds size limit" ) 171 } 172d ,err := htmlDocument (raw ,page .URL , []byte (page .HTML )) 173if err != nil { 174return Document {},err 175 } 176if err := verifyContent (d .Markdown ,d .FinalURL );err != nil { 177return Document {},err 178 } 179return f .finish (d ) 180} 181 182// meaningfulChars counts non-whitespace characters, so boilerplate whitespace 183// cannot make an empty page look full. 184func meaningfulChars (markdown string )int { 185return utf8 .RuneCountInString (strings .Join (strings .Fields (markdown ),"" )) 186} 187 188// verifyContent applies the content floor. A document with too little readable 189// text is a failed extraction, not content, and is reported with a distinct tag 190// so the caller can retry through the other fetch path. 191func verifyContent (markdown ,finalURL string )error { 192if n := meaningfulChars (markdown );n < minMeaningfulChars { 193return failure (502 ,incompleteTag ,fmt .Sprintf ("%s produced only %d meaningful characters (minimum %d)" ,finalURL ,n ,minMeaningfulChars )) 194 } 195return nil 196} 197 198func (f * Fetcher )fetchStatic (ctx context.Context ,raw string ,proxy * url.URL ) (Document ,error ) { 199req ,err := http .NewRequestWithContext (ctx ,"GET" ,raw ,nil ) 200if err != nil { 201return Document {},err 202 } 203req .Header .Set ("User-Agent" ,f .userAgent ()) 204res ,err := f .net .clientFor (proxy ).Do (req ) 205if err != nil { 206return Document {},fmt .Errorf ("page fetch failed: %w" ,err ) 207 } 208defer res .Body .Close () 209if res .StatusCode >= 400 { 210return Document {},failure (res .StatusCode ,"CRAWL_HTTP_ERROR" ,fmt .Sprintf ("page returned http %d" ,res .StatusCode )) 211 } 212// A PDF is recognized from the URL or declared type before the body is 213// read, so the larger PDF cap can apply to the read itself. 214pdfLike := isPDFURL (raw )|| isPDFContentType (res .Header .Get ("Content-Type" )) 215limit := f .maxBytes 216if pdfLike { 217limit = max (limit ,maxPDFBytes ) 218 } 219// Reject an oversized body from its declared length before reading it, 220// instead of reading up to the limit and failing after the transfer. 221if res .ContentLength > int64 (limit ) { 222return Document {},fmt .Errorf ("response exceeds %d bytes" ,limit ) 223 } 224b ,err := readBounded (res .Body ,limit ) 225if err != nil { 226return Document {},err 227 } 228final := res .Request .URL .String () 229var d Document 230switch { 231case pdfLike || looksLikePDF (b ): 232text ,err := extractPDFText (b ) 233if err != nil { 234return Document {},err 235 } 236d = Document {URL :raw ,FinalURL :final ,Title :filepath .Base (res .Request .URL .Path ),Markdown :text } 237case strings .HasPrefix (res .Header .Get ("Content-Type" ),"text/plain" )|| strings .HasPrefix (res .Header .Get ("Content-Type" ),"text/markdown" ): 238d = Document {URL :raw ,FinalURL :final ,Markdown :string (b )} 239default : 240d ,err = htmlDocument (raw ,final ,b ) 241if err != nil { 242return Document {},err 243 } 244 } 245if err := verifyContent (d .Markdown ,final );err != nil { 246return Document {},err 247 } 248return f .finish (d ) 249} 250 251// userAgent is the identity a plain fetch presents. It matches the browser 252// identity so a site cannot distinguish the two paths, and because many sites 253// serve their text only to a browser user agent. 254func (f * Fetcher )userAgent ()string { 255if f .browser != nil { 256return f .browser .userAgent () 257 } 258return chromeUserAgent (probeChromeVersion ("" )) 259} 260 261// finish applies the size gate and stamps the document identity. The content 262// floor is applied by verifyContent before this point. 263func (f * Fetcher )finish (d Document ) (Document ,error ) { 264if len (d .Markdown )> f .maxBytes { 265return Document {},fmt .Errorf ("converted document exceeds size limit" ) 266 } 267d .Hash = fmt .Sprintf ("%x" ,sha256 .Sum256 ([]byte (d .Markdown ))) 268d .Fetched = time .Now () 269return d ,nil 270} 271 272// isPDFURL reports whether a URL path ends in .pdf, ignoring its query string 273// and fragment. Both are common on PDF links: "report.pdf?download=1" and 274// "report.pdf#page=2". 275func isPDFURL (raw string )bool { 276path := raw 277if i := strings .IndexAny (path ,"?#" );i >= 0 { 278path = path [:i ] 279 } 280return len (path )>= 4 && strings .EqualFold (path [len (path )- 4 :],".pdf" ) 281} 282 283// isPDFContentType reports whether a declared content type is a PDF. 284func isPDFContentType (contentType string )bool { 285return strings .Contains (strings .ToLower (contentType ),"pdf" ) 286} 287 288// looksLikePDF reports whether bytes begin with a PDF header after any leading 289// whitespace or byte-order marks. Some servers prepend either. 290func looksLikePDF (b []byte )bool { 291i := 0 292for { 293for i < len (b )&& isASCIIWhitespace (b [i ]) { 294i ++ 295 } 296if bytes .HasPrefix (b [i :], []byte ("\xef\xbb\xbf" )) { 297i += 3 298continue 299 } 300break 301 } 302return bytes .HasPrefix (b [i :], []byte ("%PDF-" )) 303} 304 305func isASCIIWhitespace (c byte )bool { 306return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r' 307} 308 309func htmlDocument (raw ,final string ,b []byte ) (Document ,error ) { 310h ,err := goquery .NewDocumentFromReader (bytes .NewReader (b )) 311if err != nil { 312return Document {},err 313 } 314d := Document {URL :raw ,FinalURL :final ,Title :strings .TrimSpace (h .Find ("title" ).First ().Text ())} 315d .Author ,_ = h .Find (`meta[name="author"]` ).Attr ("content" ) 316date ,_ := h .Find (`meta[property="article:published_time"]` ).Attr ("content" ) 317if t ,err := time .Parse (time .RFC3339 ,date );err == nil { 318d .Published = t .Format (time .RFC3339 ) 319 } 320if isAccessChallenge (d .Title ,string (b )) { 321return d ,failure (403 ,"CRAWL_CHALLENGE" ,"page is an access challenge, not source evidence" ) 322 } 323h .Find ("script,style,noscript,nav,footer,form,iframe,svg,[hidden],[aria-hidden=true]" ).Remove () 324h .Find ("img" ).Each (func (_ int ,s * goquery.Selection ) {alt ,_ := s .Attr ("alt" );s .ReplaceWithHtml (escapeHTML (alt )) }) 325content := h .Find ("main" ).First () 326if content .Length ()== 0 { 327content = h .Find ("article" ).First () 328 } 329if content .Length ()== 0 { 330content = h .Find ("body" ).First () 331 } 332markup ,err := content .Html () 333if err != nil { 334return d ,err 335 } 336c := converter .NewConverter (converter .WithPlugins (base .NewBasePlugin (),commonmark .NewCommonmarkPlugin (),table .NewTablePlugin ())) 337d .Markdown ,err = c .ConvertString (markup ,converter .WithDomain (final )) 338d .Markdown = strings .TrimSpace (d .Markdown ) 339return d ,err 340} 341 342func escapeHTML (s string )string { 343return strings .NewReplacer ("&" ,"&" ,"<" ,"<" ,">" ,">" ,"\"" ,""" ).Replace (s ) 344} 345 346type limitedBuffer struct { 347 bytes.Buffer 348limit int 349} 350 351func (b * limitedBuffer )Write (p []byte ) (int ,error ) { 352if b .Len ()+ len (p )> b .limit { 353return 0 ,fmt .Errorf ("output exceeds limit" ) 354 } 355return b .Buffer .Write (p ) 356}