okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
5a90548
main
1package app 2 3import ( 4"bytes" 5"context" 6"net/http" 7"net/http/httptest" 8"os" 9"path/filepath" 10"strings" 11"testing" 12) 13 14func TestIsPDFURLIgnoresQueryAndFragment (t * testing.T ) { 15for _ ,tc := range []struct { 16raw string 17want bool 18 }{ 19 {"https://example.test/report.pdf?download=1" ,true }, 20 {"https://example.test/report.PDF" ,true }, 21 {"https://example.test/a/b/Report.Pdf#page=2" ,true }, 22 {"https://example.test/report.pdf.html" ,false }, 23 {"https://example.test/pdf" ,false }, 24 {"https://example.test/" ,false }, 25 } { 26if got := isPDFURL (tc .raw );got != tc .want { 27t .Errorf ("isPDFURL(%q) = %v, want %v" ,tc .raw ,got ,tc .want ) 28 } 29 } 30} 31 32func TestLooksLikePDFSkipsLeadingWhitespaceAndBOM (t * testing.T ) { 33for _ ,tc := range []struct { 34body []byte 35want bool 36 }{ 37 {[]byte ("%PDF-1.7" ),true }, 38 {[]byte (" \n\t%PDF-1.4" ),true }, 39 {[]byte ("\xef\xbb\xbf%PDF-1.3" ),true }, 40 {[]byte (" \xef\xbb\xbf\n%PDF-1.5" ),true }, 41 {[]byte ("not a PDF" ),false }, 42 {[]byte ("%PDX-1.7" ),false }, 43 {[]byte ("%PDF" ),false }, 44 {[]byte ("" ),false }, 45 } { 46if got := looksLikePDF (tc .body );got != tc .want { 47t .Errorf ("looksLikePDF(%q) = %v, want %v" ,tc .body ,got ,tc .want ) 48 } 49 } 50} 51 52func TestIsPDFContentType (t * testing.T ) { 53for _ ,tc := range []struct { 54contentType string 55want bool 56 }{ 57 {"application/pdf; charset=binary" ,true }, 58 {"APPLICATION/PDF" ,true }, 59 {"text/html" ,false }, 60 {"" ,false }, 61 } { 62if got := isPDFContentType (tc .contentType );got != tc .want { 63t .Errorf ("isPDFContentType(%q) = %v, want %v" ,tc .contentType ,got ,tc .want ) 64 } 65 } 66} 67 68// PDF text is extracted in-process. The deployment host has no pdftotext, so 69// shelling out turns every PDF request into a hard failure. 70func TestExtractPDFTextReadsRealDocument (t * testing.T ) { 71text ,err := extractPDFText (fixturePDF (t ,"dummy.pdf" )) 72if err != nil { 73t .Fatal (err ) 74 } 75if ! strings .Contains (text ,"Dummy PDF file" ) { 76t .Fatalf ("unexpected extraction: %.120q" ,text ) 77 } 78} 79 80// Malformed input must surface as an error, never as a panic: this runs on 81// arbitrary caller-supplied URLs. 82func TestExtractPDFTextRejectsMalformedInput (t * testing.T ) { 83for _ ,body := range [][]byte { 84nil , 85 []byte ("not a pdf at all" ), 86 []byte ("%PDF-1.7\ntruncated" ), 87bytes .Repeat ([]byte ("x" ),4096 ), 88 } { 89if _ ,err := extractPDFText (body );err == nil { 90t .Errorf ("malformed pdf of %d bytes was accepted" ,len (body )) 91 } 92 } 93} 94 95// PDFs legitimately exceed the HTML page cap. Before the separate PDF limit a 96// document larger than the page cap was rejected outright with a size error. 97// The page cap is set below the fixture's real size to cross the boundary. 98func TestFetchStaticAcceptsPDFLargerThanPageCap (t * testing.T ) { 99:= fixturePDF (t ,"report.pdf" ) 100if len (<= pageCapForTest { 101t .Fatalf ("fixture is %d bytes, must exceed the %d byte test cap" ,len (pageCapForTest ) 102 } 103srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 104w .Header ().Set ("Content-Type" ,"application/pdf" ) 105w .Write (defer srv .Close () 108 109n := newNetwork (true ) 110defer n .transport .CloseIdleConnections () 111f := & Fetcher {net :n ,maxBytes :pageCapForTest } 112d ,err := f .fetchStatic (context .Background (),srv .URL + "/large.pdf" ,nil ) 113if err != nil { 114t .Fatalf ("large pdf rejected: %v" ,err ) 115 } 116if ! strings .Contains (d .Markdown ,"evidence" ) { 117t .Fatalf ("unexpected extraction: %.120q" ,d .Markdown ) 118 } 119} 120 121// The larger limit is specific to PDFs; HTML over the page cap is still 122// rejected. 123func TestFetchStaticStillRejectsOversizedHTML (t * testing.T ) { 124body := append ([]byte ("<html><body><main>" ),bytes .Repeat ([]byte ("x" ),pageCapForTest )... ) 125srv := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter ,r * http.Request ) { 126w .Header ().Set ("Content-Type" ,"text/html" ) 127w .Write (body ) 128 })) 129defer srv .Close () 130 131n := newNetwork (true ) 132defer n .transport .CloseIdleConnections () 133f := & Fetcher {net :n ,maxBytes :pageCapForTest } 134if _ ,err := f .fetchStatic (context .Background (),srv .URL + "/big" ,nil );err == nil { 135t .Fatal ("oversized html page was accepted" ) 136 } 137} 138 139func TestMeaningfulCharsIgnoresWhitespaceLayout (t * testing.T ) { 140if got := meaningfulChars ("a b\n\nc\t d" );got != 4 { 141t .Fatalf ("meaningfulChars = %d, want 4" ,got ) 142 } 143if meaningfulChars (strings .Repeat ("\n \t" ,500 ))>= minMeaningfulChars { 144t .Fatal ("whitespace-only page passed the meaningful-text floor" ) 145 } 146} 147 148// pageCapForTest is a deliberately small page cap so the oversized-content 149// tests cross the boundary without multi-megabyte fixtures. 150const pageCapForTest = 4096 151 152func fixturePDF (t * testing.T ,name string ) []byte { 153t .Helper () 154b ,err := os .ReadFile (filepath .Join ("testdata" ,name )) 155if err != nil { 156t .Fatal (err ) 157 } 158return b 159}