okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petweb-extract: the Go service running on the Mac mini (vassago)5a90548

main
4.6 KiB159 linesraw
1package app
2
3import (
4	"bytes"
5	"context"
6	"net/http"
7	"net/http/httptest"
8	"os"
9	"path/filepath"
10	"strings"
11	"testing"
12)
13
14func TestIsPDFURLIgnoresQueryAndFragment(t *testing.T) {
15	for _, tc := range []struct {
16		raw  string
17		want bool
18	}{
19		{"https://example.test/report.pdf?download=1", true},
20		{"https://example.test/report.PDF", true},
21		{"https://example.test/a/b/Report.Pdf#page=2", true},
22		{"https://example.test/report.pdf.html", false},
23		{"https://example.test/pdf", false},
24		{"https://example.test/", false},
25	} {
26		if got := isPDFURL(tc.raw); got != tc.want {
27			t.Errorf("isPDFURL(%q) = %v, want %v", tc.raw, got, tc.want)
28		}
29	}
30}
31
32func TestLooksLikePDFSkipsLeadingWhitespaceAndBOM(t *testing.T) {
33	for _, tc := range []struct {
34		body []byte
35		want bool
36	}{
37		{[]byte("%PDF-1.7"), true},
38		{[]byte(" \n\t%PDF-1.4"), true},
39		{[]byte("\xef\xbb\xbf%PDF-1.3"), true},
40		{[]byte(" \xef\xbb\xbf\n%PDF-1.5"), true},
41		{[]byte("not a PDF"), false},
42		{[]byte("%PDX-1.7"), false},
43		{[]byte("%PDF"), false},
44		{[]byte(""), false},
45	} {
46		if got := looksLikePDF(tc.body); got != tc.want {
47			t.Errorf("looksLikePDF(%q) = %v, want %v", tc.body, got, tc.want)
48		}
49	}
50}
51
52func TestIsPDFContentType(t *testing.T) {
53	for _, tc := range []struct {
54		contentType string
55		want        bool
56	}{
57		{"application/pdf; charset=binary", true},
58		{"APPLICATION/PDF", true},
59		{"text/html", false},
60		{"", false},
61	} {
62		if got := isPDFContentType(tc.contentType); got != tc.want {
63			t.Errorf("isPDFContentType(%q) = %v, want %v", tc.contentType, got, tc.want)
64		}
65	}
66}
67
68// PDF text is extracted in-process. The deployment host has no pdftotext, so
69// shelling out turns every PDF request into a hard failure.
70func TestExtractPDFTextReadsRealDocument(t *testing.T) {
71	text, err := extractPDFText(fixturePDF(t, "dummy.pdf"))
72	if err != nil {
73		t.Fatal(err)
74	}
75	if !strings.Contains(text, "Dummy PDF file") {
76		t.Fatalf("unexpected extraction: %.120q", text)
77	}
78}
79
80// Malformed input must surface as an error, never as a panic: this runs on
81// arbitrary caller-supplied URLs.
82func TestExtractPDFTextRejectsMalformedInput(t *testing.T) {
83	for _, body := range [][]byte{
84		nil,
85		[]byte("not a pdf at all"),
86		[]byte("%PDF-1.7\ntruncated"),
87		bytes.Repeat([]byte("x"), 4096),
88	} {
89		if _, err := extractPDFText(body); err == nil {
90			t.Errorf("malformed pdf of %d bytes was accepted", len(body))
91		}
92	}
93}
94
95// PDFs legitimately exceed the HTML page cap. Before the separate PDF limit a
96// document larger than the page cap was rejected outright with a size error.
97// The page cap is set below the fixture's real size to cross the boundary.
98func TestFetchStaticAcceptsPDFLargerThanPageCap(t *testing.T) {
99	pdf := fixturePDF(t, "report.pdf")
100	if len(pdf) <= pageCapForTest {
101		t.Fatalf("fixture is %d bytes, must exceed the %d byte test cap", len(pdf), pageCapForTest)
102	}
103	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
104		w.Header().Set("Content-Type", "application/pdf")
105		w.Write(pdf)
106	}))
107	defer srv.Close()
108
109	n := newNetwork(true)
110	defer n.transport.CloseIdleConnections()
111	f := &Fetcher{net: n, maxBytes: pageCapForTest}
112	d, err := f.fetchStatic(context.Background(), srv.URL+"/large.pdf", nil)
113	if err != nil {
114		t.Fatalf("large pdf rejected: %v", err)
115	}
116	if !strings.Contains(d.Markdown, "evidence") {
117		t.Fatalf("unexpected extraction: %.120q", d.Markdown)
118	}
119}
120
121// The larger limit is specific to PDFs; HTML over the page cap is still
122// rejected.
123func TestFetchStaticStillRejectsOversizedHTML(t *testing.T) {
124	body := append([]byte("<html><body><main>"), bytes.Repeat([]byte("x"), pageCapForTest)...)
125	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
126		w.Header().Set("Content-Type", "text/html")
127		w.Write(body)
128	}))
129	defer srv.Close()
130
131	n := newNetwork(true)
132	defer n.transport.CloseIdleConnections()
133	f := &Fetcher{net: n, maxBytes: pageCapForTest}
134	if _, err := f.fetchStatic(context.Background(), srv.URL+"/big", nil); err == nil {
135		t.Fatal("oversized html page was accepted")
136	}
137}
138
139func TestMeaningfulCharsIgnoresWhitespaceLayout(t *testing.T) {
140	if got := meaningfulChars("a b\n\nc\t d"); got != 4 {
141		t.Fatalf("meaningfulChars = %d, want 4", got)
142	}
143	if meaningfulChars(strings.Repeat("\n \t", 500)) >= minMeaningfulChars {
144		t.Fatal("whitespace-only page passed the meaningful-text floor")
145	}
146}
147
148// pageCapForTest is a deliberately small page cap so the oversized-content
149// tests cross the boundary without multi-megabyte fixtures.
150const pageCapForTest = 4096
151
152func fixturePDF(t *testing.T, name string) []byte {
153	t.Helper()
154	b, err := os.ReadFile(filepath.Join("testdata", name))
155	if err != nil {
156		t.Fatal(err)
157	}
158	return b
159}