okami.mom/albedo

git clone https://git.nkp.pet/okami.mom/albedo

Aly Raffaufrefactor(tests): share fixtures and move workspace checks to E2Eeaef427

main
13.7 KiB345 linesraw
1// Connection snapshot races, refresh wait cancellation, and pooled transport
2// retries need deterministic gates that a live daemon cannot reliably expose.
3package daemon
4
5import (
6	"context"
7	"encoding/json"
8	"errors"
9	"io"
10	"net"
11	"net/http"
12	"net/http/httptest"
13	"strings"
14	"sync"
15	"sync/atomic"
16	"testing"
17
18	"albedo/cli/internal/daemon/protocol"
19)
20
21type captureTransport struct {
22	entered chan *http.Request
23	release chan struct{}
24	body    string
25	status  int
26}
27
28func (transport captureTransport) RoundTrip(request *http.Request) (*http.Response, error) {
29	transport.entered <- request
30	<-transport.release
31	body, status := transport.body, transport.status
32	if body == "" {
33		body = `{"result":"accepted"}`
34	}
35	if status == 0 {
36		status = http.StatusOK
37	}
38	return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
39}
40
41func TestOperationUsesCapturedAddressAndToken(t *testing.T) {
42	conn := NewConnection(ConnectionSnapshot{Port: 34567, Token: "old"}, nil)
43	transport := captureTransport{entered: make(chan *http.Request), release: make(chan struct{})}
44	conn.HTTPClient().Transport = transport
45	finished := make(chan error, 1)
46	go func() {
47		_, err := requestBytes(context.Background(), conn, operation{Name: "submit", Method: http.MethodPost, Path: "/sessions/s/inputs/i", Body: map[string]string{"text": "hello"}, Policy: authRecovery}, responseLimits{bodyBytes: 50 * 1024 * 1024, errorBytes: 64 * 1024})
48		finished <- err
49	}()
50	request := <-transport.entered
51	conn.Update(NewConnection(ConnectionSnapshot{Port: 45678, Token: "new"}, nil))
52	if request.URL.Host != "127.0.0.1:34567" || request.Header.Get("Authorization") != "Bearer old" {
53		t.Errorf("request mixed connection snapshots: %s, %s", request.URL.Host, request.Header.Get("Authorization"))
54	}
55	close(transport.release)
56	if err := <-finished; err != nil {
57		t.Fatal(err)
58	}
59}
60
61func TestPooledMutationAcknowledgementLossIsNotReplayed(t *testing.T) {
62	for _, test := range []struct {
63		body   any
64		name   string
65		method string
66	}{
67		{name: "submission", method: http.MethodPost, body: map[string]string{"text": "hello"}},
68		{name: "sign-in start", method: http.MethodPost},
69		{name: "delete", method: http.MethodDelete},
70	} {
71		t.Run(test.name, func(t *testing.T) {
72			var calls atomic.Int32
73			var connections atomic.Int32
74			server := httptest.NewUnstartedServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
75				_, _ = io.Copy(io.Discard, request.Body)
76				if calls.Add(1) == 1 {
77					_, _ = writer.Write([]byte(`{"result":"accepted"}`))
78					return
79				}
80				connection, _, err := writer.(http.Hijacker).Hijack()
81				if err != nil {
82					t.Error(err)
83					return
84				}
85				_ = connection.Close()
86			}))
87			server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
88				if state == http.StateNew {
89					connections.Add(1)
90				}
91			}
92			server.Start()
93			defer server.Close()
94			conn := NewConnection(ConnectionSnapshot{Port: server.Listener.Addr().(*net.TCPAddr).Port}, nil)
95			defer conn.HTTPClient().CloseIdleConnections()
96			operation := operation{Name: "submit", Method: test.method, Path: "/sessions/s/inputs/i", Body: test.body, Policy: authRecovery}
97			if _, err := requestBytes(context.Background(), conn, operation, responseLimits{bodyBytes: 50 * 1024 * 1024, errorBytes: 64 * 1024}); err != nil {
98				t.Fatal(err)
99			}
100			_, err := requestBytes(context.Background(), conn, operation, responseLimits{bodyBytes: 50 * 1024 * 1024, errorBytes: 64 * 1024})
101			if _, ok := errors.AsType[*UncertainOutcomeError](err); !ok {
102				t.Fatalf("lost response returned %v", err)
103			}
104			if calls.Load() != 2 || connections.Load() != 1 {
105				t.Fatalf("mutation was replayed or fixture missed reuse: calls=%d connections=%d", calls.Load(), connections.Load())
106			}
107		})
108	}
109}
110
111func TestConcurrentOperationsNeverMixConnectionSnapshots(t *testing.T) {
112	var failures atomic.Int32
113	newServer := func(token string) *httptest.Server {
114		return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
115			if request.Header.Get("Authorization") != "Bearer "+token {
116				failures.Add(1)
117			}
118			_, _ = writer.Write([]byte(`{}`))
119		}))
120	}
121	first, second := newServer("first"), newServer("second")
122	defer first.Close()
123	defer second.Close()
124	firstConn := NewConnection(ConnectionSnapshot{Port: first.Listener.Addr().(*net.TCPAddr).Port, Token: "first"}, nil)
125	secondConn := NewConnection(ConnectionSnapshot{Port: second.Listener.Addr().(*net.TCPAddr).Port, Token: "second"}, nil)
126	conn := NewConnection(firstConn.Snapshot(), nil)
127	defer conn.HTTPClient().CloseIdleConnections()
128	var readers sync.WaitGroup
129	for range 4 {
130		readers.Go(func() {
131			for range 50 {
132				_, err := requestBytes(context.Background(), conn, operation{Name: "read", Method: http.MethodGet, Path: "/server", Policy: readRecovery}, responseLimits{bodyBytes: 50 * 1024 * 1024, errorBytes: 64 * 1024})
133				if err != nil {
134					t.Error(err)
135				}
136			}
137		})
138	}
139	for range 100 {
140		conn.Update(secondConn)
141		conn.Update(firstConn)
142	}
143	readers.Wait()
144	if failures.Load() != 0 {
145		t.Fatalf("%d requests paired address with another daemon's token", failures.Load())
146	}
147}
148
149// Auth refusal must prove non-admission before a mutation can be dispatched again.
150func TestMutationAuthRecoveryBudget(t *testing.T) {
151	for _, test := range []struct {
152		name       string
153		token      string
154		status     int
155		expected   int32
156		marker     bool
157		rediscover bool
158		repeated   bool
159	}{
160		{name: "changed credentials", marker: true, status: 401, token: "new", rediscover: true, expected: 2},
161		{name: "unchanged credentials", marker: true, status: 401, token: "old", rediscover: true, expected: 1},
162		{name: "unmarked refusal", status: 403, token: "new", rediscover: true, expected: 1},
163		{name: "unmarked unauthorized", status: 401, token: "new", rediscover: true, expected: 1},
164		{name: "marked unauthorized", marker: true, status: 401, token: "new", rediscover: true, expected: 2},
165		{name: "no discovery", marker: true, status: 401, token: "new", expected: 1},
166		{name: "repeated refusal", marker: true, status: 401, token: "new", rediscover: true, repeated: true, expected: 2},
167	} {
168		t.Run(test.name, func(t *testing.T) {
169			var calls atomic.Int32
170			server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
171				if request.URL.Path == "/server" {
172					_, _ = writer.Write([]byte(readyServer))
173					return
174				}
175				count := calls.Add(1)
176				if count == 1 || test.repeated {
177					writer.WriteHeader(test.status)
178					if test.marker {
179						_, _ = writer.Write([]byte(`{"type":"about:blank","title":"Unauthorized","status":401,"code":"authentication_required","detail":"invalid bearer"}`))
180					} else {
181						_, _ = writer.Write([]byte(`{"detail":"forbidden"}`))
182					}
183					return
184				}
185				if request.Header.Get("Authorization") != "Bearer new" {
186					t.Errorf("recovery reused old credentials")
187				}
188				_, _ = writer.Write([]byte(`{"result":"accepted"}`))
189			}))
190			defer server.Close()
191			port := server.Listener.Addr().(*net.TCPAddr).Port
192			var rediscover Rediscovery
193			if test.rediscover {
194				rediscover = func(context.Context) (ConnectionSnapshot, error) {
195					return ConnectionSnapshot{Port: port, Token: test.token, Version: ProtocolVersion}, nil
196				}
197			}
198			conn := NewConnection(ConnectionSnapshot{Port: port, Token: "old", Version: ProtocolVersion}, rediscover)
199			defer conn.HTTPClient().CloseIdleConnections()
200			_, err := requestBytes(context.Background(), conn, operation{Name: "mutate", Method: http.MethodPost, Path: "/sessions/s/inputs/i", Policy: authRecovery}, responseLimits{bodyBytes: 50 * 1024 * 1024, errorBytes: 64 * 1024})
201			if calls.Load() != test.expected {
202				t.Fatalf("got %d dispatches, expected %d", calls.Load(), test.expected)
203			}
204			success := test.expected == 2 && !test.repeated
205			if (err == nil) != success {
206				t.Fatalf("unexpected result %v", err)
207			}
208			if !success {
209				if _, uncertain := errors.AsType[*UncertainOutcomeError](err); uncertain {
210					t.Fatalf("pre-admission rejection reported uncertain: %v", err)
211				}
212			}
213		})
214	}
215}
216
217func TestReadRecoveryIncludesTruncatedBody(t *testing.T) {
218	var calls atomic.Int32
219	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
220		if request.URL.Path == "/server" {
221			_, _ = writer.Write([]byte(readyServer))
222			return
223		}
224		if calls.Add(1) == 1 {
225			writer.Header().Set("Content-Length", "100")
226			_, _ = writer.Write([]byte(`{"ok":`))
227			return
228		}
229		_, _ = writer.Write([]byte(`{"items":[],"next":null}`))
230	}))
231	defer server.Close()
232	snapshot := ConnectionSnapshot{Port: server.Listener.Addr().(*net.TCPAddr).Port, Token: "token", Version: ProtocolVersion}
233	conn := NewConnection(snapshot, func(context.Context) (ConnectionSnapshot, error) { return snapshot, nil })
234	defer conn.HTTPClient().CloseIdleConnections()
235	result, err := ListSessions(context.Background(), conn)
236	if err != nil || len(result) != 0 || calls.Load() != 2 {
237		t.Fatalf("truncated read did not recover once: %v, %v, %d", result, err, calls.Load())
238	}
239}
240
241func TestMalformedMutationAcknowledgementIsUncertain(t *testing.T) {
242	var calls atomic.Int32
243	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { calls.Add(1); _, _ = writer.Write([]byte(`{"ok":`)) }))
244	defer server.Close()
245	conn := NewConnection(ConnectionSnapshot{Port: server.Listener.Addr().(*net.TCPAddr).Port}, nil)
246	defer conn.HTTPClient().CloseIdleConnections()
247	err := executeMutation(context.Background(), conn, operation{Name: "interrupt session", Method: http.MethodPost, Path: "/sessions/s/interrupt", Body: struct{}{}, Policy: noRecovery}, []int{200}, func(body []byte, _ int) error { var result protocol.Interruption; return json.Unmarshal(body, &result) })
248	if _, ok := errors.AsType[*UncertainOutcomeError](err); !ok {
249		t.Fatalf("invalid acknowledgement returned %v", err)
250	}
251	if _, ok := errors.AsType[*json.SyntaxError](err); !ok {
252		t.Fatalf("decode cause lost: %v", err)
253	}
254	if calls.Load() != 1 {
255		t.Fatalf("invalid acknowledgement replayed %d times", calls.Load())
256	}
257}
258
259func TestRecordOpenDoesNotRecoverAuthentication(t *testing.T) {
260	var calls atomic.Int32
261	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
262		if request.URL.Path == "/server" {
263			_, _ = writer.Write([]byte(readyServer))
264			return
265		}
266		calls.Add(1)
267		writer.WriteHeader(http.StatusUnauthorized)
268		_, _ = writer.Write([]byte(`{"type":"about:blank","title":"Unauthorized","status":401,"code":"authentication_required","detail":"invalid bearer"}`))
269	}))
270	defer server.Close()
271	snapshot := ConnectionSnapshot{Port: server.Listener.Addr().(*net.TCPAddr).Port, Token: "new", Version: ProtocolVersion}
272	replacement := snapshot
273	snapshot.Token = "old"
274	conn := NewConnection(snapshot, func(context.Context) (ConnectionSnapshot, error) { return replacement, nil })
275	defer conn.HTTPClient().CloseIdleConnections()
276	_, err := RecordOpen(context.Background(), conn, "test")
277	if err == nil || calls.Load() != 1 || conn.Snapshot().Token != "old" {
278		t.Fatalf("open recovered authentication: %v, %d, %s", err, calls.Load(), conn.Snapshot().Token)
279	}
280}
281
282type trackedResponseBody struct {
283	io.Reader
284	closed bool
285}
286
287func (body *trackedResponseBody) Close() error { body.closed = true; return nil }
288
289type responseTransport struct{ response *http.Response }
290
291func (transport responseTransport) RoundTrip(_ *http.Request) (*http.Response, error) {
292	return transport.response, nil
293}
294
295func TestOperationClosesResponsesAndBoundsReads(t *testing.T) {
296	for _, test := range []struct {
297		name    string
298		content string
299		status  int
300		success bool
301	}{
302		{name: "success", status: 200, content: "ok", success: true},
303		{name: "oversized success", status: 200, content: "123456"},
304		{name: "rejection", status: 403, content: `{"detail":"no"}`},
305		{name: "server failure", status: 503, content: "bad"},
306	} {
307		t.Run(test.name, func(t *testing.T) {
308			body := &trackedResponseBody{Reader: strings.NewReader(test.content)}
309			conn := NewConnection(ConnectionSnapshot{Port: 12345}, nil)
310			conn.HTTPClient().Transport = responseTransport{response: &http.Response{StatusCode: test.status, Body: body, Header: make(http.Header)}}
311			_, err := requestBytes(context.Background(), conn, operation{Name: "mutate", Method: http.MethodPost, Path: "/sessions/s/inputs/i", Policy: authRecovery}, responseLimits{bodyBytes: 4, errorBytes: 4})
312			if !body.closed {
313				t.Fatal("operation leaked response body")
314			}
315			if (err == nil) != test.success {
316				t.Fatalf("unexpected result %v", err)
317			}
318			if test.name == "oversized success" {
319				if _, uncertain := errors.AsType[*UncertainOutcomeError](err); !uncertain {
320					t.Fatalf("lost acknowledgement not uncertain: %v", err)
321				}
322			}
323		})
324	}
325}
326
327func TestMutationRedirectIsNotFollowed(t *testing.T) {
328	var admissions atomic.Int32
329	target := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
330		admissions.Add(1)
331		_, _ = writer.Write([]byte(`{}`))
332	}))
333	defer target.Close()
334	source := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
335		http.Redirect(writer, request, target.URL, http.StatusTemporaryRedirect)
336	}))
337	defer source.Close()
338	conn := NewConnection(ConnectionSnapshot{Port: source.Listener.Addr().(*net.TCPAddr).Port}, nil)
339	defer conn.HTTPClient().CloseIdleConnections()
340	_, err := requestBytes(context.Background(), conn, operation{Name: "mutate", Method: http.MethodPost, Path: "/sessions/s/inputs/i", Body: map[string]string{"text": "hello"}, Policy: authRecovery}, responseLimits{bodyBytes: 50 * 1024 * 1024, errorBytes: 64 * 1024})
341	failure, ok := errors.AsType[*APIError](err)
342	if !ok || failure.StatusCode != 307 || admissions.Load() != 0 {
343		t.Fatalf("redirect policy changed: %v admissions=%d", err, admissions.Load())
344	}
345}