okami.mom/albedo

git clone https://git.nkp.pet/okami.mom/albedo

Aly Raffaufrefactor(tests): share repeated E2E fixtures5bdc0ee

main
34.1 KiB967 linesraw
1"""Shared fixtures for offline end-to-end tests, run through ``run.py``.
2
3A provider is a scripted model: its callback receives each decoded request and
4returns ``text(...)``, ``python(...)``, ``error(...)``, or a custom ``Reply``.
5Both OpenAI streaming protocols are served. ``Provider(catalog=...)`` also serves
6a local models catalog. ``Albedo`` gives each fixture a separate workspace and
7provider route on one shared HTTP server, on the daemon the runner gave its
8test. Tests share one daemon and keep to their own profile; an ``@exclusive``
9test, which may change global settings or restart the daemon, gets a fresh one
10that boots after its fixture's ``prepare(app)`` and is discarded afterwards, so
11it never restores what it changed. ``providers={}`` leaves it unconfigured.
12Setting ``app.daemon.open_files = (soft, hard)`` in ``prepare`` boots that
13daemon under an open-file limit.
14``write_extensions(settings)`` replaces extensions.json without letting the
15daemon reach the network, and ``store_secrets(section, value)`` writes a
16creds.json section such as the OAuth ``accounts`` or ``mcp`` server secrets.
17A response ``Reply(..., usage=None)`` omits provider usage.
18A ``python(...)`` cell that starts with ``LOOPBACK_REMOTE`` has ``rem`` bound to
19a remote connection whose ssh runs everything on this machine.
20
21Example::
22
23    from harness import Albedo, Provider, text
24
25    provider = Provider(lambda request: text("hello"))
26    try:
27        with Albedo(provider) as app:
28            session = app.session()
29            app.prompt(session, "say hello").close()
30            app.idle(session)
31            assert provider.requests
32    finally:
33        provider.close()
34"""
35
36from __future__ import annotations
37
38from collections.abc import Iterable
39from typing import Any
40
41import atexit
42import contextlib
43import errno
44from dataclasses import dataclass, field
45import http.client
46import http.server
47import io
48import itertools
49import json
50import os
51from pathlib import Path
52import shutil
53import secrets
54import signal
55import subprocess
56import sys
57import tempfile
58import threading
59import time
60import urllib.error
61import urllib.parse
62import uuid
63
64from provider_events import chat_events, responses_events
65
66ROOT = Path(__file__).resolve().parents[2]
67sys.path.insert(0, str(ROOT / "test"))
68import scratch  # noqa: E402
69
70scratch.claim("e2e")
71# A test daemon needs two schedulers, and busy-waiting ones make every boot
72# pin all cores of the machine.
73TEST_VM_FLAGS = "+S 2:2 +SDcpu 2:2 +sbwt none +sbwtdcpu none +sbwtdio none"
74# Credentials a provider reads from the environment. The daemon must not see
75# the developer's, or a catalog reload reaches the live provider.
76PROVIDER_ENVIRONMENT = ("ALIBABA_API_KEY", "DASHSCOPE_API_KEY", "ALIBABA_BASE_URL")
77# Extension settings that keep a daemon off the network: the models catalog
78# never refreshes by itself and the cache-TTL table fetches no remote copy.
79OFFLINE = {"models": {"refreshHours": 0}, "cacheTtl": {"url": None}}
80
81
82@dataclass
83class Reply:
84    """A scripted assistant response and optional transport behavior."""
85
86    kind: str
87    value: str = ""
88    status: int = 200
89    reasoning: str | None = None
90    usage: dict | None = field(
91        default_factory=lambda: {"input_tokens": 10, "output_tokens": 20}
92    )
93    delay: float = 0
94    hang: bool = False
95    tool_name: str = "python"
96    tool_arguments: dict | None = None
97    events: Iterable[dict] | None = None
98
99
100def text(
101    value: str,
102    *,
103    reasoning: str | None = None,
104    usage: dict | None = None,
105    delay: float = 0,
106    hang: bool = False,
107) -> Reply:
108    return Reply(
109        "text",
110        value,
111        reasoning=reasoning,
112        usage=usage or {"input_tokens": 10, "output_tokens": 20},
113        delay=delay,
114        hang=hang,
115    )
116
117
118def python(
119    code: str, *, reasoning: str | None = None, tool_name: str = "python"
120) -> Reply:
121    return Reply("python", code, reasoning=reasoning, tool_name=tool_name)
122
123
124def error(status: int, body: str = "fixture error") -> Reply:
125    return Reply("error", body, status=status)
126
127
128# Guards the provider registry and the shared provider server.
129_config_lock = threading.RLock()
130
131
132def operation_id(timestamp=None):
133    """Generate one UUIDv7 action identity, optionally at a fixture time."""
134    milliseconds = int(time.time() * 1000) if timestamp is None else timestamp
135    bits = (milliseconds << 80) | (7 << 76) | secrets.randbits(76)
136    bits = (bits & ~(3 << 62)) | (2 << 62)
137    return str(uuid.UUID(int=bits))
138
139
140def exclusive(target):
141    """Run the test, or every test of the class, on a daemon of its own that
142    boots after the fixture's prepare step and is discarded afterwards: it may
143    change global settings or restart the daemon."""
144    target._e2e_exclusive = True
145    return target
146
147
148_providers = {}
149_provider_ids = itertools.count(1)
150_server = None
151
152
153def _provider_server():
154    global _server
155    if _server is None:
156
157        class Handler(http.server.BaseHTTPRequestHandler):
158            def log_message(self, format, *args):
159                pass
160
161            def do_GET(self):
162                self.provider()._get(self)
163
164            def do_POST(self):
165                self.provider()._post(self)
166
167            def provider(self):
168                return _providers[self.path.split("/", 3)[2]]
169
170        _server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
171        _server.daemon_threads = True
172        threading.Thread(target=_server.serve_forever, daemon=True).start()
173    return _server
174
175
176class Provider:
177    """Threaded fake provider that records requests and emits protocol SSE."""
178
179    def __init__(self, script, *, chunk_size: int = 20, catalog=None):
180        self.catalog = catalog
181        self.script = script
182        self.chunk_size = chunk_size
183        self.requests: list[dict] = []
184        self._lock = threading.Lock()
185        self.route = str(next(_provider_ids))
186        with _config_lock:
187            _providers[self.route] = self
188            self.server = _provider_server()
189        self.url = f"http://127.0.0.1:{self.server.server_port}/t/{self.route}"
190
191    def _get(owner, self):
192        payload = json.dumps(owner.catalog).encode()
193        self.send_response(200)
194        self.send_header("Content-Type", "application/json")
195        self.end_headers()
196        self.wfile.write(payload)
197
198    def _post(owner, self):
199        request = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
200        record = {
201            "path": self.path,
202            "authorization": self.headers.get("Authorization"),
203            "model": request.get("model"),
204            "request": request,
205        }
206        with owner._lock:
207            owner.requests.append(record)
208            index = len(owner.requests) - 1
209        reply = owner.script(request)
210        if isinstance(reply, (list, tuple)):
211            reply = reply[min(index, len(reply) - 1)]
212        if reply.kind == "error":
213            payload = json.dumps({"error": {"message": reply.value}}).encode()
214            self.send_response(reply.status)
215            self.send_header("Content-Type", "application/json")
216            self.end_headers()
217            self.wfile.write(payload)
218            return
219        self.send_response(200)
220        self.send_header("Content-Type", "text/event-stream")
221        self.end_headers()
222
223        def emit(value):
224            if reply.delay:
225                time.sleep(reply.delay)
226            self.wfile.write(("data: " + json.dumps(value) + "\n\n").encode())
227            self.wfile.flush()
228
229        try:
230            if reply.hang:
231                time.sleep(60)
232                return
233            if reply.events is not None:
234                for event in reply.events:
235                    emit(event)
236                self.wfile.write(b"data: [DONE]\n\n")
237                self.wfile.flush()
238                return
239            call_id = f"fixture-call-{index + 1}"
240            arguments = json.dumps(
241                reply.tool_arguments
242                if reply.tool_arguments is not None
243                else {"pattern": reply.value}
244                if reply.tool_name == "lcm_grep"
245                else {"code": reply.value, "timeout_ms": 60000}
246            )
247            events = (
248                chat_events(reply, arguments, call_id, owner.chunk_size)
249                if self.path.endswith("chat/completions")
250                else responses_events(reply, arguments, call_id, owner.chunk_size)
251            )
252            for event in events:
253                emit(event)
254            self.wfile.write(b"data: [DONE]\n\n")
255            self.wfile.flush()
256        except (BrokenPipeError, ConnectionResetError):
257            pass
258
259    def close(self):
260        # Sessions can finish background turns after their test returns. Keep
261        # their route alive until the shared daemon has stopped.
262        pass
263
264
265_daemons: list[Daemon] = []
266# The launcher of this run's daemon build; see snapshot_daemon().
267_executable = None
268_daemons_lock = threading.RLock()
269_discarding: list[threading.Thread] = []
270_current = threading.local()
271daemon_boots = 0
272restart_seconds = 0.0
273# mist closes a keep-alive connection after 10 idle seconds; one idle for less
274# than this is reused, so a request never races the daemon closing it.
275KEEP_ALIVE_SECONDS = 5
276
277
278class _Connection(http.client.HTTPConnection):
279    """A daemon connection that notes when it can carry the next request."""
280
281    def __init__(self, port, token):
282        super().__init__("127.0.0.1", port, timeout=20)
283        self.token = token
284        self.response_class = _Response
285        self.used_at = time.monotonic()
286        self.reusable = True
287        self.response = None
288
289    def idle(self):
290        """Whether the next request may go out on this connection: its last
291        response is done with and the daemon has not dropped it yet."""
292        return (
293            self.sock is not None
294            and self.reusable
295            and (self.response is None or self.response.isclosed())
296            and time.monotonic() - self.used_at < KEEP_ALIVE_SECONDS
297        )
298
299
300class _Response(http.client.HTTPResponse):
301    """Closing drains a short unread body so the connection can be reused;
302    an event stream or a long body gives the connection up instead."""
303
304    connection = None
305
306    def close(self):
307        if self.headers is None:
308            # begin() died before any status line: there is no body to drain,
309            # and reading headers here would mask the error that stopped it.
310            super().close()
311            return
312        connection = self.connection
313        streaming = self.getheader("content-type", "").startswith("text/event-stream")
314        if connection is not None and streaming:
315            connection.reusable = False
316        if connection is not None and not self.isclosed():
317            if streaming or self.length is None or self.length > 1 << 20:
318                connection.reusable = False
319            else:
320                try:
321                    self.read()
322                except (OSError, http.client.HTTPException):
323                    connection.reusable = False
324        super().close()
325        if connection is not None:
326            connection.used_at = time.monotonic()
327            if self.will_close or not connection.reusable:
328                connection.close()
329
330
331class Daemon:
332    """One daemon and its hermetic home. A concurrent daemon is shared by
333    tests that each keep to their own provider profile; any other serves one
334    test, which may change its global state or restart it."""
335
336    def __init__(self, name, *, concurrent=False):
337        if not hasattr(sys.modules["__main__"], "__file__"):
338            # A daemon lives until the process that booted it exits, which a
339            # long-lived interpreter or agent kernel never does.
340            raise RuntimeError(
341                "run E2E tests through test/e2e/run.py, not an interactive interpreter"
342            )
343        self.concurrent = concurrent
344        self.lock = threading.RLock()
345        # Resolved, so a path the daemon reports through a symlink (macOS /tmp) matches.
346        self.root = Path(tempfile.mkdtemp(prefix=f"{name}-")).resolve()
347        self.home = self.root / "home"
348        self.home.mkdir()
349        (self.root / "user-home").mkdir()
350        # Nothing of the developer's albedo setup reaches the daemon.
351        self.env = dict(
352            {
353                key: value
354                for key, value in os.environ.items()
355                if key not in PROVIDER_ENVIRONMENT
356                and key not in ("HOME", "ERL_FLAGS")
357                and not key.startswith("ALBEDO_")
358            },
359            HOME=str(self.root / "user-home"),
360            ALBEDO_HOME=str(self.home),
361            ALBEDO_ROOT=str(ROOT),
362            ALBEDO_NO_BROWSER="1",
363            ALBEDO_PARENT_PID=str(os.getpid()),
364            ERL_FLAGS=TEST_VM_FLAGS,
365            ALBEDO_IDLE_SECONDS="10",
366            # Kernels outlive a stopped daemon only this long; one whose home
367            # is removed leaves at once.
368            ALBEDO_KERNEL_GRACE_SECONDS="30",
369            ALBEDO_MCP_SECRET="configured-secret",
370            ALBEDO_MCP_CLOSED=str(self.root / "closed"),
371            ALBEDO_MCP_AMBIENT="must-not-reach-the-server",
372        )
373        if _executable:
374            self.env["ALBEDO_DAEMON"] = _executable
375        if "ALBEDO_TEST_CLI" in os.environ:
376            self.env["ALBEDO_TEST_CLI"] = os.environ["ALBEDO_TEST_CLI"]
377        # A (soft, hard) open-file limit the CLI, and so the daemon it boots,
378        # starts under; None passes on the test process's own.
379        self.open_files = None
380        self.connection = None
381        self.base = None
382        self._pid = None
383        self._local = threading.local()
384        self.boot_action = None
385        self._launchers = []
386        self._commands = []
387        with _daemons_lock:
388            _daemons.append(self)
389
390    @property
391    def booted(self):
392        return self._pid is not None
393
394    def boot(self):
395        """Start the daemon; the first fixture boots it after its prepare."""
396        extensions = self.home / "extensions.json"
397        if not extensions.exists():
398            extensions.write_text(json.dumps(OFFLINE))
399        if self.boot_action is not None:
400            self.boot_action(self)
401        self.cli("sessions")
402        self._refresh()
403
404    def start_launcher(self):
405        """Run the real launcher directly, retaining its log and process owner."""
406        with (self.home / "daemon.log").open("ab") as log:
407            process = subprocess.Popen(
408                [self.env["ALBEDO_DAEMON"]],
409                cwd=ROOT,
410                env=self.env,
411                stdout=log,
412                stderr=log,
413            )
414        self._launchers.append(process)
415        # Direct launches are our children. Reap them as soon as they exit so
416        # another CLI can observe that the old daemon is gone during upgrade.
417        threading.Thread(target=process.wait, daemon=True).start()
418        return process
419
420    def await_ready(self, timeout=20):
421        """Adopt the published endpoint once its authenticated API answers."""
422        deadline = time.monotonic() + timeout
423        while time.monotonic() < deadline:
424            try:
425                self._refresh()
426                with self.api("/server") as response:
427                    if json.load(response)["state"] == "ready":
428                        return
429            except (OSError, ValueError, http.client.HTTPException):
430                pass
431            time.sleep(0.02)
432        self._fail("launcher did not publish an authenticated endpoint")
433
434    def start_cli(self, *args, **streams):
435        """Start a CLI command whose input or cancellation the fixture controls."""
436        process = subprocess.Popen(
437            [self.env.get("ALBEDO_TEST_CLI", str(ROOT / "cli/bin/albedo")), *args],
438            cwd=ROOT,
439            env=self.env,
440            **streams,
441        )
442        self._commands.append(process)
443        return process
444
445    def _refresh(self):
446        global daemon_boots
447        self.connection = json.loads((self.home / "daemon.json").read_text())
448        self.base = f"http://127.0.0.1:{self.connection['port']}"
449        if self.connection["pid"] != self._pid:
450            # A new PID is valid only after the old daemon has exited.
451            if self._pid is not None and alive(self._pid):
452                raise AssertionError(f"a second daemon started in {self.home}")
453            self._pid = self.connection["pid"]
454            with _daemons_lock:
455                daemon_boots += 1
456
457    def restart(self, *, crash=False, prepare=None):
458        """Replace the daemon under the same home, preparing storage between."""
459        global restart_seconds
460        started = time.monotonic()
461        if crash:
462            assert self._pid is not None
463            os.kill(self._pid, signal.SIGKILL)
464            # SIGKILL is asynchronous: a CLI started before the process is gone
465            # reads the dead daemon's record and finds nothing listening.
466            if not self._gone(timeout=15):
467                self._fail("daemon did not die")
468        elif not self._stop(timeout=15):
469            self._fail("daemon did not stop")
470        if prepare is not None:
471            prepare()
472        self.cli("sessions")
473        self._refresh()
474        with _daemons_lock:
475            restart_seconds += time.monotonic() - started
476
477    def _stop(self, timeout):
478        try:
479            with self.api("/server") as response:
480                instance_id = json.load(response)["instance_id"]
481            self.api(
482                "/server/shutdown",
483                {
484                    "instance_id": instance_id,
485                    "timeout_ms": min(timeout * 1000, 30000),
486                },
487            ).close()
488        except (OSError, http.client.HTTPException):
489            pass
490        return self._gone(timeout)
491
492    def _gone(self, timeout):
493        deadline = time.monotonic() + timeout
494        while alive(self._pid):
495            if time.monotonic() > deadline:
496                return False
497            time.sleep(0.05)
498        return True
499
500    def shutdown(self):
501        """Stop the daemon, killing it if it will not stop, and drop the home."""
502        with self.lock:
503            # A failed CLI boot may already have launched this private home's daemon.
504            path = self.home / "daemon.json"
505            if path.exists():
506                self.connection = json.loads(path.read_text())
507                self.base = f"http://127.0.0.1:{self.connection['port']}"
508                self._pid = self.connection["pid"]
509            if self._pid is not None:
510                if alive(self._pid) and not self._stop(timeout=10):
511                    try:
512                        os.kill(self._pid, signal.SIGKILL)
513                    except ProcessLookupError:
514                        pass
515                self._pid = None
516            for launcher in self._launchers:
517                if launcher.poll() is None:
518                    launcher.kill()
519                launcher.wait()
520            for command in self._commands:
521                if command.poll() is None:
522                    command.kill()
523                command.wait()
524            shutil.rmtree(self.root, ignore_errors=True)
525        with _daemons_lock:
526            if self in _daemons:
527                _daemons.remove(self)
528
529    def kill(self):
530        """SIGKILL the daemon without waiting for its lock, even mid-stop."""
531        pid = self._pid
532        if pid is not None:
533            try:
534                os.kill(pid, signal.SIGKILL)
535            except ProcessLookupError:
536                pass
537
538    def discard(self):
539        """Shut the daemon down in the background; the test that used it is
540        done, and nothing waits on its teardown until the run ends."""
541        thread = threading.Thread(target=self.shutdown)
542        thread.start()
543        with _daemons_lock:
544            _discarding.append(thread)
545
546    def _fail(self, message):
547        log = self.home / "daemon.log"
548        raise AssertionError(message + ("\n" + log.read_text() if log.exists() else ""))
549
550    def _connection(self):
551        connection = getattr(self._local, "connection", None)
552        assert self.connection is not None
553        port, token = self.connection["port"], self.connection["token"]
554        # A restarted daemon has a new token and may be back on the same port.
555        if connection is None or connection.token != token or not connection.idle():
556            if connection is not None:
557                # A response still being read, such as an event stream, keeps
558                # its socket; closing it gives the connection up.
559                connection.reusable = False
560                if connection.response is None or connection.response.isclosed():
561                    connection.close()
562            connection = self._local.connection = _Connection(port, token)
563        return connection
564
565    def api(self, path, body=None, *, method=None, headers=None):
566        """One daemon request over this thread's kept-alive connection. A
567        socket per request, polled at test speed, leaves thousands of ports in
568        TIME_WAIT; the host runs out and unrelated connections fail."""
569        connection = self._connection()
570        assert self.connection is not None
571        connection.used_at = time.monotonic()
572        payload = None if body is None else json.dumps(body).encode()
573        try:
574            connection.request(
575                method or ("GET" if payload is None else "POST"),
576                path,
577                payload,
578                {
579                    "Authorization": "Bearer " + self.connection["token"],
580                    "Content-Type": "application/merge-patch+json"
581                    if method == "PATCH"
582                    else "application/json",
583                    **(headers or {}),
584                },
585            )
586            response = connection.getresponse()
587        except BaseException:
588            connection.close()
589            raise
590        response.connection, connection.response = connection, response
591        if response.status < 400:
592            return response
593        payload = response.read()
594        response.close()
595        raise urllib.error.HTTPError(
596            self.base + path,
597            response.status,
598            f"{response.reason} ({method} {path}): " + payload.decode(errors="replace"),
599            response.headers,
600            io.BytesIO(payload),
601        )
602
603    def cli(self, *args):
604        command = [self.env.get("ALBEDO_TEST_CLI", str(ROOT / "cli/bin/albedo")), *args]
605        if self.open_files:
606            command = [
607                sys.executable,
608                "-c",
609                _WITH_OPEN_FILES,
610                *map(str, self.open_files),
611                *command,
612            ]
613        result = subprocess.run(
614            command,
615            cwd=ROOT,
616            env=self.env,
617            text=True,
618            capture_output=True,
619            timeout=45,
620        )
621        if result.returncode:
622            self._fail(result.stdout + result.stderr)
623        return result.stdout
624
625    def install_shared_features(self):
626        """Enable the additive global gateways concurrent tests rely on, once,
627        before they start: none of them may change global state itself."""
628        settings = json.loads((self.home / "extensions.json").read_text())
629        settings.setdefault("enabled", {})["proxy"] = True
630        (self.home / "extensions.json").write_text(json.dumps(settings))
631        with self.api("/settings?group=extensions") as response:
632            validator = response.getheader("ETag")
633            response.read()
634        self.api(
635            "/settings?group=extensions",
636            {"defaults": {"webhooks": True}},
637            method="PATCH",
638            headers={"If-Match": validator},
639        ).close()
640
641
642# Runs a command under a (soft, hard) open-file limit, as a shell started with
643# that `ulimit -n` would.
644_WITH_OPEN_FILES = """import os, resource, sys
645resource.setrlimit(resource.RLIMIT_NOFILE, (int(sys.argv[1]), int(sys.argv[2])))
646os.execv(sys.argv[3], sys.argv[3:])"""
647
648
649def latest_user(request):
650    """Return the last user's raw content from either provider protocol."""
651    messages = request.get("messages", request.get("input", []))
652    return next(
653        (
654            item.get("content", "")
655            for item in reversed(messages)
656            if item.get("role") == "user"
657        ),
658        "",
659    )
660
661
662def wait_until(condition, seconds, message):
663    deadline = time.monotonic() + seconds
664    while time.monotonic() < deadline:
665        if condition():
666            return
667        time.sleep(0.1)
668    raise AssertionError(message)
669
670
671def release_fifo(path):
672    """Let a job blocked reading one byte from the fifo at `path` go on. True
673    once a reader had it open and got its byte; False while none has it open
674    yet, so callers retry: `wait_until(lambda: release_fifo(gate), ...)`."""
675    try:
676        descriptor = os.open(path, os.O_WRONLY | os.O_NONBLOCK)
677    except OSError as error:
678        if error.errno == errno.ENXIO:
679            return False
680        raise
681    try:
682        os.write(descriptor, b"x")
683    finally:
684        os.close(descriptor)
685    return True
686
687
688# An ssh that runs its command right here, under a login shell that does the
689# same, so a cell's remote.connect("loopback") boots, stages and talks to a
690# real remote kernel.
691_LOOPBACK_SSH = """#!/bin/sh
692while [ $# -gt 0 ]; do
693  case "$1" in
694    -o) shift 2 ;;
695    -*) shift ;;
696    *) break ;;
697  esac
698done
699shift
700exec sh -c "$*"
701"""
702_LOOPBACK_LOGIN = '#!/bin/sh\n[ "$1" = -l ] && shift\nexec /bin/sh "$@"\n'
703# The start of a cell that binds `rem` to a loopback remote connection.
704LOOPBACK_REMOTE = f"""import os
705os.makedirs("bin", exist_ok=True)
706for name, script in (("ssh", {_LOOPBACK_SSH!r}), ("loginsh", {_LOOPBACK_LOGIN!r})):
707    with open(f"bin/{{name}}", "w") as file:
708        file.write(script)
709    os.chmod(f"bin/{{name}}", 0o755)
710os.environ["SHELL"] = os.path.abspath("bin/loginsh")
711os.environ["PATH"] = os.path.abspath("bin") + os.pathsep + os.environ["PATH"]
712rem = await remote.connect("loopback")
713"""
714
715
716def alive(pid):
717    try:
718        os.kill(pid, 0)
719    except ProcessLookupError:
720        return False
721    return True
722
723
724def snapshot_daemon():
725    """Compile the daemon once for the run and boot every test daemon from a
726    copy, not through `gleam run` (see test/snapshot-daemon.sh). test.sh
727    snapshots once for every suite and passes it as ALBEDO_TEST_DAEMON."""
728    global _executable
729    _executable = os.environ.get("ALBEDO_TEST_DAEMON")
730    if _executable:
731        return
732    result = subprocess.run(
733        [str(ROOT / "test/snapshot-daemon.sh"), tempfile.mkdtemp(prefix="daemon-")],
734        text=True,
735        capture_output=True,
736    )
737    if result.returncode:
738        raise RuntimeError("building the daemon failed:\n" + result.stderr)
739    _executable = result.stdout.strip()
740
741
742def current_daemon():
743    """The daemon the runner gave this thread."""
744    daemon = getattr(_current, "daemon", None)
745    if daemon is None:
746        raise RuntimeError("run E2E tests through test/e2e/run.py")
747    return daemon
748
749
750@contextlib.contextmanager
751def on_daemon(daemon):
752    """Run fixtures created on this thread against daemon."""
753    previous = getattr(_current, "daemon", None)
754    _current.daemon = daemon
755    try:
756        yield daemon
757    finally:
758        _current.daemon = previous
759
760
761def shutdown():
762    """Stop every daemon, then the provider server."""
763    global _server
764    # The run is over and no test watches these daemons stop, so none is
765    # drained: a graceful stop sits out OTP's one-second pause for buffered
766    # output (user_sup:terminate/2), and the shared daemon's drain of every
767    # session takes longer still. Their kernels exit with them.
768    for daemon in list(_daemons):
769        daemon.kill()
770    for thread in list(_discarding):
771        thread.join()
772    _discarding.clear()
773    for daemon in list(_daemons):
774        daemon.shutdown()
775    if _server is not None:
776        _server.shutdown()
777        _server.server_close()
778        _server = None
779    with _config_lock:
780        _providers.clear()
781
782
783atexit.register(shutdown)
784
785
786def _write_config(path, value):
787    with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as output:
788        json.dump(value, output)
789        temporary = output.name
790    os.replace(temporary, path)
791
792
793class Albedo:
794    """A test's workspace and provider profile on the daemon the runner gave it."""
795
796    def __init__(
797        self,
798        provider: Provider | None = None,
799        *,
800        protocol="chat_completions",
801        providers=None,
802        prepare=None,
803    ):
804        self.provider = provider or Provider(lambda _request: text("ok"))
805        self._owns_provider = provider is None
806        self.protocol = protocol
807        self.providers = providers
808        self.prepare = prepare
809
810    def __enter__(self):
811        self.daemon = daemon = current_daemon()
812        with daemon.lock:
813            self.root, self.home, self.env = daemon.root, daemon.home, daemon.env
814            self.workspace = Path(tempfile.mkdtemp(prefix="workspace-", dir=self.root))
815            self._concurrent = daemon.concurrent
816            if self.prepare:
817                self.prepare(self)
818            if not (self.home / "extensions.json").exists():
819                self.write_extensions({})
820            default_name = (
821                f"fixture-{self.provider.route}" if self._concurrent else "fixture"
822            )
823            configured: dict[str, Any] = (
824                self.providers
825                if self.providers is not None
826                else {
827                    default_name: {
828                        "extension": "openai",
829                        "baseUrl": self.provider.url,
830                        "apiKey": "fixture-key",
831                        "model": "fixture-model",
832                        "protocol": self.protocol,
833                    }
834                }
835            )
836            self.profile = next(iter(configured), None)
837            if configured:
838                current: dict[str, Any] = (
839                    json.loads((self.home / "config.json").read_text())
840                    if self._concurrent and (self.home / "config.json").exists()
841                    else {"providers": {}}
842                )
843                current["providers"].update(configured)
844                current["active"] = (
845                    current.get("active", self.profile)
846                    if self._concurrent
847                    else self.profile
848                )
849                _write_config(self.home / "config.json", current)
850            elif not self._concurrent and (self.home / "config.json").exists():
851                (self.home / "config.json").unlink()
852            if not daemon.booted:
853                daemon.boot()
854            return self
855
856    @property
857    def connection(self):
858        return self.daemon.connection
859
860    @property
861    def base(self):
862        return self.daemon.base
863
864    def restart(self, *, crash=False, prepare=None):
865        """Restart the daemon; optionally prepare offline storage fixtures."""
866        self.daemon.restart(
867            crash=crash, prepare=None if prepare is None else lambda: prepare(self)
868        )
869
870    def _fail(self, message):
871        self.daemon._fail(message)
872
873    def api(self, path, body=None, *, method=None, headers=None):
874        return self.daemon.api(path, body, method=method, headers=headers)
875
876    def cli(self, *args):
877        return self.daemon.cli(*args)
878
879    def write_extensions(self, settings):
880        """Replace extensions.json, keeping the daemon offline in whatever
881        catalog settings leave out."""
882        merged = {
883            name: {**OFFLINE.get(name, {}), **value}
884            if name in OFFLINE and isinstance(value, dict)
885            else value
886            for name, value in {**OFFLINE, **settings}.items()
887        }
888        (self.home / "extensions.json").write_text(json.dumps(merged))
889
890    def store_secrets(self, section, value):
891        """Replaces one creds.json section, where the daemon keeps every secret."""
892        path = self.home / "creds.json"
893        creds = json.loads(path.read_text()) if path.exists() else {}
894        creds[section] = value
895        path.write_text(json.dumps(creds))
896        path.chmod(0o600)
897
898    def session(self, workspace=None):
899        session_id = operation_id()
900        with self.api(
901            f"/sessions/{session_id}",
902            {
903                "kind": "new",
904                "workspace": str(workspace or self.workspace),
905                "provider_profile": self.profile,
906            },
907            method="PUT",
908            headers={"If-None-Match": "*"},
909        ) as response:
910            created = json.load(response)
911        assert created["id"] == session_id, created
912        assert created["provider_profile"] == self.profile, created
913        return session_id
914
915    def prompt(self, session_id, content):
916        return self.api(
917            f"/sessions/{session_id}/inputs/{operation_id()}",
918            {"kind": "message", "text": content},
919            method="PUT",
920        )
921
922    def idle(self, session_id, timeout=30):
923        deadline = time.monotonic() + timeout
924        snapshot = None
925        while time.monotonic() < deadline:
926            try:
927                with self.api(f"/sessions/{session_id}?tail=0") as response:
928                    snapshot = json.load(response)
929            except urllib.error.HTTPError as error:
930                if error.code != 503:
931                    raise
932                snapshot = json.load(error)
933            else:
934                if (
935                    snapshot["status"]["phase"] == "idle"
936                    and not snapshot["pending_inputs"]
937                ):
938                    return snapshot
939            time.sleep(0.05)
940        self._fail(f"session did not settle: {snapshot}")
941
942    def stream_page(self, session_id, after=None, *, tail=None):
943        query = {}
944        if after is not None:
945            query = {
946                "after_generation": after["generation"],
947                "after_seq": after["cursor"],
948            }
949        if tail is not None:
950            query["tail"] = tail
951        suffix = "?" + urllib.parse.urlencode(query) if query else ""
952        with self.api(
953            f"/sessions/{session_id}{suffix}",
954            headers={"Accept": "text/event-stream"},
955        ) as response:
956            for line in response:
957                if line.startswith(b"data: "):
958                    return json.loads(line[6:])
959        self._fail("stream closed before its first batch")
960
961    def history(self, session_id):
962        with self.api(f"/sessions/{session_id}/history") as response:
963            return json.load(response)
964
965    def __exit__(self, *_exc):
966        if self._owns_provider:
967            self.provider.close()