okami.mom/albedo

git clone https://git.nkp.pet/okami.mom/albedo

Aly Raffaufrefactor(tests): share repeated E2E fixtures5bdc0ee2026-10-12

"""A pending cell-completion wake must survive the idle swap of a stale kernel.

The incident behind this suite: a session ran a long background cell on a
kernel left stale by a daemon restart. At the turn's end a park nobody kept
asked for a kernel, so the runtime started the swap; the arriving replacement
was dropped with no notice, the old kernel (still holding the cell's pending
completion wake) was stopped, and the model was never told anything. The wake
must instead be admitted before the swap, ride it, and carry the upgrade
notice; a swap that does end a running cell must journal it interrupted and
say so.
"""

import json
import os
from pathlib import Path
import time
import unittest

import harness
from harness import Provider, exclusive, latest_user, python, text
from kernel_support import MARKER, ModelCells, editable_daemon


# exclusive: boots an editable python bundle and restarts the daemon
@exclusive
class SwapWakeTests(ModelCells, unittest.TestCase):
    bundle: Path

    def setUp(self):
        self.cells = []

        def script(request):
            user = latest_user(request)
            if "background python cells finished" in user:
                return text("wake seen")
            if user.endswith("hold"):
                return text("holding", delay=4.0)
            if request["input"][-1].get("role") == "user" and self.cells:
                return python(self.cells.pop(0))
            return text("ok")

        def prepare(app):
            launcher, self.bundle = editable_daemon(app.root)
            app.env["ALBEDO_DAEMON"] = str(launcher)
            app.env["ALBEDO_CELL_BACKGROUND_SECONDS"] = "0.5"

        self.provider = Provider(script)
        self.addCleanup(self.provider.close)
        self.app = harness.Albedo(self.provider, protocol="responses", prepare=prepare)
        self.app.__enter__()
        self.addCleanup(self.app.__exit__, None, None, None)
        self.session = self.app.session()

    def kernel(self):
        with self.app.api(f"/sessions/{self.session}?tail=0") as response:
            return json.load(response)["kernel"]

    def gate(self):
        gate = self.app.workspace / "cell-gate"
        os.mkfifo(gate)
        self.addCleanup(harness.release_fifo, gate)
        return gate

    def held_cell(self, *, protocol_skew=False):
        """Cell code whose backgrounded job runs until the gate is released:
        the cell finishes only when the test decides, like the incident's
        build, and the kernel's loop stays free to answer the daemon."""
        gate = self.gate()
        program = f"open({str(gate)!r}, 'rb').read(1)"
        prefix = (
            "import albedo_link, os, sys\nalbedo_link.PROTOCOL = 2\n"
            if protocol_skew
            else "import os, sys\n"
        )
        code = (
            f"{prefix}survivor = 41\n"
            f"j = run(sys.executable, '-c', {program!r})\n"
            "await j\n('gate done',)"
        )
        return gate, code

    def wait_for_wake(self):
        deadline = time.monotonic() + 40
        while time.monotonic() < deadline:
            for record in reversed(self.provider.requests):
                if "background python cells finished" in latest_user(record["request"]):
                    return latest_user(record["request"])
            time.sleep(0.1)
        self.fail("the cell completion wake never reached the model")

    def test_a_pending_cell_wake_survives_the_idle_kernel_swap(self):
        gate, code = self.held_cell()
        first, _ = self.cell(code)
        self.assertEqual(first["status"], "backgrounded", first)

        with (self.bundle / "albedo_bundle.py").open("a") as source:
            source.write(MARKER)
        self.app.restart()

        # One slow turn on the still-stale kernel (its live job holds the
        # swap). The gate is released mid-turn, so the background cell
        # finishes and its wake retries while the session is busy, exactly as
        # in the incident; the spurious park used to fire when this turn
        # ended, starting a swap that killed the wake with the old kernel.
        self.app.prompt(self.session, "hold").close()
        time.sleep(0.8)
        harness.wait_until(
            lambda: harness.release_fifo(gate), 10, "the held job never opened its gate"
        )
        self.app.idle(self.session)

        wake = self.wait_for_wake()
        self.app.idle(self.session)
        # The wake was admitted before the swap, so it rides the swapped
        # kernel's turn together with the upgrade notice.
        self.assertIn("The python kernel was upgraded to the new python bundle", wake)
        self.assertRegex(wake, r"Restored: [^.]*\bsurvivor\b")

        swapped, _ = self.cell(
            "import albedo_bundle, os\n"
            "(survivor, getattr(albedo_bundle, 'MARKER', None), os.getpid())"
        )
        self.assertEqual(swapped.get("status"), "ok", swapped)
        survivor, marker, _pid = swapped["value"].strip("()").split(", ")
        self.assertEqual((survivor, marker), ("41", "'new bundle'"), swapped)
        self.assertFalse(self.kernel()["stale"])

    def test_a_swap_past_a_running_cell_ends_it_as_interrupted(self):
        # A kernel on another protocol is swapped regardless of live work, so
        # the swap ends the running cell: it must be journaled interrupted and
        # named, not left `started` with its effects unknown. (The explicit
        # upgrade refuses while a cell runs: "a cell is still running".)
        gate, code = self.held_cell(protocol_skew=True)
        first, _ = self.cell(code)
        self.assertEqual(first["status"], "backgrounded", first)
        cell_id = first["cell_id"]

        self.app.restart()
        ended, prompt = self.cell(f"print(await cells.info({cell_id!r}))")
        self.assertIn("'status': 'interrupted'", ended["output"], ended)
        self.assertIn(
            "The python kernel was upgraded to the current kernel protocol", prompt
        )
        self.assertIn(f"Cells ended by the swap: {cell_id}", prompt)


if __name__ == "__main__":
    unittest.main()