okami.mom/albedo

git clone https://git.nkp.pet/okami.mom/albedo

Aly Raffaufrefactor(config): share model and effort changesf0fd5bd

main
branchesmainall refs →
8.9 KiB234 linesraw
1"""Command mutations share conditional edits' checks, persistence, and context reset."""
2
3import json
4import os
5from pathlib import Path
6import threading
7import unittest
8
9from harness import Albedo, Provider, ROOT, exclusive, release_fifo, text, wait_until
10from inspect_support import Inspection
11from kernel_support import editable_daemon
12
13
14@exclusive
15class SessionConfigureTest(unittest.TestCase):
16    """Isolated global defaults, catalog, inspection, and copied kernel startup."""
17
18    marker: Path
19
20    def setUp(self):
21        self.provider = Provider(lambda _request: text("done"))
22        self.addCleanup(self.provider.close)
23        self.gate = None
24
25        def prepare(app):
26            app.daemon.env["ALBEDO_INSPECT"] = "1"
27            catalog = {
28                "fixture-cloud": {
29                    "api": self.provider.url,
30                    "models": {
31                        model: {
32                            "id": model,
33                            "reasoning_options": [
34                                {"type": "effort", "values": efforts}
35                            ],
36                        }
37                        for model, efforts in {
38                            "first": ["low", "high"],
39                            "second": ["medium", "high"],
40                            "third": ["low", "medium"],
41                        }.items()
42                    },
43                }
44            }
45            (app.home / "models.json").write_text(json.dumps(catalog))
46            app.write_extensions({"models": {"refreshHours": 0}})
47            if (
48                self._testMethodName
49                == "test_commands_reject_changes_during_kernel_boot"
50            ):
51                launcher, bundle = editable_daemon(app.root)
52                app.env["ALBEDO_DAEMON"] = str(launcher)
53                self.gate = app.workspace / "kernel-gate"
54                self.marker = app.workspace / "kernel-started"
55                os.mkfifo(self.gate)
56                gate = self.gate
57                self.addCleanup(lambda: gate.exists() and release_fifo(gate))
58                (bundle / "albedo_kernel.py").write_text(
59                    "from pathlib import Path\n"
60                    f"Path({str(self.marker)!r}).touch()\n"
61                    f"open({str(self.gate)!r}, 'rb').read(1) if Path({str(self.gate)!r}).exists() else None\n"
62                    "import runpy\n"
63                    "runpy.run_module('albedo_cells', run_name='__main__', alter_sys=True)\n"
64                )
65
66        self.app = Albedo(
67            self.provider,
68            prepare=prepare,
69            providers={
70                "fixture": {
71                    "extension": "openai",
72                    "baseUrl": self.provider.url,
73                    "apiKey": "fixture-key",
74                    "model": "first",
75                    "effort": "high",
76                    "protocol": "responses",
77                }
78            },
79        ).__enter__()
80        self.addCleanup(self.app.__exit__, None, None, None)
81        self.session = self.app.session()
82        self.inspection = Inspection(
83            self.app, ROOT / "test/e2e/albedo_session_configure_probe.erl"
84        )
85
86    def read(self, path):
87        with self.app.api(path) as response:
88            return json.load(response)
89
90    def configuration(self):
91        return self.read(f"/sessions/{self.session}?view=configuration")
92
93    def defaults(self):
94        return self.read("/settings?group=providers")
95
96    def invoke(self, method, *values):
97        arguments = [self.session, *values]
98        erl = ", ".join(
99            "null" if value is None else f"<<{json.dumps(value)}>>"
100            for value in arguments
101        )
102        return self.inspection.call_json(method, f"[{erl}]")
103
104    def model(self, model, provider=None, effort=None):
105        return self.invoke("model", model, provider, effort)
106
107    def turn(self):
108        self.app.prompt(self.session, "prepare a request").close()
109        self.app.idle(self.session)
110
111    def context(self):
112        return self.read(f"/sessions/{self.session}/context")
113
114    def test_commands_save_defaults_resolve_effort_and_reset_prepared_context(self):
115        self.turn()
116        self.assertEqual(self.context()["state"], "ready")
117        cursor = self.app.stream_page(self.session)
118        selected = self.model("second")["ok"]
119        self.assertEqual((selected["model"], selected["effort"]), ("second", "high"))
120        self.assertEqual(self.context()["state"], "pending")
121        changes = self.app.stream_page(self.session, cursor)["events"]
122        invalidated = {
123            event["data"]["kind"] for event in changes if event["type"] == "invalidate"
124        }
125        self.assertTrue({"session", "context"}.issubset(invalidated), changes)
126        new = self.read(f"/sessions/{self.app.session()}")
127        self.assertEqual(new["model"], "second")
128
129        self.turn()
130        defaults = self.defaults()
131        self.assertEqual(
132            self.invoke("effort", " LOW ")["error"],
133            "unsupported reasoning effort: low; available: medium, high",
134        )
135        self.assertEqual(self.context()["state"], "ready")
136        changed = self.invoke("effort", " MEDIUM ")["ok"]
137        self.assertEqual(changed["effort"], "medium")
138        self.assertEqual(self.defaults(), defaults)
139        self.assertEqual(self.context()["state"], "pending")
140
141        before = self.configuration()
142        rejected = self.model("third", effort="high")
143        self.assertIn("unsupported reasoning effort", rejected["error"])
144        self.assertEqual(self.configuration(), before)
145        self.assertEqual(self.defaults(), defaults)
146        self.invoke("effort", "high")
147        self.assertEqual(self.model("third")["ok"]["effort"], "medium")
148        self.assertEqual(self.configuration()["effort"], "medium")
149
150        # HTTP edits share mutation validation, but never remember defaults.
151        route = f"/sessions/{self.session}?view=configuration"
152        with self.app.api(route) as response:
153            json.load(response)
154            etag = response.headers["ETag"]
155        defaults = self.defaults()
156        with self.app.api(
157            route,
158            {"model": "second", "effort": "high"},
159            method="PATCH",
160            headers={"If-Match": etag},
161        ) as response:
162            json.load(response)
163        self.assertEqual(self.defaults(), defaults)
164        self.turn()
165        self.assertEqual(
166            self.provider.requests[-1]["request"]["reasoning"]["effort"], "high"
167        )
168
169    def test_commands_reject_even_unchanged_values_during_a_turn(self):
170        entered, release = threading.Event(), threading.Event()
171        self.addCleanup(release.set)
172
173        def script(_request):
174            entered.set()
175            release.wait(20)
176            return text("done")
177
178        self.provider.script = script
179        self.app.prompt(self.session, "hold the turn").close()
180        self.assertTrue(entered.wait(10))
181        before, defaults = self.configuration(), self.defaults()
182        try:
183            self.assertEqual(
184                self.model("first")["error"], "model must be nonempty and session idle"
185            )
186            self.assertEqual(
187                self.invoke("effort", "high")["error"],
188                "switching effort requires an idle session",
189            )
190            self.assertEqual(self.configuration(), before)
191            self.assertEqual(self.defaults(), defaults)
192            route = f"/sessions/{self.session}?view=configuration"
193            with self.app.api(route) as response:
194                json.load(response)
195                etag = response.headers["ETag"]
196            with self.app.api(
197                route,
198                {"model": "first", "effort": "high"},
199                method="PATCH",
200                headers={"If-Match": etag},
201            ) as response:
202                json.load(response)
203        finally:
204            release.set()
205            self.app.idle(self.session)
206
207    def test_commands_reject_changes_during_kernel_boot(self):
208        self.app.prompt(self.session, "wait for the kernel").close()
209        wait_until(
210            lambda: self.marker.exists(), 10, "kernel startup did not reach its gate"
211        )
212        before, defaults = self.configuration(), self.defaults()
213        try:
214            self.assertEqual(
215                self.model("second")["error"], "model must be nonempty and session idle"
216            )
217            self.assertEqual(
218                self.invoke("effort", "low")["error"],
219                "switching effort requires an idle session",
220            )
221            self.assertEqual(self.configuration(), before)
222            self.assertEqual(self.defaults(), defaults)
223        finally:
224            assert self.gate is not None
225            wait_until(
226                lambda: release_fifo(self.gate), 10, "kernel did not open its gate"
227            )
228            self.gate.unlink()
229        self.app.idle(self.session)
230        self.assertEqual(self.provider.requests[-1]["request"]["model"], "first")
231
232
233if __name__ == "__main__":
234    unittest.main()