okami.mom/albedo

git clone https://git.nkp.pet/okami.mom/albedo

nekomimi.petgemini: field-reader fast paths for vertex and cloud code assist text chunks, gemini mock routes12f1695

main
branchesmainall refs →
6.2 KiB207 linesraw
1//// Run explicitly with: gleam run -m manual/stream_rate_benchmark
2//// Streams from test/manual/mock_openai_server.py (start it first; its origin
3//// from ALBEDO_BENCH_ORIGIN, default http://127.0.0.1:8765, and for https the
4//// CA to trust from ALBEDO_BENCH_CA) through the real transport and reducers,
5//// and prints VM CPU and reductions per token and send-to-callback latency.
6//// Run with the daemon's ERL_FLAGS (the vm_defaults in
7//// priv/bin/albedo-daemon: two schedulers, no busy waiting), so scheduler
8//// spinning does not count as work.
9////
10//// By default each protocol streams 2000 deltas at 500 per second, then
11//// 20000 unthrottled (ALBEDO_BENCH_UNTHROTTLED=0 skips those).
12//// ALBEDO_BENCH_PARALLEL=N instead streams N at once, each at
13//// ALBEDO_BENCH_RATE (default 1000) per second for four seconds.
14//// ALBEDO_BENCH_PROTOCOL=responses|chat_completions|claude|vertex|antigravity
15//// runs one protocol, ALBEDO_BENCH_TOOLS=P streams P percent of the deltas as tool
16//// call arguments, and ALBEDO_BENCH_MSACC=1 prints microstate accounting.
17
18import albedo/harness/extensions/antigravity/catalog
19import albedo/harness/extensions/antigravity/stream as antigravity
20import albedo/harness/extensions/claude/stream as claude
21import albedo/harness/extensions/vertex/stream as vertex
22import albedo/openai_api as openai
23import albedo/openai_api/types
24import gleam/float
25import gleam/int
26import gleam/io
27import gleam/json
28import gleam/list
29import gleam/option.{None}
30import gleam/result
31import gleam/string
32import gleam/string_tree
33
34@external(erlang, "albedo_openai_rate_bench", "env")
35fn env(name: String, default: String) -> String
36
37@external(erlang, "albedo_openai_rate_bench", "token")
38fn token() -> Nil
39
40@external(erlang, "albedo_openai_rate_bench", "trust")
41fn trust(ca_file: String) -> Nil
42
43@external(erlang, "albedo_openai_rate_bench", "measure")
44fn measure(
45  origin: String,
46  runs: List(#(String, fn() -> Result(types.Turn, types.Error))),
47) -> #(List(Result(types.Turn, types.Error)), List(#(String, Float)))
48
49const runs = 3
50
51type Protocol {
52  OpenAI(types.Protocol)
53  Claude
54  Vertex
55  Antigravity
56}
57
58pub fn main() -> Nil {
59  let origin = env("ALBEDO_BENCH_ORIGIN", "http://127.0.0.1:8765")
60  trust(env("ALBEDO_BENCH_CA", ""))
61  let only = env("ALBEDO_BENCH_PROTOCOL", "")
62  let assert Ok(tools) = int.parse(env("ALBEDO_BENCH_TOOLS", "0"))
63  let assert Ok(parallel) = int.parse(env("ALBEDO_BENCH_PARALLEL", "0"))
64  let assert Ok(rate) = int.parse(env("ALBEDO_BENCH_RATE", "1000"))
65  let unthrottled = env("ALBEDO_BENCH_UNTHROTTLED", "1") == "1"
66  [
67    OpenAI(types.Responses),
68    OpenAI(types.ChatCompletions),
69    Claude,
70    Vertex,
71    Antigravity,
72  ]
73  |> list.filter(fn(protocol) { only == "" || only == name(protocol) })
74  |> list.each(fn(protocol) {
75    case parallel, unthrottled {
76      0, True -> {
77        bench(origin, protocol, tools, 1, 500, 2000)
78        bench(origin, protocol, tools, 1, 0, 20_000)
79      }
80      0, False -> bench(origin, protocol, tools, 1, 500, 2000)
81      streams, _ -> bench(origin, protocol, tools, streams, rate, rate * 4)
82    }
83  })
84}
85
86fn name(protocol: Protocol) -> String {
87  case protocol {
88    OpenAI(protocol) -> types.protocol_name(protocol)
89    Claude -> "claude"
90    Vertex -> "vertex"
91    Antigravity -> "antigravity"
92  }
93}
94
95fn bench(
96  origin: String,
97  protocol: Protocol,
98  tools: Int,
99  streams: Int,
100  rate: Int,
101  tokens: Int,
102) -> Nil {
103  let runs_once =
104    int.range(0, streams, [], list.prepend)
105    |> list.map(fn(i) {
106      let id = int.to_string(i)
107      let base =
108        string.join(
109          [
110            origin,
111            int.to_string(rate),
112            int.to_string(tokens),
113            "t" <> int.to_string(tools),
114            "s" <> id,
115            "v1",
116          ],
117          "/",
118        )
119      #(id, fn() { stream(protocol, base) })
120    })
121  // one warm-up run loads modules and opens the connections the runs reuse
122  let _ = measure(origin, runs_once)
123  let results =
124    list.repeat(Nil, runs) |> list.map(fn(_) { measure(origin, runs_once) })
125  let label =
126    name(protocol)
127    <> case tools {
128      0 -> ""
129      _ -> " (" <> int.to_string(tools) <> "% tool args)"
130    }
131    <> case streams {
132      1 -> ""
133      _ -> " " <> int.to_string(streams) <> " streams"
134    }
135    <> " @ "
136    <> case rate {
137      0 -> "unthrottled"
138      _ -> int.to_string(rate) <> " tok/s"
139    }
140  let failure =
141    list.flat_map(results, fn(run) { run.0 }) |> list.find(result.is_error)
142  case failure {
143    Ok(Error(error)) -> io.println(label <> ": " <> string.inspect(error))
144    _ -> io.println(label <> " " <> median(list.map(results, fn(r) { r.1 })))
145  }
146}
147
148fn stream(protocol: Protocol, base: String) -> Result(types.Turn, types.Error) {
149  let on_event = fn(event) {
150    case event {
151      types.ArgumentsDelta(_, _, "") -> Nil
152      types.TextDelta(..) | types.ArgumentsDelta(..) -> token()
153      _ -> Nil
154    }
155    types.Continue
156  }
157  case protocol {
158    OpenAI(protocol) ->
159      openai.stream(
160        openai.client(protocol, base, ""),
161        openai.request("mock", [types.User("go")]),
162        on_event,
163      )
164    Claude ->
165      openai.exchange(
166        exchange(base <> "/messages"),
167        claude.reducer("mock", []),
168        on_event,
169      )
170    Vertex ->
171      openai.exchange(exchange(base <> "/vertex"), vertex.reducer(), on_event)
172    Antigravity -> {
173      let model = catalog.model("/nonexistent", "gemini-3-pro", None)
174      openai.exchange(
175        exchange(base <> "/antigravity"),
176        antigravity.reducer(model),
177        on_event,
178      )
179    }
180  }
181}
182
183fn exchange(url: String) -> openai.Exchange {
184  openai.Exchange(
185    url,
186    [#("content-type", "application/json")],
187    string_tree.from_string("{}"),
188    60_000,
189    8 * 1024 * 1024,
190    require_event_stream: True,
191  )
192}
193
194/// Each metric's median across runs, as one JSON object.
195fn median(runs: List(List(#(String, Float)))) -> String {
196  let assert [first, ..] = runs
197  list.map(first, fn(metric) {
198    let values =
199      list.filter_map(runs, fn(run) { list.key_find(run, metric.0) })
200      |> list.sort(float.compare)
201    let assert Ok(middle) =
202      list.drop(values, list.length(values) / 2) |> list.first
203    #(metric.0, json.float(float.to_precision(middle, 2)))
204  })
205  |> json.object
206  |> json.to_string
207}