okami.mom/albedo

git clone https://git.nkp.pet/okami.mom/albedo

nekomimi.petawa3846f03

main
branchesmainall refs →
3.8 KiB131 linesraw
1//// Run explicitly with: gleam run -m manual/benchmark
2//// ALBEDO_BENCH_URL, ALBEDO_BENCH_MODEL, ALBEDO_BENCH_KEY are environment variables.
3//// Makes at most three requests per protocol; stops a protocol on its first error.
4
5import albedo/openai_api as openai
6import albedo/openai_api/types
7import gleam/io
8import gleam/json
9import gleam/list
10import gleam/option.{type Option, None, Some}
11import gleam/string
12
13pub type Metrics {
14  Metrics(
15    elapsed_ms: Float,
16    first_text_ms: Option(Float),
17    text_bytes: Int,
18    chunks: Int,
19    peak_process_bytes: Int,
20    peak_vm_binary_growth_bytes: Int,
21    reductions: Int,
22  )
23}
24
25@external(erlang, "albedo_openai_bench", "env")
26fn env(name: String) -> String
27
28@external(erlang, "albedo_openai_bench", "measure")
29fn measure(run: fn() -> a) -> #(a, Metrics)
30
31@external(erlang, "albedo_openai_bench", "text")
32fn text(bytes: Int) -> Nil
33
34pub fn main() -> Nil {
35  let key = env("ALBEDO_BENCH_KEY")
36  let url = env("ALBEDO_BENCH_URL")
37  let model = env("ALBEDO_BENCH_MODEL")
38  case key == "" || url == "" || model == "" {
39    True ->
40      io.println("missing benchmark environment variables; no requests made")
41    False -> {
42      let protocols = case env("ALBEDO_BENCH_PROTOCOL") {
43        "responses" -> [types.Responses]
44        "chat_completions" -> [types.ChatCompletions]
45        _ -> [types.Responses, types.ChatCompletions]
46      }
47      list.each(protocols, fn(protocol) {
48        run(openai.client(protocol, url, key), model, 1)
49      })
50    }
51  }
52}
53
54fn run(client: types.Client, model: String, iteration: Int) -> Nil {
55  let client = types.Client(..client, timeout_ms: 90_000)
56  let request =
57    openai.request(model, [
58      types.User(
59        "Write the integers from 1 through 80, separated by spaces. Do not include any other text.",
60      ),
61    ])
62  let request = types.Request(..request, max_output_tokens: Some(256))
63  let #(result, metrics) =
64    measure(fn() {
65      openai.stream(client, request, fn(event) {
66        case event {
67          types.TextDelta(_, _, value) -> text(string.byte_size(value))
68          _ -> Nil
69        }
70        types.Continue
71      })
72    })
73  let protocol = case client.protocol {
74    types.Responses -> "responses"
75    types.ChatCompletions -> "chat_completions"
76  }
77  let fields = [
78    #("protocol", json.string(protocol)),
79    #("iteration", json.int(iteration)),
80    #("elapsed_ms", json.float(metrics.elapsed_ms)),
81    #("first_text_ms", json.nullable(metrics.first_text_ms, json.float)),
82    #("text_bytes", json.int(metrics.text_bytes)),
83    #("text_chunks", json.int(metrics.chunks)),
84    #("sampled_peak_process_bytes", json.int(metrics.peak_process_bytes)),
85    #(
86      "sampled_peak_vm_binary_growth_bytes",
87      json.int(metrics.peak_vm_binary_growth_bytes),
88    ),
89    #("worker_reductions", json.int(metrics.reductions)),
90  ]
91  let fields = case result {
92    Ok(turn) -> [
93      #("ok", json.bool(True)),
94      #("finish", json.string(string.inspect(turn.finish))),
95      #(
96        "input_tokens",
97        json.nullable(option_tokens(turn.usage, True), json.int),
98      ),
99      #(
100        "output_tokens",
101        json.nullable(option_tokens(turn.usage, False), json.int),
102      ),
103      ..fields
104    ]
105    Error(error) -> [
106      #("ok", json.bool(False)),
107      #(
108        "error",
109        json.string(
110          string.inspect(error)
111          |> string.replace(client.api_key, "[redacted]")
112          |> string.slice(0, 2000),
113        ),
114      ),
115      ..fields
116    ]
117  }
118  fields |> json.object |> json.to_string |> io.println
119  case result, iteration < 3 {
120    Ok(_), True -> run(client, model, iteration + 1)
121    _, _ -> Nil
122  }
123}
124
125fn option_tokens(usage: Option(types.Usage), input: Bool) -> Option(Int) {
126  case usage, input {
127    None, _ -> None
128    Some(usage), True -> Some(usage.input_tokens)
129    Some(usage), False -> Some(usage.output_tokens)
130  }
131}