okami.mom/albedo
git clone https://git.nkp.pet/okami.mom/albedo
3a0caba
main
1// Account pins and 429 handoff depend on OAuth identities and provider responses unavailable in E2E. 2import albedo/harness/extensions/codex/extension as codex 3import albedo/harness/rotation 4import albedo/openai_api 5import albedo/openai_api/types 6import gleam/erlang/process 7import gleam/int 8import gleam/list 9import gleam/option .{ None , Some } 10import gleam/string 11import gleeunit/should 12 13const credentials = 14"{\"accounts\":{\"openai-codex\":[{\"type\":\"oauth\",\"access\":\"h.eyJodHRwczovL2FwaS5vcGVuYWkuY29tL2F1dGgiOnsiY2hhdGdwdF9hY2NvdW50X2lkIjoiYWNjb3VudC0xIiwiY2hhdGdwdF9hY2NvdW50X3VzZXJfaWQiOiJzZWF0LTEifX0.s\",\"refresh\":\"refresh-1\",\"expires\":9999999999999,\"accountId\":\"account-1\",\"accountUserId\":\"seat-1\"},{\"type\":\"oauth\",\"access\":\"h.eyJodHRwczovL2FwaS5vcGVuYWkuY29tL2F1dGgiOnsiY2hhdGdwdF9hY2NvdW50X2lkIjoiYWNjb3VudC0yIiwiY2hhdGdwdF9hY2NvdW50X3VzZXJfaWQiOiJzZWF0LTIifX0.s\",\"refresh\":\"refresh-2\",\"expires\":9999999999999,\"accountId\":\"account-2\",\"accountUserId\":\"seat-2\"}]}}" 15 16const reversed_credentials = 17"{\"accounts\":{\"openai-codex\":[{\"type\":\"oauth\",\"access\":\"h.eyJodHRwczovL2FwaS5vcGVuYWkuY29tL2F1dGgiOnsiY2hhdGdwdF9hY2NvdW50X2lkIjoiYWNjb3VudC0yIiwiY2hhdGdwdF9hY2NvdW50X3VzZXJfaWQiOiJzZWF0LTIifX0.s\",\"refresh\":\"refresh-2\",\"expires\":9999999999999,\"accountId\":\"account-2\",\"accountUserId\":\"seat-2\"},{\"type\":\"oauth\",\"access\":\"h.eyJodHRwczovL2FwaS5vcGVuYWkuY29tL2F1dGgiOnsiY2hhdGdwdF9hY2NvdW50X2lkIjoiYWNjb3VudC0xIiwiY2hhdGdwdF9hY2NvdW50X3VzZXJfaWQiOiJzZWF0LTEifX0.s\",\"refresh\":\"refresh-1\",\"expires\":9999999999999,\"accountId\":\"account-1\",\"accountUserId\":\"seat-1\"}]}}" 18 19pub fn sessions_stick_to_and_distribute_across_codex_accounts_test () -> Nil { 20let #( root , _ , home ) = fixture () 21let _ = write ( home , "creds.json" , credentials ) 22let first = access ( home , "session-1" ) 23access ( home , "session-1" ) |> should . equal ( first ) 24let accounts = 25[ access ( home , "session-1" ), access ( home , "session-2" )] 26|> list . unique 27list . length ( accounts ) |> should . equal ( 2 ) 28cleanup ( root ) 29} 30 31pub fn session_pin_survives_credential_reordering_test () -> Nil { 32let #( root , _ , home ) = fixture () 33let _ = write ( home , "creds.json" , credentials ) 34let selected = access ( home , "durable-session" ) 35let _ = write ( home , "creds.json" , reversed_credentials ) 36access ( home , "durable-session" ) |> should . equal ( selected ) 37cleanup ( root ) 38} 39 40pub fn revoked_codex_account_is_removed_test () -> Nil { 41let #( root , _ , home ) = fixture () 42let _ = write ( home , "creds.json" , credentials ) 43let #( revoked , client ) = codex_client ( home , "revoked-session" ) 44let assert Some ( message ) = 45codex . account_failure ( home , client , types . HttpError ( 401 , "token_revoked" )) 46string . ends_with ( message , "run /login" ) |> should . be_true 47codex . account_failure ( home , client , types . HttpError ( 500 , "" )) 48|> should . equal ( None ) 49let remaining = access ( home , "revoked-session" ) 50{ remaining == revoked } |> should . be_false 51access ( home , "another-session" ) |> should . equal ( remaining ) 52let #( _ , last ) = codex_client ( home , "revoked-session" ) 53let assert Some ( _ ) = 54codex . account_failure ( home , last , types . HttpError ( 401 , "token_revoked" )) 55let assert Error ( _ ) = native_access ( home , "revoked-session" ) 56cleanup ( root ) 57} 58 59pub fn usage_limit_moves_the_session_to_a_sibling_account_test () -> Nil { 60let #( root , _ , home ) = fixture () 61let _ = write ( home , "creds.json" , credentials ) 62let #( limited , client ) = codex_client ( home , "busy-session" ) 63 64// Ordinary rate limiting is transient and must not move the session. 65codex . account_failure ( home , client , types . HttpError ( 429 , "slow down" )) 66|> should . equal ( None ) 67access ( home , "busy-session" ) |> should . equal ( limited ) 68 69let body = 70"{\"error\":{\"type\":\"usage_limit_reached\",\"resets_in_seconds\":3600}}" 71let assert Some ( message ) = 72codex . account_failure ( home , client , types . HttpError ( 429 , body )) 73string . contains ( message , "usage limit reached" ) |> should . be_true 74string . contains ( message , "the next turn will use" ) |> should . be_true 75let next = access ( home , "busy-session" ) 76{ next == limited } |> should . be_false 77// Every session avoids the limited account while a sibling has usage. 78access ( home , "fresh-session" ) |> should . equal ( next ) 79 80// With every account limited the message says so instead of naming one. 81let #( _ , sibling ) = codex_client ( home , "busy-session" ) 82let assert Some ( exhausted ) = 83codex . account_failure ( home , sibling , types . HttpError ( 429 , body )) 84string . contains ( exhausted , "no other ChatGPT account" ) |> should . be_true 85cleanup ( root ) 86} 87 88/// Replays a stream where the named tokens answer 429 with `body` and every 89/// other account fails plainly, recording which accounts were tried. 90fn limited_stream ( 91limited : List ( String ), 92body : String , 93tried : process . Subject ( String ), 94) -> fn ( types . Client , a, b) -> Result ( c, types . Error ) { 95fn ( client : types . Client , _request , _on_event ) { 96process . send ( tried , client . api_key ) 97case list . contains ( limited , client . api_key ) { 98True -> Error ( types . HttpError ( 429 , body )) 99False -> Error ( types . HttpError ( 500 , "second account answered" )) 100} 101} 102} 103 104fn rotate ( 105home : String , 106session : String , 107first : types . Client , 108request : types . Request , 109stream : fn ( types . Client , types . Request , fn ( types . Event ) -> types . Control ) -> 110Result ( types . Turn , types . Error ), 111pause : fn ( Int ) -> Nil , 112) -> Result ( types . Turn , types . Error ) { 113rotation . stream ( 114codex . pool ( home , session , stream ), 115first , 116request , 117fn ( _ ) { types . Continue }, 118rotation . Budget ( 8 , 0 , pause ), 119fn ( _ ) { Nil }, 120) 121} 122 123fn drain ( tried : process . Subject ( String ), found : List ( String )) -> List ( String ) { 124case process . receive ( tried , 0 ) { 125Ok ( token ) -> drain ( tried , [ token , .. found ]) 126Error ( _ ) -> list . reverse ( found ) 127} 128} 129 130pub fn a_limited_account_hands_the_same_request_to_the_next_test () -> Nil { 131let #( root , _ , home ) = fixture () 132let _ = write ( home , "creds.json" , credentials ) 133let #( _ , first ) = codex_client ( home , "swarm-session" ) 134let tried = process . new_subject () 135let body = 136"{\"error\":{\"type\":\"usage_limit_reached\",\"resets_in_seconds\":3600}}" 137let request = openai_api . request ( "gpt-5.5" , [ types . User ( "hi" )]) 138rotate ( 139home , 140"swarm-session" , 141first , 142request , 143limited_stream ([ first . api_key ], body , tried ), 144fn ( _ ) { panic as "a sibling had room; nothing should wait" }, 145) 146|> should . equal ( Error ( types . HttpError ( 500 , "second account answered" ))) 147let attempts = drain ( tried , []) 148list . length ( attempts ) |> should . equal ( 2 ) 149list . first ( attempts ) |> should . equal ( Ok ( first . api_key )) 150cleanup ( root ) 151} 152 153pub fn a_burst_rate_limit_body_rotates_test () -> Nil { 154let #( root , _ , home ) = fixture () 155let _ = write ( home , "creds.json" , credentials ) 156let #( _ , first ) = codex_client ( home , "burst-session" ) 157let tried = process . new_subject () 158let request = openai_api . request ( "gpt-6-luna" , [ types . User ( "hi" )]) 159rotate ( 160home , 161"burst-session" , 162first , 163request , 164limited_stream ( 165[ first . api_key ], 166"{\"detail\":\"Rate limit exceeded\"}" , 167tried , 168), 169fn ( _ ) { panic as "a sibling had room; nothing should wait" }, 170) 171|> should . equal ( Error ( types . HttpError ( 500 , "second account answered" ))) 172list . length ( drain ( tried , [])) |> should . equal ( 2 ) 173cleanup ( root ) 174} 175 176pub fn a_rate_limit_rotates_too_and_stops_when_all_are_busy_test () -> Nil { 177let #( root , _ , home ) = fixture () 178let _ = write ( home , "creds.json" , credentials ) 179let #( _ , first ) = codex_client ( home , "busy-swarm" ) 180let tried = process . new_subject () 181let body = 182"{\"error\":{\"type\":\"rate_limit_exceeded\",\"resets_in_seconds\":20}}" 183let #( _ , second ) = codex_client ( home , "other-busy-swarm" ) 184let both = list . unique ([ first . api_key , second . api_key ]) 185let request = openai_api . request ( "gpt-5.5" , [ types . User ( "hi" )]) 186let paused = process . new_subject () 187rotate ( 188home , 189"busy-swarm" , 190first , 191request , 192limited_stream ( both , body , tried ), 193fn ( delay ) { process . send ( paused , int . to_string ( delay )) }, 194) 195|> should . equal ( Error ( types . HttpError ( 429 , body ))) 196// Each account once, then four waits with a retry after each, then it stops. 197list . length ( drain ( tried , [])) |> should . equal ( 6 ) 198drain ( paused , []) |> should . equal ([ "4000" , "8000" , "15000" , "30000" ]) 199cleanup ( root ) 200} 201 202pub fn a_usage_limit_on_every_account_does_not_wait_test () -> Nil { 203let #( root , _ , home ) = fixture () 204let _ = write ( home , "creds.json" , credentials ) 205let #( _ , first ) = codex_client ( home , "spent-swarm" ) 206let #( _ , second ) = codex_client ( home , "other-spent-swarm" ) 207let tried = process . new_subject () 208let body = 209"{\"error\":{\"type\":\"usage_limit_reached\",\"resets_in_seconds\":3600}}" 210let request = openai_api . request ( "gpt-6-luna" , [ types . User ( "hi" )]) 211rotate ( 212home , 213"spent-swarm" , 214first , 215request , 216limited_stream ( list . unique ([ first . api_key , second . api_key ]), body , tried ), 217fn ( _ ) { panic as "a usage limit lasts hours; it must not be waited on" }, 218) 219|> should . equal ( Error ( types . HttpError ( 429 , body ))) 220list . length ( drain ( tried , [])) |> should . equal ( 2 ) 221cleanup ( root ) 222} 223 224pub fn selected_account_overrides_the_session_pin_test () -> Nil { 225let #( root , _ , home ) = fixture () 226let _ = write ( home , "creds.json" , credentials ) 227let pinned = access ( home , "pinned-session" ) 228let other = case pinned { 229"account-1" -> "account-2" 230_ -> "account-1" 231} 232let _ = 233write ( 234home , 235"creds.json" , 236string . replace ( 237credentials , 238"\"accountId\":\"" <> other <> "\"" , 239"\"accountId\":\"" <> other <> "\",\"selected\":true" , 240), 241) 242access ( home , "pinned-session" ) |> should . equal ( other ) 243access ( home , "any-session" ) |> should . equal ( other ) 244cleanup ( root ) 245} 246 247fn codex_client ( home : String , session : String ) -> #( String , types . Client ) { 248let assert Ok ( access ) = native_access ( home , session ) 249#( 250access . account_id , 251openai_api . codex_client ( 252"http://127.0.0.1" , 253access . token , 254access . account_id , 255session , 256), 257) 258} 259 260fn access ( home : String , session : String ) -> String { 261let assert Ok ( access ) = native_access ( home , session ) 262access . account_id 263} 264 265@external ( erlang , "albedo_openai_auth" , "codex_access" ) 266fn native_access ( home : String , session : String ) -> Result ( codex . Access , String ) 267 268@external ( erlang , "albedo_skills_test_support" , "fixture" ) 269fn fixture () -> #( String , String , String ) 270 271@external ( erlang , "albedo_skills_test_support" , "write" ) 272fn write ( base : String , relative : String , content : String ) -> String 273 274@external ( erlang , "albedo_skills_test_support" , "cleanup" ) 275fn cleanup ( root : String ) -> Nil