A-Mahla HF Staff commited on
Commit
da67b51
·
1 Parent(s): 1f28297

Add waiting-queue UI and session-queue proxy

Browse files

When the compute pool is busy the load balancer now hands back a queue ticket
instead of a grant. The Space relays it end to end:

- /api/session returns a grant or a ticket; /api/queue/{id} polls position and
reserves the daily budget only at claim (re-checking it there), so waiting in
line never costs usage time. Leave endpoints (DELETE + sendBeacon) drop the
ticket; a full queue relays 503 {state:"at_capacity"}.
- The client waits in the queue polling position, and uses a hybrid mic flow:
permission is primed in the tap gesture, but the capture stream is acquired
only once a slot is granted, so the mic indicator never glows while waiting.
- New calm "queued" orb state with a "Leave queue" button; queue-full and
ticket-expired land on a friendly retry state, not an error.

CONTEXT.md gains the queue vocabulary (Queue, Ticket, Position, Claim, At
capacity).

Files changed (6) hide show
  1. CONTEXT.md +27 -0
  2. index.html +4 -0
  3. main.js +106 -15
  4. server.py +142 -2
  5. style.css +34 -0
  6. ws/s2s-ws-client.js +146 -10
CONTEXT.md CHANGED
@@ -79,3 +79,30 @@ than hidden behind a click.
79
  The popup opened from the (i) icon to the right of the identity block. Holds the
80
  general introduction to the speech-to-speech project (with a repo link) and the
81
  pipeline. Identity itself now lives in the corner, not here.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  The popup opened from the (i) icon to the right of the identity block. Holds the
80
  general introduction to the speech-to-speech project (with a repo link) and the
81
  pipeline. Identity itself now lives in the corner, not here.
82
+
83
+ ## Queue
84
+ The line of users waiting for a free conversation slot when every compute is
85
+ busy. You join the queue instead of being turned away; you leave it by reaching
86
+ the front or by giving up. Distinct from a *session* (an actual live
87
+ conversation) — being in the queue is not yet talking, and time spent waiting
88
+ never counts against your usage limit.
89
+
90
+ ## Ticket
91
+ Your held place in the queue. Created when you join, it is what the demo checks
92
+ to tell you your position and to notice if you have left. A ticket is not a
93
+ session: it only promises a spot in line, not a compute.
94
+
95
+ ## Position
96
+ How many people are ahead of you in the queue, shown while you wait ("You're #3
97
+ in line"). It only ever counts down. Distinct from an *estimated wait* — the
98
+ demo shows position, never a time, because wait time is unpredictable.
99
+
100
+ ## Claim
101
+ The moment you reach the front and a free slot becomes yours — the queue hands
102
+ off to a real session and the conversation begins. This is also the point where
103
+ your usage limit first starts to matter (never while waiting).
104
+
105
+ ## At capacity
106
+ The state where the queue itself is full, so new users can't even join the line
107
+ and are asked to try again shortly. Distinct from simply *busy* (all computes
108
+ taken but the queue still has room to wait in).
index.html CHANGED
@@ -131,6 +131,10 @@
131
  </div>
132
 
133
  <p id="circle-caption" class="circle-caption" role="status">Tap to start</p>
 
 
 
 
134
  </main>
135
 
136
  <footer class="footer">
 
131
  </div>
132
 
133
  <p id="circle-caption" class="circle-caption" role="status">Tap to start</p>
134
+
135
+ <button id="leave-queue-btn" class="leave-queue-btn" type="button" hidden>
136
+ Leave queue
137
+ </button>
138
  </main>
139
 
140
  <footer class="footer">
main.js CHANGED
@@ -13,7 +13,7 @@
13
  * client owns its own AudioContext (no `attachOutputTrack`), so we hand
14
  * it the MediaStream directly.
15
  *
16
- * @typedef {"idle" | "connecting" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState
17
  */
18
 
19
  import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js";
@@ -151,6 +151,7 @@ function saveTools() {
151
  const STATE_VIEWS = {
152
  idle: { caption: "Tap to start", disabled: false },
153
  connecting: { caption: "Connecting", disabled: true },
 
154
  listening: { caption: "", disabled: false },
155
  "user-speaking": { caption: "", disabled: false },
156
  processing: { caption: "", disabled: false },
@@ -162,6 +163,7 @@ const STATE_VIEWS = {
162
  const STATE_CLASS = {
163
  idle: "state-idle",
164
  connecting: "state-connecting",
 
165
  listening: "state-listening",
166
  "user-speaking": "state-user-speaking",
167
  processing: "state-processing",
@@ -182,6 +184,8 @@ const orbWrap = $(".orb-wrap");
182
  const micBtn = $("#mic-btn");
183
  /** @type {HTMLButtonElement} */
184
  const stopBtn = $("#stop-btn");
 
 
185
 
186
  /** @type {HTMLButtonElement} */
187
  const settingsBtn = $("#settings-btn");
@@ -314,6 +318,9 @@ let limiterOn = false;
314
  let heartbeatTimer = 0;
315
  let trackedSessionId = "";
316
  let trackedTier = "";
 
 
 
317
 
318
  /** @type {S2sWsRealtimeClient | null} */
319
  let client = null;
@@ -336,13 +343,19 @@ function setState(next) {
336
  micBtn.tabIndex = live ? 0 : -1;
337
  stopBtn.tabIndex = live ? 0 : -1;
338
 
 
 
 
 
 
339
  updateRestartAvailability();
340
  }
341
 
342
  function updateRestartAvailability() {
343
  // Restart works from any settled state — it tears down a live call (if any)
344
- // and reconnects with the current settings. Only block while mid-connect.
345
- restartBtn.disabled = currentState === "connecting";
 
346
  restartHint.hidden = false;
347
  restartHint.textContent = LIVE_STATES.has(currentState)
348
  ? "Reconnects now with the settings above."
@@ -973,6 +986,20 @@ async function handleStartError(err) {
973
  account.showLimit(err.tier);
974
  return;
975
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
976
  onFatalError(err);
977
  }
978
 
@@ -992,6 +1019,43 @@ stopBtn.addEventListener("click", async () => {
992
  await teardown();
993
  });
994
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
995
  /**
996
  * Start a conversation. Pass a pre-created AudioContext when the caller already
997
  * made one inside the tap/click gesture (required on iOS); otherwise one is
@@ -1014,19 +1078,15 @@ async function doStart(audioContext = null) {
1014
  // suspended and the whole pipeline would be silent.
1015
  if (!audioContext) audioContext = createResumedAudioContext();
1016
 
 
 
 
 
1017
  try {
1018
- micStream = await navigator.mediaDevices.getUserMedia({
1019
- audio: {
1020
- echoCancellation: true,
1021
- noiseSuppression: true,
1022
- autoGainControl: true,
1023
- },
1024
- });
1025
  } catch (err) {
1026
  if (audioContext) void audioContext.close().catch(() => {});
1027
- throw new Error(
1028
- `Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`,
1029
- );
1030
  }
1031
 
1032
  // The webcam is started on arrival (autoStartCamera), so nothing to do here;
@@ -1036,13 +1096,19 @@ async function doStart(audioContext = null) {
1036
  ...target,
1037
  voice: settings.voice,
1038
  instructions: effectiveInstructions(),
1039
- micStream,
1040
  tools: activeToolDefs(),
1041
  noiseGate: gateParams(settings.noiseGate),
1042
  ...(audioContext ? { audioContext } : {}),
1043
  });
1044
  client = c;
1045
 
 
 
 
 
 
 
1046
  c.addEventListener("status", (e) => {
1047
  const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail;
1048
  onClientStatus(detail.status);
@@ -1080,6 +1146,9 @@ async function doStart(audioContext = null) {
1080
  c.addEventListener("session", (e) => {
1081
  const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info;
1082
  console.log("[ws] session created:", info.sessionId);
 
 
 
1083
  // A metered tier (anon / free): heartbeat so the server can extend the
1084
  // reservation and tell us when the daily budget runs out. PRO isn't limited.
1085
  if (info.limited && info.sessionId) {
@@ -1163,6 +1232,24 @@ function endTrackedSession() {
1163
  trackedTier = "";
1164
  }
1165
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1166
  /** @param {string} status */
1167
  function onClientStatus(status) {
1168
  switch (status) {
@@ -1170,6 +1257,9 @@ function onClientStatus(status) {
1170
  case "connecting":
1171
  setState("connecting");
1172
  break;
 
 
 
1173
  case "connected":
1174
  setState("listening");
1175
  break;
@@ -1194,6 +1284,7 @@ function onClientStatus(status) {
1194
  async function teardown() {
1195
  stopHeartbeat();
1196
  endTrackedSession();
 
1197
  chat.reset({ dismiss: true });
1198
  if (client) {
1199
  try {
@@ -1238,7 +1329,7 @@ void autoStartCamera();
1238
  void watchCameraPermission();
1239
 
1240
  // Reconcile a live session if the tab is closed/hidden mid-call (no teardown).
1241
- window.addEventListener("pagehide", () => endTrackedSession());
1242
 
1243
  requestAnimationFrame(() => {
1244
  document.body.classList.remove("booting");
 
13
  * client owns its own AudioContext (no `attachOutputTrack`), so we hand
14
  * it the MediaStream directly.
15
  *
16
+ * @typedef {"idle" | "connecting" | "queued" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState
17
  */
18
 
19
  import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js";
 
151
  const STATE_VIEWS = {
152
  idle: { caption: "Tap to start", disabled: false },
153
  connecting: { caption: "Connecting", disabled: true },
154
+ queued: { caption: "Waiting…", disabled: true },
155
  listening: { caption: "", disabled: false },
156
  "user-speaking": { caption: "", disabled: false },
157
  processing: { caption: "", disabled: false },
 
163
  const STATE_CLASS = {
164
  idle: "state-idle",
165
  connecting: "state-connecting",
166
+ queued: "state-queued",
167
  listening: "state-listening",
168
  "user-speaking": "state-user-speaking",
169
  processing: "state-processing",
 
184
  const micBtn = $("#mic-btn");
185
  /** @type {HTMLButtonElement} */
186
  const stopBtn = $("#stop-btn");
187
+ /** @type {HTMLButtonElement} */
188
+ const leaveQueueBtn = $("#leave-queue-btn");
189
 
190
  /** @type {HTMLButtonElement} */
191
  const settingsBtn = $("#settings-btn");
 
318
  let heartbeatTimer = 0;
319
  let trackedSessionId = "";
320
  let trackedTier = "";
321
+ // The waiting-queue ticket id while we're in line (else ""). Used to leave the
322
+ // queue on teardown / tab-close so we don't hold a phantom place.
323
+ let queuedTicketId = "";
324
 
325
  /** @type {S2sWsRealtimeClient | null} */
326
  let client = null;
 
343
  micBtn.tabIndex = live ? 0 : -1;
344
  stopBtn.tabIndex = live ? 0 : -1;
345
 
346
+ // The "Leave queue" button shows only while waiting in line.
347
+ const queued = next === "queued";
348
+ leaveQueueBtn.hidden = !queued;
349
+ leaveQueueBtn.tabIndex = queued ? 0 : -1;
350
+
351
  updateRestartAvailability();
352
  }
353
 
354
  function updateRestartAvailability() {
355
  // Restart works from any settled state — it tears down a live call (if any)
356
+ // and reconnects with the current settings. Only block while mid-connect or
357
+ // while waiting in the queue (restarting from there would just re-queue).
358
+ restartBtn.disabled = currentState === "connecting" || currentState === "queued";
359
  restartHint.hidden = false;
360
  restartHint.textContent = LIVE_STATES.has(currentState)
361
  ? "Reconnects now with the settings above."
 
986
  account.showLimit(err.tier);
987
  return;
988
  }
989
+ // The user left the queue (close() aborted the wait): teardown already reset
990
+ // the UI to idle, so there's nothing to report.
991
+ if (err && err.code === "aborted") return;
992
+ // The queue was full or our ticket timed out — recoverable, not a fault. Land
993
+ // on the retry state with a plain-language reason instead of an error dump.
994
+ if (err && (err.code === "queue-full" || err.code === "queue-expired")) {
995
+ await teardown();
996
+ setState("error");
997
+ setCaption(
998
+ err.code === "queue-full" ? "All lines busy — tap to retry" : "Queue timed out — tap to retry",
999
+ "error",
1000
+ );
1001
+ return;
1002
+ }
1003
  onFatalError(err);
1004
  }
1005
 
 
1019
  await teardown();
1020
  });
1021
 
1022
+ // "Leave queue": tear down the pending connect (aborts the poll wait) and drop
1023
+ // our place in line. Same teardown path as stopping a live call.
1024
+ leaveQueueBtn.addEventListener("click", async () => {
1025
+ await teardown();
1026
+ });
1027
+
1028
+ const MIC_CONSTRAINTS = {
1029
+ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
1030
+ };
1031
+
1032
+ /** Prompt for mic permission up front, then immediately release the tracks so no
1033
+ * recording indicator lingers during a queue wait. Throws a friendly error if the
1034
+ * user denies. */
1035
+ async function primeMicPermission() {
1036
+ try {
1037
+ const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
1038
+ for (const track of s.getTracks()) track.stop();
1039
+ } catch (err) {
1040
+ throw new Error(
1041
+ `Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`,
1042
+ );
1043
+ }
1044
+ }
1045
+
1046
+ /** Acquire the live capture stream once a slot is granted. Permission was primed
1047
+ * in the tap gesture, so this is silent. Stored module-side for mute + teardown. */
1048
+ async function acquireMicStream() {
1049
+ micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
1050
+ return micStream;
1051
+ }
1052
+
1053
+ /** @param {number} position Update the queued caption ("You're #N in line"). */
1054
+ function onQueuePosition(position) {
1055
+ const n = Number(position) || 0;
1056
+ setCaption(n > 0 ? `You're #${n} in line` : "Waiting for a free slot…", "muted");
1057
+ }
1058
+
1059
  /**
1060
  * Start a conversation. Pass a pre-created AudioContext when the caller already
1061
  * made one inside the tap/click gesture (required on iOS); otherwise one is
 
1078
  // suspended and the whole pipeline would be silent.
1079
  if (!audioContext) audioContext = createResumedAudioContext();
1080
 
1081
+ // Prime the mic permission now (get the prompt out of the way up front), then
1082
+ // release it. The real capture stream is acquired only once a slot is granted
1083
+ // (see acquireMicStream), so the mic 'in use' indicator never lights while we
1084
+ // sit in the queue. Permission persists, so the later acquire is silent.
1085
  try {
1086
+ await primeMicPermission();
 
 
 
 
 
 
1087
  } catch (err) {
1088
  if (audioContext) void audioContext.close().catch(() => {});
1089
+ throw err;
 
 
1090
  }
1091
 
1092
  // The webcam is started on arrival (autoStartCamera), so nothing to do here;
 
1096
  ...target,
1097
  voice: settings.voice,
1098
  instructions: effectiveInstructions(),
1099
+ acquireMic: acquireMicStream,
1100
  tools: activeToolDefs(),
1101
  noiseGate: gateParams(settings.noiseGate),
1102
  ...(audioContext ? { audioContext } : {}),
1103
  });
1104
  client = c;
1105
 
1106
+ c.addEventListener("queue", (e) => {
1107
+ const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail;
1108
+ if (queueId) queuedTicketId = queueId;
1109
+ onQueuePosition(position);
1110
+ });
1111
+
1112
  c.addEventListener("status", (e) => {
1113
  const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail;
1114
  onClientStatus(detail.status);
 
1146
  c.addEventListener("session", (e) => {
1147
  const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info;
1148
  console.log("[ws] session created:", info.sessionId);
1149
+ // A slot was granted — we're out of the queue; drop the ticket reference so
1150
+ // teardown doesn't try to leave a line we already left.
1151
+ queuedTicketId = "";
1152
  // A metered tier (anon / free): heartbeat so the server can extend the
1153
  // reservation and tell us when the daily budget runs out. PRO isn't limited.
1154
  if (info.limited && info.sessionId) {
 
1232
  trackedTier = "";
1233
  }
1234
 
1235
+ /** Leave the waiting queue so the LB frees our place. sendBeacon so it still
1236
+ * fires on tab close; the LB also reaps the ticket on TTL as a backstop. */
1237
+ function endQueueTicket() {
1238
+ if (!queuedTicketId) return;
1239
+ const body = JSON.stringify({ queueId: queuedTicketId });
1240
+ try {
1241
+ const blob = new Blob([body], { type: "application/json" });
1242
+ if (!navigator.sendBeacon("api/queue/end", blob)) {
1243
+ void fetch("api/queue/end", {
1244
+ method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true,
1245
+ }).catch(() => {});
1246
+ }
1247
+ } catch {
1248
+ // Best-effort; the LB reaps the ticket on TTL anyway.
1249
+ }
1250
+ queuedTicketId = "";
1251
+ }
1252
+
1253
  /** @param {string} status */
1254
  function onClientStatus(status) {
1255
  switch (status) {
 
1257
  case "connecting":
1258
  setState("connecting");
1259
  break;
1260
+ case "queued":
1261
+ setState("queued");
1262
+ break;
1263
  case "connected":
1264
  setState("listening");
1265
  break;
 
1284
  async function teardown() {
1285
  stopHeartbeat();
1286
  endTrackedSession();
1287
+ endQueueTicket();
1288
  chat.reset({ dismiss: true });
1289
  if (client) {
1290
  try {
 
1329
  void watchCameraPermission();
1330
 
1331
  // Reconcile a live session if the tab is closed/hidden mid-call (no teardown).
1332
+ window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); });
1333
 
1334
  requestAnimationFrame(() => {
1335
  document.body.classList.remove("booting");
server.py CHANGED
@@ -22,10 +22,18 @@ Endpoints:
22
  GET /api/config -> { search, lb, allowDirect, auth }
23
  GET /api/me -> login + tier + remaining budget (LB mode only)
24
  POST /api/search -> { results, answer } Google via Serper.dev
25
- POST /api/session -> proxies <LB>/session, reserves the first time chunk
 
 
 
26
  POST /api/session/heartbeat-> extend the reservation; { expired }
27
  POST /api/session/end -> reconcile + refund (sendBeacon on teardown)
28
  /* -> static files (index.html, main.js, ...)
 
 
 
 
 
29
  """
30
 
31
  import asyncio
@@ -212,7 +220,8 @@ async def session(request: Request):
212
  # nothing is tracked. Within metering, unlimited tiers (pro, org) aren't either.
213
  tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None
214
 
215
- # Refuse before troubling the LB if the day's budget is already gone.
 
216
  if tracked:
217
  rem = await asyncio.to_thread(limiter.remaining, keys, tier)
218
  if rem is not None and rem <= 0:
@@ -231,6 +240,16 @@ async def session(request: Request):
231
  logger.warning("Load balancer unreachable: %r", exc)
232
  raise HTTPException(status_code=502, detail="Speech service unreachable.")
233
 
 
 
 
 
 
 
 
 
 
 
234
  if lb.status_code != 200:
235
  # The LB's error body may name the reason (e.g. capacity); it carries no
236
  # secret, so relay a trimmed copy.
@@ -238,6 +257,100 @@ async def session(request: Request):
238
  raise HTTPException(status_code=502, detail=f"Session handshake failed ({lb.status_code}).")
239
 
240
  data = lb.json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
  remaining = None
242
  if tracked and data.get("session_id"):
243
  await asyncio.to_thread(limiter.begin, data["session_id"], keys, tier)
@@ -255,6 +368,33 @@ async def session(request: Request):
255
  return resp
256
 
257
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
  async def _session_id(request: Request) -> str:
259
  """Pull `sessionId` from a JSON body, tolerating sendBeacon's blob posts."""
260
  try:
 
22
  GET /api/config -> { search, lb, allowDirect, auth }
23
  GET /api/me -> login + tier + remaining budget (LB mode only)
24
  POST /api/search -> { results, answer } Google via Serper.dev
25
+ POST /api/session -> proxies <LB>/session: a grant, or a queue ticket
26
+ GET /api/queue/{id} -> proxies <LB>/queue/{id}: position, or a grant on claim
27
+ DELETE /api/queue/{id} -> leave the queue (explicit "Leave queue" button)
28
+ POST /api/queue/end -> leave the queue (sendBeacon on teardown)
29
  POST /api/session/heartbeat-> extend the reservation; { expired }
30
  POST /api/session/end -> reconcile + refund (sendBeacon on teardown)
31
  /* -> static files (index.html, main.js, ...)
32
+
33
+ When every compute slot is busy the load balancer hands back a queue ticket
34
+ instead of a grant; the browser polls /api/queue/{id} until it reaches the front
35
+ and a slot frees. Waiting reserves nothing — the daily budget is only reserved at
36
+ the moment a slot is actually claimed (a grant), never while queued.
37
  """
38
 
39
  import asyncio
 
220
  # nothing is tracked. Within metering, unlimited tiers (pro, org) aren't either.
221
  tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None
222
 
223
+ # Refuse before troubling the LB if the day's budget is already gone. Done
224
+ # here (at enqueue) so we never put a user who can't talk into the queue.
225
  if tracked:
226
  rem = await asyncio.to_thread(limiter.remaining, keys, tier)
227
  if rem is not None and rem <= 0:
 
240
  logger.warning("Load balancer unreachable: %r", exc)
241
  raise HTTPException(status_code=502, detail="Speech service unreachable.")
242
 
243
+ # The queue is full: the LB replies 503 {state:"at_capacity"}. Relay it as-is
244
+ # so the client shows a soft "try again shortly", not a hard error.
245
+ if lb.status_code == 503:
246
+ body = _safe_json(lb)
247
+ if body.get("state") == "at_capacity":
248
+ resp = JSONResponse({"state": "at_capacity"}, status_code=503)
249
+ if set_cookie:
250
+ auth.set_anon_cookie(resp, set_cookie)
251
+ return resp
252
+
253
  if lb.status_code != 200:
254
  # The LB's error body may name the reason (e.g. capacity); it carries no
255
  # secret, so relay a trimmed copy.
 
257
  raise HTTPException(status_code=502, detail=f"Session handshake failed ({lb.status_code}).")
258
 
259
  data = lb.json()
260
+
261
+ # Busy pool: the LB queued us. Relay the ticket untouched — crucially with NO
262
+ # reservation, so waiting in line never costs the day's budget.
263
+ if data.get("state") == "queued":
264
+ data["tier"] = tier
265
+ resp = JSONResponse(data)
266
+ if set_cookie:
267
+ auth.set_anon_cookie(resp, set_cookie)
268
+ return resp
269
+
270
+ # A slot was free: reserve the first chunk now and return the grant.
271
+ return await _finalize_grant(data, keys, tier, tracked, set_cookie)
272
+
273
+
274
+ @app.get("/api/queue/{queue_id}")
275
+ async def queue_status(queue_id: str, request: Request):
276
+ """Poll a waiting ticket: relay the position, or — when the head of the line
277
+ claims a freed slot — reserve the budget now and return the grant. Re-checks the
278
+ daily budget at claim, since a multi-minute wait could have spent it elsewhere."""
279
+ if not LOAD_BALANCER_URL:
280
+ raise HTTPException(status_code=404, detail="Not found.")
281
+
282
+ tier, keys, set_cookie = auth.resolve_identity(request)
283
+ tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None
284
+
285
+ url = f"{LOAD_BALANCER_URL.rstrip('/')}/queue/{queue_id}"
286
+ try:
287
+ async with httpx.AsyncClient(timeout=15.0) as http:
288
+ lb = await http.get(url)
289
+ except httpx.RequestError as exc:
290
+ logger.warning("Load balancer unreachable: %r", exc)
291
+ raise HTTPException(status_code=502, detail="Speech service unreachable.")
292
+
293
+ if lb.status_code == 404:
294
+ # Ticket unknown/expired (reaped after we stopped polling). Tell the client
295
+ # to start over rather than spin.
296
+ resp = JSONResponse({"state": "expired"}, status_code=404)
297
+ if set_cookie:
298
+ auth.set_anon_cookie(resp, set_cookie)
299
+ return resp
300
+
301
+ if lb.status_code != 200:
302
+ logger.warning("Queue poll failed %s: %s", lb.status_code, lb.text[:300])
303
+ raise HTTPException(status_code=502, detail=f"Queue poll failed ({lb.status_code}).")
304
+
305
+ data = lb.json()
306
+
307
+ if data.get("state") == "queued":
308
+ data["tier"] = tier
309
+ resp = JSONResponse(data)
310
+ if set_cookie:
311
+ auth.set_anon_cookie(resp, set_cookie)
312
+ return resp
313
+
314
+ # Claimed a slot. Re-check the budget: it may have been spent in another tab
315
+ # during the wait. If so, refuse — the just-claimed slot is now a pending
316
+ # session on the LB and its pending-timeout reaper reclaims it shortly.
317
+ if tracked:
318
+ rem = await asyncio.to_thread(limiter.remaining, keys, tier)
319
+ if rem is not None and rem <= 0:
320
+ resp = JSONResponse(
321
+ {"tier": tier, "reason": "limit", "remainingSec": 0}, status_code=402
322
+ )
323
+ if set_cookie:
324
+ auth.set_anon_cookie(resp, set_cookie)
325
+ return resp
326
+
327
+ return await _finalize_grant(data, keys, tier, tracked, set_cookie)
328
+
329
+
330
+ @app.delete("/api/queue/{queue_id}")
331
+ async def queue_leave(queue_id: str):
332
+ """Leave the queue from the explicit 'Leave queue' button (a real fetch)."""
333
+ if not LOAD_BALANCER_URL:
334
+ raise HTTPException(status_code=404, detail="Not found.")
335
+ await _lb_leave(queue_id)
336
+ return {"ok": True}
337
+
338
+
339
+ @app.post("/api/queue/end")
340
+ async def queue_end(request: Request):
341
+ """Leave the queue on teardown/tab-close (navigator.sendBeacon, which can only
342
+ POST). Body: { queueId }. Best-effort; the LB reaps the ticket on TTL anyway."""
343
+ if not LOAD_BALANCER_URL:
344
+ raise HTTPException(status_code=404, detail="Not found.")
345
+ qid = await _queue_id(request)
346
+ if qid:
347
+ await _lb_leave(qid)
348
+ return {"ok": True}
349
+
350
+
351
+ async def _finalize_grant(data, keys, tier, tracked, set_cookie):
352
+ """Shared grant tail (fast path or queue claim): reserve the first chunk, attach
353
+ the metering fields the client needs, and set the anon cookie."""
354
  remaining = None
355
  if tracked and data.get("session_id"):
356
  await asyncio.to_thread(limiter.begin, data["session_id"], keys, tier)
 
368
  return resp
369
 
370
 
371
+ async def _lb_leave(queue_id: str) -> None:
372
+ """Best-effort: tell the LB to drop a waiting ticket."""
373
+ url = f"{LOAD_BALANCER_URL.rstrip('/')}/queue/{queue_id}"
374
+ try:
375
+ async with httpx.AsyncClient(timeout=5.0) as http:
376
+ await http.delete(url)
377
+ except httpx.RequestError as exc:
378
+ logger.warning("Queue leave failed: %r", exc)
379
+
380
+
381
+ def _safe_json(response) -> dict:
382
+ try:
383
+ body = response.json()
384
+ except Exception:
385
+ return {}
386
+ return body if isinstance(body, dict) else {}
387
+
388
+
389
+ async def _queue_id(request: Request) -> str:
390
+ """Pull `queueId` from a JSON body, tolerating sendBeacon's blob posts."""
391
+ try:
392
+ data = await request.json()
393
+ except Exception:
394
+ return ""
395
+ return (data or {}).get("queueId", "") if isinstance(data, dict) else ""
396
+
397
+
398
  async def _session_id(request: Request) -> str:
399
  """Pull `sessionId` from a JSON body, tolerating sendBeacon's blob posts."""
400
  try:
style.css CHANGED
@@ -778,6 +778,7 @@ a:hover {
778
  .state-connected .ind-spinner,
779
  .state-auto-selecting .ind-spinner,
780
  .state-starting .ind-spinner,
 
781
  .state-processing .ind-thinking,
782
  .state-listening .ind-bars,
783
  .state-user-speaking .ind-bars,
@@ -846,6 +847,34 @@ a:hover {
846
  letter-spacing: 0.08em;
847
  }
848
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
849
  /* ─── Tool-call toaster ───────────────────────────────────────────────── */
850
 
851
  /* A small, non-interactive pill that appears below the circle when the
@@ -1091,6 +1120,11 @@ body.booting *::after {
1091
  .circle.state-connected,
1092
  .circle.state-auto-selecting,
1093
  .circle.state-starting { --glow: #facc15; }
 
 
 
 
 
1094
  .circle.state-listening,
1095
  .circle.state-user-speaking { --glow: var(--listening); }
1096
  .circle.state-processing { --glow: var(--processing); }
 
778
  .state-connected .ind-spinner,
779
  .state-auto-selecting .ind-spinner,
780
  .state-starting .ind-spinner,
781
+ .state-queued .ind-spinner,
782
  .state-processing .ind-thinking,
783
  .state-listening .ind-bars,
784
  .state-user-speaking .ind-bars,
 
847
  letter-spacing: 0.08em;
848
  }
849
 
850
+ /* "Leave queue": a quiet outlined pill under the caption, shown only while
851
+ * waiting. Understated to match the caption — it's an escape hatch, not a CTA. */
852
+ .leave-queue-btn {
853
+ margin-top: 14px;
854
+ padding: 7px 16px;
855
+ font-family: var(--font-mono);
856
+ font-size: 11px;
857
+ font-weight: 500;
858
+ letter-spacing: 0.12em;
859
+ text-transform: uppercase;
860
+ color: var(--text-faint);
861
+ background: transparent;
862
+ border: 1px solid color-mix(in srgb, var(--text-faint) 35%, transparent);
863
+ border-radius: 999px;
864
+ cursor: pointer;
865
+ transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease;
866
+ }
867
+ .leave-queue-btn:hover {
868
+ color: var(--text);
869
+ border-color: color-mix(in srgb, var(--text-faint) 60%, transparent);
870
+ background: color-mix(in srgb, var(--text-faint) 8%, transparent);
871
+ }
872
+ .leave-queue-btn:focus-visible {
873
+ outline: 2px solid var(--accent);
874
+ outline-offset: 2px;
875
+ }
876
+ .leave-queue-btn[hidden] { display: none; }
877
+
878
  /* ─── Tool-call toaster ───────────────────────────────────────────────── */
879
 
880
  /* A small, non-interactive pill that appears below the circle when the
 
1120
  .circle.state-connected,
1121
  .circle.state-auto-selecting,
1122
  .circle.state-starting { --glow: #facc15; }
1123
+ /* Queued: a calm slate glow, distinct from connecting's active yellow — this is
1124
+ * waiting, not working. The spinner turns slowly and the ring breathes. */
1125
+ .circle.state-queued { --glow: #94a3b8; }
1126
+ .circle.state-queued .ind-spinner { animation-duration: 2.4s; }
1127
+ .circle.state-queued .circle-ring { animation: breathe 2.6s ease-in-out infinite; }
1128
  .circle.state-listening,
1129
  .circle.state-user-speaking { --glow: var(--listening); }
1130
  .circle.state-processing { --glow: var(--processing); }
ws/s2s-ws-client.js CHANGED
@@ -23,7 +23,7 @@
23
  * sees high-level lifecycle events (`status`, `transcript`, `error`,
24
  * `session`), the same shape as the WebRTC client.
25
  *
26
- * @typedef {"idle" | "creating-session" | "connecting" | "connected" |
27
  * "user-speaking" | "processing" | "ai-speaking" | "closed" | "error"
28
  * } WsStatus
29
  *
@@ -50,7 +50,11 @@
50
  * session POST and dials it directly — no load balancer in between.
51
  * @property {string} voice
52
  * @property {string} instructions
53
- * @property {MediaStream} micStream
 
 
 
 
54
  * @property {AudioContext} [audioContext] Pre-created (and resumed) context.
55
  * iOS Safari only lets an AudioContext start from within a user gesture, so
56
  * the caller creates/resumes it synchronously on the orb tap and hands it
@@ -85,6 +89,16 @@ import {
85
  } from "./codec.js";
86
  import { OrbVisualiser, VIS_FFT_SIZE } from "./orb-visualizer.js";
87
 
 
 
 
 
 
 
 
 
 
 
88
  // The s2s pipeline runs internally at 16 kHz mono PCM. The WebRTC transport
89
  // resamples to 48 kHz for Opus, but the WebSocket transport emits the
90
  // native pipeline rate. We don't (can't) override it via `audio.output.format`
@@ -111,6 +125,16 @@ export class S2sWsRealtimeClient extends EventTarget {
111
  : options.loadBalancerUrl
112
  ? `${trimTrailingSlash(options.loadBalancerUrl)}/session`
113
  : "";
 
 
 
 
 
 
 
 
 
 
114
  /** @type {NoiseGate} Mic noise gate; off by default. */
115
  this._noiseGate = options.noiseGate ?? { enabled: false, thresholdDb: -45 };
116
  /** @type {WebSocket | null} */
@@ -213,12 +237,20 @@ export class S2sWsRealtimeClient extends EventTarget {
213
  throw new Error("No session endpoint or direct URL configured");
214
  }
215
  this._setStatus("creating-session");
216
- const session = await this._createSession();
 
217
  this.dispatchEvent(new CustomEvent("session", { detail: { info: session } }));
218
  connectUrl = session.connectUrl;
219
  this._setStatus("connecting");
220
  }
221
 
 
 
 
 
 
 
 
222
  // Spin up the AudioContext + worklets in parallel with the WS dial.
223
  const audioReady = this._setupAudio();
224
  const wsReady = this._openWebSocket(connectUrl);
@@ -226,9 +258,24 @@ export class S2sWsRealtimeClient extends EventTarget {
226
  }
227
 
228
  /**
 
 
229
  * @returns {Promise<WsSessionInfo>}
230
  */
231
- async _createSession() {
 
 
 
 
 
 
 
 
 
 
 
 
 
232
  const url = this._sessionUrl;
233
  console.log("[ws] POST", url);
234
  const response = await fetch(url, {
@@ -240,18 +287,98 @@ export class S2sWsRealtimeClient extends EventTarget {
240
  // The session proxy refused: today's per-tier time budget is spent. Surface
241
  // it as a typed error so the UI shows the limit modal, not a crash.
242
  const body = await response.json().catch(() => ({}));
243
- const err = /** @type {Error & { code?: string; tier?: string }} */ (
244
- new Error("Daily conversation limit reached")
245
- );
246
- err.code = "limit";
247
- err.tier = body?.tier;
248
- throw err;
 
 
249
  }
250
  if (!response.ok) {
251
  const text = await response.text().catch(() => "");
252
  throw new Error(`/session failed (${response.status}): ${text}`);
253
  }
254
  const json = await response.json();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
255
  return {
256
  sessionId: json.session_id,
257
  connectUrl: json.connect_url,
@@ -842,6 +969,15 @@ export class S2sWsRealtimeClient extends EventTarget {
842
  }
843
 
844
  async close() {
 
 
 
 
 
 
 
 
 
845
  this._visualiser?.stop();
846
  this._visualiser = null;
847
  try {
 
23
  * sees high-level lifecycle events (`status`, `transcript`, `error`,
24
  * `session`), the same shape as the WebRTC client.
25
  *
26
+ * @typedef {"idle" | "creating-session" | "queued" | "connecting" | "connected" |
27
  * "user-speaking" | "processing" | "ai-speaking" | "closed" | "error"
28
  * } WsStatus
29
  *
 
50
  * session POST and dials it directly — no load balancer in between.
51
  * @property {string} voice
52
  * @property {string} instructions
53
+ * @property {MediaStream} [micStream] Live mic stream. Provide this OR `acquireMic`.
54
+ * @property {() => Promise<MediaStream>} [acquireMic] Lazily obtain the mic stream,
55
+ * called only once a session is actually granted (after any queue wait). Lets the
56
+ * caller prime mic permission up front but not hold the mic 'in use' indicator on
57
+ * while waiting in line. Ignored if `micStream` is already set.
58
  * @property {AudioContext} [audioContext] Pre-created (and resumed) context.
59
  * iOS Safari only lets an AudioContext start from within a user gesture, so
60
  * the caller creates/resumes it synchronously on the orb tap and hands it
 
89
  } from "./codec.js";
90
  import { OrbVisualiser, VIS_FFT_SIZE } from "./orb-visualizer.js";
91
 
92
+ /** Build an Error carrying a `code` (and optional extra fields) so callers can
93
+ * branch on the failure kind: "limit" | "queue-full" | "queue-expired" | "aborted".
94
+ * @param {string} message @param {string} code @param {object} [extra] */
95
+ function _codedError(message, code, extra) {
96
+ const err = /** @type {Error & { code?: string }} */ (new Error(message));
97
+ err.code = code;
98
+ if (extra) Object.assign(err, extra);
99
+ return err;
100
+ }
101
+
102
  // The s2s pipeline runs internally at 16 kHz mono PCM. The WebRTC transport
103
  // resamples to 48 kHz for Opus, but the WebSocket transport emits the
104
  // native pipeline rate. We don't (can't) override it via `audio.output.format`
 
125
  : options.loadBalancerUrl
126
  ? `${trimTrailingSlash(options.loadBalancerUrl)}/session`
127
  : "";
128
+ /** @type {(() => Promise<MediaStream>) | null} Lazy mic acquisition (post-grant). */
129
+ this._acquireMic = options.acquireMic ?? null;
130
+ /** @type {boolean} Set by close() to abort a queue wait in progress. */
131
+ this._closed = false;
132
+ /** @type {string} The active queue ticket id while waiting (else ""). */
133
+ this._queueId = "";
134
+ /** @type {(() => void) | null} Wakes the queue poll sleep early on close(). */
135
+ this._queueWake = null;
136
+ /** @type {ReturnType<typeof setTimeout> | 0} */
137
+ this._queueTimer = 0;
138
  /** @type {NoiseGate} Mic noise gate; off by default. */
139
  this._noiseGate = options.noiseGate ?? { enabled: false, thresholdDb: -45 };
140
  /** @type {WebSocket | null} */
 
237
  throw new Error("No session endpoint or direct URL configured");
238
  }
239
  this._setStatus("creating-session");
240
+ const session = await this._createSessionOrQueue();
241
+ if (this._closed) throw _codedError("connect aborted", "aborted");
242
  this.dispatchEvent(new CustomEvent("session", { detail: { info: session } }));
243
  connectUrl = session.connectUrl;
244
  this._setStatus("connecting");
245
  }
246
 
247
+ // Acquire the mic now — only once a slot is actually ours. The caller primed
248
+ // permission up front, so this is silent and the 'in use' indicator lights
249
+ // only for a real, connecting session (never during a queue wait).
250
+ if (!this.options.micStream && this._acquireMic) {
251
+ this.options.micStream = await this._acquireMic();
252
+ }
253
+
254
  // Spin up the AudioContext + worklets in parallel with the WS dial.
255
  const audioReady = this._setupAudio();
256
  const wsReady = this._openWebSocket(connectUrl);
 
258
  }
259
 
260
  /**
261
+ * POST the session handshake; if the pool is busy, wait in the queue (polling
262
+ * position) until a slot is claimed. Resolves to a grant either way.
263
  * @returns {Promise<WsSessionInfo>}
264
  */
265
+ async _createSessionOrQueue() {
266
+ const first = await this._postSession();
267
+ if (first.state === "queued") {
268
+ this._setStatus("queued");
269
+ return await this._pollQueue(first);
270
+ }
271
+ return first.grant;
272
+ }
273
+
274
+ /**
275
+ * POST /session once. Returns either a granted session or a queue ticket.
276
+ * @returns {Promise<{ state: "granted", grant: WsSessionInfo } | { state: "queued", queueId: string, position: number, pollIntervalS: number }>}
277
+ */
278
+ async _postSession() {
279
  const url = this._sessionUrl;
280
  console.log("[ws] POST", url);
281
  const response = await fetch(url, {
 
287
  // The session proxy refused: today's per-tier time budget is spent. Surface
288
  // it as a typed error so the UI shows the limit modal, not a crash.
289
  const body = await response.json().catch(() => ({}));
290
+ throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier });
291
+ }
292
+ if (response.status === 503) {
293
+ const body = await response.json().catch(() => ({}));
294
+ if (body?.state === "at_capacity") {
295
+ throw _codedError("The queue is full — try again shortly.", "queue-full");
296
+ }
297
+ throw new Error("/session failed (503)");
298
  }
299
  if (!response.ok) {
300
  const text = await response.text().catch(() => "");
301
  throw new Error(`/session failed (${response.status}): ${text}`);
302
  }
303
  const json = await response.json();
304
+ if (json.state === "queued") {
305
+ return {
306
+ state: "queued",
307
+ queueId: json.queue_id,
308
+ position: json.position,
309
+ pollIntervalS: json.poll_interval_s,
310
+ };
311
+ }
312
+ return { state: "granted", grant: this._parseGrant(json) };
313
+ }
314
+
315
+ /**
316
+ * Poll the waiting queue until this ticket claims a slot. Emits `queue` events
317
+ * ({ position }) as the line advances. Throws on limit (402), expiry (404), or
318
+ * close(). Transient network/5xx blips are ignored and retried next tick.
319
+ * @param {{ queueId: string, position: number, pollIntervalS: number }} ticket
320
+ * @returns {Promise<WsSessionInfo>}
321
+ */
322
+ async _pollQueue(ticket) {
323
+ const intervalMs = Math.max(1, ticket.pollIntervalS || 2) * 1000;
324
+ this._queueId = ticket.queueId;
325
+ this._emitQueue(ticket.position);
326
+
327
+ while (true) {
328
+ await this._queueSleep(intervalMs);
329
+ if (this._closed) throw _codedError("queue wait aborted", "aborted");
330
+
331
+ let response;
332
+ try {
333
+ response = await fetch(`api/queue/${encodeURIComponent(this._queueId)}`, {
334
+ headers: { "Content-Type": "application/json" },
335
+ });
336
+ } catch {
337
+ continue; // network blip — keep our place, retry next tick
338
+ }
339
+
340
+ if (response.status === 402) {
341
+ const body = await response.json().catch(() => ({}));
342
+ throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier });
343
+ }
344
+ if (response.status === 404) {
345
+ throw _codedError("Queue timed out", "queue-expired");
346
+ }
347
+ if (!response.ok) continue; // 502/503 — transient, retry
348
+
349
+ const json = await response.json().catch(() => null);
350
+ if (!json) continue;
351
+ if (json.state === "queued") {
352
+ this._emitQueue(json.position);
353
+ continue;
354
+ }
355
+ // Reached the front and claimed a slot.
356
+ this._queueId = "";
357
+ return this._parseGrant(json);
358
+ }
359
+ }
360
+
361
+ /** @param {number} position */
362
+ _emitQueue(position) {
363
+ this.dispatchEvent(
364
+ new CustomEvent("queue", { detail: { position, queueId: this._queueId } }),
365
+ );
366
+ }
367
+
368
+ /** A sleep that close() can cut short so a queued client tears down promptly.
369
+ * @param {number} ms */
370
+ _queueSleep(ms) {
371
+ return new Promise((resolve) => {
372
+ this._queueWake = resolve;
373
+ this._queueTimer = setTimeout(() => {
374
+ this._queueWake = null;
375
+ resolve();
376
+ }, ms);
377
+ });
378
+ }
379
+
380
+ /** @param {any} json @returns {WsSessionInfo} */
381
+ _parseGrant(json) {
382
  return {
383
  sessionId: json.session_id,
384
  connectUrl: json.connect_url,
 
969
  }
970
 
971
  async close() {
972
+ // Abort a queue wait in progress: flag it and wake the poll sleep so
973
+ // `_pollQueue` throws "aborted" and connect() unwinds cleanly.
974
+ this._closed = true;
975
+ if (this._queueWake) {
976
+ clearTimeout(this._queueTimer);
977
+ const wake = this._queueWake;
978
+ this._queueWake = null;
979
+ wake();
980
+ }
981
  this._visualiser?.stop();
982
  this._visualiser = null;
983
  try {