Skip to main content

hyprstream_core/server/routes/
openai.rs

1//! OpenAI-compatible API endpoints
2
3use axum::{
4    extract::{Json, State},
5    http::{header, HeaderMap, StatusCode},
6    response::{IntoResponse, Sse},
7    routing::{get, post},
8    Extension, Router,
9};
10use futures::stream::StreamExt;
11use hyprstream_rpc_std::inference_client::GenerationRequest;
12use hyprstream_rpc_std::model_client::ModelClient;
13use std::convert::Infallible;
14use tokio::sync::mpsc;
15use tokio_stream::wrappers::ReceiverStream;
16use tracing::{error, info, trace};
17
18use crate::{
19    api::{
20        openai_compat::{
21            ChatChoice, ChatCompletionRequest, ChatCompletionResponse, ChatMessage,
22            CompletionChoice, CompletionRequest, CompletionResponse, EmbeddingRequest,
23            ListModelsResponse, Model, OnlineTrainingDetails, ToolChoice, Usage,
24        },
25        tools,
26    },
27    archetypes::capabilities::Infer,
28    auth::Operation,
29    runtime::CacheOwner,
30    server::{state::ServerState, AuthenticatedUser},
31};
32
33/// RAII guard for metrics cleanup
34struct MetricsGuard<'a> {
35    metrics: &'a crate::server::state::Metrics,
36    decremented: bool,
37}
38
39impl<'a> MetricsGuard<'a> {
40    fn new(metrics: &'a crate::server::state::Metrics) -> Self {
41        Self {
42            metrics,
43            decremented: false,
44        }
45    }
46}
47
48impl<'a> Drop for MetricsGuard<'a> {
49    fn drop(&mut self) {
50        if !self.decremented {
51            self.metrics
52                .active_requests
53                .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
54        }
55    }
56}
57
58/// Standard error response format
59#[derive(serde::Serialize)]
60struct ErrorResponse {
61    error: ErrorDetails,
62}
63
64#[derive(serde::Serialize)]
65struct ErrorDetails {
66    message: String,
67    #[serde(rename = "type")]
68    error_type: String,
69    code: String,
70    #[serde(skip_serializing_if = "Option::is_none")]
71    param: Option<&'static str>,
72}
73
74impl ErrorResponse {
75    fn new(
76        message: impl Into<String>,
77        error_type: impl Into<String>,
78        code: impl Into<String>,
79    ) -> Self {
80        Self {
81            error: ErrorDetails {
82                message: message.into(),
83                error_type: error_type.into(),
84                code: code.into(),
85                param: None,
86            },
87        }
88    }
89
90    fn tool_choice(failure: ToolChoiceFailure) -> Self {
91        let (message, code) = match failure {
92            ToolChoiceFailure::RequiredUnsupported => (
93                "tool_choice=required is not supported for this model",
94                "unsupported_tool_choice",
95            ),
96            ToolChoiceFailure::NamedUnsupported => (
97                "Named tool_choice is not supported for this model",
98                "unsupported_tool_choice",
99            ),
100            ToolChoiceFailure::Invalid => ("Invalid tool_choice", "invalid_tool_choice"),
101        };
102        Self {
103            error: ErrorDetails {
104                message: message.to_owned(),
105                error_type: "invalid_request_error".to_owned(),
106                code: code.to_owned(),
107                param: Some("tool_choice"),
108            },
109        }
110    }
111}
112
113#[derive(Debug, Clone, Copy, PartialEq, Eq)]
114enum ToolChoiceFailure {
115    RequiredUnsupported,
116    NamedUnsupported,
117    Invalid,
118}
119
120/// Normalize supported tool choices before selecting either completion path.
121/// The backend cannot force a required or named call, so those requests fail
122/// explicitly instead of returning a plain-text `stop` response.
123fn prepare_chat_request(
124    mut request: ChatCompletionRequest,
125) -> Result<(ChatCompletionRequest, bool), ToolChoiceFailure> {
126    match request.tool_choice.as_ref() {
127        None => {}
128        Some(ToolChoice::String(choice)) => match choice.as_str() {
129            "auto" => {}
130            "none" => request.tools = None,
131            "required" => return Err(ToolChoiceFailure::RequiredUnsupported),
132            _ => return Err(ToolChoiceFailure::Invalid),
133        },
134        Some(ToolChoice::Specific { .. }) => return Err(ToolChoiceFailure::NamedUnsupported),
135    }
136    let is_streaming = request.stream.unwrap_or(false);
137    Ok((request, is_streaming))
138}
139
140fn tool_choice_error_response(failure: ToolChoiceFailure) -> axum::response::Response {
141    (
142        StatusCode::BAD_REQUEST,
143        Json(ErrorResponse::tool_choice(failure)),
144    )
145        .into_response()
146}
147
148/// A stream has already sent HTTP 200, so report a subsequent failure in the
149/// same structured OpenAI error envelope used by non-streaming responses.
150fn stream_error_payload(message: &str) -> serde_json::Value {
151    serde_json::json!({
152        "error": {
153            "message": message,
154            "type": "generation_error",
155            "code": "internal_error",
156            "param": null
157        }
158    })
159}
160
161/// Result of collecting a stream into a single response.
162struct CollectedResult {
163    text: String,
164    stats: crate::services::rpc_types::InferenceComplete,
165}
166
167/// Collect a stream to a single result for non-streaming endpoints.
168///
169/// This replaces the removed sync `infer()` method. Using streaming internally
170/// fixes the bug where `infer()` hardcoded `ttt_metrics: None` — the completion
171/// metadata from the stream includes all fields including TTT metrics.
172async fn collect_stream_to_result(
173    model_client: &ModelClient,
174    model_ref: &str,
175    request: &GenerationRequest,
176) -> anyhow::Result<CollectedResult> {
177    use hyprstream_rpc_std::model_client::InferRpc;
178    let mut handle = InferRpc::generate_stream(&model_client.infer(model_ref), request).await?;
179
180    use futures::StreamExt;
181    use hyprstream_rpc::streaming::StreamPayload;
182
183    let mut text = String::new();
184    loop {
185        match handle.next().await {
186            Some(Ok(payload)) => match payload {
187                StreamPayload::Data(data) => {
188                    text.push_str(&String::from_utf8_lossy(&data));
189                }
190                StreamPayload::Complete(meta) => {
191                    let stats: crate::services::rpc_types::InferenceComplete =
192                        serde_json::from_slice(&meta).unwrap_or_else(|e| {
193                            tracing::warn!("Failed to parse InferenceComplete: {e}");
194                            crate::services::rpc_types::InferenceComplete::empty()
195                        });
196                    return Ok(CollectedResult { text, stats });
197                }
198                StreamPayload::Error(msg) => {
199                    anyhow::bail!("Generation error: {msg}");
200                }
201                StreamPayload::Tagged { .. } => {
202                    // encrypted event payload, skip
203                    continue;
204                }
205            },
206            Some(Err(e)) => {
207                anyhow::bail!("Stream error: {e}");
208            }
209            None => {
210                anyhow::bail!("Stream ended without completion");
211            }
212        }
213    }
214}
215
216// validate_chat_request removed - validation now handled by streaming pipeline
217// The TextStream and sampling code handle all parameter edge cases safely:
218// - Empty messages → empty response (safe)
219// - Invalid temperature/top_p → clamped or falls back to greedy (safe)
220// - max_tokens → enforced by generation loop (safe)
221// Rate limiting should be handled at middleware layer, not per-endpoint
222
223/// Extract cache owner from request headers.
224///
225/// Session management:
226/// - `x-session-id` header: Uses session-based caching (context preserved)
227/// - No header: Generates a stateless request ID (context not preserved)
228///
229/// The session ID is used to route the request to the appropriate KV cache,
230/// allowing multiple concurrent sessions to have isolated context.
231fn extract_cache_owner(headers: &HeaderMap) -> CacheOwner {
232    if let Some(session_id) = headers
233        .get("x-session-id")
234        .and_then(|v| v.to_str().ok())
235        .filter(|s| !s.is_empty())
236    {
237        trace!("Using session-based caching for session: {}", session_id);
238        CacheOwner::Session(session_id.to_owned())
239    } else {
240        // Stateless request - generate unique ID
241        let request_id = std::time::SystemTime::now()
242            .duration_since(std::time::UNIX_EPOCH)
243            .map(|d| d.as_nanos() as u64)
244            .unwrap_or(0);
245        trace!("Using stateless caching for request: {}", request_id);
246        CacheOwner::Stateless(request_id)
247    }
248}
249
250/// Extract auth info, including the authority-verified hosted-account tenant.
251fn extract_auth(
252    auth_user: Option<&AuthenticatedUser>,
253) -> Result<(String, String, Option<String>, Option<i64>), &'static str> {
254    let user = auth_user.ok_or("authenticated identity missing")?;
255    trace!("Using authenticated user: {}", user.user);
256    Ok((
257        user.user.clone(),
258        user.authorization_domain()?,
259        user.token.clone(),
260        user.exp,
261    ))
262}
263
264/// Build Claims from auth info, using the JWT's real exp for timeout enforcement.
265fn claims_from_auth(
266    user: &str,
267    domain: &str,
268    jwt_token: Option<&str>,
269    jwt_exp: Option<i64>,
270) -> hyprstream_rpc::auth::Claims {
271    let now = chrono::Utc::now().timestamp();
272    let exp = jwt_exp.unwrap_or(now + 3600);
273    let mut claims = hyprstream_rpc::auth::Claims::new(user.to_owned(), now, exp);
274    if domain != "*" {
275        claims = claims.with_tenant(domain.to_owned());
276    }
277    if let Some(token) = jwt_token {
278        claims = claims.with_token(token.to_owned());
279    }
280    claims
281}
282
283/// Preserve the verified HTTP user's bearer under OAI's own service signer.
284/// Model must independently verify both the current relay key and original
285/// token. Missing user or service credentials never fall back to OAI authority.
286fn model_client_for_request(
287    state: &ServerState,
288    jwt_token: Option<&str>,
289) -> anyhow::Result<ModelClient> {
290    let bearer = jwt_token
291        .filter(|token| !token.trim().is_empty())
292        .ok_or_else(|| anyhow::anyhow!("verified OAI user bearer is required"))?;
293    let signer = (*state.signing_key).clone();
294    let service_token = crate::services::factories::service_token(&signer)
295        .ok_or_else(|| anyhow::anyhow!("OAI service credential is unavailable"))?;
296    let client = ModelClient::from_provider(
297        &hyprstream_discovery::ProductionRpcClientProvider,
298        signer,
299        Some(service_token),
300    )?;
301    Ok(client.with_delegated_bearer(bearer.to_owned()))
302}
303
304/// Helper: Resolve model name to filesystem path (also validates inference capability)
305async fn resolve_model_path(
306    state: &ServerState,
307    model_name: &str,
308) -> Result<std::path::PathBuf, impl IntoResponse> {
309    let model_ref = match crate::storage::ModelRef::parse(model_name) {
310        Ok(r) => r,
311        Err(e) => {
312            return Err((
313                StatusCode::BAD_REQUEST,
314                Json(ErrorResponse::new(
315                    format!("Invalid model reference: {e}"),
316                    "invalid_model_ref",
317                    "parse_error",
318                )),
319            )
320                .into_response());
321        }
322    };
323
324    // Inline model path resolution
325    let path_result: Result<std::path::PathBuf, _> = async {
326        let tracked = state.registry.get_by_name(model_ref.name()).await?;
327        let repo = state.registry.repo(&tracked.id);
328        let branch = match &model_ref.git_ref {
329            crate::storage::GitRef::Branch(name) => name.clone(),
330            _ => repo.get_head().await?,
331        };
332        // Verify worktree exists
333        let wts = repo.list_worktrees().await?;
334        if !wts.iter().any(|wt| wt.branch_name == branch) {
335            anyhow::bail!("worktree for {}:{} not found", model_ref.name(), branch);
336        }
337        // Derive path locally
338        let storage_paths = crate::storage::StoragePaths::new()?;
339        storage_paths.worktree_path(model_ref.name(), &branch)
340    }
341    .await;
342
343    match path_result {
344        Ok(path) => {
345            // Check if model has INFERENCE capability
346            let archetype_registry = crate::archetypes::global_registry();
347            let detected = archetype_registry.detect(&path);
348
349            let domains = detected.to_detected_domains();
350            if !domains.has::<Infer>() {
351                return Err((
352                    StatusCode::BAD_REQUEST,
353                    Json(ErrorResponse::new(
354                        format!(
355                            "Model '{}' does not support inference. Detected domains: {:?}",
356                            model_name, domains.domains
357                        ),
358                        "capability_error",
359                        "inference_not_supported",
360                    )),
361                )
362                    .into_response());
363            }
364
365            Ok(path)
366        }
367        Err(e) => Err((
368            StatusCode::NOT_FOUND,
369            Json(ErrorResponse::new(
370                format!("Model path not found: {e}"),
371                "model_not_found",
372                "path_error",
373            )),
374        )
375            .into_response()),
376    }
377}
378
379/// Helper: Add no-cache headers to response
380fn add_no_cache_headers(response: &mut axum::response::Response) {
381    use axum::http::HeaderValue;
382    let headers = response.headers_mut();
383    headers.insert(
384        header::CACHE_CONTROL,
385        HeaderValue::from_static("no-cache, no-store, must-revalidate"),
386    );
387    headers.insert(header::PRAGMA, HeaderValue::from_static("no-cache"));
388}
389
390/// Create OpenAI API router
391pub fn create_router() -> Router<ServerState> {
392    Router::new()
393        .route("/chat/completions", post(chat_completions))
394        .route("/completions", post(completions))
395        .route("/embeddings", post(embeddings))
396        .route("/models", get(list_models))
397}
398
399/// Handle chat completion requests
400async fn chat_completions(
401    State(state): State<ServerState>,
402    auth_user: Option<Extension<AuthenticatedUser>>,
403    headers: HeaderMap,
404    Json(request): Json<ChatCompletionRequest>,
405) -> impl IntoResponse {
406    // Extract user identity from JWT (via middleware)
407    let (user, domain, jwt_token, jwt_exp) =
408        match extract_auth(auth_user.as_ref().map(|Extension(u)| u)) {
409            Ok(identity) => identity,
410            Err(_) => {
411                return (
412                    StatusCode::FORBIDDEN,
413                    Json(ErrorResponse::new(
414                        "Verified hosted-account tenant binding required",
415                        "permission_denied",
416                        "missing_tenant",
417                    )),
418                )
419                    .into_response();
420            }
421        };
422
423    // Check permission for inference on this model via ZMQ
424    let resource = format!("model:{}", request.model);
425    let policy_request = hyprstream_rpc_std::policy_client::PolicyCheck {
426        subject: user.clone(),
427        domain: domain.clone(),
428        resource: resource.clone(),
429        operation: Operation::Infer.as_str().to_owned(),
430    };
431    match crate::services::policy::check_with_verified_bearer(
432        &state.policy_client,
433        &policy_request,
434        jwt_token.as_deref(),
435        &hyprstream_rpc::envelope::Subject::new(user.clone()),
436    )
437    .await
438    {
439        Ok(allowed) if !allowed => {
440            return (
441                StatusCode::FORBIDDEN,
442                Json(ErrorResponse::new(
443                    format!(
444                        "Permission denied: user '{}' cannot infer on '{}'",
445                        user, request.model
446                    ),
447                    "permission_denied",
448                    "insufficient_permissions",
449                )),
450            )
451                .into_response();
452        }
453        Err(e) => {
454            error!("Policy check failed: {}", e);
455            return (
456                StatusCode::INTERNAL_SERVER_ERROR,
457                Json(ErrorResponse::new(
458                    format!("Policy check failed: {e}"),
459                    "policy_error",
460                    "internal_error",
461                )),
462            )
463                .into_response();
464        }
465        _ => {} // allowed
466    }
467
468    let (request, is_streaming) = match prepare_chat_request(request) {
469        Ok(prepared) => prepared,
470        Err(failure) => return tool_choice_error_response(failure),
471    };
472
473    // Extract session/request ID for KV cache routing
474    let cache_owner = extract_cache_owner(&headers);
475
476    // Debug log the incoming request
477    info!(
478        "Chat completion request - model: {}, stream: {:?}, messages: {} msgs, cache: {:?}",
479        request.model,
480        request.stream,
481        request.messages.len(),
482        cache_owner
483    );
484
485    // Log if streaming is defaulting
486    info!(
487        "Streaming mode: {} (explicit: {:?})",
488        is_streaming, request.stream
489    );
490
491    if is_streaming {
492        info!("Handling streaming request");
493        return stream_chat(state, headers, request, user, domain, jwt_token, jwt_exp)
494            .await
495            .into_response();
496    }
497    info!("Handling non-streaming request");
498
499    state
500        .metrics
501        .active_requests
502        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
503    state
504        .metrics
505        .total_requests
506        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
507    let _metrics_guard = MetricsGuard::new(&state.metrics);
508
509    let start_time = std::time::Instant::now();
510
511    // Resolve model path (also validates inference capability)
512    let model_path = match resolve_model_path(&state, &request.model).await {
513        Ok(path) => path,
514        Err(response) => return response.into_response(),
515    };
516
517    // Apply chat template via ZMQ ModelService
518    info!("Applying chat template via ModelService...");
519    let model_client = match model_client_for_request(&state, jwt_token.as_deref()) {
520        Ok(client) => client,
521        Err(e) => {
522            tracing::error!("Failed to create authenticated ModelClient: {e}");
523            return (StatusCode::INTERNAL_SERVER_ERROR, "Client creation failed").into_response();
524        }
525    };
526
527    // Serialize tools to JSON Value for the template engine (if provided)
528    let tools_json: Option<serde_json::Value> = request.tools.as_ref().map(|t| {
529        info!("Passing {} tools to template engine", t.len());
530        serde_json::to_value(t).unwrap_or_default()
531    });
532
533    // Determine tool call format from model architecture (for response parsing)
534    let tool_call_format = if request.tools.is_some() {
535        tools::ToolCallFormat::from_architecture(
536            &crate::runtime::model_config::ModelConfig::detect_architecture(&model_path),
537        )
538    } else {
539        tools::ToolCallFormat::None
540    };
541
542    let rpc_messages = to_rpc_messages(&request.messages);
543    let tools_str = tools_json
544        .as_ref()
545        .map(|t| serde_json::to_string(t).unwrap_or_default())
546        .unwrap_or_default();
547    let templated_prompt = match model_client
548        .infer(&request.model)
549        .apply_chat_template(&hyprstream_rpc_std::model_client::ChatTemplateRequest {
550            messages: rpc_messages.clone(),
551            add_generation_prompt: true,
552            tools_json: Some(tools_str.clone()).filter(|s| !s.is_empty()),
553            max_tokens: request.max_tokens.map(|v| v as u32),
554            enable_thinking: None,
555            template_vars_json: None,
556        })
557        .await
558    {
559        Ok(prompt_str) => crate::config::TemplatedPrompt::new(prompt_str),
560        Err(e) => {
561            error!("Failed to apply chat template: {}", e);
562            return (
563                StatusCode::INTERNAL_SERVER_ERROR,
564                Json(ErrorResponse::new(
565                    format!("Failed to apply chat template: {e}"),
566                    "template_error",
567                    "internal_error",
568                )),
569            )
570                .into_response();
571        }
572    };
573    info!(
574        "Template applied, prompt length: {}",
575        templated_prompt.len()
576    );
577
578    // Build generation request with templated prompt
579    let gen_params = crate::config::SamplingParams::from(&state.config.sampling_defaults)
580        .merge(
581            crate::config::SamplingParams::from_model_path(&model_path)
582                .await
583                .unwrap_or_default(),
584        )
585        .merge((&request).into());
586    let gen_request = gen_params.into_generation_request(templated_prompt.into_inner());
587
588    info!(
589        "Using generation config: max_tokens={:?}, temp={:?}, top_p={:?}, top_k={:?}, repeat_penalty={:?}",
590        gen_request.max_tokens, gen_request.temperature, gen_request.top_p, gen_request.top_k, gen_request.repeat_penalty
591    );
592
593    info!(
594        "Starting generation with prompt length: {}",
595        gen_request.prompt.len()
596    );
597
598    let _claims = claims_from_auth(&user, &domain, jwt_token.as_deref(), jwt_exp);
599    // The request-local client carries the verified HTTP JWT through both
600    // template formatting and inference dispatch.
601    let result = collect_stream_to_result(&model_client, &request.model, &gen_request).await;
602
603    info!("Generation completed - success: {}", result.is_ok());
604
605    match result {
606        Ok(collected) => {
607            let tokens_generated = collected.stats.tokens_generated;
608            state.metrics.total_tokens.fetch_add(
609                tokens_generated as u64,
610                std::sync::atomic::Ordering::Relaxed,
611            );
612
613            let latency_ms = start_time.elapsed().as_millis() as f64;
614            let mut avg_latency = state.metrics.avg_latency_ms.write().await;
615            *avg_latency = (*avg_latency * 0.9) + (latency_ms * 0.1);
616
617            let finish_reason_str = &collected.stats.finish_reason;
618
619            // Parse tool calls from response using format-aware detection
620            let (content, tool_calls, finish_reason) =
621                if tools::has_tool_calls_for_format(tool_call_format, &collected.text) {
622                    info!(
623                        "Detected tool calls in response (format: {:?})",
624                        tool_call_format
625                    );
626                    match tools::parse_tool_calls_for_format(tool_call_format, &collected.text) {
627                        Ok(parsed_tool_calls) if !parsed_tool_calls.is_empty() => {
628                            let content_text = tools::extract_text_content_for_format(
629                                tool_call_format,
630                                &collected.text,
631                            );
632                            let content = if content_text.is_empty() {
633                                None
634                            } else {
635                                Some(content_text)
636                            };
637                            info!("Parsed {} tool calls", parsed_tool_calls.len());
638                            (content, Some(parsed_tool_calls), "tool_calls")
639                        }
640                        Ok(_) | Err(_) => {
641                            let fr = if finish_reason_str == "max_tokens" {
642                                "length"
643                            } else {
644                                "stop"
645                            };
646                            (Some(collected.text), None, fr)
647                        }
648                    }
649                } else {
650                    let fr = if finish_reason_str == "max_tokens" {
651                        "length"
652                    } else {
653                        "stop"
654                    };
655                    (Some(collected.text), None, fr)
656                };
657
658            let response = ChatCompletionResponse {
659                id: format!("chatcmpl-{}", uuid::Uuid::new_v4()),
660                object: "chat.completion".to_owned(),
661                created: chrono::Utc::now().timestamp(),
662                model: request.model.clone(),
663                choices: vec![ChatChoice {
664                    index: 0,
665                    message: ChatMessage {
666                        role: "assistant".to_owned(),
667                        content,
668                        tool_calls,
669                        tool_call_id: None,
670                    },
671                    finish_reason: Some(finish_reason.to_owned()),
672                }],
673                usage: Some(Usage {
674                    prompt_tokens: 0,
675                    completion_tokens: tokens_generated,
676                    total_tokens: tokens_generated,
677                    online_training_details: collected
678                        .stats
679                        .ttt_metrics
680                        .as_ref()
681                        .map(OnlineTrainingDetails::from),
682                }),
683            };
684
685            let mut response = Json(response).into_response();
686            add_no_cache_headers(&mut response);
687            response
688        }
689        Err(e) => {
690            error!("Generation failed: {}", e);
691            (
692                StatusCode::INTERNAL_SERVER_ERROR,
693                Json(serde_json::json!({
694                    "error": {
695                        "message": format!("Generation failed: {}", e),
696                        "type": "generation_error",
697                        "code": "internal_error"
698                    }
699                })),
700            )
701                .into_response()
702        }
703    }
704}
705
706/// Handle streaming chat completions via ZMQ PUB/SUB
707async fn stream_chat(
708    state: ServerState,
709    _headers: HeaderMap,
710    request: ChatCompletionRequest,
711    user: String,
712    domain: String,
713    jwt_token: Option<String>,
714    jwt_exp: Option<i64>,
715) -> impl IntoResponse {
716    // Create channel for SSE events
717    let (tx, rx) = mpsc::channel::<Result<serde_json::Value, anyhow::Error>>(100);
718
719    // Clone state for metrics cleanup
720    let state_clone = state.clone();
721
722    // Spawn generation task with configured defaults
723    let defaults = state.config.sampling_defaults.clone();
724    let model_name = request.model.clone();
725    let messages = request.messages.clone();
726    let stop_sequences = request.stop.clone().unwrap_or_default();
727    let tools = request.tools.clone();
728
729    // Track active request for proper cleanup
730    state_clone
731        .metrics
732        .active_requests
733        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
734
735    tokio::spawn(async move {
736        // Ensure metrics are decremented on all exit paths
737        let _metrics_guard = MetricsGuard::new(&state.metrics);
738
739        // Get model path for config loading
740        let model_ref = match crate::storage::ModelRef::parse(&model_name) {
741            Ok(r) => r,
742            Err(e) => {
743                let _ = tx
744                    .send(Err(anyhow::anyhow!("Invalid model reference: {}", e)))
745                    .await;
746                return;
747            }
748        };
749
750        let model_path = match async {
751            let tracked = state.registry.get_by_name(model_ref.name()).await?;
752            let repo = state.registry.repo(&tracked.id);
753            let branch = match &model_ref.git_ref {
754                crate::storage::GitRef::Branch(name) => name.clone(),
755                _ => repo.get_head().await?,
756            };
757            // Verify worktree exists
758            let wts = repo.list_worktrees().await?;
759            if !wts.iter().any(|wt| wt.branch_name == branch) {
760                anyhow::bail!("worktree not found");
761            }
762            // Derive path locally
763            let storage_paths = crate::storage::StoragePaths::new()?;
764            storage_paths.worktree_path(model_ref.name(), &branch)
765        }
766        .await
767        {
768            Ok(path) => path,
769            Err(e) => {
770                let _ = tx
771                    .send(Err(anyhow::anyhow!("Could not get model path: {}", e)))
772                    .await;
773                return;
774            }
775        };
776
777        // Serialize tools to JSON Value for the template engine
778        let tools_json: Option<serde_json::Value> = tools.as_ref().map(|t| {
779            info!("Passing {} tools to streaming template engine", t.len());
780            serde_json::to_value(t).unwrap_or_default()
781        });
782
783        // Determine tool call format from model architecture
784        let tool_call_format = if tools.is_some() {
785            tools::ToolCallFormat::from_architecture(
786                &crate::runtime::model_config::ModelConfig::detect_architecture(&model_path),
787            )
788        } else {
789            tools::ToolCallFormat::None
790        };
791
792        // Apply chat template via ZMQ ModelService (tools passed to template)
793        info!("Applying chat template for streaming...");
794        let model_client = match model_client_for_request(&state, jwt_token.as_deref()) {
795            Ok(client) => client,
796            Err(e) => {
797                error!("Failed to create authenticated ModelClient: {}", e);
798                let _ = tx
799                    .send(Err(anyhow::anyhow!("Client creation failed: {}", e)))
800                    .await;
801                return;
802            }
803        };
804        let rpc_messages = to_rpc_messages(&messages);
805        let tools_str = tools_json
806            .as_ref()
807            .map(|t| serde_json::to_string(t).unwrap_or_default())
808            .unwrap_or_default();
809        let templated_prompt = match model_client
810            .infer(&model_name)
811            .apply_chat_template(&hyprstream_rpc_std::model_client::ChatTemplateRequest {
812                messages: rpc_messages.clone(),
813                add_generation_prompt: true,
814                tools_json: Some(tools_str.clone()).filter(|s| !s.is_empty()),
815                max_tokens: request.max_tokens.map(|v| v as u32),
816                enable_thinking: None,
817                template_vars_json: None,
818            })
819            .await
820        {
821            Ok(prompt_str) => {
822                let prompt = crate::config::TemplatedPrompt::new(prompt_str);
823                info!(
824                    "Streaming template applied, prompt length: {}",
825                    prompt.len()
826                );
827                prompt
828            }
829            Err(e) => {
830                error!("Template formatting failed in streaming: {}", e);
831                let _ = tx
832                    .send(Err(anyhow::anyhow!("Template formatting failed: {}", e)))
833                    .await;
834                return;
835            }
836        };
837
838        // Build generation request
839        let mut gen_params = crate::config::SamplingParams::from(&defaults)
840            .merge(
841                crate::config::SamplingParams::from_model_path(&model_path)
842                    .await
843                    .unwrap_or_default(),
844            )
845            .merge((&request).into());
846        if !stop_sequences.is_empty() {
847            gen_params.stop_tokens = Some(stop_sequences);
848        }
849        let gen_request = gen_params.into_generation_request(templated_prompt.into_inner());
850
851        info!(
852            "Streaming: max_tokens={:?}, temp={:?}, top_p={:?}, top_k={:?}, repeat_penalty={:?}",
853            gen_request.max_tokens,
854            gen_request.temperature,
855            gen_request.top_p,
856            gen_request.top_k,
857            gen_request.repeat_penalty
858        );
859
860        let _claims = claims_from_auth(&user, &domain, jwt_token.as_deref(), jwt_exp);
861        use hyprstream_rpc_std::model_client::InferRpc;
862        let mut stream_handle =
863            match InferRpc::generate_stream(&model_client.infer(&model_name), &gen_request).await {
864                Ok(h) => h,
865                Err(e) => {
866                    error!("Failed to start ZMQ stream: {}", e);
867                    let _ = tx
868                        .send(Err(anyhow::anyhow!("Generation failed: {}", e)))
869                        .await;
870                    return;
871                }
872            };
873
874        // OpenAI-style stream ID for SSE responses
875        let sse_stream_id = format!("chatcmpl-{}", uuid::Uuid::new_v4());
876
877        // Send initial role message
878        let initial_msg = serde_json::json!({
879            "id": sse_stream_id,
880            "object": "chat.completion.chunk",
881            "created": chrono::Utc::now().timestamp(),
882            "model": model_name,
883            "choices": [{
884                "index": 0,
885                "delta": {
886                    "role": "assistant",
887                    "content": ""
888                },
889                "finish_reason": null
890            }]
891        });
892        let _ = tx.send(Ok(initial_msg)).await;
893
894        // ZMQ receive loop - forward StreamBlock payloads to SSE
895        info!("Starting ZMQ streaming receive loop (StreamBlock format)...");
896
897        // Accumulate full response text for tool call parsing
898        let mut accumulated_text = String::new();
899
900        use futures::StreamExt;
901
902        'outer: loop {
903            tokio::select! {
904                payload = stream_handle.next() => {
905                    match payload {
906                        Some(Ok(payload)) => {
907                            use crate::services::rpc_types::{InferenceStreamPayload, StreamPayloadExt};
908                            match payload.to_inference() {
909                                Ok(InferenceStreamPayload::Token(text)) => {
910                                    // Accumulate text for tool call parsing
911                                    accumulated_text.push_str(&text);
912                                    let sse_chunk = serde_json::json!({
913                                        "id": sse_stream_id,
914                                        "object": "chat.completion.chunk",
915                                        "created": chrono::Utc::now().timestamp(),
916                                        "model": model_name,
917                                        "choices": [{
918                                            "index": 0,
919                                            "delta": {
920                                                "content": text
921                                            },
922                                            "finish_reason": null
923                                        }]
924                                    });
925
926                                    if tx.send(Ok(sse_chunk)).await.is_err() {
927                                        info!("Client disconnected during streaming");
928                                        break 'outer;
929                                    }
930                                }
931                                Ok(InferenceStreamPayload::Complete(stats)) => {
932                                    info!(
933                                        "Streaming complete: {} tokens in {}ms ({:.2} tok/s)",
934                                        stats.tokens_generated,
935                                        stats.generation_time_ms,
936                                        stats.tokens_per_second
937                                    );
938
939                                    state.metrics.total_tokens.fetch_add(
940                                        stats.tokens_generated as u64,
941                                        std::sync::atomic::Ordering::Relaxed,
942                                    );
943
944                                    // Map finish_reason to OpenAI format
945                                    let oai_finish_reason = match stats.finish_reason.as_str() {
946                                        "max_tokens" | "MaxTokens" | "length" => "length",
947                                        _ => "stop",
948                                    };
949
950                                    let usage = Usage {
951                                        prompt_tokens: stats.prefill_tokens,
952                                        completion_tokens: stats.tokens_generated,
953                                        total_tokens: stats.prefill_tokens + stats.tokens_generated,
954                                        online_training_details: stats.ttt_metrics.as_ref()
955                                            .map(OnlineTrainingDetails::from),
956                                    };
957
958                                    // Check if response contains tool calls (format-aware)
959                                    let oai_finish_reason = if tools::has_tool_calls_for_format(tool_call_format, &accumulated_text) {
960                                        info!("Detected tool calls in streaming response (format: {:?})", tool_call_format);
961                                        match tools::parse_tool_calls_for_format(tool_call_format, &accumulated_text) {
962                                            Ok(parsed_tool_calls) if !parsed_tool_calls.is_empty() => {
963                                                info!("Parsed {} tool calls", parsed_tool_calls.len());
964                                                let tool_calls_chunk = serde_json::json!({
965                                                    "id": sse_stream_id,
966                                                    "object": "chat.completion.chunk",
967                                                    "created": chrono::Utc::now().timestamp(),
968                                                    "model": model_name,
969                                                    "choices": [{
970                                                        "index": 0,
971                                                        "delta": {
972                                                            "tool_calls": parsed_tool_calls
973                                                        },
974                                                        "finish_reason": null
975                                                    }]
976                                                });
977                                                let _ = tx.send(Ok(tool_calls_chunk)).await;
978                                                "tool_calls"
979                                            }
980                                            _ => oai_finish_reason,
981                                        }
982                                    } else {
983                                        oai_finish_reason
984                                    };
985
986                                    let completion_msg = serde_json::json!({
987                                        "id": sse_stream_id,
988                                        "object": "chat.completion.chunk",
989                                        "created": chrono::Utc::now().timestamp(),
990                                        "model": model_name,
991                                        "choices": [{
992                                            "index": 0,
993                                            "delta": {},
994                                            "finish_reason": oai_finish_reason
995                                        }],
996                                        "usage": usage
997                                    });
998                                    let _ = tx.send(Ok(completion_msg)).await;
999
1000                                    let _ = tx.send(Ok(serde_json::json!({"done": true}))).await;
1001                                    break 'outer;
1002                                }
1003                                Ok(InferenceStreamPayload::Error(message)) => {
1004                                    error!("Stream error: {}", message);
1005                                    let _ = tx.send(Err(anyhow::anyhow!("Generation error: {}", message))).await;
1006                                    break 'outer;
1007                                }
1008                                Err(e) => {
1009                                    error!("Failed to parse stream payload: {}", e);
1010                                    continue 'outer;
1011                                }
1012                            }
1013                        }
1014                        Some(Err(e)) => {
1015                            error!("Stream receive error: {}", e);
1016                            let _ = tx.send(Err(anyhow::anyhow!("Stream receive error: {}", e))).await;
1017                            break 'outer;
1018                        }
1019                        None => {
1020                            break 'outer;
1021                        }
1022                    }
1023                }
1024                _ = tx.closed() => {
1025                    info!("Client disconnected, stopping stream");
1026                    stream_handle.cancel();
1027                    break 'outer;
1028                }
1029            }
1030        }
1031
1032        // StreamHandle owns the socket and cleans up on drop
1033        // Metrics are automatically decremented by MetricsGuard drop
1034    });
1035
1036    // Convert channel to SSE stream
1037    let stream = ReceiverStream::new(rx).map(|result| {
1038        match result {
1039            Ok(json) => {
1040                if json.get("done").is_some() {
1041                    // Send [DONE] message
1042                    Ok::<_, Infallible>(axum::response::sse::Event::default().data("[DONE]"))
1043                } else {
1044                    // Send data chunk
1045                    Ok(axum::response::sse::Event::default()
1046                        .data(serde_json::to_string(&json).unwrap_or_else(|_| "{}".to_owned())))
1047                }
1048            }
1049            Err(e) => {
1050                // Keep the failure visible to OpenAI-compatible stream clients.
1051                Ok(axum::response::sse::Event::default()
1052                    .data(stream_error_payload(&e.to_string()).to_string()))
1053            }
1054        }
1055    });
1056
1057    // Set up SSE response with keep-alive and no-cache headers
1058    let mut response = Sse::new(stream)
1059        .keep_alive(axum::response::sse::KeepAlive::new())
1060        .into_response();
1061
1062    // Add cache control headers (using helper)
1063    add_no_cache_headers(&mut response);
1064    response
1065        .headers_mut()
1066        .insert(header::EXPIRES, axum::http::HeaderValue::from_static("0"));
1067
1068    response
1069}
1070
1071/// Handle text completion requests
1072async fn completions(
1073    State(state): State<ServerState>,
1074    auth_user: Option<Extension<AuthenticatedUser>>,
1075    _headers: HeaderMap,
1076    Json(request): Json<CompletionRequest>,
1077) -> impl IntoResponse {
1078    // Extract user identity from JWT (via middleware)
1079    let (user, domain, jwt_token, jwt_exp) =
1080        match extract_auth(auth_user.as_ref().map(|Extension(u)| u)) {
1081            Ok(identity) => identity,
1082            Err(_) => {
1083                return (
1084                    StatusCode::FORBIDDEN,
1085                    Json(ErrorResponse::new(
1086                        "Verified hosted-account tenant binding required",
1087                        "permission_denied",
1088                        "missing_tenant",
1089                    )),
1090                )
1091                    .into_response();
1092            }
1093        };
1094
1095    // Check permission for inference on this model
1096    let resource = format!("model:{}", request.model);
1097    let policy_request = hyprstream_rpc_std::policy_client::PolicyCheck {
1098        subject: user.clone(),
1099        domain: domain.clone(),
1100        resource: resource.clone(),
1101        operation: Operation::Infer.as_str().to_owned(),
1102    };
1103    match crate::services::policy::check_with_verified_bearer(
1104        &state.policy_client,
1105        &policy_request,
1106        jwt_token.as_deref(),
1107        &hyprstream_rpc::envelope::Subject::new(user.clone()),
1108    )
1109    .await
1110    {
1111        Ok(allowed) if !allowed => {
1112            return (
1113                StatusCode::FORBIDDEN,
1114                Json(ErrorResponse::new(
1115                    format!(
1116                        "Permission denied: user '{}' cannot infer on '{}'",
1117                        user, request.model
1118                    ),
1119                    "permission_denied",
1120                    "insufficient_permissions",
1121                )),
1122            )
1123                .into_response();
1124        }
1125        Err(e) => {
1126            error!("Policy check failed: {}", e);
1127            return (
1128                StatusCode::INTERNAL_SERVER_ERROR,
1129                Json(ErrorResponse::new(
1130                    format!("Policy check failed: {e}"),
1131                    "policy_error",
1132                    "internal_error",
1133                )),
1134            )
1135                .into_response();
1136        }
1137        _ => {} // allowed
1138    }
1139
1140    // Update metrics (use RAII guard for automatic cleanup)
1141    state
1142        .metrics
1143        .active_requests
1144        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1145    state
1146        .metrics
1147        .total_requests
1148        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1149    let _metrics_guard = MetricsGuard::new(&state.metrics);
1150
1151    // Resolve model path (also validates inference capability)
1152    let model_path = match resolve_model_path(&state, &request.model).await {
1153        Ok(path) => path,
1154        Err(response) => return response.into_response(),
1155    };
1156
1157    // Convert raw prompt to chat format and apply template via ZMQ ModelService
1158    // The completions endpoint expects raw text, but modern models need templated input
1159    let model_client = match model_client_for_request(&state, jwt_token.as_deref()) {
1160        Ok(client) => client,
1161        Err(e) => {
1162            tracing::error!("Failed to create authenticated ModelClient: {e}");
1163            return (StatusCode::INTERNAL_SERVER_ERROR, "Client creation failed").into_response();
1164        }
1165    };
1166    let rpc_messages = vec![hyprstream_rpc_std::inference_client::ChatMessage {
1167        role: "user".to_owned(),
1168        content: request.prompt.clone(),
1169        tool_calls: vec![],
1170        tool_call_id: String::new(),
1171    }];
1172
1173    let templated_prompt = match model_client
1174        .infer(&request.model)
1175        .apply_chat_template(&hyprstream_rpc_std::model_client::ChatTemplateRequest {
1176            messages: rpc_messages.clone(),
1177            add_generation_prompt: true,
1178            tools_json: None,
1179            max_tokens: request.max_tokens.map(|v| v as u32),
1180            enable_thinking: None,
1181            template_vars_json: None,
1182        })
1183        .await
1184    {
1185        Ok(prompt_str) => crate::config::TemplatedPrompt::new(prompt_str),
1186        Err(e) => {
1187            error!("Template formatting failed for completions endpoint: {}", e);
1188            return (
1189                StatusCode::INTERNAL_SERVER_ERROR,
1190                Json(ErrorResponse::new(
1191                    format!("Template formatting failed: {e}"),
1192                    "template_error",
1193                    "template_formatting_failed",
1194                )),
1195            )
1196                .into_response();
1197        }
1198    };
1199
1200    let gen_params = crate::config::SamplingParams::from(&state.config.sampling_defaults)
1201        .merge(
1202            crate::config::SamplingParams::from_model_path(&model_path)
1203                .await
1204                .unwrap_or_default(),
1205        )
1206        .merge((&request).into());
1207    let gen_request = gen_params.into_generation_request(templated_prompt.into_inner());
1208
1209    info!(
1210        "Completions: max_tokens={:?}, temp={:?}, top_p={:?}, top_k={:?}, repeat_penalty={:?}",
1211        gen_request.max_tokens,
1212        gen_request.temperature,
1213        gen_request.top_p,
1214        gen_request.top_k,
1215        gen_request.repeat_penalty
1216    );
1217
1218    let _claims = claims_from_auth(&user, &domain, jwt_token.as_deref(), jwt_exp);
1219    let result = collect_stream_to_result(&model_client, &request.model, &gen_request).await;
1220
1221    // Metrics automatically decremented by MetricsGuard on drop
1222
1223    match result {
1224        Ok(collected) => {
1225            let response = CompletionResponse {
1226                id: format!("cmpl-{}", uuid::Uuid::new_v4()),
1227                object: "text_completion".to_owned(),
1228                created: chrono::Utc::now().timestamp(),
1229                model: request.model.clone(),
1230                choices: vec![CompletionChoice {
1231                    text: collected.text,
1232                    index: 0,
1233                    logprobs: None,
1234                    finish_reason: Some(
1235                        if collected.stats.finish_reason == "max_tokens" {
1236                            "length"
1237                        } else {
1238                            "stop"
1239                        }
1240                        .to_owned(),
1241                    ),
1242                }],
1243                usage: Some(Usage {
1244                    prompt_tokens: request.prompt.len() / 4, // Rough estimate: 4 chars per token
1245                    completion_tokens: collected.stats.tokens_generated,
1246                    total_tokens: request.prompt.len() / 4 + collected.stats.tokens_generated,
1247                    online_training_details: collected
1248                        .stats
1249                        .ttt_metrics
1250                        .as_ref()
1251                        .map(OnlineTrainingDetails::from),
1252                }),
1253            };
1254
1255            let mut response = Json(response).into_response();
1256            add_no_cache_headers(&mut response);
1257            response
1258        }
1259        Err(e) => (
1260            StatusCode::INTERNAL_SERVER_ERROR,
1261            Json(serde_json::json!({
1262                "error": {
1263                    "message": format!("Generation failed: {}", e),
1264                    "type": "generation_error",
1265                    "code": "internal_error"
1266                }
1267            })),
1268        )
1269            .into_response(),
1270    }
1271}
1272
1273/// Handle embedding requests
1274async fn embeddings(
1275    State(_state): State<ServerState>,
1276    Json(_request): Json<EmbeddingRequest>,
1277) -> impl IntoResponse {
1278    (
1279        StatusCode::NOT_IMPLEMENTED,
1280        Json(serde_json::json!({
1281            "error": {
1282                "message": "Embeddings not yet implemented",
1283                "type": "not_implemented",
1284                "code": "feature_not_available"
1285            }
1286        })),
1287    )
1288        .into_response()
1289}
1290
1291/// List available models
1292///
1293/// Returns all worktrees as model:branch references.
1294/// Models are only accessible via their worktree branches.
1295async fn list_models(
1296    State(state): State<ServerState>,
1297    auth_user: Option<Extension<AuthenticatedUser>>,
1298) -> impl IntoResponse {
1299    let (user, domain, jwt_token, _) = match extract_auth(auth_user.as_ref().map(|Extension(u)| u))
1300    {
1301        Ok(identity) => identity,
1302        Err(_) => {
1303            return (
1304                StatusCode::FORBIDDEN,
1305                Json(ErrorResponse::new(
1306                    "Verified hosted-account tenant binding required",
1307                    "permission_denied",
1308                    "missing_tenant",
1309                )),
1310            )
1311                .into_response();
1312        }
1313    };
1314    let policy_request = hyprstream_rpc_std::policy_client::PolicyCheck {
1315        subject: user.clone(),
1316        domain,
1317        resource: "registry:*".to_owned(),
1318        operation: Operation::Query.as_str().to_owned(),
1319    };
1320    if !crate::services::policy::check_with_verified_bearer(
1321        &state.policy_client,
1322        &policy_request,
1323        jwt_token.as_deref(),
1324        &hyprstream_rpc::envelope::Subject::new(user),
1325    )
1326    .await
1327    .unwrap_or(false)
1328    {
1329        return (
1330            StatusCode::FORBIDDEN,
1331            Json(ErrorResponse::new(
1332                "Permission denied: cannot query registry",
1333                "permission_denied",
1334                "insufficient_permissions",
1335            )),
1336        )
1337            .into_response();
1338    }
1339
1340    let mut models = vec![];
1341
1342    // Get all worktrees from registry (formatted as model:branch)
1343    let result: Result<(), anyhow::Error> = async {
1344        let repos = state.registry.list().await?;
1345        for repo in repos {
1346            if repo.name.is_empty() {
1347                continue;
1348            }
1349            let name = &repo.name;
1350            match state.registry.repo(&repo.id).list_worktrees().await {
1351                Ok(worktrees) => {
1352                    for wt in worktrees {
1353                        if wt.branch_name.is_empty() {
1354                            continue;
1355                        }
1356                        let display = format!("{}:{}", name, wt.branch_name);
1357                        models.push(Model {
1358                            id: display,
1359                            object: "model".to_owned(),
1360                            created: chrono::Utc::now().timestamp(),
1361                            owned_by: "system".to_owned(),
1362                        });
1363                    }
1364                }
1365                Err(e) => {
1366                    error!("Failed to list worktrees for {}: {}", name, e);
1367                }
1368            }
1369        }
1370        Ok(())
1371    }
1372    .await;
1373
1374    if let Err(e) = result {
1375        error!("Failed to list models from storage: {}", e);
1376    }
1377
1378    // Add no-cache headers
1379    let mut response = Json(ListModelsResponse {
1380        object: "list".to_owned(),
1381        data: models,
1382    })
1383    .into_response();
1384    add_no_cache_headers(&mut response);
1385    response
1386}
1387
1388/// Convert HTTP-boundary ChatMessage to the Cap'n Proto RPC type.
1389///
1390/// `inference_client::ToolCall` is already the same type as `openai_compat::ToolCall`
1391/// (re-exported via `pub use`), so only `content` and `tool_call_id` need unwrapping.
1392fn to_rpc_messages(msgs: &[ChatMessage]) -> Vec<hyprstream_rpc_std::inference_client::ChatMessage> {
1393    use hyprstream_rpc_std::inference_client::ChatMessage as RpcMsg;
1394    msgs.iter()
1395        .map(|m| RpcMsg {
1396            role: m.role.clone(),
1397            content: m.content.as_deref().unwrap_or("").to_owned(),
1398            tool_calls: m.tool_calls.clone().unwrap_or_default(),
1399            tool_call_id: m.tool_call_id.as_deref().unwrap_or("").to_owned(),
1400        })
1401        .collect()
1402}
1403
1404#[cfg(test)]
1405mod tests {
1406    use super::{
1407        prepare_chat_request, stream_error_payload, tool_choice_error_response, ErrorResponse,
1408        ToolChoiceFailure,
1409    };
1410    use crate::api::openai_compat::ChatCompletionRequest;
1411    use axum::{body::to_bytes, http::StatusCode};
1412
1413    fn tool_choice_request(
1414        stream: bool,
1415        choice: Option<serde_json::Value>,
1416    ) -> ChatCompletionRequest {
1417        let mut value = serde_json::json!({
1418            "model": "model:main",
1419            "messages": [{"role": "user", "content": "hello"}],
1420            "stream": stream,
1421            "tools": [{
1422                "type": "function",
1423                "function": {"name": "status", "parameters": {"type": "object"}}
1424            }]
1425        });
1426        if let Some(choice) = choice {
1427            value["tool_choice"] = choice;
1428        }
1429        match serde_json::from_value(value) {
1430            Ok(request) => request,
1431            Err(error) => panic!("tool choice fixture must deserialize: {error}"),
1432        }
1433    }
1434
1435    #[test]
1436    fn absent_and_auto_preserve_tools_for_both_completion_paths() {
1437        for stream in [false, true] {
1438            for choice in [None, Some(serde_json::json!("auto"))] {
1439                let (request, selected_stream) =
1440                    match prepare_chat_request(tool_choice_request(stream, choice)) {
1441                        Ok(prepared) => prepared,
1442                        Err(error) => panic!("supported tool choice was rejected: {error:?}"),
1443                    };
1444                assert_eq!(selected_stream, stream);
1445                assert_eq!(request.tools.as_ref().map(Vec::len), Some(1));
1446            }
1447        }
1448    }
1449
1450    #[test]
1451    fn none_omits_tool_definitions_for_both_completion_paths() {
1452        for stream in [false, true] {
1453            let (request, selected_stream) = match prepare_chat_request(tool_choice_request(
1454                stream,
1455                Some(serde_json::json!("none")),
1456            )) {
1457                Ok(prepared) => prepared,
1458                Err(error) => panic!("none tool choice was rejected: {error:?}"),
1459            };
1460            assert_eq!(selected_stream, stream);
1461            // Both paths derive template tools and response parser mode from
1462            // request.tools; None disables both without altering auto behavior.
1463            assert!(request.tools.is_none());
1464        }
1465    }
1466
1467    #[tokio::test]
1468    async fn required_named_and_unknown_choices_fail_before_either_completion_path() {
1469        for stream in [false, true] {
1470            for (choice, expected) in [
1471                (
1472                    serde_json::json!("required"),
1473                    ToolChoiceFailure::RequiredUnsupported,
1474                ),
1475                (
1476                    serde_json::json!({"type": "function", "function": {"name": "status"}}),
1477                    ToolChoiceFailure::NamedUnsupported,
1478                ),
1479                (
1480                    serde_json::json!("sentinel-unsupported-choice"),
1481                    ToolChoiceFailure::Invalid,
1482                ),
1483            ] {
1484                let failure = match prepare_chat_request(tool_choice_request(stream, Some(choice)))
1485                {
1486                    Ok(_) => panic!("unsupported tool choice was accepted"),
1487                    Err(failure) => failure,
1488                };
1489                assert_eq!(failure, expected);
1490                let response = tool_choice_error_response(failure);
1491                assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1492                let body = match to_bytes(response.into_body(), 1024).await {
1493                    Ok(bytes) => bytes,
1494                    Err(error) => panic!("tool choice error body must be readable: {error}"),
1495                };
1496                let response: serde_json::Value = match serde_json::from_slice(&body) {
1497                    Ok(value) => value,
1498                    Err(error) => panic!("tool choice error must be JSON: {error}"),
1499                };
1500                assert_eq!(response["error"]["type"], "invalid_request_error");
1501                assert_eq!(response["error"]["param"], "tool_choice");
1502                assert_eq!(
1503                    response["error"]["code"],
1504                    if failure == ToolChoiceFailure::Invalid {
1505                        "invalid_tool_choice"
1506                    } else {
1507                        "unsupported_tool_choice"
1508                    }
1509                );
1510                assert!(!response.to_string().contains("sentinel-unsupported-choice"));
1511            }
1512        }
1513    }
1514
1515    #[test]
1516    fn ordinary_errors_keep_their_existing_shape() {
1517        let value = match serde_json::to_value(ErrorResponse::new(
1518            "ordinary error",
1519            "existing_type",
1520            "existing_code",
1521        )) {
1522            Ok(value) => value,
1523            Err(error) => panic!("existing error must serialize: {error}"),
1524        };
1525        assert!(value["error"].get("param").is_none());
1526    }
1527
1528    #[test]
1529    fn streamed_failure_has_openai_error_object_and_valid_json() {
1530        let message = "Stream receive error: stream idle: no group for 30s; \"retry\"\nnext line";
1531        let serialized = stream_error_payload(message).to_string();
1532        let decoded: serde_json::Value = match serde_json::from_str(&serialized) {
1533            Ok(value) => value,
1534            Err(error) => panic!("stream error envelope is invalid JSON: {error}"),
1535        };
1536
1537        assert_eq!(decoded["error"]["message"], message);
1538        assert_eq!(decoded["error"]["type"], "generation_error");
1539        assert_eq!(decoded["error"]["code"], "internal_error");
1540        assert!(decoded["error"]["param"].is_null());
1541        assert!(decoded["error"].is_object());
1542        assert!(decoded.get("done").is_none());
1543    }
1544}