1use axum::{
4 extract::{Json, State},
5 http::{header, HeaderMap, StatusCode},
6 response::{IntoResponse, Sse},
7 routing::{get, post},
8 Extension, Router,
9};
10use futures::stream::StreamExt;
11use hyprstream_rpc_std::inference_client::GenerationRequest;
12use hyprstream_rpc_std::model_client::ModelClient;
13use std::convert::Infallible;
14use tokio::sync::mpsc;
15use tokio_stream::wrappers::ReceiverStream;
16use tracing::{error, info, trace};
17
18use crate::{
19 api::{
20 openai_compat::{
21 ChatChoice, ChatCompletionRequest, ChatCompletionResponse, ChatMessage,
22 CompletionChoice, CompletionRequest, CompletionResponse, EmbeddingRequest,
23 ListModelsResponse, Model, OnlineTrainingDetails, ToolChoice, Usage,
24 },
25 tools,
26 },
27 archetypes::capabilities::Infer,
28 auth::Operation,
29 runtime::CacheOwner,
30 server::{state::ServerState, AuthenticatedUser},
31};
32
33struct MetricsGuard<'a> {
35 metrics: &'a crate::server::state::Metrics,
36 decremented: bool,
37}
38
39impl<'a> MetricsGuard<'a> {
40 fn new(metrics: &'a crate::server::state::Metrics) -> Self {
41 Self {
42 metrics,
43 decremented: false,
44 }
45 }
46}
47
48impl<'a> Drop for MetricsGuard<'a> {
49 fn drop(&mut self) {
50 if !self.decremented {
51 self.metrics
52 .active_requests
53 .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
54 }
55 }
56}
57
58#[derive(serde::Serialize)]
60struct ErrorResponse {
61 error: ErrorDetails,
62}
63
64#[derive(serde::Serialize)]
65struct ErrorDetails {
66 message: String,
67 #[serde(rename = "type")]
68 error_type: String,
69 code: String,
70 #[serde(skip_serializing_if = "Option::is_none")]
71 param: Option<&'static str>,
72}
73
74impl ErrorResponse {
75 fn new(
76 message: impl Into<String>,
77 error_type: impl Into<String>,
78 code: impl Into<String>,
79 ) -> Self {
80 Self {
81 error: ErrorDetails {
82 message: message.into(),
83 error_type: error_type.into(),
84 code: code.into(),
85 param: None,
86 },
87 }
88 }
89
90 fn tool_choice(failure: ToolChoiceFailure) -> Self {
91 let (message, code) = match failure {
92 ToolChoiceFailure::RequiredUnsupported => (
93 "tool_choice=required is not supported for this model",
94 "unsupported_tool_choice",
95 ),
96 ToolChoiceFailure::NamedUnsupported => (
97 "Named tool_choice is not supported for this model",
98 "unsupported_tool_choice",
99 ),
100 ToolChoiceFailure::Invalid => ("Invalid tool_choice", "invalid_tool_choice"),
101 };
102 Self {
103 error: ErrorDetails {
104 message: message.to_owned(),
105 error_type: "invalid_request_error".to_owned(),
106 code: code.to_owned(),
107 param: Some("tool_choice"),
108 },
109 }
110 }
111}
112
113#[derive(Debug, Clone, Copy, PartialEq, Eq)]
114enum ToolChoiceFailure {
115 RequiredUnsupported,
116 NamedUnsupported,
117 Invalid,
118}
119
120fn prepare_chat_request(
124 mut request: ChatCompletionRequest,
125) -> Result<(ChatCompletionRequest, bool), ToolChoiceFailure> {
126 match request.tool_choice.as_ref() {
127 None => {}
128 Some(ToolChoice::String(choice)) => match choice.as_str() {
129 "auto" => {}
130 "none" => request.tools = None,
131 "required" => return Err(ToolChoiceFailure::RequiredUnsupported),
132 _ => return Err(ToolChoiceFailure::Invalid),
133 },
134 Some(ToolChoice::Specific { .. }) => return Err(ToolChoiceFailure::NamedUnsupported),
135 }
136 let is_streaming = request.stream.unwrap_or(false);
137 Ok((request, is_streaming))
138}
139
140fn tool_choice_error_response(failure: ToolChoiceFailure) -> axum::response::Response {
141 (
142 StatusCode::BAD_REQUEST,
143 Json(ErrorResponse::tool_choice(failure)),
144 )
145 .into_response()
146}
147
148fn stream_error_payload(message: &str) -> serde_json::Value {
151 serde_json::json!({
152 "error": {
153 "message": message,
154 "type": "generation_error",
155 "code": "internal_error",
156 "param": null
157 }
158 })
159}
160
161struct CollectedResult {
163 text: String,
164 stats: crate::services::rpc_types::InferenceComplete,
165}
166
167async fn collect_stream_to_result(
173 model_client: &ModelClient,
174 model_ref: &str,
175 request: &GenerationRequest,
176) -> anyhow::Result<CollectedResult> {
177 use hyprstream_rpc_std::model_client::InferRpc;
178 let mut handle = InferRpc::generate_stream(&model_client.infer(model_ref), request).await?;
179
180 use futures::StreamExt;
181 use hyprstream_rpc::streaming::StreamPayload;
182
183 let mut text = String::new();
184 loop {
185 match handle.next().await {
186 Some(Ok(payload)) => match payload {
187 StreamPayload::Data(data) => {
188 text.push_str(&String::from_utf8_lossy(&data));
189 }
190 StreamPayload::Complete(meta) => {
191 let stats: crate::services::rpc_types::InferenceComplete =
192 serde_json::from_slice(&meta).unwrap_or_else(|e| {
193 tracing::warn!("Failed to parse InferenceComplete: {e}");
194 crate::services::rpc_types::InferenceComplete::empty()
195 });
196 return Ok(CollectedResult { text, stats });
197 }
198 StreamPayload::Error(msg) => {
199 anyhow::bail!("Generation error: {msg}");
200 }
201 StreamPayload::Tagged { .. } => {
202 continue;
204 }
205 },
206 Some(Err(e)) => {
207 anyhow::bail!("Stream error: {e}");
208 }
209 None => {
210 anyhow::bail!("Stream ended without completion");
211 }
212 }
213 }
214}
215
216fn extract_cache_owner(headers: &HeaderMap) -> CacheOwner {
232 if let Some(session_id) = headers
233 .get("x-session-id")
234 .and_then(|v| v.to_str().ok())
235 .filter(|s| !s.is_empty())
236 {
237 trace!("Using session-based caching for session: {}", session_id);
238 CacheOwner::Session(session_id.to_owned())
239 } else {
240 let request_id = std::time::SystemTime::now()
242 .duration_since(std::time::UNIX_EPOCH)
243 .map(|d| d.as_nanos() as u64)
244 .unwrap_or(0);
245 trace!("Using stateless caching for request: {}", request_id);
246 CacheOwner::Stateless(request_id)
247 }
248}
249
250fn extract_auth(
252 auth_user: Option<&AuthenticatedUser>,
253) -> Result<(String, String, Option<String>, Option<i64>), &'static str> {
254 let user = auth_user.ok_or("authenticated identity missing")?;
255 trace!("Using authenticated user: {}", user.user);
256 Ok((
257 user.user.clone(),
258 user.authorization_domain()?,
259 user.token.clone(),
260 user.exp,
261 ))
262}
263
264fn claims_from_auth(
266 user: &str,
267 domain: &str,
268 jwt_token: Option<&str>,
269 jwt_exp: Option<i64>,
270) -> hyprstream_rpc::auth::Claims {
271 let now = chrono::Utc::now().timestamp();
272 let exp = jwt_exp.unwrap_or(now + 3600);
273 let mut claims = hyprstream_rpc::auth::Claims::new(user.to_owned(), now, exp);
274 if domain != "*" {
275 claims = claims.with_tenant(domain.to_owned());
276 }
277 if let Some(token) = jwt_token {
278 claims = claims.with_token(token.to_owned());
279 }
280 claims
281}
282
283fn model_client_for_request(
287 state: &ServerState,
288 jwt_token: Option<&str>,
289) -> anyhow::Result<ModelClient> {
290 let bearer = jwt_token
291 .filter(|token| !token.trim().is_empty())
292 .ok_or_else(|| anyhow::anyhow!("verified OAI user bearer is required"))?;
293 let signer = (*state.signing_key).clone();
294 let service_token = crate::services::factories::service_token(&signer)
295 .ok_or_else(|| anyhow::anyhow!("OAI service credential is unavailable"))?;
296 let client = ModelClient::from_provider(
297 &hyprstream_discovery::ProductionRpcClientProvider,
298 signer,
299 Some(service_token),
300 )?;
301 Ok(client.with_delegated_bearer(bearer.to_owned()))
302}
303
304async fn resolve_model_path(
306 state: &ServerState,
307 model_name: &str,
308) -> Result<std::path::PathBuf, impl IntoResponse> {
309 let model_ref = match crate::storage::ModelRef::parse(model_name) {
310 Ok(r) => r,
311 Err(e) => {
312 return Err((
313 StatusCode::BAD_REQUEST,
314 Json(ErrorResponse::new(
315 format!("Invalid model reference: {e}"),
316 "invalid_model_ref",
317 "parse_error",
318 )),
319 )
320 .into_response());
321 }
322 };
323
324 let path_result: Result<std::path::PathBuf, _> = async {
326 let tracked = state.registry.get_by_name(model_ref.name()).await?;
327 let repo = state.registry.repo(&tracked.id);
328 let branch = match &model_ref.git_ref {
329 crate::storage::GitRef::Branch(name) => name.clone(),
330 _ => repo.get_head().await?,
331 };
332 let wts = repo.list_worktrees().await?;
334 if !wts.iter().any(|wt| wt.branch_name == branch) {
335 anyhow::bail!("worktree for {}:{} not found", model_ref.name(), branch);
336 }
337 let storage_paths = crate::storage::StoragePaths::new()?;
339 storage_paths.worktree_path(model_ref.name(), &branch)
340 }
341 .await;
342
343 match path_result {
344 Ok(path) => {
345 let archetype_registry = crate::archetypes::global_registry();
347 let detected = archetype_registry.detect(&path);
348
349 let domains = detected.to_detected_domains();
350 if !domains.has::<Infer>() {
351 return Err((
352 StatusCode::BAD_REQUEST,
353 Json(ErrorResponse::new(
354 format!(
355 "Model '{}' does not support inference. Detected domains: {:?}",
356 model_name, domains.domains
357 ),
358 "capability_error",
359 "inference_not_supported",
360 )),
361 )
362 .into_response());
363 }
364
365 Ok(path)
366 }
367 Err(e) => Err((
368 StatusCode::NOT_FOUND,
369 Json(ErrorResponse::new(
370 format!("Model path not found: {e}"),
371 "model_not_found",
372 "path_error",
373 )),
374 )
375 .into_response()),
376 }
377}
378
379fn add_no_cache_headers(response: &mut axum::response::Response) {
381 use axum::http::HeaderValue;
382 let headers = response.headers_mut();
383 headers.insert(
384 header::CACHE_CONTROL,
385 HeaderValue::from_static("no-cache, no-store, must-revalidate"),
386 );
387 headers.insert(header::PRAGMA, HeaderValue::from_static("no-cache"));
388}
389
390pub fn create_router() -> Router<ServerState> {
392 Router::new()
393 .route("/chat/completions", post(chat_completions))
394 .route("/completions", post(completions))
395 .route("/embeddings", post(embeddings))
396 .route("/models", get(list_models))
397}
398
399async fn chat_completions(
401 State(state): State<ServerState>,
402 auth_user: Option<Extension<AuthenticatedUser>>,
403 headers: HeaderMap,
404 Json(request): Json<ChatCompletionRequest>,
405) -> impl IntoResponse {
406 let (user, domain, jwt_token, jwt_exp) =
408 match extract_auth(auth_user.as_ref().map(|Extension(u)| u)) {
409 Ok(identity) => identity,
410 Err(_) => {
411 return (
412 StatusCode::FORBIDDEN,
413 Json(ErrorResponse::new(
414 "Verified hosted-account tenant binding required",
415 "permission_denied",
416 "missing_tenant",
417 )),
418 )
419 .into_response();
420 }
421 };
422
423 let resource = format!("model:{}", request.model);
425 let policy_request = hyprstream_rpc_std::policy_client::PolicyCheck {
426 subject: user.clone(),
427 domain: domain.clone(),
428 resource: resource.clone(),
429 operation: Operation::Infer.as_str().to_owned(),
430 };
431 match crate::services::policy::check_with_verified_bearer(
432 &state.policy_client,
433 &policy_request,
434 jwt_token.as_deref(),
435 &hyprstream_rpc::envelope::Subject::new(user.clone()),
436 )
437 .await
438 {
439 Ok(allowed) if !allowed => {
440 return (
441 StatusCode::FORBIDDEN,
442 Json(ErrorResponse::new(
443 format!(
444 "Permission denied: user '{}' cannot infer on '{}'",
445 user, request.model
446 ),
447 "permission_denied",
448 "insufficient_permissions",
449 )),
450 )
451 .into_response();
452 }
453 Err(e) => {
454 error!("Policy check failed: {}", e);
455 return (
456 StatusCode::INTERNAL_SERVER_ERROR,
457 Json(ErrorResponse::new(
458 format!("Policy check failed: {e}"),
459 "policy_error",
460 "internal_error",
461 )),
462 )
463 .into_response();
464 }
465 _ => {} }
467
468 let (request, is_streaming) = match prepare_chat_request(request) {
469 Ok(prepared) => prepared,
470 Err(failure) => return tool_choice_error_response(failure),
471 };
472
473 let cache_owner = extract_cache_owner(&headers);
475
476 info!(
478 "Chat completion request - model: {}, stream: {:?}, messages: {} msgs, cache: {:?}",
479 request.model,
480 request.stream,
481 request.messages.len(),
482 cache_owner
483 );
484
485 info!(
487 "Streaming mode: {} (explicit: {:?})",
488 is_streaming, request.stream
489 );
490
491 if is_streaming {
492 info!("Handling streaming request");
493 return stream_chat(state, headers, request, user, domain, jwt_token, jwt_exp)
494 .await
495 .into_response();
496 }
497 info!("Handling non-streaming request");
498
499 state
500 .metrics
501 .active_requests
502 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
503 state
504 .metrics
505 .total_requests
506 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
507 let _metrics_guard = MetricsGuard::new(&state.metrics);
508
509 let start_time = std::time::Instant::now();
510
511 let model_path = match resolve_model_path(&state, &request.model).await {
513 Ok(path) => path,
514 Err(response) => return response.into_response(),
515 };
516
517 info!("Applying chat template via ModelService...");
519 let model_client = match model_client_for_request(&state, jwt_token.as_deref()) {
520 Ok(client) => client,
521 Err(e) => {
522 tracing::error!("Failed to create authenticated ModelClient: {e}");
523 return (StatusCode::INTERNAL_SERVER_ERROR, "Client creation failed").into_response();
524 }
525 };
526
527 let tools_json: Option<serde_json::Value> = request.tools.as_ref().map(|t| {
529 info!("Passing {} tools to template engine", t.len());
530 serde_json::to_value(t).unwrap_or_default()
531 });
532
533 let tool_call_format = if request.tools.is_some() {
535 tools::ToolCallFormat::from_architecture(
536 &crate::runtime::model_config::ModelConfig::detect_architecture(&model_path),
537 )
538 } else {
539 tools::ToolCallFormat::None
540 };
541
542 let rpc_messages = to_rpc_messages(&request.messages);
543 let tools_str = tools_json
544 .as_ref()
545 .map(|t| serde_json::to_string(t).unwrap_or_default())
546 .unwrap_or_default();
547 let templated_prompt = match model_client
548 .infer(&request.model)
549 .apply_chat_template(&hyprstream_rpc_std::model_client::ChatTemplateRequest {
550 messages: rpc_messages.clone(),
551 add_generation_prompt: true,
552 tools_json: Some(tools_str.clone()).filter(|s| !s.is_empty()),
553 max_tokens: request.max_tokens.map(|v| v as u32),
554 enable_thinking: None,
555 template_vars_json: None,
556 })
557 .await
558 {
559 Ok(prompt_str) => crate::config::TemplatedPrompt::new(prompt_str),
560 Err(e) => {
561 error!("Failed to apply chat template: {}", e);
562 return (
563 StatusCode::INTERNAL_SERVER_ERROR,
564 Json(ErrorResponse::new(
565 format!("Failed to apply chat template: {e}"),
566 "template_error",
567 "internal_error",
568 )),
569 )
570 .into_response();
571 }
572 };
573 info!(
574 "Template applied, prompt length: {}",
575 templated_prompt.len()
576 );
577
578 let gen_params = crate::config::SamplingParams::from(&state.config.sampling_defaults)
580 .merge(
581 crate::config::SamplingParams::from_model_path(&model_path)
582 .await
583 .unwrap_or_default(),
584 )
585 .merge((&request).into());
586 let gen_request = gen_params.into_generation_request(templated_prompt.into_inner());
587
588 info!(
589 "Using generation config: max_tokens={:?}, temp={:?}, top_p={:?}, top_k={:?}, repeat_penalty={:?}",
590 gen_request.max_tokens, gen_request.temperature, gen_request.top_p, gen_request.top_k, gen_request.repeat_penalty
591 );
592
593 info!(
594 "Starting generation with prompt length: {}",
595 gen_request.prompt.len()
596 );
597
598 let _claims = claims_from_auth(&user, &domain, jwt_token.as_deref(), jwt_exp);
599 let result = collect_stream_to_result(&model_client, &request.model, &gen_request).await;
602
603 info!("Generation completed - success: {}", result.is_ok());
604
605 match result {
606 Ok(collected) => {
607 let tokens_generated = collected.stats.tokens_generated;
608 state.metrics.total_tokens.fetch_add(
609 tokens_generated as u64,
610 std::sync::atomic::Ordering::Relaxed,
611 );
612
613 let latency_ms = start_time.elapsed().as_millis() as f64;
614 let mut avg_latency = state.metrics.avg_latency_ms.write().await;
615 *avg_latency = (*avg_latency * 0.9) + (latency_ms * 0.1);
616
617 let finish_reason_str = &collected.stats.finish_reason;
618
619 let (content, tool_calls, finish_reason) =
621 if tools::has_tool_calls_for_format(tool_call_format, &collected.text) {
622 info!(
623 "Detected tool calls in response (format: {:?})",
624 tool_call_format
625 );
626 match tools::parse_tool_calls_for_format(tool_call_format, &collected.text) {
627 Ok(parsed_tool_calls) if !parsed_tool_calls.is_empty() => {
628 let content_text = tools::extract_text_content_for_format(
629 tool_call_format,
630 &collected.text,
631 );
632 let content = if content_text.is_empty() {
633 None
634 } else {
635 Some(content_text)
636 };
637 info!("Parsed {} tool calls", parsed_tool_calls.len());
638 (content, Some(parsed_tool_calls), "tool_calls")
639 }
640 Ok(_) | Err(_) => {
641 let fr = if finish_reason_str == "max_tokens" {
642 "length"
643 } else {
644 "stop"
645 };
646 (Some(collected.text), None, fr)
647 }
648 }
649 } else {
650 let fr = if finish_reason_str == "max_tokens" {
651 "length"
652 } else {
653 "stop"
654 };
655 (Some(collected.text), None, fr)
656 };
657
658 let response = ChatCompletionResponse {
659 id: format!("chatcmpl-{}", uuid::Uuid::new_v4()),
660 object: "chat.completion".to_owned(),
661 created: chrono::Utc::now().timestamp(),
662 model: request.model.clone(),
663 choices: vec![ChatChoice {
664 index: 0,
665 message: ChatMessage {
666 role: "assistant".to_owned(),
667 content,
668 tool_calls,
669 tool_call_id: None,
670 },
671 finish_reason: Some(finish_reason.to_owned()),
672 }],
673 usage: Some(Usage {
674 prompt_tokens: 0,
675 completion_tokens: tokens_generated,
676 total_tokens: tokens_generated,
677 online_training_details: collected
678 .stats
679 .ttt_metrics
680 .as_ref()
681 .map(OnlineTrainingDetails::from),
682 }),
683 };
684
685 let mut response = Json(response).into_response();
686 add_no_cache_headers(&mut response);
687 response
688 }
689 Err(e) => {
690 error!("Generation failed: {}", e);
691 (
692 StatusCode::INTERNAL_SERVER_ERROR,
693 Json(serde_json::json!({
694 "error": {
695 "message": format!("Generation failed: {}", e),
696 "type": "generation_error",
697 "code": "internal_error"
698 }
699 })),
700 )
701 .into_response()
702 }
703 }
704}
705
706async fn stream_chat(
708 state: ServerState,
709 _headers: HeaderMap,
710 request: ChatCompletionRequest,
711 user: String,
712 domain: String,
713 jwt_token: Option<String>,
714 jwt_exp: Option<i64>,
715) -> impl IntoResponse {
716 let (tx, rx) = mpsc::channel::<Result<serde_json::Value, anyhow::Error>>(100);
718
719 let state_clone = state.clone();
721
722 let defaults = state.config.sampling_defaults.clone();
724 let model_name = request.model.clone();
725 let messages = request.messages.clone();
726 let stop_sequences = request.stop.clone().unwrap_or_default();
727 let tools = request.tools.clone();
728
729 state_clone
731 .metrics
732 .active_requests
733 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
734
735 tokio::spawn(async move {
736 let _metrics_guard = MetricsGuard::new(&state.metrics);
738
739 let model_ref = match crate::storage::ModelRef::parse(&model_name) {
741 Ok(r) => r,
742 Err(e) => {
743 let _ = tx
744 .send(Err(anyhow::anyhow!("Invalid model reference: {}", e)))
745 .await;
746 return;
747 }
748 };
749
750 let model_path = match async {
751 let tracked = state.registry.get_by_name(model_ref.name()).await?;
752 let repo = state.registry.repo(&tracked.id);
753 let branch = match &model_ref.git_ref {
754 crate::storage::GitRef::Branch(name) => name.clone(),
755 _ => repo.get_head().await?,
756 };
757 let wts = repo.list_worktrees().await?;
759 if !wts.iter().any(|wt| wt.branch_name == branch) {
760 anyhow::bail!("worktree not found");
761 }
762 let storage_paths = crate::storage::StoragePaths::new()?;
764 storage_paths.worktree_path(model_ref.name(), &branch)
765 }
766 .await
767 {
768 Ok(path) => path,
769 Err(e) => {
770 let _ = tx
771 .send(Err(anyhow::anyhow!("Could not get model path: {}", e)))
772 .await;
773 return;
774 }
775 };
776
777 let tools_json: Option<serde_json::Value> = tools.as_ref().map(|t| {
779 info!("Passing {} tools to streaming template engine", t.len());
780 serde_json::to_value(t).unwrap_or_default()
781 });
782
783 let tool_call_format = if tools.is_some() {
785 tools::ToolCallFormat::from_architecture(
786 &crate::runtime::model_config::ModelConfig::detect_architecture(&model_path),
787 )
788 } else {
789 tools::ToolCallFormat::None
790 };
791
792 info!("Applying chat template for streaming...");
794 let model_client = match model_client_for_request(&state, jwt_token.as_deref()) {
795 Ok(client) => client,
796 Err(e) => {
797 error!("Failed to create authenticated ModelClient: {}", e);
798 let _ = tx
799 .send(Err(anyhow::anyhow!("Client creation failed: {}", e)))
800 .await;
801 return;
802 }
803 };
804 let rpc_messages = to_rpc_messages(&messages);
805 let tools_str = tools_json
806 .as_ref()
807 .map(|t| serde_json::to_string(t).unwrap_or_default())
808 .unwrap_or_default();
809 let templated_prompt = match model_client
810 .infer(&model_name)
811 .apply_chat_template(&hyprstream_rpc_std::model_client::ChatTemplateRequest {
812 messages: rpc_messages.clone(),
813 add_generation_prompt: true,
814 tools_json: Some(tools_str.clone()).filter(|s| !s.is_empty()),
815 max_tokens: request.max_tokens.map(|v| v as u32),
816 enable_thinking: None,
817 template_vars_json: None,
818 })
819 .await
820 {
821 Ok(prompt_str) => {
822 let prompt = crate::config::TemplatedPrompt::new(prompt_str);
823 info!(
824 "Streaming template applied, prompt length: {}",
825 prompt.len()
826 );
827 prompt
828 }
829 Err(e) => {
830 error!("Template formatting failed in streaming: {}", e);
831 let _ = tx
832 .send(Err(anyhow::anyhow!("Template formatting failed: {}", e)))
833 .await;
834 return;
835 }
836 };
837
838 let mut gen_params = crate::config::SamplingParams::from(&defaults)
840 .merge(
841 crate::config::SamplingParams::from_model_path(&model_path)
842 .await
843 .unwrap_or_default(),
844 )
845 .merge((&request).into());
846 if !stop_sequences.is_empty() {
847 gen_params.stop_tokens = Some(stop_sequences);
848 }
849 let gen_request = gen_params.into_generation_request(templated_prompt.into_inner());
850
851 info!(
852 "Streaming: max_tokens={:?}, temp={:?}, top_p={:?}, top_k={:?}, repeat_penalty={:?}",
853 gen_request.max_tokens,
854 gen_request.temperature,
855 gen_request.top_p,
856 gen_request.top_k,
857 gen_request.repeat_penalty
858 );
859
860 let _claims = claims_from_auth(&user, &domain, jwt_token.as_deref(), jwt_exp);
861 use hyprstream_rpc_std::model_client::InferRpc;
862 let mut stream_handle =
863 match InferRpc::generate_stream(&model_client.infer(&model_name), &gen_request).await {
864 Ok(h) => h,
865 Err(e) => {
866 error!("Failed to start ZMQ stream: {}", e);
867 let _ = tx
868 .send(Err(anyhow::anyhow!("Generation failed: {}", e)))
869 .await;
870 return;
871 }
872 };
873
874 let sse_stream_id = format!("chatcmpl-{}", uuid::Uuid::new_v4());
876
877 let initial_msg = serde_json::json!({
879 "id": sse_stream_id,
880 "object": "chat.completion.chunk",
881 "created": chrono::Utc::now().timestamp(),
882 "model": model_name,
883 "choices": [{
884 "index": 0,
885 "delta": {
886 "role": "assistant",
887 "content": ""
888 },
889 "finish_reason": null
890 }]
891 });
892 let _ = tx.send(Ok(initial_msg)).await;
893
894 info!("Starting ZMQ streaming receive loop (StreamBlock format)...");
896
897 let mut accumulated_text = String::new();
899
900 use futures::StreamExt;
901
902 'outer: loop {
903 tokio::select! {
904 payload = stream_handle.next() => {
905 match payload {
906 Some(Ok(payload)) => {
907 use crate::services::rpc_types::{InferenceStreamPayload, StreamPayloadExt};
908 match payload.to_inference() {
909 Ok(InferenceStreamPayload::Token(text)) => {
910 accumulated_text.push_str(&text);
912 let sse_chunk = serde_json::json!({
913 "id": sse_stream_id,
914 "object": "chat.completion.chunk",
915 "created": chrono::Utc::now().timestamp(),
916 "model": model_name,
917 "choices": [{
918 "index": 0,
919 "delta": {
920 "content": text
921 },
922 "finish_reason": null
923 }]
924 });
925
926 if tx.send(Ok(sse_chunk)).await.is_err() {
927 info!("Client disconnected during streaming");
928 break 'outer;
929 }
930 }
931 Ok(InferenceStreamPayload::Complete(stats)) => {
932 info!(
933 "Streaming complete: {} tokens in {}ms ({:.2} tok/s)",
934 stats.tokens_generated,
935 stats.generation_time_ms,
936 stats.tokens_per_second
937 );
938
939 state.metrics.total_tokens.fetch_add(
940 stats.tokens_generated as u64,
941 std::sync::atomic::Ordering::Relaxed,
942 );
943
944 let oai_finish_reason = match stats.finish_reason.as_str() {
946 "max_tokens" | "MaxTokens" | "length" => "length",
947 _ => "stop",
948 };
949
950 let usage = Usage {
951 prompt_tokens: stats.prefill_tokens,
952 completion_tokens: stats.tokens_generated,
953 total_tokens: stats.prefill_tokens + stats.tokens_generated,
954 online_training_details: stats.ttt_metrics.as_ref()
955 .map(OnlineTrainingDetails::from),
956 };
957
958 let oai_finish_reason = if tools::has_tool_calls_for_format(tool_call_format, &accumulated_text) {
960 info!("Detected tool calls in streaming response (format: {:?})", tool_call_format);
961 match tools::parse_tool_calls_for_format(tool_call_format, &accumulated_text) {
962 Ok(parsed_tool_calls) if !parsed_tool_calls.is_empty() => {
963 info!("Parsed {} tool calls", parsed_tool_calls.len());
964 let tool_calls_chunk = serde_json::json!({
965 "id": sse_stream_id,
966 "object": "chat.completion.chunk",
967 "created": chrono::Utc::now().timestamp(),
968 "model": model_name,
969 "choices": [{
970 "index": 0,
971 "delta": {
972 "tool_calls": parsed_tool_calls
973 },
974 "finish_reason": null
975 }]
976 });
977 let _ = tx.send(Ok(tool_calls_chunk)).await;
978 "tool_calls"
979 }
980 _ => oai_finish_reason,
981 }
982 } else {
983 oai_finish_reason
984 };
985
986 let completion_msg = serde_json::json!({
987 "id": sse_stream_id,
988 "object": "chat.completion.chunk",
989 "created": chrono::Utc::now().timestamp(),
990 "model": model_name,
991 "choices": [{
992 "index": 0,
993 "delta": {},
994 "finish_reason": oai_finish_reason
995 }],
996 "usage": usage
997 });
998 let _ = tx.send(Ok(completion_msg)).await;
999
1000 let _ = tx.send(Ok(serde_json::json!({"done": true}))).await;
1001 break 'outer;
1002 }
1003 Ok(InferenceStreamPayload::Error(message)) => {
1004 error!("Stream error: {}", message);
1005 let _ = tx.send(Err(anyhow::anyhow!("Generation error: {}", message))).await;
1006 break 'outer;
1007 }
1008 Err(e) => {
1009 error!("Failed to parse stream payload: {}", e);
1010 continue 'outer;
1011 }
1012 }
1013 }
1014 Some(Err(e)) => {
1015 error!("Stream receive error: {}", e);
1016 let _ = tx.send(Err(anyhow::anyhow!("Stream receive error: {}", e))).await;
1017 break 'outer;
1018 }
1019 None => {
1020 break 'outer;
1021 }
1022 }
1023 }
1024 _ = tx.closed() => {
1025 info!("Client disconnected, stopping stream");
1026 stream_handle.cancel();
1027 break 'outer;
1028 }
1029 }
1030 }
1031
1032 });
1035
1036 let stream = ReceiverStream::new(rx).map(|result| {
1038 match result {
1039 Ok(json) => {
1040 if json.get("done").is_some() {
1041 Ok::<_, Infallible>(axum::response::sse::Event::default().data("[DONE]"))
1043 } else {
1044 Ok(axum::response::sse::Event::default()
1046 .data(serde_json::to_string(&json).unwrap_or_else(|_| "{}".to_owned())))
1047 }
1048 }
1049 Err(e) => {
1050 Ok(axum::response::sse::Event::default()
1052 .data(stream_error_payload(&e.to_string()).to_string()))
1053 }
1054 }
1055 });
1056
1057 let mut response = Sse::new(stream)
1059 .keep_alive(axum::response::sse::KeepAlive::new())
1060 .into_response();
1061
1062 add_no_cache_headers(&mut response);
1064 response
1065 .headers_mut()
1066 .insert(header::EXPIRES, axum::http::HeaderValue::from_static("0"));
1067
1068 response
1069}
1070
1071async fn completions(
1073 State(state): State<ServerState>,
1074 auth_user: Option<Extension<AuthenticatedUser>>,
1075 _headers: HeaderMap,
1076 Json(request): Json<CompletionRequest>,
1077) -> impl IntoResponse {
1078 let (user, domain, jwt_token, jwt_exp) =
1080 match extract_auth(auth_user.as_ref().map(|Extension(u)| u)) {
1081 Ok(identity) => identity,
1082 Err(_) => {
1083 return (
1084 StatusCode::FORBIDDEN,
1085 Json(ErrorResponse::new(
1086 "Verified hosted-account tenant binding required",
1087 "permission_denied",
1088 "missing_tenant",
1089 )),
1090 )
1091 .into_response();
1092 }
1093 };
1094
1095 let resource = format!("model:{}", request.model);
1097 let policy_request = hyprstream_rpc_std::policy_client::PolicyCheck {
1098 subject: user.clone(),
1099 domain: domain.clone(),
1100 resource: resource.clone(),
1101 operation: Operation::Infer.as_str().to_owned(),
1102 };
1103 match crate::services::policy::check_with_verified_bearer(
1104 &state.policy_client,
1105 &policy_request,
1106 jwt_token.as_deref(),
1107 &hyprstream_rpc::envelope::Subject::new(user.clone()),
1108 )
1109 .await
1110 {
1111 Ok(allowed) if !allowed => {
1112 return (
1113 StatusCode::FORBIDDEN,
1114 Json(ErrorResponse::new(
1115 format!(
1116 "Permission denied: user '{}' cannot infer on '{}'",
1117 user, request.model
1118 ),
1119 "permission_denied",
1120 "insufficient_permissions",
1121 )),
1122 )
1123 .into_response();
1124 }
1125 Err(e) => {
1126 error!("Policy check failed: {}", e);
1127 return (
1128 StatusCode::INTERNAL_SERVER_ERROR,
1129 Json(ErrorResponse::new(
1130 format!("Policy check failed: {e}"),
1131 "policy_error",
1132 "internal_error",
1133 )),
1134 )
1135 .into_response();
1136 }
1137 _ => {} }
1139
1140 state
1142 .metrics
1143 .active_requests
1144 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1145 state
1146 .metrics
1147 .total_requests
1148 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1149 let _metrics_guard = MetricsGuard::new(&state.metrics);
1150
1151 let model_path = match resolve_model_path(&state, &request.model).await {
1153 Ok(path) => path,
1154 Err(response) => return response.into_response(),
1155 };
1156
1157 let model_client = match model_client_for_request(&state, jwt_token.as_deref()) {
1160 Ok(client) => client,
1161 Err(e) => {
1162 tracing::error!("Failed to create authenticated ModelClient: {e}");
1163 return (StatusCode::INTERNAL_SERVER_ERROR, "Client creation failed").into_response();
1164 }
1165 };
1166 let rpc_messages = vec![hyprstream_rpc_std::inference_client::ChatMessage {
1167 role: "user".to_owned(),
1168 content: request.prompt.clone(),
1169 tool_calls: vec![],
1170 tool_call_id: String::new(),
1171 }];
1172
1173 let templated_prompt = match model_client
1174 .infer(&request.model)
1175 .apply_chat_template(&hyprstream_rpc_std::model_client::ChatTemplateRequest {
1176 messages: rpc_messages.clone(),
1177 add_generation_prompt: true,
1178 tools_json: None,
1179 max_tokens: request.max_tokens.map(|v| v as u32),
1180 enable_thinking: None,
1181 template_vars_json: None,
1182 })
1183 .await
1184 {
1185 Ok(prompt_str) => crate::config::TemplatedPrompt::new(prompt_str),
1186 Err(e) => {
1187 error!("Template formatting failed for completions endpoint: {}", e);
1188 return (
1189 StatusCode::INTERNAL_SERVER_ERROR,
1190 Json(ErrorResponse::new(
1191 format!("Template formatting failed: {e}"),
1192 "template_error",
1193 "template_formatting_failed",
1194 )),
1195 )
1196 .into_response();
1197 }
1198 };
1199
1200 let gen_params = crate::config::SamplingParams::from(&state.config.sampling_defaults)
1201 .merge(
1202 crate::config::SamplingParams::from_model_path(&model_path)
1203 .await
1204 .unwrap_or_default(),
1205 )
1206 .merge((&request).into());
1207 let gen_request = gen_params.into_generation_request(templated_prompt.into_inner());
1208
1209 info!(
1210 "Completions: max_tokens={:?}, temp={:?}, top_p={:?}, top_k={:?}, repeat_penalty={:?}",
1211 gen_request.max_tokens,
1212 gen_request.temperature,
1213 gen_request.top_p,
1214 gen_request.top_k,
1215 gen_request.repeat_penalty
1216 );
1217
1218 let _claims = claims_from_auth(&user, &domain, jwt_token.as_deref(), jwt_exp);
1219 let result = collect_stream_to_result(&model_client, &request.model, &gen_request).await;
1220
1221 match result {
1224 Ok(collected) => {
1225 let response = CompletionResponse {
1226 id: format!("cmpl-{}", uuid::Uuid::new_v4()),
1227 object: "text_completion".to_owned(),
1228 created: chrono::Utc::now().timestamp(),
1229 model: request.model.clone(),
1230 choices: vec![CompletionChoice {
1231 text: collected.text,
1232 index: 0,
1233 logprobs: None,
1234 finish_reason: Some(
1235 if collected.stats.finish_reason == "max_tokens" {
1236 "length"
1237 } else {
1238 "stop"
1239 }
1240 .to_owned(),
1241 ),
1242 }],
1243 usage: Some(Usage {
1244 prompt_tokens: request.prompt.len() / 4, completion_tokens: collected.stats.tokens_generated,
1246 total_tokens: request.prompt.len() / 4 + collected.stats.tokens_generated,
1247 online_training_details: collected
1248 .stats
1249 .ttt_metrics
1250 .as_ref()
1251 .map(OnlineTrainingDetails::from),
1252 }),
1253 };
1254
1255 let mut response = Json(response).into_response();
1256 add_no_cache_headers(&mut response);
1257 response
1258 }
1259 Err(e) => (
1260 StatusCode::INTERNAL_SERVER_ERROR,
1261 Json(serde_json::json!({
1262 "error": {
1263 "message": format!("Generation failed: {}", e),
1264 "type": "generation_error",
1265 "code": "internal_error"
1266 }
1267 })),
1268 )
1269 .into_response(),
1270 }
1271}
1272
1273async fn embeddings(
1275 State(_state): State<ServerState>,
1276 Json(_request): Json<EmbeddingRequest>,
1277) -> impl IntoResponse {
1278 (
1279 StatusCode::NOT_IMPLEMENTED,
1280 Json(serde_json::json!({
1281 "error": {
1282 "message": "Embeddings not yet implemented",
1283 "type": "not_implemented",
1284 "code": "feature_not_available"
1285 }
1286 })),
1287 )
1288 .into_response()
1289}
1290
1291async fn list_models(
1296 State(state): State<ServerState>,
1297 auth_user: Option<Extension<AuthenticatedUser>>,
1298) -> impl IntoResponse {
1299 let (user, domain, jwt_token, _) = match extract_auth(auth_user.as_ref().map(|Extension(u)| u))
1300 {
1301 Ok(identity) => identity,
1302 Err(_) => {
1303 return (
1304 StatusCode::FORBIDDEN,
1305 Json(ErrorResponse::new(
1306 "Verified hosted-account tenant binding required",
1307 "permission_denied",
1308 "missing_tenant",
1309 )),
1310 )
1311 .into_response();
1312 }
1313 };
1314 let policy_request = hyprstream_rpc_std::policy_client::PolicyCheck {
1315 subject: user.clone(),
1316 domain,
1317 resource: "registry:*".to_owned(),
1318 operation: Operation::Query.as_str().to_owned(),
1319 };
1320 if !crate::services::policy::check_with_verified_bearer(
1321 &state.policy_client,
1322 &policy_request,
1323 jwt_token.as_deref(),
1324 &hyprstream_rpc::envelope::Subject::new(user),
1325 )
1326 .await
1327 .unwrap_or(false)
1328 {
1329 return (
1330 StatusCode::FORBIDDEN,
1331 Json(ErrorResponse::new(
1332 "Permission denied: cannot query registry",
1333 "permission_denied",
1334 "insufficient_permissions",
1335 )),
1336 )
1337 .into_response();
1338 }
1339
1340 let mut models = vec![];
1341
1342 let result: Result<(), anyhow::Error> = async {
1344 let repos = state.registry.list().await?;
1345 for repo in repos {
1346 if repo.name.is_empty() {
1347 continue;
1348 }
1349 let name = &repo.name;
1350 match state.registry.repo(&repo.id).list_worktrees().await {
1351 Ok(worktrees) => {
1352 for wt in worktrees {
1353 if wt.branch_name.is_empty() {
1354 continue;
1355 }
1356 let display = format!("{}:{}", name, wt.branch_name);
1357 models.push(Model {
1358 id: display,
1359 object: "model".to_owned(),
1360 created: chrono::Utc::now().timestamp(),
1361 owned_by: "system".to_owned(),
1362 });
1363 }
1364 }
1365 Err(e) => {
1366 error!("Failed to list worktrees for {}: {}", name, e);
1367 }
1368 }
1369 }
1370 Ok(())
1371 }
1372 .await;
1373
1374 if let Err(e) = result {
1375 error!("Failed to list models from storage: {}", e);
1376 }
1377
1378 let mut response = Json(ListModelsResponse {
1380 object: "list".to_owned(),
1381 data: models,
1382 })
1383 .into_response();
1384 add_no_cache_headers(&mut response);
1385 response
1386}
1387
1388fn to_rpc_messages(msgs: &[ChatMessage]) -> Vec<hyprstream_rpc_std::inference_client::ChatMessage> {
1393 use hyprstream_rpc_std::inference_client::ChatMessage as RpcMsg;
1394 msgs.iter()
1395 .map(|m| RpcMsg {
1396 role: m.role.clone(),
1397 content: m.content.as_deref().unwrap_or("").to_owned(),
1398 tool_calls: m.tool_calls.clone().unwrap_or_default(),
1399 tool_call_id: m.tool_call_id.as_deref().unwrap_or("").to_owned(),
1400 })
1401 .collect()
1402}
1403
1404#[cfg(test)]
1405mod tests {
1406 use super::{
1407 prepare_chat_request, stream_error_payload, tool_choice_error_response, ErrorResponse,
1408 ToolChoiceFailure,
1409 };
1410 use crate::api::openai_compat::ChatCompletionRequest;
1411 use axum::{body::to_bytes, http::StatusCode};
1412
1413 fn tool_choice_request(
1414 stream: bool,
1415 choice: Option<serde_json::Value>,
1416 ) -> ChatCompletionRequest {
1417 let mut value = serde_json::json!({
1418 "model": "model:main",
1419 "messages": [{"role": "user", "content": "hello"}],
1420 "stream": stream,
1421 "tools": [{
1422 "type": "function",
1423 "function": {"name": "status", "parameters": {"type": "object"}}
1424 }]
1425 });
1426 if let Some(choice) = choice {
1427 value["tool_choice"] = choice;
1428 }
1429 match serde_json::from_value(value) {
1430 Ok(request) => request,
1431 Err(error) => panic!("tool choice fixture must deserialize: {error}"),
1432 }
1433 }
1434
1435 #[test]
1436 fn absent_and_auto_preserve_tools_for_both_completion_paths() {
1437 for stream in [false, true] {
1438 for choice in [None, Some(serde_json::json!("auto"))] {
1439 let (request, selected_stream) =
1440 match prepare_chat_request(tool_choice_request(stream, choice)) {
1441 Ok(prepared) => prepared,
1442 Err(error) => panic!("supported tool choice was rejected: {error:?}"),
1443 };
1444 assert_eq!(selected_stream, stream);
1445 assert_eq!(request.tools.as_ref().map(Vec::len), Some(1));
1446 }
1447 }
1448 }
1449
1450 #[test]
1451 fn none_omits_tool_definitions_for_both_completion_paths() {
1452 for stream in [false, true] {
1453 let (request, selected_stream) = match prepare_chat_request(tool_choice_request(
1454 stream,
1455 Some(serde_json::json!("none")),
1456 )) {
1457 Ok(prepared) => prepared,
1458 Err(error) => panic!("none tool choice was rejected: {error:?}"),
1459 };
1460 assert_eq!(selected_stream, stream);
1461 assert!(request.tools.is_none());
1464 }
1465 }
1466
1467 #[tokio::test]
1468 async fn required_named_and_unknown_choices_fail_before_either_completion_path() {
1469 for stream in [false, true] {
1470 for (choice, expected) in [
1471 (
1472 serde_json::json!("required"),
1473 ToolChoiceFailure::RequiredUnsupported,
1474 ),
1475 (
1476 serde_json::json!({"type": "function", "function": {"name": "status"}}),
1477 ToolChoiceFailure::NamedUnsupported,
1478 ),
1479 (
1480 serde_json::json!("sentinel-unsupported-choice"),
1481 ToolChoiceFailure::Invalid,
1482 ),
1483 ] {
1484 let failure = match prepare_chat_request(tool_choice_request(stream, Some(choice)))
1485 {
1486 Ok(_) => panic!("unsupported tool choice was accepted"),
1487 Err(failure) => failure,
1488 };
1489 assert_eq!(failure, expected);
1490 let response = tool_choice_error_response(failure);
1491 assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1492 let body = match to_bytes(response.into_body(), 1024).await {
1493 Ok(bytes) => bytes,
1494 Err(error) => panic!("tool choice error body must be readable: {error}"),
1495 };
1496 let response: serde_json::Value = match serde_json::from_slice(&body) {
1497 Ok(value) => value,
1498 Err(error) => panic!("tool choice error must be JSON: {error}"),
1499 };
1500 assert_eq!(response["error"]["type"], "invalid_request_error");
1501 assert_eq!(response["error"]["param"], "tool_choice");
1502 assert_eq!(
1503 response["error"]["code"],
1504 if failure == ToolChoiceFailure::Invalid {
1505 "invalid_tool_choice"
1506 } else {
1507 "unsupported_tool_choice"
1508 }
1509 );
1510 assert!(!response.to_string().contains("sentinel-unsupported-choice"));
1511 }
1512 }
1513 }
1514
1515 #[test]
1516 fn ordinary_errors_keep_their_existing_shape() {
1517 let value = match serde_json::to_value(ErrorResponse::new(
1518 "ordinary error",
1519 "existing_type",
1520 "existing_code",
1521 )) {
1522 Ok(value) => value,
1523 Err(error) => panic!("existing error must serialize: {error}"),
1524 };
1525 assert!(value["error"].get("param").is_none());
1526 }
1527
1528 #[test]
1529 fn streamed_failure_has_openai_error_object_and_valid_json() {
1530 let message = "Stream receive error: stream idle: no group for 30s; \"retry\"\nnext line";
1531 let serialized = stream_error_payload(message).to_string();
1532 let decoded: serde_json::Value = match serde_json::from_str(&serialized) {
1533 Ok(value) => value,
1534 Err(error) => panic!("stream error envelope is invalid JSON: {error}"),
1535 };
1536
1537 assert_eq!(decoded["error"]["message"], message);
1538 assert_eq!(decoded["error"]["type"], "generation_error");
1539 assert_eq!(decoded["error"]["code"], "internal_error");
1540 assert!(decoded["error"]["param"].is_null());
1541 assert!(decoded["error"].is_object());
1542 assert!(decoded.get("done").is_none());
1543 }
1544}