1use crate::config::TrainingMode;
37use crate::runtime::kv_cache::CacheOwner;
38use crate::runtime::model_config::ModelConfig;
39use crate::runtime::{create_engine, RuntimeConfig, RuntimeEngine, TorchEngine};
40use hyprstream_rpc_std::inference_client::GenerationRequest;
41use hyprstream_rpc_std::inference_client::ModelInfo;
42use hyprstream_rpc_std::policy_client::PolicyClient;
43
44use crate::services::EnvelopeContext;
45use crate::services::WorktreeClientExt;
46use crate::training::serialize_state_dict_to_bytes;
47use crate::training::{DeltaPool, TTTConfig, TenantDeltaConfig, TestTimeTrainer};
48use anyhow::{anyhow, Result};
49use hyprstream_rpc::prelude::*;
50use hyprstream_rpc::StreamChannel;
51use hyprstream_rpc::Subject;
52use hyprstream_rpc_std::registry_client::WorktreeClient;
53use parking_lot::Mutex;
54use std::path::{Path, PathBuf};
55use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
56use std::sync::Arc;
57use tokenizers::Tokenizer;
58use tokio::runtime::Handle;
59use tracing::{debug, error, info, trace, warn};
60
61#[cfg(feature = "ledger")]
64use crate::services::ledger::{
65 observe_spend_result, InferenceSpendEmitter, SpendInput, SpendResult,
66};
67
68enum PendingWork {
76 Generation {
78 stream_ctx: hyprstream_rpc::StreamContext,
80 request: GenerationRequest,
82 subject: Subject,
84 ttt_overrides: crate::training::ttt::TTTOverrides,
86 #[cfg_attr(not(feature = "ledger"), allow(dead_code))]
92 owner_did: Option<String>,
93 },
94 Training {
96 stream_ctx: hyprstream_rpc::StreamContext,
98 subject: Subject,
100 input: String,
102 gradient_steps: Option<u32>,
104 learning_rate: Option<f32>,
106 adaptation_strategy: crate::training::adaptation_state::AdaptationStrategy,
108 },
109}
110
111pub const INFERENCE_ENDPOINT: &str = "inproc://hyprstream/inference";
113
114pub struct InferenceServiceInner {
142 engine: parking_lot::RwLock<TorchEngine>,
143 model_path: PathBuf,
145 state_path: PathBuf,
147 _pinned_artifact: Option<Arc<crate::storage::pinned_model::PinnedModelArtifact>>,
149 session_id: parking_lot::RwLock<Option<String>>,
151 #[allow(dead_code)] runtime_handle: Handle,
154 stream_channel: Option<StreamChannel>,
157 #[allow(dead_code)]
159 server_pubkey: VerifyingKey,
160 #[allow(dead_code)]
162 signing_key: SigningKey,
163 #[allow(dead_code)]
165 nonce_cache: Arc<InMemoryNonceCache>,
166 policy_client: PolicyClient,
168 ttt_trainer: Option<Arc<TestTimeTrainer>>,
170 tokenizer: Option<Arc<Tokenizer>>,
172 delta_pool: Option<Arc<DeltaPool>>,
174 base_delta: Mutex<Option<Arc<Mutex<crate::training::TenantDelta>>>>,
177 fs: Option<WorktreeClient>,
180 #[allow(dead_code)] transport: hyprstream_rpc::transport::TransportConfig,
183 lora_generation: Arc<AtomicU64>,
186 tenant_domain: String,
188 controller_pubkey: VerifyingKey,
191 network_ready: Arc<AtomicBool>,
193 draining: Arc<AtomicBool>,
195 shutdown: Arc<tokio::sync::Notify>,
197 #[cfg(feature = "ledger")]
203 ledger: parking_lot::RwLock<Option<Arc<InferenceSpendEmitter>>>,
204 object_label: hyprstream_rpc::auth::mac::SecurityLabel,
205}
206
207pub struct InferenceService {
212 inner: Arc<InferenceServiceInner>,
213}
214
215impl Clone for InferenceService {
216 fn clone(&self) -> Self {
217 Self {
218 inner: Arc::clone(&self.inner),
219 }
220 }
221}
222
223impl std::ops::Deref for InferenceService {
224 type Target = InferenceServiceInner;
225 fn deref(&self) -> &Self::Target {
226 &self.inner
227 }
228}
229
230unsafe impl Send for InferenceServiceInner {}
235unsafe impl Sync for InferenceServiceInner {}
236
237fn resolve_instance_domain(
238 tenant_domain: &str,
239 controller_pubkey: &VerifyingKey,
240 ctx: &EnvelopeContext,
241) -> Result<String> {
242 let domain = ctx.domain()?;
243 if domain == tenant_domain {
244 return Ok(domain);
245 }
246 if domain == "local" && ctx.cnf == controller_pubkey.to_bytes() {
247 return Ok(tenant_domain.to_owned());
248 }
249 anyhow::bail!(
250 "authorization denied: inference instance is bound to a different verified tenant"
251 )
252}
253pub(crate) fn enforce_inference_mac(
254 ctx: &EnvelopeContext,
255 object_label: &hyprstream_rpc::auth::mac::SecurityLabel,
256) -> Result<()> {
257 enforce_inference_security_context(ctx.security_context(), object_label)
258}
259pub(crate) fn inference_object_label() -> hyprstream_rpc::auth::mac::SecurityLabel {
261 crate::mac::genesis::SitePolicy::conservative().label_for("/srv/inference")
262}
263fn enforce_inference_security_context(
264 security_context: Option<hyprstream_rpc::auth::mac::SecurityContext>,
265 object_label: &hyprstream_rpc::auth::mac::SecurityLabel,
266) -> Result<()> {
267 let security_context = security_context.ok_or_else(|| {
268 anyhow!("authorization denied: inference caller has no verified MAC clearance")
269 })?;
270 anyhow::ensure!(
271 security_context.can_access(object_label),
272 "authorization denied: inference caller clearance does not dominate engine label {object_label}"
273 );
274 Ok(())
275}
276
277#[derive(Debug)]
294enum GenLoopOutcome {
295 Exhausted,
297 Cancelled,
299 StreamError(anyhow::Error),
301 PublishFailed(anyhow::Error),
303}
304
305trait GenSink {
310 fn publish_chunk(
312 &mut self,
313 data: &str,
314 rate: f32,
315 ) -> impl std::future::Future<Output = Result<()>>;
316}
317
318impl GenSink for hyprstream_rpc::moq_stream::AnyStreamPublisher {
319 fn publish_chunk(
320 &mut self,
321 data: &str,
322 rate: f32,
323 ) -> impl std::future::Future<Output = Result<()>> {
324 self.publish_data_with_rate(data.as_bytes(), rate)
325 }
326}
327
328trait GenOutput: futures::Stream<Item = anyhow::Result<String>> + Unpin {
331 fn ema_rate(&self) -> f32;
333 fn completion_stats(&self) -> crate::runtime::GenerationStats;
335}
336
337impl<'a> GenOutput for crate::runtime::TextStream<'a> {
338 fn ema_rate(&self) -> f32 {
339 self.stats().inference_tokens_per_sec_ema
340 }
341 fn completion_stats(&self) -> crate::runtime::GenerationStats {
342 self.stats()
343 }
344}
345
346async fn drive_generation_loop<S: GenOutput>(
353 stream: &mut S,
354 cancel: &tokio_util::sync::CancellationToken,
355 sink: &mut impl GenSink,
356) -> GenLoopOutcome {
357 use futures::StreamExt;
358 loop {
359 let next = tokio::select! {
360 biased;
361 _ = cancel.cancelled() => return GenLoopOutcome::Cancelled,
362 n = stream.next() => n,
363 };
364 match next {
365 None => return GenLoopOutcome::Exhausted,
366 Some(Err(e)) => return GenLoopOutcome::StreamError(e),
367 Some(Ok(text)) => {
368 let rate = stream.ema_rate();
369 if let Err(e) = sink.publish_chunk(&text, rate).await {
370 return GenLoopOutcome::PublishFailed(e);
371 }
372 }
373 }
374 }
375}
376
377#[cfg_attr(not(feature = "ledger"), allow(dead_code))] const LEDGER_DENY_NO_IDENTITY: &str =
382 "generation denied: no verified billing identity for token-spend accounting";
383
384#[cfg_attr(not(feature = "ledger"), allow(dead_code))] fn ledger_requires_identity(emitter_attached: bool, owner_did: Option<&str>) -> bool {
395 emitter_attached && owner_did.is_none()
396}
397
398#[cfg(feature = "ledger")]
407async fn post_completion_spend(
408 emitter: &InferenceSpendEmitter,
409 stats: Option<&crate::runtime::GenerationStats>,
410 owner_did: Option<&str>,
411 stream_id: &str,
412) -> Option<SpendResult> {
413 let stats = stats?;
414 let owner = owner_did?;
415 let res = emitter
416 .post_generation_spend(SpendInput {
417 owner_did: owner,
418 stream_id,
419 prompt_tokens: stats.prefill_tokens as u64,
420 generated_tokens: stats.tokens_generated as u64,
421 })
422 .await;
423 Some(res)
424}
425
426impl InferenceService {
427 pub async fn drain_export(&self) -> anyhow::Result<usize> {
439 let Some(pool) = self.delta_pool.as_ref() else {
440 return Ok(0);
441 };
442 let exported = pool.export_all().await?;
443 let persisted = exported.iter().filter(|(_, p)| p.is_some()).count();
444 tracing::info!(
445 "drain_export: persisted {}/{} resident delta(s) to snapshots",
446 persisted,
447 exported.len()
448 );
449 Ok(persisted)
450 }
451
452 #[cfg(feature = "ledger")]
461 pub fn attach_ledger_spend(&self, emitter: Arc<InferenceSpendEmitter>) {
462 *self.ledger.write() = Some(emitter);
463 }
464
465 #[allow(dead_code)] fn ttt_guard(&self, subject: &Subject) -> crate::training::GuardStatus {
470 let (expired, at_capacity) = if let Some(pool) = &self.delta_pool {
471 if let Some(delta_arc) = pool.get(subject) {
472 let delta = delta_arc.lock();
473 (delta.adaptation_state.is_expired(), delta.is_at_capacity())
474 } else {
475 (false, false)
476 }
477 } else {
478 (false, false)
479 };
480 crate::training::GuardStatus {
481 expired,
482 at_capacity,
483 lora_generation: self
484 .lora_generation
485 .load(std::sync::atomic::Ordering::Acquire),
486 }
487 }
488
489 async fn initialize(
491 model_path: PathBuf,
492 state_path: PathBuf,
493 pinned_artifact: Option<Arc<crate::storage::pinned_model::PinnedModelArtifact>>,
494 config: RuntimeConfig,
495 server_pubkey: VerifyingKey,
496 signing_key: SigningKey,
497 nonce_cache: Arc<InMemoryNonceCache>,
498 policy_client: PolicyClient,
499 fs: Option<WorktreeClient>,
500 tenant_domain: String,
501 controller_pubkey: VerifyingKey,
502 producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
503 moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
504 network_ready: Arc<AtomicBool>,
505 draining: Arc<AtomicBool>,
506 shutdown: Arc<tokio::sync::Notify>,
507 ) -> Result<Self> {
508 let runtime_handle = Handle::current();
510
511 #[cfg(test)]
512 crate::services::restart_diag::phase("engine.create.enter");
513 let mut engine = create_engine(&config)?;
514 #[cfg(test)]
515 crate::services::restart_diag::phase("engine.create.done");
516 #[cfg(test)]
517 crate::services::restart_diag::phase("engine.load_model.enter");
518 if let Some(ref artifact) = pinned_artifact {
519 artifact.ensure_current_process()?;
520 }
521 RuntimeEngine::load_model(&mut engine, &model_path).await?;
522 #[cfg(test)]
523 crate::services::restart_diag::phase("engine.load_model.done");
524
525 let model_info = RuntimeEngine::model_info(&engine);
527 let num_layers = model_info.num_hidden_layers.unwrap_or(32) as usize;
528 let max_seq_len = config.max_context.unwrap_or(model_info.context_length) as usize;
529 let kv_budget = engine.compute_kv_budget();
530 engine.initialize_kv_registry(num_layers, max_seq_len, config.kv_quant_type, kv_budget);
531
532 info!(
533 "KV cache registry initialized: {} layers, max_seq_len={}",
534 num_layers, max_seq_len
535 );
536
537 let training_config = ModelConfig::load_training_config(&model_path);
539
540 let (ttt_trainer, tokenizer): (Option<Arc<TestTimeTrainer>>, Option<Arc<Tokenizer>>) =
543 if let Some(ref tc) = training_config {
544 if tc.is_enabled() && tc.mode == TrainingMode::TestTimeTraining {
545 info!(
546 "Test-Time Training enabled: lr={}, steps={}",
547 tc.ttt.learning_rate, tc.ttt.gradient_steps
548 );
549
550 let ttt_config = TTTConfig {
551 learning_rate: tc.ttt.learning_rate,
552 gradient_steps: tc.ttt.gradient_steps,
553 max_grad_norm: tc.ttt.max_grad_norm,
554 min_input_length: tc.ttt.min_input_length,
555 max_ttt_context: tc.ttt.max_ttt_context,
556 enabled: true,
557 ..TTTConfig::default()
558 };
559
560 let device = engine.device();
561 let mut trainer = TestTimeTrainer::new(ttt_config, device);
562 if let Some(ref gating) = tc.ttt.gradient_gating {
564 trainer = trainer.with_gradient_gating(gating.clone());
565 }
566
567 let tokenizer = engine.get_tokenizer().ok().map(Arc::new);
569 (Some(Arc::new(trainer)), tokenizer)
570 } else {
571 (None, None)
572 }
573 } else {
574 (None, None)
575 };
576
577 let delta_pool = if ttt_trainer.is_some() {
579 let module_dims = engine.get_lora_module_dims().unwrap_or_default();
580 let per_layer_dims = engine.get_per_layer_lora_dims();
581 let device = engine.device();
582
583 let base_delta_config = if let Some(ref tc) = training_config {
584 let alpha = tc.lora_alpha.unwrap_or(tc.lora_rank as f32);
585 TenantDeltaConfig {
586 rank: tc.lora_rank,
587 alpha,
588 target_modules: tc.target_modules.clone(),
589 learning_rate: tc.ttt.learning_rate,
590 ..TenantDeltaConfig::default()
591 }
592 } else {
593 TenantDeltaConfig::default()
594 };
595
596 let model_config_result = crate::runtime::model_config::ModelConfig::load(
598 &model_path,
599 &std::collections::HashMap::new(),
600 );
601 let delta_config = if let Ok(ref mc) = model_config_result {
602 match crate::runtime::ttn_profile::get_layer_profile(&model_path, mc, None) {
603 Ok(profile) => {
604 let cfg = TenantDeltaConfig::from_profile(&base_delta_config, &profile);
605 info!(
606 "Delta pool: using TTN profile ({} layer overrides)",
607 cfg.layer_overrides
608 .as_ref()
609 .map_or(0, std::collections::HashMap::len)
610 );
611 cfg
612 }
613 Err(e) => {
614 info!("TTN profile unavailable ({}), using uniform config", e);
615 base_delta_config
616 }
617 }
618 } else {
619 base_delta_config
620 };
621
622 let kv_reg = engine.kv_registry();
623 let snapshots_dir = state_path.join("adapters").join(".snapshots");
624 let num_layers = engine.get_num_layers().unwrap_or(32);
625
626 info!(
627 "Delta pool initialized: rank={}, alpha={:.1}, modules={:?}, lr={:.1e}",
628 delta_config.rank,
629 delta_config.alpha,
630 delta_config.target_modules,
631 delta_config.learning_rate
632 );
633 let mut pool = DeltaPool::new(
634 delta_config,
635 module_dims,
636 device,
637 kv_reg,
638 snapshots_dir,
639 fs.clone(),
640 num_layers,
641 );
642 if let Some(pld) = per_layer_dims {
643 pool = pool.with_per_layer_dims(pld);
644 }
645 if let Some(ref tc) = training_config {
647 if let Some(ref oracle_config) = tc.ttt.rank_oracle {
648 pool = pool.with_rank_oracle(oracle_config.clone());
649 info!(
650 "Rank oracle enabled: interval={}, auto_adapt={}",
651 oracle_config.adaptation_interval, oracle_config.auto_adapt
652 );
653 }
654 }
655
656 Some(Arc::new(pool))
657 } else {
658 None
659 };
660
661 let stream_channel = StreamChannel::new(signing_key.clone())
663 .with_reach_config_handle(producer_reach_config)
664 .with_moq_origin_handle(moq_origin);
665 let object_label = inference_object_label();
666
667 Ok(InferenceService {
668 inner: Arc::new(InferenceServiceInner {
669 engine: parking_lot::RwLock::new(engine),
670 model_path,
671 state_path,
672 _pinned_artifact: pinned_artifact,
673 session_id: parking_lot::RwLock::new(None),
674 runtime_handle,
675 stream_channel: Some(stream_channel),
676 server_pubkey,
677 signing_key,
678 nonce_cache,
679 policy_client,
680 ttt_trainer,
681 tokenizer,
682 delta_pool,
683 base_delta: Mutex::new(None),
684 fs,
685 transport: hyprstream_rpc::transport::TransportConfig::inproc("inference-unset"),
686 lora_generation: Arc::new(AtomicU64::new(0)),
687 tenant_domain,
688 controller_pubkey,
689 network_ready,
690 draining,
691 shutdown,
692 #[cfg(feature = "ledger")]
693 ledger: parking_lot::RwLock::new(None),
694 object_label,
695 }),
696 })
697 }
698
699 fn authorization_domain(&self, ctx: &EnvelopeContext) -> Result<String> {
706 resolve_instance_domain(&self.tenant_domain, &self.controller_pubkey, ctx)
707 }
708
709 fn resolve_delta(
724 &self,
725 subject: &hyprstream_rpc::Subject,
726 ) -> (
727 Option<Arc<Mutex<crate::training::TenantDelta>>>,
728 Option<String>,
729 u64,
730 ) {
731 let base = self.base_delta.lock().clone();
732 let tenant = self.delta_pool.as_ref().and_then(|pool| pool.get(subject));
733
734 let tenant_id = tenant.as_ref().map(|_| subject.to_string());
737 let weight_epoch = tenant
741 .as_ref()
742 .map(|d| d.lock().weight_epoch())
743 .unwrap_or(0);
744
745 let delta = match (base, tenant) {
746 (Some(base), Some(tenant)) => {
747 Some(crate::training::TenantDelta::compose(&base, &tenant))
749 }
750 (Some(base), None) => Some(base),
751 (None, Some(tenant)) => Some(tenant),
752 (None, None) => None,
753 };
754
755 (delta, tenant_id, weight_epoch)
756 }
757
758 async fn apply_ttt_adaptation_with_overrides(
769 &self,
770 prompt: &str,
771 subject: &Subject,
772 overrides: &crate::training::ttt::TTTOverrides,
773 ) -> Result<Option<crate::training::ttt::TTTResult>> {
774 use anyhow::anyhow;
775
776 let ttt_trainer = match self.ttt_trainer.as_ref() {
777 Some(t) => t,
778 None => return Ok(None), };
780
781 let pool = match self.delta_pool.as_ref() {
782 Some(p) => p,
783 None => return Ok(None), };
785
786 let tokenizer = match self.tokenizer.as_ref() {
787 Some(t) => t,
788 None => return Ok(None), };
790
791 let encoding = tokenizer
792 .encode(prompt, false)
793 .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
794 let input_tokens: Vec<u32> = encoding.get_ids().to_vec();
795
796 pool.ensure_capacity().await?;
799
800 let delta_arc = pool.get_or_hydrate(subject).await?;
803 let engine = self.engine.read();
804
805 let mut delta = delta_arc.lock();
807
808 let pre_muon = crate::training::muon::snapshot_muon_states(&delta.muon_states);
815 let pre_eff_ranks = delta.effective_ranks.clone();
816
817 let adapt_result = ttt_trainer.adapt_tenant(&engine, &mut delta, &input_tokens, overrides);
818
819 drop(engine);
822
823 match adapt_result {
824 Ok((mut result, pre_snapshot)) => {
825 if !result.skipped {
826 debug!(
827 "TTT (subject {}): steps={}, improvement={:.4}, time={}ms, ppl={:.1}->{:.1}, rec={}",
828 subject,
829 result.steps_performed,
830 result.loss_improvement,
831 result.adaptation_time_ms,
832 result.initial_perplexity,
833 result.final_perplexity,
834 result.recommendation,
835 );
836 }
837
838 let strategy = overrides.adaptation_strategy.clone();
840
841 let guard = crate::training::GuardStatus {
843 expired: delta.adaptation_state.is_expired(),
844 at_capacity: delta.is_at_capacity(),
845 lora_generation: self
846 .lora_generation
847 .load(std::sync::atomic::Ordering::Acquire),
848 };
849
850 let outcome = delta.adaptation_state.resolve(
851 strategy,
852 &guard,
853 &result,
854 pre_snapshot,
855 pre_muon,
856 pre_eff_ranks,
857 );
858
859 match outcome {
860 crate::training::ResolveOutcome::WrittenBack => {
861 delta.accumulated_steps += result.steps_performed as u64;
862 delta.request_count += 1;
863 let n = delta.request_count as f64;
864 delta.avg_loss_improvement = delta.avg_loss_improvement * ((n - 1.0) / n)
865 + result.loss_improvement as f64 / n;
866 result.pending = false;
867 debug!("TTT: auto-committed adaptation for subject {}", subject);
868 }
869 crate::training::ResolveOutcome::Evicted {
870 snapshot,
871 muon,
872 eff_ranks,
873 } => {
874 let _ = delta.load_state_dict(&snapshot);
879 crate::training::muon::restore_muon_states(&mut delta.muon_states, &muon);
880 delta.effective_ranks = eff_ranks;
881 result.pending = false;
882 debug!("TTT: auto-rolled back adaptation for subject {}", subject);
883 }
884 crate::training::ResolveOutcome::StoredPending => {
885 result.pending = true;
887 debug!("TTT: stored pending adaptation for subject {}", subject);
888 }
889 crate::training::ResolveOutcome::Skipped { reason } => {
890 debug!("TTT: skipped for subject {}: {}", subject, reason);
891 }
892 }
893
894 Ok(Some(result))
895 }
896 Err(e) => {
897 warn!("TTT adaptation failed for subject {}: {}", subject, e);
898 Err(e)
899 }
900 }
901 }
902
903 async fn prepare_stream(
921 &self,
922 request: GenerationRequest,
923 client_ephemeral_pubkey: Option<&[u8]>,
924 claims: Option<hyprstream_rpc::auth::Claims>,
925 expiry_secs: i64,
926 subject: &Subject,
927 ttt_overrides: crate::training::ttt::TTTOverrides,
928 owner_did: Option<String>,
929 ) -> Result<(
930 String,
931 [u8; 32],
932 String,
933 Vec<hyprstream_rpc::stream_info::Destination>,
934 PendingWork,
935 )> {
936 anyhow::ensure!(
937 !self.draining.load(Ordering::Acquire),
938 "inference service is draining and refuses new streams"
939 );
940 let client_pub_bytes = client_ephemeral_pubkey.ok_or_else(|| {
945 anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
946 })?;
947
948 let stream_channel = self
950 .stream_channel
951 .as_ref()
952 .ok_or_else(|| anyhow!("StreamChannel not initialized"))?;
953
954 let stream_ctx = stream_channel
955 .prepare_third_party_interop_stream_with_claims(client_pub_bytes, expiry_secs, claims)
956 .await?
957 .with_qos_preset::<hyprstream_rpc::stream_info::Job>();
958
959 debug!(
960 stream_id = %stream_ctx.stream_id(),
961 topic = %stream_ctx.topic(),
962 "Stream prepared via StreamChannel (DH + pre-authorization)"
963 );
964
965 let stream_id = stream_ctx.stream_id().to_owned();
966 let server_pubkey = *stream_ctx.server_pubkey();
967 let broadcast_path = stream_channel.broadcast_path(stream_ctx.topic())?;
968 let reach = stream_ctx.reach();
971
972 let pending = PendingWork::Generation {
974 stream_ctx,
975 request,
976 subject: subject.clone(),
977 ttt_overrides,
978 owner_did,
979 };
980
981 Ok((stream_id, server_pubkey, broadcast_path, reach, pending))
982 }
983
984 #[allow(clippy::await_holding_lock)] async fn execute_stream(&self, pending: PendingWork) {
1005 debug_assert!(
1008 tokio::runtime::Handle::current().runtime_flavor()
1009 == tokio::runtime::RuntimeFlavor::CurrentThread,
1010 "InferenceService must run on current_thread runtime for parking_lot safety"
1011 );
1012
1013 #[cfg_attr(not(feature = "ledger"), allow(unused_variables))]
1018 let PendingWork::Generation {
1019 stream_ctx,
1020 request,
1021 subject,
1022 ttt_overrides,
1023 owner_did,
1024 } = pending
1025 else {
1026 error!("execute_stream called with non-Generation PendingWork");
1027 return;
1028 };
1029 let stream_ctx = &stream_ctx;
1030 #[cfg(feature = "ledger")]
1033 let stream_id_str = stream_ctx.stream_id().to_owned();
1034
1035 let stream_channel = match &self.stream_channel {
1037 Some(sc) => sc,
1038 None => {
1039 error!("StreamChannel not initialized for streaming");
1040 return;
1041 }
1042 };
1043
1044 #[cfg(feature = "ledger")]
1053 {
1054 if ledger_requires_identity(self.ledger.read().is_some(), owner_did.as_deref()) {
1055 warn!(
1056 stream_id = %stream_id_str,
1057 "ledger: spend emitter attached but caller has no verified pairwise DID \
1058 — denying generation before it starts (fail-closed, #1265)"
1059 );
1060 if let Err(e) = stream_channel
1061 .run_stream(stream_ctx, |publisher| async move {
1062 (publisher, Err::<(), _>(anyhow!(LEDGER_DENY_NO_IDENTITY)))
1063 })
1064 .await
1065 {
1066 error!(
1067 stream_id = %stream_id_str,
1068 error = %e,
1069 "failed to publish ledger deny frame"
1070 );
1071 }
1072 return;
1073 }
1074 }
1075
1076 let lora_gen_before = self.lora_generation.load(Ordering::Acquire);
1078
1079 let ttt_result = match self
1083 .apply_ttt_adaptation_with_overrides(request.prompt.as_str(), &subject, &ttt_overrides)
1084 .await
1085 {
1086 Ok(Some(result)) => Some(result),
1087 Ok(None) => {
1088 if ttt_overrides.enabled == Some(true) {
1090 error!(stream_id = %stream_ctx.stream_id(), subject = %subject, "TTT explicitly enabled but not available on server");
1092 let sc = stream_channel;
1093 if let Err(send_err) = sc
1094 .run_stream(stream_ctx, |publisher| async move {
1095 (
1096 publisher,
1097 Err::<(), _>(anyhow::anyhow!(
1098 "TTT explicitly enabled but not configured on this server"
1099 )),
1100 )
1101 })
1102 .await
1103 {
1104 error!(stream_id = %stream_ctx.stream_id(), error = %send_err,
1105 "Failed to publish TTT error to stream");
1106 }
1107 return;
1108 }
1109 None }
1111 Err(e) => {
1112 if ttt_overrides.enabled == Some(true) {
1113 error!(stream_id = %stream_ctx.stream_id(), subject = %subject, error = %e, "TTT adaptation failed");
1115 let sc = stream_channel;
1116 if let Err(send_err) = sc
1117 .run_stream(stream_ctx, |publisher| async move {
1118 (
1119 publisher,
1120 Err::<(), _>(anyhow::anyhow!("TTT adaptation failed: {}", e)),
1121 )
1122 })
1123 .await
1124 {
1125 error!(stream_id = %stream_ctx.stream_id(), error = %send_err,
1126 "Failed to publish TTT error to stream");
1127 }
1128 return;
1129 }
1130 warn!(
1132 "TTT adaptation failed (server-config), continuing without: {}",
1133 e
1134 );
1135 None
1136 }
1137 };
1138
1139 let (delta, tenant_id, weight_epoch) = self.resolve_delta(&subject);
1141
1142 trace!(
1143 stream_id = %stream_ctx.stream_id(),
1144 topic = %stream_ctx.topic(),
1145 has_delta = delta.is_some(),
1146 has_ttt = ttt_result.is_some(),
1147 "Starting E2E authenticated stream via StreamChannel"
1148 );
1149
1150 let lora_gen_now = self.lora_generation.load(Ordering::Acquire);
1152 if lora_gen_now != lora_gen_before {
1153 warn!(
1154 stream_id = %stream_ctx.stream_id(),
1155 "LoRA reconfigured during TTT adaptation (gen {} -> {}), aborting stream",
1156 lora_gen_before, lora_gen_now
1157 );
1158 let sc = stream_channel;
1159 if let Err(e) = sc
1160 .run_stream(stream_ctx, |publisher| async move {
1161 (
1162 publisher,
1163 Err::<(), _>(anyhow::anyhow!(
1164 "LoRA adapter was reconfigured during adaptation — retry request"
1165 )),
1166 )
1167 })
1168 .await
1169 {
1170 error!(stream_id = %stream_ctx.stream_id(), error = %e, "Failed to publish LoRA error");
1171 }
1172 return;
1173 }
1174
1175 let engine = self.engine.read();
1178 let stream_result =
1182 engine.generate_with_delta(request, delta, tenant_id, weight_epoch, lora_gen_now);
1183
1184 #[cfg(feature = "ledger")]
1189 let completion_stats: Arc<
1190 parking_lot::Mutex<Option<crate::runtime::GenerationStats>>,
1191 > = Arc::new(parking_lot::Mutex::new(None));
1192
1193 let result = stream_channel
1194 .run_stream(stream_ctx, |mut publisher| {
1195 #[cfg(feature = "ledger")]
1196 let completion_stats = completion_stats.clone();
1197 async move {
1198 let result = match stream_result {
1199 Ok(mut stream) => {
1200 let cancel = stream_ctx.cancel_token();
1201 let outcome =
1208 drive_generation_loop(&mut stream, cancel, &mut publisher).await;
1209
1210 let stats = stream.completion_stats();
1214 #[cfg(feature = "ledger")]
1215 {
1216 *completion_stats.lock() = Some(stats.clone());
1217 }
1218
1219 match outcome {
1220 GenLoopOutcome::Exhausted => {
1221 let mut complete =
1222 crate::services::rpc_types::InferenceComplete::from(&stats);
1223 complete.ttt_metrics = ttt_result.map(std::convert::Into::into);
1225 publisher.complete_ref(&complete.to_bytes()).await
1226 }
1227 GenLoopOutcome::Cancelled => {
1228 let _ = publisher.publish_error("cancelled").await;
1229 Ok(())
1230 }
1231 GenLoopOutcome::StreamError(e) => {
1232 let _ = publisher.publish_error(&e.to_string()).await;
1233 Ok(())
1234 }
1235 GenLoopOutcome::PublishFailed(e) => Err(e),
1236 }
1237 }
1238 Err(e) => Err(e), };
1240 (publisher, result)
1241 }
1242 })
1243 .await;
1244
1245 if let Err(e) = result {
1246 error!(
1247 stream_id = %stream_ctx.stream_id(),
1248 error = %e,
1249 "Stream execution failed"
1250 );
1251 }
1252
1253 #[cfg(feature = "ledger")]
1261 {
1262 if let Some(emitter) = self.ledger.read().clone() {
1263 let stats_opt = completion_stats.lock().clone();
1264 match post_completion_spend(
1265 &emitter,
1266 stats_opt.as_ref(),
1267 owner_did.as_deref(),
1268 &stream_id_str,
1269 )
1270 .await
1271 {
1272 Some(res) => {
1273 observe_spend_result(&res, &stream_id_str, emitter.unit());
1274 }
1275 None => {
1276 }
1282 }
1283 }
1284 }
1285 }
1286
1287 async fn execute_training_stream(&self, pending: PendingWork) {
1292 let PendingWork::Training {
1293 stream_ctx,
1294 subject,
1295 input,
1296 gradient_steps,
1297 learning_rate,
1298 adaptation_strategy,
1299 } = pending
1300 else {
1301 error!("execute_training_stream called with non-Training PendingWork");
1302 return;
1303 };
1304 let stream_ctx = &stream_ctx;
1305
1306 let stream_channel = match &self.stream_channel {
1307 Some(sc) => sc,
1308 None => {
1309 error!("StreamChannel not initialized for training stream");
1310 return;
1311 }
1312 };
1313
1314 trace!(
1315 stream_id = %stream_ctx.stream_id(),
1316 "Starting training stream via StreamChannel"
1317 );
1318
1319 let result = stream_channel
1320 .run_stream(stream_ctx, |mut publisher| async move {
1321 if stream_ctx.cancel_token().is_cancelled() {
1322 let _ = publisher.publish_error("cancelled").await;
1323 return (publisher, Ok(()));
1324 }
1325 let result = match self
1326 .handle_train_step(
1327 &subject,
1328 &input,
1329 gradient_steps,
1330 learning_rate,
1331 adaptation_strategy,
1332 )
1333 .await
1334 {
1335 Ok(result) => {
1336 let payload = serde_json::to_vec(&result).unwrap_or_else(|e| {
1338 format!("{{\"error\":\"serialize failed: {e}\"}}").into_bytes()
1339 });
1340 publisher.complete_ref(&payload).await
1341 }
1342 Err(e) => Err(e), };
1344 (publisher, result)
1345 })
1346 .await;
1347
1348 if let Err(e) = result {
1349 error!(
1350 stream_id = %stream_ctx.stream_id(),
1351 error = %e,
1352 "Training stream execution failed"
1353 );
1354 }
1355 }
1356
1357 async fn handle_model_info(&self) -> ModelInfo {
1359 RuntimeEngine::model_info(&*self.engine.read())
1360 }
1361
1362 async fn handle_is_ready(&self) -> bool {
1364 self.engine.read().is_loaded()
1365 && self.network_ready.load(Ordering::Acquire)
1366 && !self.draining.load(Ordering::Acquire)
1367 }
1368
1369 async fn handle_apply_chat_template(
1371 &self,
1372 messages: Vec<crate::runtime::template_engine::ChatMessage>,
1373 add_generation_prompt: bool,
1374 tools: Option<&serde_json::Value>,
1375 enable_thinking: Option<bool>,
1376 template_vars_json: Option<&str>,
1377 ) -> Result<String> {
1378 self.engine.read().apply_chat_template_with_vars(
1379 &messages,
1380 add_generation_prompt,
1381 tools,
1382 enable_thinking,
1383 template_vars_json,
1384 )
1385 }
1386
1387 fn truncate_messages_to_budget(
1396 &self,
1397 messages: Vec<crate::runtime::template_engine::ChatMessage>,
1398 reserve_tokens: usize,
1399 ) -> Result<Vec<crate::runtime::template_engine::ChatMessage>> {
1400 let info = self.engine.read().model_info();
1401 let max_seq_len = info.context_length as usize;
1402 let budget = max_seq_len.saturating_sub(reserve_tokens);
1403 if budget == 0 {
1404 return Ok(messages);
1405 }
1406
1407 const FRAMING_OVERHEAD: usize = 6; let tokenizer = match self.engine.read().get_tokenizer() {
1410 Ok(t) => t,
1411 Err(_) => return Ok(messages), };
1413
1414 let msg_costs: Vec<usize> = messages
1415 .iter()
1416 .map(|m| {
1417 let content_tokens = m
1419 .content
1420 .as_deref()
1421 .and_then(|c| tokenizer.encode(c, false).ok())
1422 .map(|e| e.get_ids().len())
1423 .unwrap_or(0);
1424
1425 let tool_call_tokens = m.tool_calls.as_ref().map_or(0, |tcs| {
1427 tcs.iter()
1428 .map(|tc| {
1429 let call_str = format!(
1430 "{}({})",
1431 tc.function.name,
1432 serde_json::to_string(&tc.function.arguments).unwrap_or_default()
1433 );
1434 tokenizer
1435 .encode(call_str.as_str(), false)
1436 .ok()
1437 .map(|e| e.get_ids().len())
1438 .unwrap_or(10) })
1440 .sum::<usize>()
1441 });
1442
1443 content_tokens + tool_call_tokens + FRAMING_OVERHEAD
1444 })
1445 .collect();
1446
1447 let total: usize = msg_costs.iter().sum();
1448 if total <= budget {
1449 return Ok(messages); }
1451
1452 let mut groups: Vec<(usize, Vec<usize>)> = Vec::new(); let mut i = 0;
1456 while i < messages.len() {
1457 let m = &messages[i];
1458 if m.role == "assistant" && m.tool_calls.as_ref().is_some_and(|tc| !tc.is_empty()) {
1459 let tool_ids: std::collections::HashSet<&str> = m
1460 .tool_calls
1461 .as_deref()
1462 .unwrap_or_default()
1463 .iter()
1464 .map(|tc| tc.id.as_str())
1465 .collect();
1466 let mut indices = vec![i];
1467 let mut cost = msg_costs[i];
1468 let mut j = i + 1;
1469 while j < messages.len()
1470 && messages[j].role == "tool"
1471 && messages[j]
1472 .tool_call_id
1473 .as_deref()
1474 .is_some_and(|id| tool_ids.contains(id))
1475 {
1476 indices.push(j);
1477 cost += msg_costs[j];
1478 j += 1;
1479 }
1480 groups.push((cost, indices));
1481 i = j;
1482 } else {
1483 groups.push((msg_costs[i], vec![i]));
1484 i += 1;
1485 }
1486 }
1487
1488 let has_system = messages
1490 .first()
1491 .map(|m| m.role == "system")
1492 .unwrap_or(false);
1493 let system_cost = if has_system { groups[0].0 } else { 0 };
1494 let start = if has_system { 1 } else { 0 };
1495
1496 if groups.len() > start {
1498 let last_group_cost = groups.last().map(|(c, _)| *c).unwrap_or(0);
1499 if system_cost + last_group_cost > budget {
1500 return Err(anyhow::anyhow!(
1501 "Input exceeds context window: system prompt (~{} tokens) + last message (~{} tokens) = ~{} tokens, \
1502 but only {} tokens available (max_seq_len={}, reserved for generation={}). \
1503 Reduce message size or use a model with a larger context window.",
1504 system_cost, last_group_cost, system_cost + last_group_cost,
1505 budget, max_seq_len, reserve_tokens
1506 ));
1507 }
1508 }
1509
1510 let mut kept_cost = system_cost;
1511 let mut keep_from = groups.len();
1512 for i in (start..groups.len()).rev() {
1513 if kept_cost + groups[i].0 <= budget {
1514 kept_cost += groups[i].0;
1515 keep_from = i;
1516 } else {
1517 break;
1518 }
1519 }
1520
1521 let mut kept_indices: Vec<usize> = Vec::new();
1523 if has_system {
1524 kept_indices.extend(&groups[0].1);
1525 }
1526 for g in &groups[keep_from..] {
1527 kept_indices.extend(&g.1);
1528 }
1529 kept_indices.sort();
1530
1531 let dropped = messages.len() - kept_indices.len();
1532 if dropped > 0 {
1533 tracing::info!(
1534 "Context truncation: dropped {} messages ({} → ~{} tokens, budget: {}, max_seq_len: {}, reserve: {})",
1535 dropped, total, kept_cost, budget, max_seq_len, reserve_tokens
1536 );
1537 }
1538
1539 Ok(kept_indices
1540 .into_iter()
1541 .map(|i| messages[i].clone())
1542 .collect())
1543 }
1544
1545 async fn handle_create_lora(&self, config: TenantDeltaConfig) -> Result<()> {
1547 if let Some(pool) = &self.delta_pool {
1550 tracing::info!(
1551 "[TTT] Updating delta pool config: target_modules={:?}, rank={}, alpha={:.1}, lr={:.1e}",
1552 config.target_modules, config.rank, config.alpha, config.learning_rate
1553 );
1554 pool.update_config(config.clone());
1555 }
1556 let result = self.engine.write().create_lora(config);
1557 if result.is_ok() {
1558 self.lora_generation.fetch_add(1, Ordering::Release);
1559 }
1560 result
1561 }
1562
1563 async fn setup_stream(
1572 &self,
1573 ctx: &EnvelopeContext,
1574 ) -> Result<(
1575 hyprstream_rpc_std::inference_client::StreamInfo,
1576 hyprstream_rpc::streaming::StreamContext,
1577 )> {
1578 let client_pub_bytes = ctx.ephemeral_pubkey().ok_or_else(|| {
1579 anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
1580 })?;
1581 let client_pub_ref: &[u8] = &client_pub_bytes;
1582 let stream_channel = self
1583 .stream_channel
1584 .as_ref()
1585 .ok_or_else(|| anyhow!("StreamChannel not initialized"))?;
1586
1587 let expiry_secs = ctx
1588 .claims()
1589 .map(|c| c.exp - chrono::Utc::now().timestamp())
1590 .unwrap_or(600)
1591 .max(600);
1592 let claims = ctx.claims().cloned();
1593
1594 let stream_ctx = stream_channel
1595 .prepare_third_party_interop_stream_with_claims(client_pub_ref, expiry_secs, claims)
1596 .await?
1597 .with_qos_preset::<hyprstream_rpc::stream_info::Job>();
1598
1599 let stream_id = stream_ctx.stream_id().to_owned();
1600 let server_pubkey = *stream_ctx.server_pubkey();
1601 let broadcast_path = hyprstream_rpc::moq_stream::global_moq_origin()
1602 .map(|o| o.broadcast_path(stream_ctx.topic()))
1603 .unwrap_or_default();
1604
1605 let stream_info = hyprstream_rpc_std::inference_client::StreamInfo {
1606 stream_id,
1607 dh_public: server_pubkey,
1608 qos: stream_ctx.qos().clone(),
1609 broadcast_path,
1610 announced_at: stream_ctx.reach(), moql_server_identity: stream_ctx.moql_server_identity(),
1612 kem_ciphertexts: Vec::new(), };
1614
1615 Ok((stream_info, stream_ctx))
1616 }
1617
1618 async fn execute_create_lora_stream(
1623 &self,
1624 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1625 config: crate::training::TenantDeltaConfig,
1626 ) {
1627 let sc = match &self.stream_channel {
1628 Some(sc) => sc,
1629 None => {
1630 error!("StreamChannel not initialized");
1631 return;
1632 }
1633 };
1634 let _ = sc
1635 .run_stream(&stream_ctx, |mut publisher| async {
1636 let result = InferenceService::handle_create_lora(self, config).await;
1637 match &result {
1638 Ok(()) => {
1639 let _ = publisher.complete_ref(b"{}").await;
1640 }
1641 Err(e) => {
1642 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1643 }
1644 }
1645 (publisher, result.map(|_| ()))
1646 })
1647 .await;
1648 }
1649
1650 async fn execute_load_lora_stream(
1651 &self,
1652 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1653 path: String,
1654 ) {
1655 let sc = match &self.stream_channel {
1656 Some(sc) => sc,
1657 None => {
1658 error!("StreamChannel not initialized");
1659 return;
1660 }
1661 };
1662 let _ = sc
1663 .run_stream(&stream_ctx, |mut publisher| async {
1664 let result = self.load_lora(Path::new(&path)).await;
1665 match &result {
1666 Ok(()) => {
1667 let _ = publisher.complete_ref(b"{}").await;
1668 }
1669 Err(e) => {
1670 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1671 }
1672 }
1673 (publisher, result)
1674 })
1675 .await;
1676 }
1677
1678 async fn execute_save_lora_stream(
1679 &self,
1680 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1681 name: String,
1682 ) {
1683 let sc = match &self.stream_channel {
1684 Some(sc) => sc,
1685 None => {
1686 error!("StreamChannel not initialized");
1687 return;
1688 }
1689 };
1690 let _ = sc
1691 .run_stream(&stream_ctx, |mut publisher| async {
1692 let result = self.save_lora(&name).await;
1693 match &result {
1694 Ok(()) => {
1695 let _ = publisher.complete_ref(b"{}").await;
1696 }
1697 Err(e) => {
1698 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1699 }
1700 }
1701 (publisher, result)
1702 })
1703 .await;
1704 }
1705
1706 async fn execute_save_adaptation_stream(
1707 &self,
1708 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1709 subject: Subject,
1710 name: String,
1711 merge_strategy: String,
1712 merge_weight: f32,
1713 ) {
1714 let sc = match &self.stream_channel {
1715 Some(sc) => sc,
1716 None => {
1717 error!("StreamChannel not initialized");
1718 return;
1719 }
1720 };
1721 let _ = sc
1722 .run_stream(&stream_ctx, |mut publisher| async {
1723 let result = InferenceService::handle_save_adaptation(
1724 self,
1725 &subject,
1726 &name,
1727 &merge_strategy,
1728 merge_weight,
1729 )
1730 .await;
1731 match &result {
1732 Ok(info) => {
1733 let payload = serde_json::to_vec(&serde_json::json!({
1734 "adapter_name": info.adapter_name,
1735 "adapter_path": info.adapter_path,
1736 "content_hash": info.content_hash,
1737 "merge_strategy": info.merge_strategy,
1738 }))
1739 .unwrap_or_default();
1740 let _ = publisher.complete_ref(&payload).await;
1741 }
1742 Err(e) => {
1743 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1744 }
1745 }
1746 (publisher, result.map(|_| ()))
1747 })
1748 .await;
1749 }
1750
1751 async fn execute_snapshot_delta_stream(
1752 &self,
1753 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1754 subject: Subject,
1755 ) {
1756 let sc = match &self.stream_channel {
1757 Some(sc) => sc,
1758 None => {
1759 error!("StreamChannel not initialized");
1760 return;
1761 }
1762 };
1763 let _ = sc
1764 .run_stream(&stream_ctx, |mut publisher| async {
1765 let result = InferenceService::handle_snapshot_delta(self, &subject).await;
1766 match &result {
1767 Ok(info) => {
1768 let payload = serde_json::to_vec(&serde_json::json!({
1769 "content_hash": info.content_hash,
1770 "size_bytes": info.size_bytes,
1771 "accumulated_steps": info.accumulated_steps,
1772 "request_count": info.request_count,
1773 }))
1774 .unwrap_or_default();
1775 let _ = publisher.complete_ref(&payload).await;
1776 }
1777 Err(e) => {
1778 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1779 }
1780 }
1781 (publisher, result.map(|_| ()))
1782 })
1783 .await;
1784 }
1785
1786 async fn execute_export_peft_adapter_stream(
1787 &self,
1788 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1789 subject: Subject,
1790 name: String,
1791 commit_message: String,
1792 ) {
1793 let sc = match &self.stream_channel {
1794 Some(sc) => sc,
1795 None => {
1796 error!("StreamChannel not initialized");
1797 return;
1798 }
1799 };
1800 let _ = sc
1801 .run_stream(&stream_ctx, |mut publisher| async {
1802 let result = InferenceService::handle_export_peft_adapter(
1803 self,
1804 &subject,
1805 &name,
1806 &commit_message,
1807 )
1808 .await;
1809 match &result {
1810 Ok(info) => {
1811 let payload = serde_json::to_vec(&serde_json::json!({
1812 "adapter_path": info.adapter_path,
1813 "content_hash": info.content_hash,
1814 }))
1815 .unwrap_or_default();
1816 let _ = publisher.complete_ref(&payload).await;
1817 }
1818 Err(e) => {
1819 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1820 }
1821 }
1822 (publisher, result.map(|_| ()))
1823 })
1824 .await;
1825 }
1826
1827 async fn execute_merge_lora_stream(
1828 &self,
1829 stream_ctx: hyprstream_rpc::streaming::StreamContext,
1830 adapter_path: String,
1831 weight: f32,
1832 strategy: String,
1833 ) {
1834 let sc = match &self.stream_channel {
1835 Some(sc) => sc,
1836 None => {
1837 error!("StreamChannel not initialized");
1838 return;
1839 }
1840 };
1841 let _ = sc
1842 .run_stream(&stream_ctx, |mut publisher| async {
1843 let result = self
1844 .merge_lora(Path::new(&adapter_path), weight, &strategy)
1845 .await;
1846 match &result {
1847 Ok(()) => {
1848 let _ = publisher.complete_ref(b"{}").await;
1849 }
1850 Err(e) => {
1851 let _ = publisher.publish_error(&format!("{:#}", e)).await;
1852 }
1853 }
1854 (publisher, result)
1855 })
1856 .await;
1857 }
1858
1859 fn handle_writeback(&self, subject: &Subject) -> Result<()> {
1865 if let Some(pool) = &self.delta_pool {
1866 if let Some(delta_arc) = pool.get(subject) {
1867 let mut delta = delta_arc.lock();
1868 let info = delta.adaptation_state.writeback_stats()?;
1869 delta.accumulated_steps += info.steps_performed as u64;
1870 delta.request_count += 1;
1871 let n = delta.request_count as f64;
1872 delta.avg_loss_improvement =
1873 delta.avg_loss_improvement * ((n - 1.0) / n) + info.loss_improvement as f64 / n;
1874 debug!(
1875 "Wrote back adaptation for '{}': steps={}, improvement={:.4}",
1876 subject, info.steps_performed, info.loss_improvement
1877 );
1878 Ok(())
1879 } else {
1880 Err(anyhow::anyhow!("No delta found for subject '{}'", subject))
1881 }
1882 } else {
1883 Err(anyhow::anyhow!("No delta pool configured"))
1884 }
1885 }
1886
1887 fn handle_evict(&self, subject: &Subject) -> Result<()> {
1889 if let Some(pool) = &self.delta_pool {
1890 if let Some(delta_arc) = pool.get(subject) {
1891 let mut delta = delta_arc.lock();
1892 let snapshot = delta.adaptation_state.evict()?;
1893 let _ = delta.load_state_dict(&snapshot.pre_snapshot);
1894 crate::training::muon::restore_muon_states(
1895 &mut delta.muon_states,
1896 &snapshot.pre_muon,
1897 );
1898 delta.effective_ranks = snapshot.pre_eff_ranks;
1899 debug!("Evicted adaptation for subject '{}'", subject);
1900 Ok(())
1901 } else {
1902 Err(anyhow::anyhow!("No delta found for subject '{}'", subject))
1903 }
1904 } else {
1905 Err(anyhow::anyhow!("No delta pool configured"))
1906 }
1907 }
1908
1909 async fn handle_train_step(
1911 &self,
1912 subject: &Subject,
1913 input: &str,
1914 gradient_steps: Option<u32>,
1915 learning_rate: Option<f32>,
1916 strategy: crate::training::adaptation_state::AdaptationStrategy,
1917 ) -> Result<crate::training::ttt::TTTResult> {
1918 let ttt_trainer = self
1919 .ttt_trainer
1920 .as_ref()
1921 .ok_or_else(|| anyhow!("TTT not configured"))?;
1922 let tokenizer = self
1923 .tokenizer
1924 .as_ref()
1925 .ok_or_else(|| anyhow!("No tokenizer available"))?;
1926 let pool = self
1927 .delta_pool
1928 .as_ref()
1929 .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
1930
1931 pool.ensure_capacity().await?;
1933 let delta_arc = pool.get_or_hydrate(subject).await?;
1937
1938 let (input_tokens, pre_snapshot, pre_muon, pre_eff_ranks) = {
1941 let delta = delta_arc.lock();
1942 let encoding = tokenizer
1943 .encode(input, false)
1944 .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
1945 let input_tokens: Vec<u32> = encoding.get_ids().to_vec();
1946 let pre_snapshot = delta.extract_state_dict();
1947 let pre_muon = crate::training::muon::snapshot_muon_states(&delta.muon_states);
1948 let pre_eff_ranks = delta.effective_ranks.clone();
1949 (input_tokens, pre_snapshot, pre_muon, pre_eff_ranks)
1950 }; let steps = gradient_steps
1953 .map(|g| g as usize)
1954 .unwrap_or(ttt_trainer.config.gradient_steps as usize);
1955 let lr = learning_rate.map(|l| l as f64);
1956
1957 let engine = self.engine.read();
1959
1960 let mut delta = delta_arc.lock();
1962 let mut result =
1963 ttt_trainer.train_step(&engine, &mut delta, &input_tokens, steps, lr, None)?;
1964
1965 let guard = crate::training::GuardStatus {
1969 expired: delta.adaptation_state.is_expired(),
1970 at_capacity: delta.is_at_capacity(),
1971 lora_generation: self
1972 .lora_generation
1973 .load(std::sync::atomic::Ordering::Acquire),
1974 };
1975
1976 let outcome = delta.adaptation_state.resolve(
1977 strategy,
1978 &guard,
1979 &result,
1980 pre_snapshot,
1981 pre_muon,
1982 pre_eff_ranks,
1983 );
1984
1985 match outcome {
1986 crate::training::ResolveOutcome::WrittenBack => {
1987 delta.accumulated_steps += result.steps_performed as u64;
1988 delta.request_count += 1;
1989 let n = delta.request_count as f64;
1990 delta.avg_loss_improvement = delta.avg_loss_improvement * ((n - 1.0) / n)
1991 + result.loss_improvement as f64 / n;
1992 result.pending = false;
1993 debug!(
1994 "TTT: auto-committed train_step adaptation for subject {}",
1995 subject
1996 );
1997 }
1998 crate::training::ResolveOutcome::Evicted {
1999 snapshot,
2000 muon,
2001 eff_ranks,
2002 } => {
2003 let _ = delta.load_state_dict(&snapshot);
2004 crate::training::muon::restore_muon_states(&mut delta.muon_states, &muon);
2005 delta.effective_ranks = eff_ranks;
2006 result.pending = false;
2007 debug!("TTT: auto-rolled back train_step adaptation for subject {} (negative recommendation)", subject);
2008 }
2009 crate::training::ResolveOutcome::StoredPending => {
2010 result.pending = true;
2011 debug!(
2012 "TTT: stored pending train_step adaptation for subject {}",
2013 subject
2014 );
2015 }
2016 crate::training::ResolveOutcome::Skipped { reason } => {
2017 debug!(
2018 "TTT: train_step skipped for subject {}: {}",
2019 subject, reason
2020 );
2021 }
2022 }
2023
2024 Ok(result)
2025 }
2026
2027 fn handle_reset_delta(&self, subject: &Subject) -> Result<()> {
2029 if let Some(pool) = &self.delta_pool {
2030 if let Some(delta_arc) = pool.get(subject) {
2031 let mut delta = delta_arc.lock();
2032 delta.reset();
2033 debug!("Reset delta for subject '{}'", subject);
2034 } else {
2035 debug!("No delta to reset for subject '{}'", subject);
2036 }
2037 }
2038
2039 Ok(())
2040 }
2041
2042 fn handle_get_delta_status(&self, subject: &Subject) -> Result<DeltaStatusResult> {
2044 if let Some(pool) = &self.delta_pool {
2045 if let Some(delta_arc) = pool.get(subject) {
2046 let delta = delta_arc.lock();
2047 let norm_ratios = delta.delta_norm_ratio(pool.base_weight_norms());
2048 let has_pending = delta.adaptation_state.is_pending();
2049 return Ok(DeltaStatusResult {
2050 exists: true,
2051 accumulated_steps: delta.accumulated_steps,
2052 max_accumulated_steps: delta.max_accumulated_steps,
2053 request_count: delta.request_count,
2054 avg_loss_improvement: delta.avg_loss_improvement as f32,
2055 memory_bytes: delta.memory_bytes() as u64,
2056 last_snapshot_hash: delta.last_snapshot_hash.clone().unwrap_or_default(),
2057 delta_norm_ratios: norm_ratios
2058 .into_iter()
2059 .map(|(name, ratio)| ModuleNormRatio {
2060 module_name: name,
2061 ratio: ratio as f32,
2062 })
2063 .collect(),
2064 has_pending,
2065 });
2066 }
2067 }
2068
2069 Ok(DeltaStatusResult {
2070 exists: false,
2071 accumulated_steps: 0,
2072 max_accumulated_steps: 0,
2073 request_count: 0,
2074 avg_loss_improvement: 0.0,
2075 memory_bytes: 0,
2076 last_snapshot_hash: String::new(),
2077 delta_norm_ratios: vec![],
2078 has_pending: false,
2079 })
2080 }
2081
2082 async fn handle_save_adaptation(
2087 &self,
2088 subject: &Subject,
2089 name: &str,
2090 merge_strategy_name: &str,
2091 merge_weight: f32,
2092 ) -> Result<SaveAdaptationResult> {
2093 use crate::training::{merge_state_dicts, MergeStrategy};
2094
2095 let pool = self
2096 .delta_pool
2097 .as_ref()
2098 .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
2099
2100 let delta_arc = pool
2101 .get(subject)
2102 .ok_or_else(|| anyhow!("No delta for subject '{}'", subject))?;
2103 let new_state_dict = {
2104 let delta = delta_arc.lock();
2105 delta.extract_state_dict()
2106 };
2107
2108 let strategy_name = if merge_strategy_name.is_empty() {
2110 "do_merge"
2111 } else {
2112 merge_strategy_name
2113 };
2114 let weight = if merge_weight <= 0.0 || merge_weight > 1.0 {
2115 0.3
2116 } else {
2117 merge_weight as f64
2118 };
2119 let strategy = MergeStrategy::from_name(strategy_name, weight)?;
2120
2121 let adapter_mgr = crate::storage::AdapterManager::new(&self.state_path);
2123 let adapter_name = if name.is_empty() {
2124 format!("ttt_{}", subject)
2125 } else {
2126 name.to_owned()
2127 };
2128
2129 let existing_adapters = adapter_mgr.list_adapters().unwrap_or_default();
2131 let existing_state =
2132 if let Some(existing) = existing_adapters.iter().find(|a| a.name == adapter_name) {
2133 let rel_path = format!(
2134 "adapters/{}",
2135 existing
2136 .path
2137 .file_name()
2138 .and_then(|f| f.to_str())
2139 .unwrap_or("")
2140 );
2141 if let Some(ref fs) = self.fs {
2142 match fs.read_file_chunked(&rel_path).await {
2143 Ok(bytes) => crate::training::load_state_dict_from_bytes(&bytes)
2144 .map_err(|e| {
2145 tracing::debug!(
2146 "Could not load existing adapter '{}': {}",
2147 existing.name,
2148 e
2149 );
2150 e
2151 })
2152 .ok(),
2153 Err(e) => {
2154 tracing::debug!(
2155 "Could not read existing adapter '{}': {}",
2156 existing.name,
2157 e
2158 );
2159 None
2160 }
2161 }
2162 } else {
2163 None
2164 }
2165 } else {
2166 None
2167 };
2168
2169 let adapter_filename =
2171 if let Some(existing) = existing_adapters.iter().find(|a| a.name == adapter_name) {
2172 existing
2173 .path
2174 .file_name()
2175 .and_then(|f| f.to_str())
2176 .unwrap_or("adapter.safetensors")
2177 .to_owned()
2178 } else {
2179 let next_index = adapter_mgr.get_next_index().unwrap_or(0);
2180 format!("{:02}_{}.safetensors", next_index, adapter_name)
2181 };
2182
2183 let final_state = if let Some(existing) = existing_state {
2185 merge_state_dicts(&existing, &new_state_dict, &strategy)?
2186 } else {
2187 new_state_dict
2188 };
2189
2190 let fs = self.fs.as_ref().ok_or_else(|| {
2192 anyhow!("FsOps not available — cannot write without path containment")
2193 })?;
2194 let rel_path = format!("adapters/{}", adapter_filename);
2195 fs.mkdir_p("adapters")
2196 .await
2197 .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2198 let bytes = serialize_state_dict_to_bytes(&final_state)?;
2199 fs.write_file_chunked(&rel_path, &bytes)
2200 .await
2201 .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
2202 let result_path = rel_path;
2203
2204 let actual_strategy = format!("{:?}", strategy).to_lowercase();
2205
2206 info!(
2207 "Saved adaptation for subject '{}' as adapter '{}' at {} (strategy: {})",
2208 subject, adapter_name, result_path, actual_strategy
2209 );
2210
2211 Ok(SaveAdaptationResult {
2212 adapter_name: adapter_name.clone(),
2213 adapter_path: result_path,
2214 content_hash: String::new(),
2215 merge_strategy: strategy_name.to_owned(),
2216 })
2217 }
2218
2219 async fn handle_snapshot_delta(&self, subject: &Subject) -> Result<SnapshotDeltaResult> {
2221 let pool = self
2222 .delta_pool
2223 .as_ref()
2224 .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
2225
2226 let delta_arc = pool
2227 .get(subject)
2228 .ok_or_else(|| anyhow!("No delta for subject '{}'", subject))?;
2229
2230 let filename = subject.to_string();
2231 let state_dict = {
2232 let delta = delta_arc.lock();
2233 delta.extract_state_dict()
2234 };
2235
2236 let fs = self.fs.as_ref().ok_or_else(|| {
2238 anyhow!("FsOps not available — cannot write without path containment")
2239 })?;
2240 let rel_snapshot = format!("adapters/.snapshots/{}.safetensors", filename);
2241 fs.mkdir_p("adapters/.snapshots")
2242 .await
2243 .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2244 let bytes = serialize_state_dict_to_bytes(&state_dict)?;
2245 let size_bytes = bytes.len() as u64;
2246 fs.write_file_chunked(&rel_snapshot, &bytes)
2247 .await
2248 .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
2249 let path_str = rel_snapshot;
2250
2251 let mut delta = delta_arc.lock();
2253 delta.last_snapshot_hash = Some(path_str.clone());
2254
2255 Ok(SnapshotDeltaResult {
2256 content_hash: path_str,
2257 size_bytes,
2258 accumulated_steps: delta.accumulated_steps,
2259 request_count: delta.request_count,
2260 })
2261 }
2262
2263 async fn handle_export_peft_adapter(
2268 &self,
2269 subject: &Subject,
2270 name: &str,
2271 _commit_message: &str,
2272 ) -> Result<ExportPeftResult> {
2273 let pool = self
2274 .delta_pool
2275 .as_ref()
2276 .ok_or_else(|| anyhow!("Delta pool not initialized"))?;
2277
2278 let delta_arc = pool
2279 .get(subject)
2280 .ok_or_else(|| anyhow!("No delta for subject '{}'", subject))?;
2281
2282 let adapter_name = if name.is_empty() {
2283 format!("ttt_{}", subject)
2284 } else {
2285 name.to_owned()
2286 };
2287
2288 let (safetensors_bytes, config_bytes) = {
2290 let delta = delta_arc.lock();
2291
2292 let safetensors_bytes = delta.serialize_to_safetensors_bytes()?;
2294
2295 let adapter_config = serde_json::json!({
2297 "peft_type": "LORA",
2298 "auto_mapping": null,
2299 "base_model_name_or_path": "",
2300 "bias": "none",
2301 "fan_in_fan_out": false,
2302 "inference_mode": true,
2303 "init_lora_weights": true,
2304 "layers_to_transform": null,
2305 "layers_pattern": null,
2306 "lora_alpha": delta.scaling * delta.rank as f64,
2307 "lora_dropout": 0.0,
2308 "modules_to_save": null,
2309 "r": delta.rank,
2310 "rank_pattern": {},
2311 "alpha_pattern": {},
2312 "revision": null,
2313 "target_modules": delta.target_modules.clone(),
2314 "task_type": "CAUSAL_LM"
2315 });
2316 let config_bytes = serde_json::to_vec_pretty(&adapter_config)?;
2317 (safetensors_bytes, config_bytes)
2318 };
2319
2320 let fs = self.fs.as_ref().ok_or_else(|| {
2322 anyhow!("FsOps not available — cannot write without path containment")
2323 })?;
2324
2325 let dir_path = format!("adapters/{}", adapter_name);
2326 fs.mkdir_p(&dir_path)
2327 .await
2328 .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2329
2330 let safetensors_path = format!("{}/adapter_model.safetensors", dir_path);
2331 fs.write_file_chunked(&safetensors_path, &safetensors_bytes)
2332 .await
2333 .map_err(|e| anyhow!("FsOps write adapter_model.safetensors failed: {}", e))?;
2334
2335 let config_path = format!("{}/adapter_config.json", dir_path);
2336 fs.write_file_chunked(&config_path, &config_bytes)
2337 .await
2338 .map_err(|e| anyhow!("FsOps write adapter_config.json failed: {}", e))?;
2339
2340 info!(
2341 "Exported PEFT adapter for subject '{}' as '{}' ({} bytes safetensors)",
2342 subject,
2343 adapter_name,
2344 safetensors_bytes.len()
2345 );
2346
2347 Ok(ExportPeftResult {
2348 adapter_path: dir_path,
2349 content_hash: String::new(),
2350 })
2351 }
2352
2353 #[allow(dead_code)]
2355 fn get_ttt_config(&self) -> Option<crate::training::ttt::TTTConfig> {
2356 self.ttt_trainer
2357 .as_ref()
2358 .map(|trainer| trainer.config.clone())
2359 }
2360
2361 async fn handle_set_session(&self, session_id: String) -> Result<()> {
2363 *self.session_id.write() = Some(session_id.clone());
2365 self.engine
2366 .write()
2367 .set_session(CacheOwner::Session(session_id))
2368 }
2369
2370 async fn handle_clear_session(&self) {
2372 *self.session_id.write() = None;
2373 self.engine.write().clear_kv_cache();
2374 }
2375
2376 async fn handle_release_session(&self, session_id: &str) -> Result<()> {
2378 self.engine
2379 .write()
2380 .release_session(&CacheOwner::Session(session_id.to_owned()))
2381 }
2382
2383 pub async fn load_lora(&self, path: &Path) -> Result<()> {
2390 let device = self.engine.read().device();
2391 let fs = self
2393 .fs
2394 .as_ref()
2395 .ok_or_else(|| anyhow!("FsOps not available — cannot read without path containment"))?;
2396 let safetensors_path = format!("{}/adapter_model.safetensors", path.to_string_lossy());
2397 let bytes = fs.read_file_chunked(&safetensors_path).await.map_err(|e| {
2398 anyhow!(
2399 "Failed to read LoRA adapter from {}: {}",
2400 safetensors_path,
2401 e
2402 )
2403 })?;
2404 let delta = crate::training::TenantDelta::load_from_safetensors_bytes(&bytes, device)?;
2405
2406 if let Some(pool) = self.delta_pool.as_ref() {
2410 let pool_rank = pool.rank();
2411 if delta.rank != pool_rank {
2412 tracing::warn!(
2413 "LoRA adapter rank ({}) differs from delta pool rank ({}). \
2414 Composition will use effective-weight decomposition (slower but correct).",
2415 delta.rank,
2416 pool_rank
2417 );
2418 }
2419 }
2420
2421 *self.base_delta.lock() = Some(Arc::new(Mutex::new(delta)));
2422 self.lora_generation.fetch_add(1, Ordering::Release);
2423 tracing::info!("Loaded LoRA adapter as base delta from {}", path.display());
2424 Ok(())
2425 }
2426
2427 pub async fn save_lora(&self, path: &str) -> Result<()> {
2429 let base = self.base_delta.lock().clone();
2430 if let Some(delta_arc) = base {
2431 let bytes = {
2432 let delta = delta_arc.lock();
2433 delta.serialize_to_safetensors_bytes()?
2434 };
2435 let fs = self.fs.as_ref().ok_or_else(|| {
2437 anyhow!("FsOps not available — cannot write without path containment")
2438 })?;
2439 let safe_name = sanitize_adapter_name(path)?;
2440 let rel_path = format!("adapters/{}.safetensors", safe_name);
2441 fs.mkdir_p("adapters")
2442 .await
2443 .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
2444 fs.write_file_chunked(&rel_path, &bytes)
2445 .await
2446 .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
2447 Ok(())
2448 } else {
2449 Err(anyhow::anyhow!("No LoRA adapter loaded to save"))
2450 }
2451 }
2452
2453 pub async fn unload_lora(&self) -> Result<()> {
2455 let mut base = self.base_delta.lock();
2456 if base.is_some() {
2457 *base = None;
2458 self.lora_generation.fetch_add(1, Ordering::Release);
2459 tracing::info!("Unloaded base LoRA delta");
2460 Ok(())
2461 } else {
2462 Err(anyhow::anyhow!("No LoRA adapter loaded to unload"))
2463 }
2464 }
2465
2466 pub async fn has_lora(&self) -> Result<bool> {
2468 Ok(self.base_delta.lock().is_some())
2469 }
2470
2471 pub async fn merge_lora(&self, path: &Path, weight: f32, strategy_name: &str) -> Result<()> {
2477 let device = self.engine.read().device();
2478
2479 let fs = self
2481 .fs
2482 .as_ref()
2483 .ok_or_else(|| anyhow!("FsOps not available — cannot read without path containment"))?;
2484 let safetensors_path = format!("{}/adapter_model.safetensors", path.to_string_lossy());
2485 let bytes = fs
2486 .read_file_chunked(&safetensors_path)
2487 .await
2488 .map_err(|e| anyhow!("Failed to read adapter for merge: {}", e))?;
2489 let incoming = crate::training::TenantDelta::load_from_safetensors_bytes(&bytes, device)?;
2490
2491 let base_arc = self.base_delta.lock().clone().ok_or_else(|| {
2493 anyhow!("No adapter loaded in base_delta register. Load one first with adapter.load.")
2494 })?;
2495
2496 let strategy =
2498 crate::training::merge::MergeStrategy::from_name(strategy_name, weight as f64)?;
2499
2500 let merged_state = {
2502 let base = base_arc.lock();
2503 let existing = base.extract_state_dict();
2504 let new = incoming.extract_state_dict();
2505 crate::training::merge::merge_state_dicts(&existing, &new, &strategy)?
2506 };
2507
2508 {
2510 let mut base = base_arc.lock();
2511 base.load_state_dict(&merged_state)?;
2512 }
2513
2514 tracing::info!(
2515 "Merged adapter from {} into base_delta (strategy={}, weight={})",
2516 path.display(),
2517 strategy_name,
2518 weight
2519 );
2520 Ok(())
2521 }
2522}
2523
2524fn sanitize_adapter_name(name: &str) -> Result<String> {
2529 let stem = std::path::Path::new(name)
2530 .file_stem()
2531 .and_then(|s| s.to_str())
2532 .unwrap_or(name);
2533
2534 if stem.contains("..") || stem.contains('/') || stem.contains('\\') || stem.is_empty() {
2536 return Err(anyhow!("Invalid adapter name: '{}'", name));
2537 }
2538
2539 let safe: String = stem
2541 .chars()
2542 .filter(|c| c.is_alphanumeric() || *c == '_' || *c == '-')
2543 .collect();
2544
2545 if safe.is_empty() {
2546 return Err(anyhow!(
2547 "Adapter name '{}' contains no valid characters",
2548 name
2549 ));
2550 }
2551
2552 Ok(safe)
2553}
2554
2555use crate::services::generated::inference_client::{
2560 dispatch_inference, serialize_response, InferenceHandler,
2561};
2562use hyprstream_rpc_std::inference_client::{
2563 AdaptationStrategy as AdaptationStrategyEnum, ChatTemplateRequest, DeltaStatusResult,
2564 EmbedImagesRequest, EmbedImagesResponse, ErrorInfo, ExportPeftRequest, ExportPeftResult,
2565 HealthStatus, InferenceResponseVariant, LoraConfig, MergeLoraRequest, ModuleNormRatio,
2566 SaveAdaptationRequest, SaveAdaptationResult, SnapshotDeltaResult, TrainStepRequest,
2567};
2568use hyprstream_rpc_std::policy_client::PolicyCheck;
2569fn map_adaptation_strategy(
2574 capnp_strategy: AdaptationStrategyEnum,
2575 writeback_threshold: Option<f32>,
2576) -> crate::training::adaptation_state::AdaptationStrategy {
2577 match capnp_strategy {
2578 AdaptationStrategyEnum::AutoWriteback => {
2579 crate::training::adaptation_state::AdaptationStrategy::AutoWriteback
2580 }
2581 AdaptationStrategyEnum::AutoEvict => {
2582 crate::training::adaptation_state::AdaptationStrategy::AutoEvict
2583 }
2584 AdaptationStrategyEnum::Speculative => {
2585 crate::training::adaptation_state::AdaptationStrategy::Speculative
2586 }
2587 AdaptationStrategyEnum::WritebackIfAbove => {
2588 crate::training::adaptation_state::AdaptationStrategy::WritebackIfAbove {
2589 threshold: writeback_threshold.unwrap_or(0.0),
2590 }
2591 }
2592 }
2593}
2594
2595#[async_trait::async_trait(?Send)]
2596impl InferenceHandler for InferenceService {
2597 async fn authorize(
2598 &self,
2599 ctx: &EnvelopeContext,
2600 resource: &str,
2601 operation: &str,
2602 ) -> Result<()> {
2603 if let Some(work) = ctx.internal_work() {
2610 anyhow::ensure!(
2611 work.resource == resource && work.operation == operation,
2612 "internal work order does not authorize {operation} on {resource}"
2613 );
2614 enforce_inference_mac(ctx, &self.object_label)?;
2615 tracing::info!(
2616 target: "hyprstream.mac.audit",
2617 decision = "allow",
2618 subject = %work.subject,
2619 resource = %resource,
2620 action = %operation,
2621 request_id = ctx.request_id,
2622 plane = "rpc",
2623 via = "model-internal-work",
2624 controller = hex::encode(self.controller_pubkey.as_bytes()),
2625 "authorization decision"
2626 );
2627 return Ok(());
2628 }
2629 let subject = ctx.subject();
2630 let domain = self.authorization_domain(ctx)?;
2631 enforce_inference_mac(ctx, &self.object_label)?;
2632 let request = PolicyCheck {
2633 subject: subject.to_string(),
2634 domain,
2635 resource: resource.to_owned(),
2636 operation: operation.to_owned(),
2637 };
2638 let allowed = crate::services::policy::check_with_holder_evidence(
2639 &self.policy_client,
2640 &request,
2641 ctx.jwt_token(),
2642 &ctx.subject(),
2643 ctx.original_holder_evidence(),
2644 )
2645 .await
2646 .unwrap_or_else(|e| {
2647 warn!(
2648 "Policy check failed for {} on {}: {} - denying access",
2649 subject, resource, e
2650 );
2651 false
2652 });
2653 if allowed {
2654 Ok(())
2655 } else {
2656 anyhow::bail!(
2657 "Unauthorized: {} cannot {} on {}",
2658 subject,
2659 operation,
2660 resource
2661 )
2662 }
2663 }
2664
2665 async fn handle_generate_stream(
2666 &self,
2667 ctx: &EnvelopeContext,
2668 _request_id: u64,
2669 data: &hyprstream_rpc_std::inference_client::GenerationRequest,
2670 ) -> Result<(
2671 hyprstream_rpc_std::inference_client::StreamInfo,
2672 hyprstream_rpc::service::Continuation,
2673 )> {
2674 let subject = ctx.subject();
2675 let request = data.clone();
2676
2677 let ttt_overrides = crate::training::ttt::TTTOverrides {
2679 enabled: if data.ttt_enabled { Some(true) } else { None },
2680 gradient_steps: data.ttt_gradient_steps,
2681 learning_rate: data.ttt_learning_rate,
2682 adaptation_strategy: map_adaptation_strategy(
2683 data.adaptation_strategy,
2684 data.writeback_threshold,
2685 ),
2686 max_adaptation_ms: None,
2687 };
2688
2689 let expiry_secs = ctx
2691 .claims()
2692 .map(|c| c.exp - chrono::Utc::now().timestamp())
2693 .unwrap_or(600)
2694 .max(600);
2695
2696 let client_ephemeral_pubkey = ctx.ephemeral_pubkey();
2697 let claims = ctx.claims().cloned();
2698 #[cfg(feature = "ledger")]
2709 let owner_did = ctx
2710 .internal_work()
2711 .and_then(|work| work.owner_did.clone())
2712 .or_else(|| {
2713 ctx.authenticated_pairwise_did()
2714 .map(|d| d.as_str().to_owned())
2715 });
2716 #[cfg(not(feature = "ledger"))]
2717 let owner_did: Option<String> = None;
2718 let (stream_id, server_pubkey, broadcast_path, reach, pending) = self
2719 .prepare_stream(
2720 request,
2721 client_ephemeral_pubkey.as_ref().map(<[u8; 32]>::as_slice),
2722 claims,
2723 expiry_secs,
2724 &subject,
2725 ttt_overrides,
2726 owner_did,
2727 )
2728 .await?;
2729
2730 let stream_info = hyprstream_rpc_std::inference_client::StreamInfo {
2731 stream_id,
2732 dh_public: server_pubkey,
2733 qos: <hyprstream_rpc::stream_info::Job as hyprstream_rpc::stream_info::StreamOptPreset>::stream_opt(),
2734 broadcast_path,
2735 announced_at: reach,
2736 moql_server_identity: Default::default(),
2737 kem_ciphertexts: Vec::new(), };
2739
2740 let service = self.clone();
2744 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
2745 service.execute_stream(pending).await;
2746 });
2747
2748 Ok((stream_info, continuation))
2749 }
2750
2751 async fn handle_model_info(
2752 &self,
2753 _ctx: &EnvelopeContext,
2754 _request_id: u64,
2755 ) -> Result<InferenceResponseVariant> {
2756 let mut info = InferenceService::handle_model_info(self).await;
2757 info.lora_loaded = self.base_delta.lock().is_some();
2758 Ok(InferenceResponseVariant::ModelInfoResult(info))
2759 }
2760
2761 async fn handle_is_ready(
2762 &self,
2763 _ctx: &EnvelopeContext,
2764 _request_id: u64,
2765 ) -> Result<InferenceResponseVariant> {
2766 let ready = InferenceService::handle_is_ready(self).await;
2767 Ok(InferenceResponseVariant::IsReadyResult(ready))
2768 }
2769
2770 async fn handle_get_layer_profile(
2771 &self,
2772 _ctx: &EnvelopeContext,
2773 _request_id: u64,
2774 ) -> Result<InferenceResponseVariant> {
2775 use crate::runtime::ttn_profile;
2776 use hyprstream_rpc_std::inference_client::LayerProfileResult;
2777
2778 let model_config = crate::runtime::model_config::ModelConfig::load(
2779 &self.model_path,
2780 &std::collections::HashMap::new(),
2781 )?;
2782 let profile = ttn_profile::get_layer_profile(&self.model_path, &model_config, None)?;
2783 let json = serde_json::to_string_pretty(&profile)?;
2784
2785 Ok(InferenceResponseVariant::GetLayerProfileResult(
2786 LayerProfileResult { json },
2787 ))
2788 }
2789
2790 async fn handle_apply_chat_template(
2791 &self,
2792 _ctx: &EnvelopeContext,
2793 _request_id: u64,
2794 data: &ChatTemplateRequest,
2795 ) -> Result<InferenceResponseVariant> {
2796 let chat_messages: Vec<crate::runtime::template_engine::ChatMessage> = data
2797 .messages
2798 .iter()
2799 .map(|m| {
2800 let tool_calls = if m.tool_calls.is_empty() {
2801 None
2802 } else {
2803 Some(
2806 m.tool_calls
2807 .iter()
2808 .map(crate::runtime::template_engine::TemplatToolCall::from)
2809 .collect(),
2810 )
2811 };
2812 crate::runtime::template_engine::ChatMessage {
2813 role: m.role.clone(),
2814 content: if m.content.is_empty() {
2815 None
2816 } else {
2817 Some(m.content.clone())
2818 },
2819 tool_calls,
2820 tool_call_id: if m.tool_call_id.is_empty() {
2821 None
2822 } else {
2823 Some(m.tool_call_id.clone())
2824 },
2825 }
2826 })
2827 .collect();
2828 let tools: Option<serde_json::Value> = data
2830 .tools_json
2831 .as_deref()
2832 .filter(|s| !s.is_empty())
2833 .and_then(|s| serde_json::from_str(s).ok());
2834
2835 let chat_messages = if let Some(reserve) = data.max_tokens {
2839 self.truncate_messages_to_budget(chat_messages, reserve as usize)?
2840 } else {
2841 chat_messages
2842 };
2843
2844 let result = InferenceService::handle_apply_chat_template(
2845 self,
2846 chat_messages,
2847 data.add_generation_prompt,
2848 tools.as_ref(),
2849 data.enable_thinking,
2850 data.template_vars_json.as_deref(),
2851 )
2852 .await?;
2853 Ok(InferenceResponseVariant::ApplyChatTemplateResult(result))
2854 }
2855
2856 async fn handle_create_lora(
2857 &self,
2858 _ctx: &EnvelopeContext,
2859 _request_id: u64,
2860 data: &LoraConfig,
2861 ) -> Result<InferenceResponseVariant> {
2862 let defaults = crate::training::TenantDeltaConfig::default();
2863 let config = crate::training::TenantDeltaConfig {
2864 rank: data.rank as usize,
2865 alpha: data.alpha.unwrap_or(defaults.alpha),
2866 dropout: data.dropout.unwrap_or(defaults.dropout),
2867 target_modules: data.target_modules.clone(),
2868 learning_rate: data
2869 .learning_rate
2870 .map(|v| v as f64)
2871 .unwrap_or(defaults.learning_rate),
2872 ..defaults
2873 };
2874 InferenceService::handle_create_lora(self, config).await?;
2875 Ok(InferenceResponseVariant::CreateLoraResult)
2876 }
2877
2878 async fn handle_load_lora(
2879 &self,
2880 _ctx: &EnvelopeContext,
2881 _request_id: u64,
2882 value: &str,
2883 ) -> Result<InferenceResponseVariant> {
2884 self.load_lora(Path::new(value)).await?;
2885 Ok(InferenceResponseVariant::LoadLoraResult)
2886 }
2887
2888 async fn handle_save_lora(
2889 &self,
2890 _ctx: &EnvelopeContext,
2891 _request_id: u64,
2892 value: &str,
2893 ) -> Result<InferenceResponseVariant> {
2894 self.save_lora(value).await?;
2895 Ok(InferenceResponseVariant::SaveLoraResult)
2896 }
2897
2898 async fn handle_unload_lora(
2899 &self,
2900 _ctx: &EnvelopeContext,
2901 _request_id: u64,
2902 ) -> Result<InferenceResponseVariant> {
2903 self.unload_lora().await?;
2904 Ok(InferenceResponseVariant::UnloadLoraResult)
2905 }
2906
2907 async fn handle_has_lora(
2908 &self,
2909 _ctx: &EnvelopeContext,
2910 _request_id: u64,
2911 ) -> Result<InferenceResponseVariant> {
2912 let has = self.has_lora().await?;
2913 Ok(InferenceResponseVariant::HasLoraResult(has))
2914 }
2915
2916 async fn handle_set_session(
2917 &self,
2918 _ctx: &EnvelopeContext,
2919 _request_id: u64,
2920 value: &str,
2921 ) -> Result<InferenceResponseVariant> {
2922 InferenceService::handle_set_session(self, value.to_owned()).await?;
2923 Ok(InferenceResponseVariant::SetSessionResult)
2924 }
2925
2926 async fn handle_clear_session(
2927 &self,
2928 _ctx: &EnvelopeContext,
2929 _request_id: u64,
2930 ) -> Result<InferenceResponseVariant> {
2931 InferenceService::handle_clear_session(self).await;
2932 Ok(InferenceResponseVariant::ClearSessionResult)
2933 }
2934
2935 async fn handle_release_session(
2936 &self,
2937 _ctx: &EnvelopeContext,
2938 _request_id: u64,
2939 value: &str,
2940 ) -> Result<InferenceResponseVariant> {
2941 InferenceService::handle_release_session(self, value).await?;
2942 Ok(InferenceResponseVariant::ReleaseSessionResult)
2943 }
2944
2945 async fn handle_health_check(
2946 &self,
2947 _ctx: &EnvelopeContext,
2948 _request_id: u64,
2949 ) -> Result<InferenceResponseVariant> {
2950 let model_loaded = self.engine.read().is_loaded();
2951 let network_ready = self.network_ready.load(Ordering::Acquire);
2952 let draining = self.draining.load(Ordering::Acquire);
2953 let status = if draining {
2954 "draining"
2955 } else if !model_loaded {
2956 "not_loaded"
2957 } else if !network_ready {
2958 "stream_plane_unavailable"
2959 } else {
2960 "ok"
2961 };
2962 Ok(InferenceResponseVariant::HealthCheckResult(HealthStatus {
2963 status: status.into(),
2964 model_loaded,
2965 kv_cache_usage_percent: 0.0,
2966 gpu_memory_used_mb: 0,
2967 gpu_memory_total_mb: 0,
2968 }))
2969 }
2970
2971 async fn handle_shutdown(
2972 &self,
2973 _ctx: &EnvelopeContext,
2974 _request_id: u64,
2975 ) -> Result<InferenceResponseVariant> {
2976 info!("Inference service shutdown requested");
2977 self.draining.store(true, Ordering::Release);
2978 self.network_ready.store(false, Ordering::Release);
2979 let shutdown = Arc::clone(&self.shutdown);
2980 tokio::spawn(async move {
2981 tokio::task::yield_now().await;
2982 shutdown.notify_waiters();
2983 });
2984 Ok(InferenceResponseVariant::Success)
2985 }
2986
2987 async fn handle_ttt_writeback(
2988 &self,
2989 ctx: &EnvelopeContext,
2990 _request_id: u64,
2991 ) -> Result<InferenceResponseVariant> {
2992 let subject = ctx.subject();
2993 InferenceService::handle_writeback(self, &subject)?;
2994 Ok(InferenceResponseVariant::TttWritebackResult)
2995 }
2996
2997 async fn handle_ttt_evict(
2998 &self,
2999 ctx: &EnvelopeContext,
3000 _request_id: u64,
3001 ) -> Result<InferenceResponseVariant> {
3002 let subject = ctx.subject();
3003 InferenceService::handle_evict(self, &subject)?;
3004 Ok(InferenceResponseVariant::TttEvictResult)
3005 }
3006
3007 async fn handle_train_step(
3008 &self,
3009 _ctx: &EnvelopeContext,
3010 _request_id: u64,
3011 _data: &TrainStepRequest,
3012 ) -> Result<InferenceResponseVariant> {
3013 Err(anyhow!(
3017 "trainStep is deprecated — use trainStepStream for non-blocking training"
3018 ))
3019 }
3020
3021 async fn handle_train_step_stream(
3022 &self,
3023 ctx: &EnvelopeContext,
3024 _request_id: u64,
3025 data: &TrainStepRequest,
3026 ) -> Result<(
3027 hyprstream_rpc_std::inference_client::StreamInfo,
3028 hyprstream_rpc::service::Continuation,
3029 )> {
3030 let subject = ctx.subject();
3031
3032 let client_pub_bytes = ctx.ephemeral_pubkey().ok_or_else(|| {
3034 anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
3035 })?;
3036 let client_pub_ref: &[u8] = &client_pub_bytes;
3037 let stream_channel = self
3038 .stream_channel
3039 .as_ref()
3040 .ok_or_else(|| anyhow!("StreamChannel not initialized"))?;
3041
3042 let expiry_secs = ctx
3043 .claims()
3044 .map(|c| c.exp - chrono::Utc::now().timestamp())
3045 .unwrap_or(600)
3046 .max(600);
3047 let claims = ctx.claims().cloned();
3048
3049 let stream_ctx = stream_channel
3050 .prepare_third_party_interop_stream_with_claims(client_pub_ref, expiry_secs, claims)
3051 .await?
3052 .with_qos_preset::<hyprstream_rpc::stream_info::Job>();
3053
3054 let stream_id = stream_ctx.stream_id().to_owned();
3055 let server_pubkey = *stream_ctx.server_pubkey();
3056 let broadcast_path = hyprstream_rpc::moq_stream::global_moq_origin()
3057 .map(|o| o.broadcast_path(stream_ctx.topic()))
3058 .unwrap_or_default();
3059
3060 let stream_info = hyprstream_rpc_std::inference_client::StreamInfo {
3061 stream_id,
3062 dh_public: server_pubkey,
3063 qos: stream_ctx.qos().clone(),
3064 broadcast_path,
3065 announced_at: stream_ctx.reach(), moql_server_identity: stream_ctx.moql_server_identity(),
3067 kem_ciphertexts: Vec::new(), };
3069
3070 let adaptation_strategy =
3071 map_adaptation_strategy(data.adaptation_strategy, data.writeback_threshold);
3072 let pending = PendingWork::Training {
3073 stream_ctx,
3074 subject,
3075 input: data.input.clone(),
3076 gradient_steps: data.gradient_steps,
3077 learning_rate: data.learning_rate,
3078 adaptation_strategy,
3079 };
3080
3081 let service = self.clone();
3083 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3084 service.execute_training_stream(pending).await;
3085 });
3086
3087 Ok((stream_info, continuation))
3088 }
3089
3090 async fn handle_ttt_zero(
3091 &self,
3092 ctx: &EnvelopeContext,
3093 _request_id: u64,
3094 ) -> Result<InferenceResponseVariant> {
3095 let subject = ctx.subject();
3096 InferenceService::handle_reset_delta(self, &subject)?;
3097 Ok(InferenceResponseVariant::TttZeroResult)
3098 }
3099
3100 async fn handle_get_delta_status(
3101 &self,
3102 ctx: &EnvelopeContext,
3103 _request_id: u64,
3104 ) -> Result<InferenceResponseVariant> {
3105 let subject = ctx.subject();
3106 let result = InferenceService::handle_get_delta_status(self, &subject)?;
3107 Ok(InferenceResponseVariant::GetDeltaStatusResult(result))
3108 }
3109
3110 async fn handle_save_adaptation(
3111 &self,
3112 ctx: &EnvelopeContext,
3113 _request_id: u64,
3114 data: &SaveAdaptationRequest,
3115 ) -> Result<InferenceResponseVariant> {
3116 let subject = ctx.subject();
3117 let result = InferenceService::handle_save_adaptation(
3118 self,
3119 &subject,
3120 &data.name,
3121 data.merge_strategy.as_deref().unwrap_or(""),
3122 data.merge_weight.unwrap_or(0.0),
3123 )
3124 .await?;
3125 Ok(InferenceResponseVariant::SaveAdaptationResult(result))
3126 }
3127
3128 async fn handle_snapshot_delta(
3129 &self,
3130 ctx: &EnvelopeContext,
3131 _request_id: u64,
3132 ) -> Result<InferenceResponseVariant> {
3133 let subject = ctx.subject();
3134 let result = InferenceService::handle_snapshot_delta(self, &subject).await?;
3135 Ok(InferenceResponseVariant::SnapshotDeltaResult(result))
3136 }
3137
3138 async fn handle_export_peft_adapter(
3139 &self,
3140 ctx: &EnvelopeContext,
3141 _request_id: u64,
3142 data: &ExportPeftRequest,
3143 ) -> Result<InferenceResponseVariant> {
3144 let subject = ctx.subject();
3145 let result = InferenceService::handle_export_peft_adapter(
3146 self,
3147 &subject,
3148 &data.name,
3149 data.commit_message.as_deref().unwrap_or(""),
3150 )
3151 .await?;
3152 Ok(InferenceResponseVariant::ExportPeftAdapterResult(result))
3153 }
3154
3155 async fn handle_merge_lora(
3156 &self,
3157 _ctx: &EnvelopeContext,
3158 _request_id: u64,
3159 data: &MergeLoraRequest,
3160 ) -> Result<InferenceResponseVariant> {
3161 let weight = data.weight.filter(|&w| w > 0.0).unwrap_or(1.0);
3162 let strategy = data
3163 .strategy
3164 .as_deref()
3165 .filter(|s| !s.is_empty())
3166 .unwrap_or("do_merge");
3167 self.merge_lora(Path::new(&data.adapter_path), weight, strategy)
3168 .await?;
3169 Ok(InferenceResponseVariant::MergeLoraResult)
3170 }
3171
3172 async fn handle_create_lora_stream(
3181 &self,
3182 ctx: &EnvelopeContext,
3183 _request_id: u64,
3184 data: &LoraConfig,
3185 ) -> Result<(
3186 hyprstream_rpc_std::inference_client::StreamInfo,
3187 hyprstream_rpc::service::Continuation,
3188 )> {
3189 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3190 let service = self.clone();
3191 let defaults = crate::training::TenantDeltaConfig::default();
3192 let config = crate::training::TenantDeltaConfig {
3193 rank: data.rank as usize,
3194 alpha: data.alpha.unwrap_or(defaults.alpha),
3195 dropout: data.dropout.unwrap_or(defaults.dropout),
3196 target_modules: data.target_modules.clone(),
3197 learning_rate: data
3198 .learning_rate
3199 .map(|v| v as f64)
3200 .unwrap_or(defaults.learning_rate),
3201 ..defaults
3202 };
3203 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3204 service.execute_create_lora_stream(stream_ctx, config).await;
3205 });
3206 Ok((stream_info, continuation))
3207 }
3208
3209 async fn handle_load_lora_stream(
3210 &self,
3211 ctx: &EnvelopeContext,
3212 _request_id: u64,
3213 value: &str,
3214 ) -> Result<(
3215 hyprstream_rpc_std::inference_client::StreamInfo,
3216 hyprstream_rpc::service::Continuation,
3217 )> {
3218 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3219 let service = self.clone();
3220 let path = value.to_owned();
3221 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3222 service.execute_load_lora_stream(stream_ctx, path).await;
3223 });
3224 Ok((stream_info, continuation))
3225 }
3226
3227 async fn handle_save_lora_stream(
3228 &self,
3229 ctx: &EnvelopeContext,
3230 _request_id: u64,
3231 value: &str,
3232 ) -> Result<(
3233 hyprstream_rpc_std::inference_client::StreamInfo,
3234 hyprstream_rpc::service::Continuation,
3235 )> {
3236 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3237 let service = self.clone();
3238 let name = value.to_owned();
3239 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3240 service.execute_save_lora_stream(stream_ctx, name).await;
3241 });
3242 Ok((stream_info, continuation))
3243 }
3244
3245 async fn handle_save_adaptation_stream(
3246 &self,
3247 ctx: &EnvelopeContext,
3248 _request_id: u64,
3249 data: &SaveAdaptationRequest,
3250 ) -> Result<(
3251 hyprstream_rpc_std::inference_client::StreamInfo,
3252 hyprstream_rpc::service::Continuation,
3253 )> {
3254 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3255 let service = self.clone();
3256 let subject = ctx.subject();
3257 let name = data.name.clone();
3258 let merge_strategy = data.merge_strategy.clone();
3259 let merge_weight = data.merge_weight;
3260 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3261 service
3262 .execute_save_adaptation_stream(
3263 stream_ctx,
3264 subject,
3265 name,
3266 merge_strategy.unwrap_or_default(),
3267 merge_weight.unwrap_or(0.0),
3268 )
3269 .await;
3270 });
3271 Ok((stream_info, continuation))
3272 }
3273
3274 async fn handle_snapshot_delta_stream(
3275 &self,
3276 ctx: &EnvelopeContext,
3277 _request_id: u64,
3278 ) -> Result<(
3279 hyprstream_rpc_std::inference_client::StreamInfo,
3280 hyprstream_rpc::service::Continuation,
3281 )> {
3282 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3283 let service = self.clone();
3284 let subject = ctx.subject();
3285 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3286 service
3287 .execute_snapshot_delta_stream(stream_ctx, subject)
3288 .await;
3289 });
3290 Ok((stream_info, continuation))
3291 }
3292
3293 async fn handle_export_peft_adapter_stream(
3294 &self,
3295 ctx: &EnvelopeContext,
3296 _request_id: u64,
3297 data: &ExportPeftRequest,
3298 ) -> Result<(
3299 hyprstream_rpc_std::inference_client::StreamInfo,
3300 hyprstream_rpc::service::Continuation,
3301 )> {
3302 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3303 let service = self.clone();
3304 let subject = ctx.subject();
3305 let name = data.name.clone();
3306 let commit_message = data.commit_message.clone();
3307 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3308 service
3309 .execute_export_peft_adapter_stream(
3310 stream_ctx,
3311 subject,
3312 name,
3313 commit_message.unwrap_or_default(),
3314 )
3315 .await;
3316 });
3317 Ok((stream_info, continuation))
3318 }
3319
3320 async fn handle_merge_lora_stream(
3321 &self,
3322 ctx: &EnvelopeContext,
3323 _request_id: u64,
3324 data: &MergeLoraRequest,
3325 ) -> Result<(
3326 hyprstream_rpc_std::inference_client::StreamInfo,
3327 hyprstream_rpc::service::Continuation,
3328 )> {
3329 let (stream_info, stream_ctx) = self.setup_stream(ctx).await?;
3330 let service = self.clone();
3331 let adapter_path = data.adapter_path.clone();
3332 let weight = data.weight.filter(|&w| w > 0.0).unwrap_or(1.0);
3333 let strategy = data
3334 .strategy
3335 .as_deref()
3336 .filter(|s| !s.is_empty())
3337 .unwrap_or("do_merge")
3338 .to_owned();
3339 let continuation: hyprstream_rpc::service::Continuation = Box::pin(async move {
3340 service
3341 .execute_merge_lora_stream(stream_ctx, adapter_path, weight, strategy)
3342 .await;
3343 });
3344 Ok((stream_info, continuation))
3345 }
3346
3347 async fn handle_embed(
3348 &self,
3349 _ctx: &EnvelopeContext,
3350 _request_id: u64,
3351 data: &EmbedImagesRequest,
3352 ) -> Result<InferenceResponseVariant> {
3353 let engine = self.engine.read();
3354 let embeddings = engine.embed_images(&data.images)?;
3355 let dimensions = embeddings.first().map(|v| v.len() as u32).unwrap_or(0);
3356 Ok(InferenceResponseVariant::EmbedResult(EmbedImagesResponse {
3357 embeddings,
3358 dimensions,
3359 }))
3360 }
3361}
3362
3363struct InferenceZmqAdapter {
3376 service: InferenceService,
3377 transport: hyprstream_rpc::transport::TransportConfig,
3378 signing_key: SigningKey,
3379 expected_audience: Option<String>,
3380 jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
3381 instance_model_ref: Option<String>,
3384 instance_audience: String,
3388}
3389
3390#[async_trait::async_trait(?Send)]
3391impl hyprstream_rpc::service::RequestService for InferenceZmqAdapter {
3392 fn decode_request_body(
3393 &self,
3394 signed_body: &[u8],
3395 ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
3396 crate::services::generated::inference_client::decode_inference_request_body(signed_body)
3400 }
3401
3402 async fn handle_request(
3403 &self,
3404 ctx: &EnvelopeContext,
3405 body: &hyprstream_rpc::service::DecodedRequestBody,
3406 ) -> anyhow::Result<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
3407 dispatch_inference(&self.service, ctx, body).await
3408 }
3409
3410 fn name(&self) -> &str {
3411 "inference"
3414 }
3415
3416 fn transport(&self) -> &hyprstream_rpc::transport::TransportConfig {
3417 &self.transport
3418 }
3419
3420 fn signing_key(&self) -> SigningKey {
3421 self.signing_key.clone()
3422 }
3423
3424 fn expected_audience(&self) -> Option<&str> {
3425 self.expected_audience.as_deref()
3426 }
3427
3428 fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
3429 self.jwt_key_source.clone()
3430 }
3431 fn accept_delegated_bearer(&self, signer_pubkey: &[u8; 32]) -> bool {
3432 signer_pubkey == &self.service.controller_pubkey.to_bytes()
3433 }
3434
3435 async fn verify_internal_work_bearer(
3444 &self,
3445 ctx: &EnvelopeContext,
3446 token: &str,
3447 _protected: &hyprstream_rpc::auth::ProtectedHeader,
3448 ) -> anyhow::Result<hyprstream_rpc::auth::internal_work::InternalWorkClaims> {
3449 let instance_model_ref = self.instance_model_ref.as_ref().ok_or_else(|| {
3450 anyhow::anyhow!("standalone inference accepts no internal work bearer")
3451 })?;
3452 anyhow::ensure!(
3453 ctx.cnf == self.service.controller_pubkey.to_bytes(),
3454 "internal work bearer envelope was not signed by the pinned controller"
3455 );
3456 let now = chrono::Utc::now().timestamp();
3457 let claims = hyprstream_rpc::auth::internal_work::verify_internal_work(
3458 token,
3459 &self.service.controller_pubkey,
3460 &self.instance_audience,
3461 now,
3462 )?;
3463 anyhow::ensure!(
3464 claims.tenant == self.service.tenant_domain,
3465 "internal work token tenant does not match this instance binding"
3466 );
3467 anyhow::ensure!(
3468 claims.model == *instance_model_ref,
3469 "internal work token model does not match this instance binding"
3470 );
3471 anyhow::ensure!(
3472 hyprstream_rpc::auth::internal_work::admit_internal_work_jti_once(
3473 &claims.jti,
3474 claims.exp,
3475 now,
3476 ),
3477 "internal work token jti was already admitted (replay)"
3478 );
3479 Ok(claims)
3480 }
3481
3482 fn resolve_key_subject(
3491 &self,
3492 signer_pubkey: &[u8; 32],
3493 ) -> Option<hyprstream_rpc::envelope::Subject> {
3494 hyprstream_service::global_trust_store().resolve_subject(signer_pubkey)
3495 }
3496
3497 fn build_error_payload(&self, request_id: u64, error: &str) -> Vec<u8> {
3498 let variant = InferenceResponseVariant::Error(ErrorInfo {
3499 message: error.to_owned(),
3500 code: "INTERNAL".to_owned(),
3501 details: String::new(),
3502 });
3503 serialize_response(request_id, &variant).unwrap_or_else(|e| {
3504 error!("Failed to serialize error payload: {}", e);
3505 vec![]
3506 })
3507 }
3508}
3509
3510#[derive(Clone)]
3516pub struct IncarnationHandoff {
3517 pub instance_service_name: String,
3519 pub controller_pubkey: [u8; 32],
3521 pub generation: u64,
3523 pub incarnation: String,
3525 pub audience: String,
3528}
3529
3530pub const INCARNATION_HANDOFF_TIMEOUT_SECS: u64 = 30;
3533
3534pub(crate) fn internal_work_audience(service_name: &str, incarnation: &str) -> String {
3537 format!("iw1/{service_name}/{incarnation}")
3538}
3539
3540pub struct InferenceServiceConfig {
3546 service_name: String,
3547 model_path: PathBuf,
3548 state_path: PathBuf,
3549 pinned_artifact: Option<Arc<crate::storage::pinned_model::PinnedModelArtifact>>,
3550 config: RuntimeConfig,
3551 server_pubkey: VerifyingKey,
3552 signing_key: SigningKey,
3553 policy_signing_key: SigningKey,
3558 policy_transport: hyprstream_rpc::transport::TransportConfig,
3559 transport: hyprstream_rpc::transport::TransportConfig,
3560 fs: Option<WorktreeClient>,
3561 expected_audience: Option<String>,
3563 jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
3565 tenant_domain: String,
3567 model_ref: Option<String>,
3572 incarnation_handoff: Option<tokio::sync::oneshot::Sender<IncarnationHandoff>>,
3576 attempt_generation: u64,
3578 controller_pubkey: VerifyingKey,
3580 network_reach: Arc<parking_lot::RwLock<Option<hyprstream_rpc::transport::TransportConfig>>>,
3582 producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
3584 moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
3585 quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
3587 quic_advertise_addr: Option<std::net::SocketAddr>,
3589 owns_stream_plane: bool,
3591 network_ready: Arc<AtomicBool>,
3592 draining: Arc<AtomicBool>,
3593}
3594
3595impl InferenceServiceConfig {
3596 pub fn new(
3605 model_path: impl AsRef<Path>,
3606 config: RuntimeConfig,
3607 server_pubkey: VerifyingKey,
3608 signing_key: SigningKey,
3609 transport: hyprstream_rpc::transport::TransportConfig,
3610 policy_transport: hyprstream_rpc::transport::TransportConfig,
3611 fs: Option<WorktreeClient>,
3612 ) -> Self {
3613 let policy_signing_key = signing_key.clone();
3614 Self {
3615 service_name: "inference".to_owned(),
3616 model_path: model_path.as_ref().to_path_buf(),
3617 state_path: model_path.as_ref().to_path_buf(),
3618 pinned_artifact: None,
3619 config,
3620 server_pubkey,
3621 signing_key,
3622 policy_signing_key,
3623 policy_transport,
3624 transport,
3625 fs,
3626 expected_audience: None,
3627 jwt_key_source: None,
3628 tenant_domain: "local".to_owned(),
3629 model_ref: None,
3630 incarnation_handoff: None,
3631 attempt_generation: 0,
3632 controller_pubkey: server_pubkey,
3633 network_reach: Arc::new(parking_lot::RwLock::new(None)),
3634 producer_reach_config: Arc::new(parking_lot::RwLock::new(
3635 hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
3636 )),
3637 moq_origin: Arc::new(parking_lot::RwLock::new(None)),
3638 quic_config: None,
3639 quic_advertise_addr: None,
3640 owns_stream_plane: true,
3641 network_ready: Arc::new(AtomicBool::new(true)),
3645 draining: Arc::new(AtomicBool::new(false)),
3646 }
3647 }
3648
3649 #[must_use]
3652 pub fn with_pinned_artifact(
3653 mut self,
3654 artifact: Arc<crate::storage::pinned_model::PinnedModelArtifact>,
3655 ) -> Self {
3656 self.model_path = artifact.root().to_path_buf();
3657 self.pinned_artifact = Some(artifact);
3658 self
3659 }
3660
3661 #[must_use]
3663 pub fn with_instance_identity(
3664 mut self,
3665 service_name: String,
3666 tenant_domain: String,
3667 controller_pubkey: VerifyingKey,
3668 ) -> Self {
3669 self.service_name = service_name;
3670 self.tenant_domain = tenant_domain;
3671 self.controller_pubkey = controller_pubkey;
3672 self
3673 }
3674
3675 #[must_use]
3679 pub fn with_model_ref(mut self, model_ref: impl Into<String>) -> Self {
3680 self.model_ref = Some(model_ref.into());
3681 self
3682 }
3683
3684 #[must_use]
3688 pub fn with_incarnation_handoff(
3689 mut self,
3690 sender: tokio::sync::oneshot::Sender<IncarnationHandoff>,
3691 attempt_generation: u64,
3692 ) -> Self {
3693 self.incarnation_handoff = Some(sender);
3694 self.attempt_generation = attempt_generation;
3695 self
3696 }
3697
3698 #[must_use]
3700 pub fn with_stream_plane(
3701 mut self,
3702 producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
3703 moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
3704 ) -> Self {
3705 self.producer_reach_config = producer_reach_config;
3706 self.moq_origin = moq_origin;
3707 self.owns_stream_plane = false;
3708 self.network_ready.store(
3709 self.producer_reach_config.read().quic_reach.is_some(),
3710 Ordering::Release,
3711 );
3712 self
3713 }
3714
3715 #[must_use]
3717 pub fn with_quic_config(
3718 mut self,
3719 quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
3720 ) -> Self {
3721 if quic_config.is_some() {
3722 self.network_ready.store(false, Ordering::Release);
3723 }
3724 self.quic_config = quic_config;
3725 self
3726 }
3727
3728 #[must_use]
3730 pub fn with_quic_advertise_addr(
3731 mut self,
3732 advertise_addr: Option<std::net::SocketAddr>,
3733 ) -> Self {
3734 self.quic_advertise_addr = advertise_addr;
3735 self
3736 }
3737
3738 pub fn with_expected_audience(mut self, audience: String) -> Self {
3740 self.expected_audience = Some(audience);
3741 self
3742 }
3743
3744 pub fn with_jwt_key_source(
3746 mut self,
3747 src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
3748 ) -> Self {
3749 self.jwt_key_source = Some(src);
3750 self
3751 }
3752 #[must_use]
3753 pub fn network_reach_handle(
3754 &self,
3755 ) -> Arc<parking_lot::RwLock<Option<hyprstream_rpc::transport::TransportConfig>>> {
3756 Arc::clone(&self.network_reach)
3757 }
3758}
3759
3760#[allow(clippy::too_many_arguments)]
3761async fn serve_inference_bridged(
3762 service_name: &str,
3763 transport: &hyprstream_rpc::transport::TransportConfig,
3764 processor: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor>,
3765 signing_key: SigningKey,
3766 shutdown: Arc<tokio::sync::Notify>,
3767 on_ready: Option<tokio::sync::oneshot::Sender<()>>,
3768 quic_config: Option<hyprstream_rpc::service::QuicLoopConfig>,
3769 quic_advertise_addr: Option<std::net::SocketAddr>,
3770 producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
3771 moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
3772 network_ready: Arc<AtomicBool>,
3773 draining: Arc<AtomicBool>,
3774) -> hyprstream_rpc::error::Result<()> {
3775 let Some(mut qc) = quic_config else {
3776 return hyprstream_rpc::service::serve::serve_bridged(
3777 transport,
3778 processor,
3779 signing_key,
3780 shutdown,
3781 on_ready,
3782 )
3783 .await
3784 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string()));
3785 };
3786
3787 hyprstream_rpc::transport::install_pq_crypto_provider().map_err(|error| {
3788 hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC crypto provider: {error}"))
3789 })?;
3790 let chain: Vec<rustls::pki_types::CertificateDer<'static>> = qc
3791 .cert_chain
3792 .iter()
3793 .map(|der| rustls::pki_types::CertificateDer::from(der.clone()))
3794 .collect();
3795 let key = rustls::pki_types::PrivateKeyDer::try_from((*qc.key_der).clone())
3796 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC key: {e}")))?;
3797 let wt_server = web_transport_quinn::ServerBuilder::new()
3798 .with_addr(qc.bind_addr)
3799 .with_certificate(chain, key)
3800 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC bind: {e}")))?;
3801 let actual_addr = wt_server.local_addr().map_err(|e| {
3802 hyprstream_rpc::error::RpcError::SpawnFailed(format!("QUIC local_addr: {e}"))
3803 })?;
3804 let advertise_addr = quic_advertise_addr.ok_or_else(|| {
3805 hyprstream_rpc::error::RpcError::SpawnFailed(
3806 "standalone inference requires an explicit routable QUIC advertise address".to_owned(),
3807 )
3808 })?;
3809 if advertise_addr.ip().is_unspecified()
3810 || advertise_addr.ip().is_loopback()
3811 || advertise_addr.port() == 0
3812 || advertise_addr.port() != actual_addr.port()
3813 {
3814 return Err(hyprstream_rpc::error::RpcError::SpawnFailed(format!(
3815 "standalone inference advertise address {advertise_addr} is not routable or does not match bound QUIC port {}",
3816 actual_addr.port()
3817 )));
3818 }
3819 let pin = hyprstream_rpc::transport::quinn_transport::cert_sha256(&qc.cert_chain[0]);
3820 let origin = moq_origin
3821 .read()
3822 .clone()
3823 .or_else(|| hyprstream_rpc::moq_stream::global_moq_origin().cloned())
3824 .ok_or_else(|| {
3825 hyprstream_rpc::error::RpcError::SpawnFailed(
3826 "standalone inference has no MoQ origin".to_owned(),
3827 )
3828 })?;
3829
3830 let rpc_server = hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::with_capacity(
3831 wt_server,
3832 Arc::clone(&processor),
3833 signing_key.clone(),
3834 hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
3835 )
3836 .with_moq_consumer(origin.consumer().clone());
3837
3838 if let Some(registry) = hyprstream_rpc::registry::try_global() {
3839 registry.register(
3840 service_name,
3841 hyprstream_rpc::registry::SocketKind::Quic,
3842 hyprstream_rpc::transport::TransportConfig::quic_pinned(
3843 advertise_addr,
3844 &qc.server_name,
3845 pin,
3846 ),
3847 None,
3848 );
3849 }
3850 {
3851 let mut reach = producer_reach_config.write();
3852 reach.quic_reach = Some(hyprstream_rpc::moq_stream::NodeStreamReach {
3853 addr: advertise_addr,
3854 server_name: qc.server_name.clone(),
3855 cert_hashes: vec![pin],
3856 });
3857 reach.relay = qc.moq_relay.clone();
3858 }
3859 network_ready.store(true, Ordering::Release);
3860 if let Some(callback) = qc.on_quic_bound.take() {
3861 callback(
3862 service_name.to_owned(),
3863 advertise_addr,
3864 qc.server_name.clone(),
3865 );
3866 }
3867 if let Some(relay) = qc.moq_relay.take() {
3868 hyprstream_rpc::moq_stream::serve_origin_to_relay_background(
3869 origin.producer().clone(),
3870 relay,
3871 qc.moq_admission_proof.clone(),
3872 qc.moq_relay_server_identity.take(),
3873 );
3874 }
3875
3876 let (rep_armed_tx, rep_armed_rx) = tokio::sync::oneshot::channel();
3881 let rep_transport = transport.clone();
3882 let rep_processor = Arc::clone(&processor);
3883 let rep_shutdown = Arc::clone(&shutdown);
3884 let rep_task = tokio::spawn(async move {
3885 hyprstream_rpc::service::serve::serve_bridged_with_shutdown_armed_silent(
3886 &rep_transport,
3887 rep_processor,
3888 signing_key,
3889 rep_shutdown,
3890 None,
3891 Some(rep_armed_tx),
3892 )
3893 .await
3894 });
3895 let rep_owner = RepTaskOwner::new(rep_task);
3896 if rep_armed_rx.await.is_err() {
3897 network_ready.store(false, Ordering::Release);
3898 let rep_error = rep_owner.join().await;
3899 processor.close_admission();
3900 return Err(match rep_error {
3901 Err(error) => hyprstream_rpc::error::RpcError::SpawnFailed(error.to_string()),
3902 Ok(()) => {
3903 hyprstream_rpc::error::RpcError::SpawnFailed("REP waiter failed to arm".to_owned())
3904 }
3905 });
3906 }
3907 let rep_admission = Arc::clone(&processor);
3908 let rep = async move {
3909 let result = rep_owner.join().await;
3910 if result.is_err() {
3911 rep_admission.close_admission();
3915 }
3916 result
3917 };
3918
3919 let drain_limit = rpc_server.stream_limit();
3920 let drain_capacity = rpc_server.capacity();
3921 let drain_token = rpc_server.shutdown_token();
3922 let drain_shutdown = Arc::clone(&shutdown);
3923 let drain_state = Arc::clone(&draining);
3924 let drain_ready = Arc::clone(&network_ready);
3925 let (drain_armed_tx, drain_armed_rx) = tokio::sync::oneshot::channel();
3926 let drain_task = tokio::spawn(async move {
3927 let notified = drain_shutdown.notified();
3931 tokio::pin!(notified);
3932 notified.as_mut().enable();
3933 let _ = drain_armed_tx.send(());
3934 notified.await;
3935 drain_state.store(true, Ordering::Release);
3936 drain_ready.store(false, Ordering::Release);
3937 hyprstream_rpc::transport::quinn_transport::QuinnRpcServer::shutdown(
3938 &drain_limit,
3939 drain_capacity,
3940 &drain_token,
3941 )
3942 .await;
3943 });
3944 let drain_owner = QuinnDrainOwner::new(drain_task);
3945 let _ = drain_armed_rx.await;
3949 let _ = hyprstream_rpc::notify::ready();
3953 if let Some(on_ready) = on_ready {
3954 let _ = on_ready.send(());
3955 }
3956 let quic = rpc_server.run();
3957 run_quinn_lifecycle(rep, quic, shutdown, network_ready, draining, drain_owner).await
3958}
3959
3960async fn run_quinn_lifecycle<Rep, Quic>(
3961 rep: Rep,
3962 quic: Quic,
3963 shutdown: Arc<tokio::sync::Notify>,
3964 network_ready: Arc<AtomicBool>,
3965 draining: Arc<AtomicBool>,
3966 drain_owner: QuinnDrainOwner,
3967) -> hyprstream_rpc::error::Result<()>
3968where
3969 Rep: std::future::Future<Output = anyhow::Result<()>>,
3970 Quic: std::future::Future<Output = anyhow::Result<()>>,
3971{
3972 tokio::pin!(rep);
3973 tokio::pin!(quic);
3974 tokio::select! {
3975 biased;
3978 rep_result = &mut rep => {
3979 shutdown.notify_waiters();
3980 let quic_result = quic.await;
3981 drain_owner.join().await;
3982 network_ready.store(false, Ordering::Release);
3983 if let Err(error) = quic_result {
3984 tracing::warn!(%error, "standalone inference QUIC loop ended during drain");
3985 }
3986 rep_result.map_err(|e| {
3987 hyprstream_rpc::error::RpcError::SpawnFailed(e.to_string())
3988 })
3989 }
3990 quic_result = &mut quic => {
3991 network_ready.store(false, Ordering::Release);
3992 if draining.load(Ordering::Acquire) {
3993 shutdown.notify_waiters();
4000 let (rep_result, _) = tokio::join!(rep, drain_owner.join());
4001 let rep_result = rep_result.map_err(|error| {
4002 hyprstream_rpc::error::RpcError::SpawnFailed(error.to_string())
4003 });
4004 return rep_result;
4005 }
4006 shutdown.notify_waiters();
4007 let detail = match quic_result {
4008 Ok(()) => "QUIC server exited unexpectedly".to_owned(),
4009 Err(error) => format!("QUIC server failed: {error}"),
4010 };
4011 let (_rep_result, _) = tokio::join!(rep, drain_owner.join());
4016 Err(hyprstream_rpc::error::RpcError::SpawnFailed(detail))
4017 }
4018 }
4019}
4020
4021struct QuinnDrainOwner {
4027 task: Option<tokio::task::JoinHandle<()>>,
4028}
4029
4030struct RepTaskOwner {
4034 task: Option<tokio::task::JoinHandle<anyhow::Result<()>>>,
4035}
4036
4037impl RepTaskOwner {
4038 fn new(task: tokio::task::JoinHandle<anyhow::Result<()>>) -> Self {
4039 Self { task: Some(task) }
4040 }
4041
4042 async fn join(mut self) -> anyhow::Result<()> {
4043 let Some(task) = self.task.as_mut() else {
4044 return Err(anyhow::anyhow!("REP task missing"));
4045 };
4046 let result = task
4047 .await
4048 .map_err(|error| anyhow::anyhow!("REP task failed: {error}"))?;
4049 self.task.take();
4050 result
4051 }
4052}
4053
4054impl Drop for RepTaskOwner {
4055 fn drop(&mut self) {
4056 if let Some(task) = self.task.take() {
4057 task.abort();
4058 }
4059 }
4060}
4061
4062impl QuinnDrainOwner {
4063 fn new(task: tokio::task::JoinHandle<()>) -> Self {
4064 Self { task: Some(task) }
4065 }
4066
4067 async fn join(mut self) {
4068 if let Some(task) = self.task.as_mut() {
4069 if let Err(error) = task.await {
4070 tracing::warn!(%error, "standalone inference QUIC drain task ended unexpectedly");
4071 }
4072 }
4073 self.task.take();
4074 }
4075}
4076
4077impl Drop for QuinnDrainOwner {
4078 fn drop(&mut self) {
4079 if let Some(task) = self.task.take() {
4080 task.abort();
4081 }
4082 }
4083}
4084
4085async fn observe_inference_shutdown(
4086 shutdown: Arc<tokio::sync::Notify>,
4087 network_ready: Arc<AtomicBool>,
4088 draining: Arc<AtomicBool>,
4089 armed: tokio::sync::oneshot::Sender<()>,
4090) {
4091 let notified = shutdown.notified();
4092 tokio::pin!(notified);
4093 notified.as_mut().enable();
4094 let _ = armed.send(());
4095 notified.await;
4096 draining.store(true, Ordering::Release);
4097 network_ready.store(false, Ordering::Release);
4098}
4099
4100impl hyprstream_service::Spawnable for InferenceServiceConfig {
4101 fn name(&self) -> &str {
4102 &self.service_name
4103 }
4104
4105 fn registrations(
4106 &self,
4107 ) -> Vec<(
4108 hyprstream_rpc::registry::SocketKind,
4109 hyprstream_rpc::transport::TransportConfig,
4110 )> {
4111 vec![(
4112 hyprstream_rpc::registry::SocketKind::Rep,
4113 self.transport.clone(),
4114 )]
4115 }
4116
4117 fn run(
4118 self: Box<Self>,
4119 shutdown: Arc<tokio::sync::Notify>,
4120 on_ready: Option<tokio::sync::oneshot::Sender<()>>,
4121 ) -> hyprstream_rpc::error::Result<()> {
4122 let transport = self.transport.clone();
4123 let server_signing_key = self.signing_key.clone();
4124
4125 let rt = tokio::runtime::Builder::new_multi_thread()
4130 .enable_all()
4131 .build()
4132 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("runtime: {e}")))?;
4133
4134 rt.block_on(async move {
4135 let nonce_cache = Arc::new(InMemoryNonceCache::new());
4137 let bridge_nonce = Arc::clone(&nonce_cache);
4138
4139 let InferenceServiceConfig {
4141 service_name,
4142 model_path,
4143 state_path,
4144 pinned_artifact,
4145 config,
4146 server_pubkey,
4147 signing_key: svc_signing_key,
4148 policy_signing_key,
4149 policy_transport,
4150 transport: _transport,
4151 fs,
4152 expected_audience,
4153 jwt_key_source,
4154 tenant_domain,
4155 model_ref: model_ref_binding,
4156 incarnation_handoff,
4157 attempt_generation,
4158 controller_pubkey,
4159 network_reach,
4160 producer_reach_config,
4161 moq_origin,
4162 quic_config,
4163 quic_advertise_addr,
4164 owns_stream_plane,
4165 network_ready,
4166 draining,
4167 } = *self;
4168 let adapter_transport = transport.clone();
4169 let instance_service_name = service_name.clone();
4171
4172 let lifecycle_ready = Arc::clone(&network_ready);
4173 let lifecycle_draining = Arc::clone(&draining);
4174 let lifecycle_shutdown = Arc::clone(&shutdown);
4175 let (lifecycle_armed_tx, lifecycle_armed_rx) = tokio::sync::oneshot::channel();
4176 tokio::spawn(async move {
4177 observe_inference_shutdown(
4178 lifecycle_shutdown,
4179 lifecycle_ready,
4180 lifecycle_draining,
4181 lifecycle_armed_tx,
4182 )
4183 .await;
4184 });
4185 lifecycle_armed_rx.await.map_err(|_| {
4186 hyprstream_rpc::error::RpcError::SpawnFailed(
4187 "inference lifecycle shutdown observer exited before arming".to_owned(),
4188 )
4189 })?;
4190
4191 if owns_stream_plane && quic_config.is_some() {
4194 *moq_origin.write() = Some(
4195 hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone()
4196 .with_prefix(hyprstream_rpc::moq_stream::DEFAULT_PREFIX)
4197 .build(),
4198 );
4199 }
4200 let bridge_reach_config = Arc::clone(&producer_reach_config);
4201 let bridge_moq_origin = Arc::clone(&moq_origin);
4202 let bridge_network_ready = Arc::clone(&network_ready);
4203 let bridge_draining = Arc::clone(&draining);
4204 let bridge_shutdown = Arc::clone(&shutdown);
4205
4206 #[cfg(test)]
4208 crate::services::restart_diag::phase("bridge.build.enter");
4209 let (bridge, ready) = hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn_with(
4210 "inference",
4211 move || async move {
4212 let policy_vk = hyprstream_service::global_trust_store()
4213 .resolve_one("policy")
4214 .ok_or_else(|| {
4215 anyhow::anyhow!("trust store has no policy key — startup must populate it")
4216 })?;
4217 let policy_client = if hyprstream_discovery::native_network_required() {
4221 PolicyClient::from_provider(&hyprstream_discovery::ProductionRpcClientProvider, policy_signing_key, None)?
4222 } else {
4223 PolicyClient::for_local_transport_bootstrap(
4224 &policy_transport,
4225 policy_signing_key,
4226 policy_vk,
4227 None,
4228 )?
4229 };
4230 let service = InferenceService::initialize(
4231 model_path,
4232 state_path,
4233 pinned_artifact,
4234 config,
4235 server_pubkey,
4236 svc_signing_key.clone(),
4237 Arc::clone(&bridge_nonce),
4238 policy_client,
4239 fs,
4240 tenant_domain,
4241 controller_pubkey,
4242 bridge_reach_config,
4243 bridge_moq_origin,
4244 bridge_network_ready,
4245 bridge_draining,
4246 bridge_shutdown,
4247 )
4248 .await
4249 .map_err(|e| anyhow::anyhow!("inference init: {e}"))?;
4250
4251 let incarnation_bytes = [
4259 hyprstream_rpc::envelope::generate_nonce(),
4260 hyprstream_rpc::envelope::generate_nonce(),
4261 ]
4262 .concat();
4263 let incarnation = hex::encode(incarnation_bytes);
4264 let instance_audience =
4265 internal_work_audience(&instance_service_name, &incarnation);
4266 match incarnation_handoff {
4267 Some(sender) => sender
4268 .send(IncarnationHandoff {
4269 instance_service_name: instance_service_name.clone(),
4270 controller_pubkey: controller_pubkey.to_bytes(),
4271 generation: attempt_generation,
4272 incarnation,
4273 audience: instance_audience.clone(),
4274 })
4275 .map_err(|_| {
4276 anyhow::anyhow!(
4277 "incarnation handoff receiver dropped by the owner; refusing to serve internal work from an unbound worker"
4278 )
4279 })?,
4280 None if model_ref_binding.is_some() => {
4283 anyhow::bail!(
4284 "pinned instance has no incarnation handoff receiver; refusing to serve"
4285 )
4286 }
4287 None => {}
4288 }
4289 #[cfg(test)]
4290 crate::services::restart_diag::phase("incarnation.sent");
4291
4292 Ok(InferenceZmqAdapter {
4293 service,
4294 transport: adapter_transport,
4295 signing_key: svc_signing_key,
4296 expected_audience,
4297 jwt_key_source,
4298 instance_model_ref: model_ref_binding,
4299 instance_audience,
4300 })
4301 },
4302 nonce_cache,
4303 0,
4304 )
4305 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("bridge: {e}")))?;
4306
4307 #[cfg(test)]
4309 crate::services::restart_diag::phase("bridge.ready.await.enter");
4310 ready
4311 .await
4312 .map_err(|_| {
4313 hyprstream_rpc::error::RpcError::SpawnFailed(
4314 "inference bridge readiness dropped".to_owned(),
4315 )
4316 })?
4317 .map_err(|e| hyprstream_rpc::error::RpcError::SpawnFailed(format!("init: {e}")))?;
4318 #[cfg(test)]
4319 crate::services::restart_diag::phase("bridge.ready.await.done");
4320
4321 let processor: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor> =
4322 Arc::new(bridge);
4323 let purpose =
4324 crate::runtime::inference_profile::inference_iroh_key_purpose(&service_name);
4325 let iroh_key =
4326 hyprstream_rpc::node_identity::derive_purpose_key(&server_signing_key, &purpose);
4327 let rpc_handler =
4328 hyprstream_rpc::transport::iroh_rpc::IrohRpcProtocolHandler::with_stream_limit(
4329 Arc::clone(&processor),
4330 server_signing_key.clone(),
4331 hyprstream_rpc::transport::rpc_session::DEFAULT_STREAM_LIMIT,
4332 );
4333 #[cfg(test)]
4334 crate::services::restart_diag::phase("iroh.bind.enter");
4335 let substrate =
4336 hyprstream_rpc::transport::iroh_substrate::IrohSubstrate::new(
4337 iroh_key.to_bytes(),
4338 hyprstream_rpc::transport::iroh_substrate::RefuseHandler::new(
4339 "inference streaming uses the existing StreamService",
4340 ),
4341 rpc_handler,
4342 )
4343 .await
4344 .map_err(|e| {
4345 hyprstream_rpc::error::RpcError::SpawnFailed(format!(
4346 "inference iroh bind: {e}"
4347 ))
4348 })?;
4349 #[cfg(test)]
4350 crate::services::restart_diag::phase("iroh.bind.done");
4351 let node_id = *substrate.endpoint_id().as_bytes();
4352 let direct_addrs = substrate.endpoint().bound_sockets().into_iter().collect();
4353 *network_reach.write() = Some(hyprstream_rpc::transport::TransportConfig::iroh(
4354 node_id,
4355 direct_addrs,
4356 None,
4357 ));
4358 #[cfg(test)]
4359 crate::services::restart_diag::phase("inproc.serve.enter");
4360 let result = serve_inference_bridged(
4361 &service_name,
4362 &transport,
4363 processor,
4364 server_signing_key,
4365 shutdown,
4366 on_ready,
4367 quic_config,
4368 quic_advertise_addr,
4369 producer_reach_config,
4370 moq_origin,
4371 network_ready,
4372 draining,
4373 )
4374 .await;
4375 if let Err(error) = substrate.shutdown().await {
4376 tracing::warn!(%error, "inference iroh shutdown failed");
4377 }
4378 result
4379 })
4380 }
4381}
4382
4383pub enum StreamChunkMessage {
4399 Chunk {
4400 text: String,
4401 },
4402 Complete {
4403 stats: crate::runtime::GenerationStats,
4404 },
4405 Error {
4406 error: String,
4407 },
4408}
4409
4410impl StreamChunkMessage {
4411 pub fn from_stream_payload(payload: crate::services::rpc_types::StreamPayload) -> Self {
4416 use crate::services::rpc_types::{InferenceStreamPayload, StreamPayloadExt};
4417
4418 match payload.to_inference() {
4419 Ok(InferenceStreamPayload::Token(text)) => StreamChunkMessage::Chunk { text },
4420 Ok(InferenceStreamPayload::Error(message)) => {
4421 StreamChunkMessage::Error { error: message }
4422 }
4423 Ok(InferenceStreamPayload::Complete(stats)) => {
4424 let quality_metrics = if stats.perplexity.is_some() || stats.avg_entropy.is_some() {
4425 Some(
4426 crate::runtime::generation_metrics::GenerationQualityMetrics {
4427 perplexity: stats.perplexity.unwrap_or(0.0),
4428 avg_entropy: stats.avg_entropy.unwrap_or(0.0),
4429 ..Default::default()
4430 },
4431 )
4432 } else {
4433 None
4434 };
4435
4436 let gen_stats = crate::runtime::GenerationStats {
4437 tokens_generated: stats.tokens_generated,
4438 generation_time_ms: stats.generation_time_ms,
4439 tokens_per_second: stats.tokens_per_second,
4440 finish_reason: Some(match stats.finish_reason.as_str() {
4441 "length" => crate::config::FinishReason::MaxTokens,
4442 "eos" => crate::config::FinishReason::EndOfSequence,
4443 "error" => crate::config::FinishReason::Error(String::new()),
4444 _ => crate::config::FinishReason::Stop,
4445 }),
4446 quality_metrics,
4447 prefill_tokens: stats.prefill_tokens,
4448 prefill_time_ms: stats.prefill_time_ms,
4449 prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
4450 inference_tokens: stats.inference_tokens,
4451 inference_time_ms: stats.inference_time_ms,
4452 inference_tokens_per_sec: stats.inference_tokens_per_sec,
4453 inference_tokens_per_sec_ema: stats.inference_tokens_per_sec_ema,
4454 speculative_accepted: 0,
4456 speculative_rejected: 0,
4457 };
4458 StreamChunkMessage::Complete { stats: gen_stats }
4459 }
4460 Err(e) => StreamChunkMessage::Error {
4461 error: format!("Failed to parse payload: {e}"),
4462 },
4463 }
4464 }
4465
4466 pub fn is_last(&self) -> bool {
4468 matches!(
4469 self,
4470 StreamChunkMessage::Complete { .. } | StreamChunkMessage::Error { .. }
4471 )
4472 }
4473}
4474
4475#[cfg(test)]
4479mod tenant_binding_tests {
4480 use super::*;
4481 use hyprstream_rpc::auth::mac::{
4482 Assurance, CompartmentSet, Level, SecurityContext, SecurityLabel, VerifiedKeyMaterial,
4483 };
4484 use hyprstream_rpc::envelope::Subject;
4485
4486 fn key(seed: u8) -> SigningKey {
4487 SigningKey::from_bytes(&[seed; 32])
4488 }
4489
4490 #[tokio::test]
4491 async fn pinned_config_retains_base_inputs_but_not_writable_state() -> Result<()> {
4492 let dir = tempfile::tempdir()?;
4493 let repo = git2db::Repository::init(dir.path())?;
4494 std::fs::write(dir.path().join("config.json"), b"reviewed")?;
4495 let mut index = repo.index()?;
4496 index.add_path(Path::new("config.json"))?;
4497 index.write()?;
4498 let tree = repo.find_tree(index.write_tree()?)?;
4499 let sig = git2::Signature::now("test", "test@example.invalid")?;
4500 let oid = repo.commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])?;
4501 drop(tree);
4502 drop(repo);
4503 let artifact = crate::storage::pinned_model::acquire_pinned_model(dir.path(), oid).await?;
4504 let signing_key = key(3);
4505 let config = InferenceServiceConfig::new(
4506 dir.path(),
4507 RuntimeConfig::default(),
4508 signing_key.verifying_key(),
4509 signing_key,
4510 hyprstream_rpc::transport::TransportConfig::inproc("pinned-test"),
4511 hyprstream_rpc::transport::TransportConfig::inproc("policy"),
4512 None,
4513 )
4514 .with_pinned_artifact(artifact);
4515 std::fs::write(dir.path().join("config.json"), b"replaced")?;
4516 assert_eq!(
4517 std::fs::read(config.model_path.join("config.json"))?,
4518 b"reviewed"
4519 );
4520 assert_eq!(config.state_path, dir.path());
4521 assert_eq!(
4522 config
4523 .pinned_artifact
4524 .as_ref()
4525 .ok_or_else(|| anyhow!("missing pin"))?
4526 .commit(),
4527 oid
4528 );
4529 Ok(())
4530 }
4531
4532 #[test]
4533 fn inference_config_shares_outer_moq_plane_by_identity() {
4534 let reach = Arc::new(parking_lot::RwLock::new(
4535 hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
4536 ));
4537 let origin = Arc::new(parking_lot::RwLock::new(None));
4538 let signing_key = key(4);
4539 let config = InferenceServiceConfig::new(
4540 "/model",
4541 RuntimeConfig::default(),
4542 signing_key.verifying_key(),
4543 signing_key.clone(),
4544 hyprstream_rpc::transport::TransportConfig::inproc("inference-stream-plane-test"),
4545 hyprstream_rpc::transport::TransportConfig::inproc("policy"),
4546 None,
4547 )
4548 .with_stream_plane(Arc::clone(&reach), Arc::clone(&origin));
4549
4550 assert!(Arc::ptr_eq(&config.producer_reach_config, &reach));
4551 assert!(Arc::ptr_eq(&config.moq_origin, &origin));
4552 assert!(!config.owns_stream_plane);
4553 assert!(!config.network_ready.load(Ordering::Acquire));
4554 }
4555
4556 #[test]
4557 fn legacy_in_process_config_preserves_model_based_readiness() {
4558 let signing_key = key(5);
4559 let config = InferenceServiceConfig::new(
4560 "/model",
4561 RuntimeConfig::default(),
4562 signing_key.verifying_key(),
4563 signing_key,
4564 hyprstream_rpc::transport::TransportConfig::inproc("inference-legacy-readiness-test"),
4565 hyprstream_rpc::transport::TransportConfig::inproc("policy"),
4566 None,
4567 );
4568
4569 assert!(config.network_ready.load(Ordering::Acquire));
4570 }
4571
4572 #[test]
4573 fn direct_request_must_match_instance_tenant() {
4574 let controller = key(7).verifying_key();
4575 let caller = key(8).verifying_key();
4576 let same = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4577 Subject::new("alice"),
4578 "tenant-a",
4579 caller,
4580 );
4581 assert_eq!(
4582 resolve_instance_domain("tenant-a", &controller, &same)
4583 .unwrap_or_else(|e| panic!("matching tenant must pass: {e}")),
4584 "tenant-a"
4585 );
4586
4587 let other = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4588 Subject::new("mallory"),
4589 "tenant-b",
4590 caller,
4591 );
4592 assert!(resolve_instance_domain("tenant-a", &controller, &other).is_err());
4593 }
4594
4595 #[test]
4596 fn only_pinned_controller_can_bridge_local_domain() {
4597 let controller = key(7).verifying_key();
4598 let local_controller = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4599 Subject::new("system"),
4600 "local",
4601 controller,
4602 );
4603 assert_eq!(
4604 resolve_instance_domain("tenant-a", &controller, &local_controller)
4605 .unwrap_or_else(|e| panic!("pinned controller must pass: {e}")),
4606 "tenant-a"
4607 );
4608
4609 let impostor = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4610 Subject::new("system"),
4611 "local",
4612 key(9).verifying_key(),
4613 );
4614 assert!(resolve_instance_domain("tenant-a", &controller, &impostor).is_err());
4615 }
4616
4617 #[test]
4618 fn mac_denies_missing_clearance() {
4619 let object =
4620 SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
4621 assert!(enforce_inference_security_context(None, &object).is_err());
4622 }
4623
4624 #[test]
4625 fn mac_denies_insufficient_clearance() {
4626 let subject = SecurityContext::new(
4627 Level::Public,
4628 CompartmentSet::EMPTY,
4629 VerifiedKeyMaterial::Classical,
4630 );
4631 let object =
4632 SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
4633 assert!(enforce_inference_security_context(Some(subject), &object).is_err());
4634 }
4635
4636 #[test]
4637 fn mac_allows_dominating_clearance() {
4638 let subject = SecurityContext::new(
4639 Level::Secret,
4640 CompartmentSet::EMPTY,
4641 VerifiedKeyMaterial::Classical,
4642 );
4643 let object =
4644 SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
4645 assert!(enforce_inference_security_context(Some(subject), &object).is_ok());
4646 }
4647}
4648
4649#[cfg(test)]
4650#[allow(clippy::unwrap_used, clippy::expect_used)]
4651mod quinn_drain_tests {
4652 use super::*;
4653 use anyhow::Context as _;
4654 use std::future::Future as _;
4655 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
4656 use std::process::Command;
4657 use std::time::{Duration, Instant};
4658
4659 struct JoinGuard<T> {
4660 task: Option<tokio::task::JoinHandle<T>>,
4661 }
4662
4663 impl<T> JoinGuard<T> {
4664 fn new(task: tokio::task::JoinHandle<T>) -> Self {
4665 Self { task: Some(task) }
4666 }
4667
4668 fn handle_mut(&mut self) -> &mut tokio::task::JoinHandle<T> {
4669 self.task.as_mut().expect("join guard task present")
4670 }
4671
4672 async fn join(mut self) -> std::result::Result<T, tokio::task::JoinError> {
4673 let result = self.handle_mut().await;
4674 self.task.take();
4675 result
4676 }
4677 }
4678
4679 impl<T> Drop for JoinGuard<T> {
4680 fn drop(&mut self) {
4681 if let Some(task) = self.task.take() {
4682 task.abort();
4683 }
4684 }
4685 }
4686
4687 struct ReleaseOnDrop(Arc<tokio::sync::Semaphore>);
4688
4689 impl Drop for ReleaseOnDrop {
4690 fn drop(&mut self) {
4691 self.0.add_permits(1);
4692 }
4693 }
4694
4695 struct PermitInferenceDispatch;
4696
4697 impl hyprstream_rpc::auth::mac::MacDispatchPep for PermitInferenceDispatch {
4698 fn check(
4699 &self,
4700 _ctx: &EnvelopeContext,
4701 service_domain: &str,
4702 _method: Option<&[u16]>,
4703 ) -> hyprstream_rpc::auth::mac::MacDecision {
4704 if service_domain == "inference" {
4705 hyprstream_rpc::auth::mac::MacDecision::Permit
4706 } else {
4707 hyprstream_rpc::auth::mac::MacDecision::Deny(
4708 hyprstream_rpc::auth::mac::MacDenyReason::UnlabeledObject,
4709 )
4710 }
4711 }
4712 }
4713
4714 struct DrainGateService {
4715 transport: hyprstream_rpc::transport::TransportConfig,
4716 signing_key: SigningKey,
4717 handler_entered: Arc<tokio::sync::Semaphore>,
4718 handler_completed: Arc<tokio::sync::Semaphore>,
4719 release: Arc<tokio::sync::Semaphore>,
4720 }
4721
4722 #[async_trait::async_trait(?Send)]
4723 impl hyprstream_rpc::service::RequestService for DrainGateService {
4724 async fn handle_request(
4725 &self,
4726 _ctx: &EnvelopeContext,
4727 _body: &hyprstream_rpc::service::DecodedRequestBody,
4728 ) -> Result<(Vec<u8>, Option<hyprstream_rpc::service::Continuation>)> {
4729 self.handler_entered.add_permits(1);
4730 self.release.acquire().await?.forget();
4731 self.handler_completed.add_permits(1);
4732 Ok((b"held-response".to_vec(), None))
4733 }
4734
4735 fn decode_request_body(
4736 &self,
4737 signed_body: &[u8],
4738 ) -> Result<hyprstream_rpc::service::DecodedRequestBody> {
4739 crate::services::generated::inference_client::decode_inference_request_body(signed_body)
4740 }
4741
4742 async fn verify_claims(&self, _ctx: &mut EnvelopeContext) -> Result<()> {
4743 Ok(())
4744 }
4745
4746 fn build_error_payload(&self, _request_id: u64, _error: &str) -> Vec<u8> {
4747 b"error-response".to_vec()
4748 }
4749
4750 fn name(&self) -> &str {
4751 "inference"
4752 }
4753
4754 fn transport(&self) -> &hyprstream_rpc::transport::TransportConfig {
4755 &self.transport
4756 }
4757
4758 fn signing_key(&self) -> SigningKey {
4759 self.signing_key.clone()
4760 }
4761 }
4762
4763 fn test_quic_config(
4764 port: u16,
4765 cert_der: Vec<u8>,
4766 key_der: Vec<u8>,
4767 ) -> hyprstream_rpc::service::QuicLoopConfig {
4768 hyprstream_rpc::service::QuicLoopConfig {
4769 announcement_cancellation: tokio_util::sync::CancellationToken::new(),
4770 cert_chain: vec![cert_der],
4771 key_der: zeroize::Zeroizing::new(key_der),
4772 bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port),
4773 server_name: "localhost".to_owned(),
4774 protected_resource_json: None,
4775 on_quic_bound: None,
4776 iroh_enabled: false,
4777 iroh_required: false,
4778 on_iroh_bound: None,
4779 moq_relay: None,
4780 moq_relay_server_identity: None,
4781 moq_admission: None,
4782 moq_ingress_authorizer: None,
4783 moq_admission_proof: None,
4784 }
4785 }
4786
4787 fn valid_request_body() -> Result<Vec<u8>> {
4788 let mut message = capnp::message::Builder::new_default();
4789 {
4790 let mut request = message
4791 .init_root::<hyprstream_rpc_std::inference_capnp::inference_request::Builder>(
4792 );
4793 request.set_id(7);
4794 request.set_is_ready(());
4795 }
4796 let mut bytes = Vec::new();
4797 capnp::serialize::write_message(&mut bytes, &message)?;
4798 Ok(bytes)
4799 }
4800
4801 #[test]
4804 fn inference_quic_owner_joins_drain_before_returning() -> Result<()> {
4805 let executable = std::env::current_exe()?;
4806 let mut child = Command::new(executable)
4807 .args([
4808 "--exact",
4809 "services::inference::quinn_drain_tests::inference_quic_owner_joins_drain_before_returning_child",
4810 "--nocapture",
4811 ])
4812 .env("HYPRSTREAM_INFERENCE_DRAIN_CHILD", "1")
4813 .spawn()?;
4814 let deadline = Instant::now() + Duration::from_secs(30);
4815 let status = loop {
4816 if let Some(status) = child.try_wait()? {
4817 break status;
4818 }
4819 if Instant::now() >= deadline {
4820 child.kill()?;
4821 let _ = child.wait();
4822 anyhow::bail!("isolated causal child exceeded 30 second watchdog");
4823 }
4824 std::thread::sleep(Duration::from_millis(20));
4825 };
4826 anyhow::ensure!(
4827 status.success(),
4828 "isolated causal child exited with {status}"
4829 );
4830 Ok(())
4831 }
4832
4833 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4834 async fn inference_quic_owner_joins_drain_before_returning_child() -> Result<()> {
4835 if std::env::var_os("HYPRSTREAM_INFERENCE_DRAIN_CHILD").is_none() {
4836 return Ok(());
4837 }
4838 hyprstream_rpc::transport::install_pq_crypto_provider()?;
4839 let cert_key = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])?;
4840 let cert_der = cert_key.cert.der().to_vec();
4841 let key_der = cert_key.key_pair.serialize_der();
4842 let probe = UdpSocket::bind("127.0.0.1:0")?;
4843 let port = probe.local_addr()?.port();
4844 drop(probe);
4845
4846 let handler_entered = Arc::new(tokio::sync::Semaphore::new(0));
4847 let handler_completed = Arc::new(tokio::sync::Semaphore::new(0));
4848 let release = Arc::new(tokio::sync::Semaphore::new(0));
4849 let _release_cleanup = ReleaseOnDrop(Arc::clone(&release));
4850 let service_key = SigningKey::from_bytes(&[0x31; 32]);
4851 let client_key = SigningKey::from_bytes(&[0x32; 32]);
4852 let mut verify_store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
4853 let client_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&client_key);
4854 let client_pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
4855 &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(&client_pq),
4856 )?;
4857 verify_store.bind(client_key.verifying_key().to_bytes(), &client_pq_vk);
4858 verify_store.bind(
4859 service_key.verifying_key().to_bytes(),
4860 &hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
4861 &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(
4862 &hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_key),
4863 ),
4864 )?,
4865 );
4866 hyprstream_rpc::envelope::install_verify_config(
4867 hyprstream_rpc::envelope::EnvelopeVerifyConfig {
4868 policy: hyprstream_rpc::crypto::CryptoPolicy::Hybrid,
4869 pq_store: Some(Arc::new(verify_store)),
4870 },
4871 )?;
4872 hyprstream_rpc::auth::mac::install_mac_dispatch_pep(Arc::new(PermitInferenceDispatch));
4873 let transport = hyprstream_rpc::transport::TransportConfig::inproc(format!(
4874 "inference-drain-test-{port}"
4875 ));
4876 let bridge = hyprstream_rpc::transport::iroh_rpc::LocalServiceBridge::spawn(
4877 DrainGateService {
4878 transport: transport.clone(),
4879 signing_key: service_key.clone(),
4880 handler_entered: Arc::clone(&handler_entered),
4881 handler_completed: Arc::clone(&handler_completed),
4882 release: Arc::clone(&release),
4883 },
4884 Arc::new(hyprstream_rpc::envelope::InMemoryNonceCache::new()),
4885 0,
4886 )?;
4887 let processor: Arc<dyn hyprstream_rpc::transport::rpc_session::IrohRequestProcessor> =
4888 Arc::new(bridge);
4889 let shutdown = Arc::new(tokio::sync::Notify::new());
4890 let network_ready = Arc::new(AtomicBool::new(false));
4891 let draining = Arc::new(AtomicBool::new(false));
4892 let producer_reach_config = Arc::new(parking_lot::RwLock::new(
4893 hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
4894 ));
4895 let moq_origin = Arc::new(parking_lot::RwLock::new(Some(
4896 hyprstream_rpc::moq_stream::MoqStreamOrigin::standalone().build(),
4897 )));
4898 let advertised_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), port);
4899 let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
4900 let owner_transport = transport.clone();
4901 let owner_cert_der = cert_der.clone();
4902 let owner_shutdown = Arc::clone(&shutdown);
4903 let owner_draining = Arc::clone(&draining);
4904 let owner_service_key = service_key.clone();
4905 let mut owner = JoinGuard::new(tokio::spawn(async move {
4906 serve_inference_bridged(
4907 "inference",
4908 &owner_transport,
4909 Arc::clone(&processor),
4910 owner_service_key,
4911 owner_shutdown,
4912 Some(ready_tx),
4913 Some(test_quic_config(port, owner_cert_der, key_der)),
4914 Some(advertised_addr),
4915 Arc::clone(&producer_reach_config),
4916 Arc::clone(&moq_origin),
4917 Arc::clone(&network_ready),
4918 owner_draining,
4919 )
4920 .await
4921 }));
4922
4923 tokio::time::timeout(Duration::from_secs(5), ready_rx)
4924 .await
4925 .context("inference owner readiness timeout")??;
4926 let session = tokio::time::timeout(
4927 Duration::from_secs(5),
4928 hyprstream_rpc::transport::quinn_transport::connect_pinned(
4929 SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
4930 &cert_der,
4931 ),
4932 )
4933 .await
4934 .context("QUIC connect timed out")??;
4935 let service_vk = service_key.verifying_key();
4936 let mut kem_store = hyprstream_rpc::crypto::hybrid_kem::KeyedKemTrustStore::new();
4937 kem_store.bind(
4938 service_vk.to_bytes(),
4939 hyprstream_rpc::node_identity::derive_mesh_kem_recipient(&service_key)?.public(),
4940 );
4941 let service_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&service_key);
4942 let service_pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
4943 &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(&service_pq),
4944 )?;
4945 let mut pq_store = hyprstream_rpc::envelope::KeyedPqTrustStore::new();
4946 pq_store.bind(service_vk.to_bytes(), &service_pq_vk);
4947 let client = hyprstream_rpc::rpc_client::RpcClientImpl::new(
4948 hyprstream_rpc::signer::LocalSigner::new(client_key),
4949 hyprstream_rpc::transport::quinn_transport::QuinnTransport::new(session),
4950 Some(service_vk),
4951 )
4952 .with_request_kem_store(Arc::new(kem_store))
4953 .with_response_pq_store(Arc::new(pq_store));
4954 let response = JoinGuard::new(tokio::spawn(async move {
4955 client
4956 .call_for_service("inference", valid_request_body()?)
4957 .await
4958 }));
4959 let entered_permit =
4960 match tokio::time::timeout(Duration::from_secs(5), handler_entered.acquire()).await {
4961 Ok(Ok(permit)) => permit,
4962 Ok(Err(error)) => return Err(error.into()),
4963 Err(_) => {
4964 let response_result =
4965 tokio::time::timeout(Duration::from_secs(5), response.join())
4966 .await
4967 .context("held request task cleanup timed out")?
4968 .context("held request task panicked")?;
4969 anyhow::bail!(
4970 "QUIC request did not enter service; response result: {response_result:?}"
4971 );
4972 }
4973 };
4974 entered_permit.forget();
4975
4976 shutdown.notify_waiters();
4977 tokio::time::timeout(Duration::from_secs(5), async {
4978 while !draining.load(Ordering::Acquire) {
4979 tokio::task::yield_now().await;
4980 }
4981 })
4982 .await
4983 .context("inference drain did not start")?;
4984 let owner_done = tokio::time::timeout(Duration::from_millis(100), owner.handle_mut()).await;
4985 assert!(
4986 owner_done.is_err(),
4987 "inference owner returned before QUIC drain"
4988 );
4989
4990 release.add_permits(1);
4991 let response_result = tokio::time::timeout(Duration::from_secs(5), response.join())
4992 .await
4993 .context("held QUIC response did not complete")??;
4994 assert_eq!(response_result?, b"held-response");
4995 tokio::time::timeout(Duration::from_secs(5), handler_completed.acquire())
4996 .await
4997 .context("handler completion marker timed out")??
4998 .forget();
4999 let owner_result = tokio::time::timeout(Duration::from_secs(5), owner.join())
5000 .await
5001 .context("inference owner did not join QUIC drain")??;
5002 owner_result?;
5003 Ok(())
5004 }
5005
5006 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5007 async fn unexpected_quic_exit_notifies_unpolled_rep_and_joins_drain() -> Result<()> {
5008 let shutdown = Arc::new(tokio::sync::Notify::new());
5009 let drain_shutdown = Arc::clone(&shutdown);
5010 let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>();
5011 let (drain_started_tx, drain_started_rx) = tokio::sync::oneshot::channel();
5012 let (drain_release_tx, drain_release_rx) = tokio::sync::oneshot::channel();
5013 let drain_finished = Arc::new(AtomicBool::new(false));
5014 let drain_finished_task = Arc::clone(&drain_finished);
5015 let drain_task = tokio::spawn(async move {
5016 let notified = drain_shutdown.notified();
5017 tokio::pin!(notified);
5018 notified.as_mut().enable();
5019 let _ = armed_tx.send(());
5020 notified.await;
5021 let _ = drain_started_tx.send(());
5022 let _ = drain_release_rx.await;
5023 drain_finished_task.store(true, Ordering::Release);
5024 });
5025 tokio::time::timeout(Duration::from_secs(1), armed_rx)
5026 .await
5027 .context("test drain waiter did not arm")??;
5028
5029 let rep_shutdown_observed = Arc::new(AtomicBool::new(false));
5030 let rep_shutdown_observed_task = Arc::clone(&rep_shutdown_observed);
5031 let rep_shutdown = Arc::clone(&shutdown);
5032 let rep = async move {
5033 rep_shutdown.notified().await;
5034 rep_shutdown_observed_task.store(true, Ordering::Release);
5035 Ok::<(), anyhow::Error>(())
5036 };
5037 let quic = async { Err::<(), _>(anyhow::anyhow!("accept-loop sentinel")) };
5038 let lifecycle = tokio::spawn(run_quinn_lifecycle(
5039 rep,
5040 quic,
5041 shutdown,
5042 Arc::new(AtomicBool::new(true)),
5043 Arc::new(AtomicBool::new(false)),
5044 QuinnDrainOwner::new(drain_task),
5045 ));
5046 tokio::time::timeout(Duration::from_secs(1), drain_started_rx)
5047 .await
5048 .context("test drain did not start")??;
5049 tokio::time::timeout(Duration::from_millis(200), async {
5050 while !rep_shutdown_observed.load(Ordering::Acquire) {
5051 tokio::task::yield_now().await;
5052 }
5053 })
5054 .await
5055 .context("REP shutdown was not polled while QUIC drain was blocked")?;
5056 drain_release_tx
5057 .send(())
5058 .expect("drain waiter remains live");
5059 let result = tokio::time::timeout(Duration::from_secs(1), lifecycle)
5060 .await
5061 .context("unexpected QUIC lifecycle cleanup timed out")??;
5062 let error = result.expect_err("unexpected QUIC exit must remain the primary error");
5063 assert!(error.to_string().contains("accept-loop sentinel"));
5064 assert!(rep_shutdown_observed.load(Ordering::Acquire));
5065 assert!(drain_finished.load(Ordering::Acquire));
5066 Ok(())
5067 }
5068
5069 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5070 async fn draining_quic_exit_renotifies_rep_before_join() -> Result<()> {
5071 let shutdown = Arc::new(tokio::sync::Notify::new());
5072 let drain_shutdown = Arc::clone(&shutdown);
5073 let (drain_armed_tx, drain_armed_rx) = tokio::sync::oneshot::channel();
5074 let (drain_started_tx, drain_started_rx) = tokio::sync::oneshot::channel();
5075 let (drain_release_tx, drain_release_rx) = tokio::sync::oneshot::channel();
5076 let drain_task = tokio::spawn(async move {
5077 let notified = drain_shutdown.notified();
5078 tokio::pin!(notified);
5079 notified.as_mut().enable();
5080 let _ = drain_armed_tx.send(());
5081 notified.await;
5082 let _ = drain_started_tx.send(());
5083 let _ = drain_release_rx.await;
5084 });
5085
5086 let rep_shutdown = Arc::clone(&shutdown);
5087 let rep_observed = Arc::new(AtomicBool::new(false));
5088 let rep_observed_task = Arc::clone(&rep_observed);
5089 let rep = async move {
5090 let notified = rep_shutdown.notified();
5093 tokio::pin!(notified);
5094 notified.as_mut().enable();
5095 tokio::task::yield_now().await;
5096 notified.await;
5097 rep_observed_task.store(true, Ordering::Release);
5098 Ok::<(), anyhow::Error>(())
5099 };
5100 tokio::time::timeout(Duration::from_secs(1), drain_armed_rx)
5101 .await
5102 .context("drain waiter did not arm")??;
5103 let quic = async { Err::<(), _>(anyhow::anyhow!("draining accept-loop sentinel")) };
5104 let lifecycle = tokio::spawn(run_quinn_lifecycle(
5105 rep,
5106 quic,
5107 Arc::clone(&shutdown),
5108 Arc::new(AtomicBool::new(true)),
5109 Arc::new(AtomicBool::new(true)),
5110 QuinnDrainOwner::new(drain_task),
5111 ));
5112
5113 tokio::time::timeout(Duration::from_secs(1), drain_started_rx)
5114 .await
5115 .context("drain did not start")??;
5116 tokio::time::timeout(Duration::from_secs(1), async {
5117 while !rep_observed.load(Ordering::Acquire) {
5118 tokio::task::yield_now().await;
5119 }
5120 })
5121 .await
5122 .context("draining QUIC exit did not re-notify REP")?;
5123 drain_release_tx
5124 .send(())
5125 .expect("drain waiter remains live");
5126 let result = tokio::time::timeout(Duration::from_secs(1), lifecycle)
5127 .await
5128 .context("draining lifecycle cleanup timed out")??;
5129 result?;
5130 Ok(())
5131 }
5132
5133 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5134 async fn cancelling_drain_join_aborts_owned_waiter() -> Result<()> {
5135 struct DropMarker(Arc<AtomicBool>);
5136
5137 impl Drop for DropMarker {
5138 fn drop(&mut self) {
5139 self.0.store(true, Ordering::Release);
5140 }
5141 }
5142
5143 let dropped = Arc::new(AtomicBool::new(false));
5144 let marker = Arc::clone(&dropped);
5145 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
5146 let task = tokio::spawn(async move {
5147 let _marker = DropMarker(marker);
5148 let _ = started_tx.send(());
5149 std::future::pending::<()>().await;
5150 });
5151 let owner = QuinnDrainOwner::new(task);
5152 tokio::time::timeout(Duration::from_secs(1), started_rx)
5153 .await
5154 .context("drain child did not start")??;
5155 let mut join = Box::pin(owner.join());
5156 let mut join_polled = false;
5157 let join_was_pending = std::future::poll_fn(|cx| {
5158 join_polled = true;
5159 match join.as_mut().poll(cx) {
5160 std::task::Poll::Pending => std::task::Poll::Ready(true),
5161 std::task::Poll::Ready(()) => std::task::Poll::Ready(false),
5162 }
5163 })
5164 .await;
5165 assert!(join_polled, "owner join was not polled");
5166 assert!(join_was_pending, "owner join unexpectedly completed");
5167 drop(join);
5168 tokio::time::timeout(Duration::from_secs(1), async {
5169 while !dropped.load(Ordering::Acquire) {
5170 tokio::task::yield_now().await;
5171 }
5172 })
5173 .await
5174 .context("cancellation did not abort the owned drain waiter")?;
5175 Ok(())
5176 }
5177
5178 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5179 async fn cancelling_rep_join_aborts_owned_task() -> Result<()> {
5180 struct DropMarker(Arc<AtomicBool>);
5181
5182 impl Drop for DropMarker {
5183 fn drop(&mut self) {
5184 self.0.store(true, Ordering::Release);
5185 }
5186 }
5187
5188 let dropped = Arc::new(AtomicBool::new(false));
5189 let marker = Arc::clone(&dropped);
5190 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
5191 let task = tokio::spawn(async move {
5192 let _marker = DropMarker(marker);
5193 let _ = started_tx.send(());
5194 std::future::pending::<()>().await;
5195 Ok::<(), anyhow::Error>(())
5196 });
5197 let owner = RepTaskOwner::new(task);
5198 tokio::time::timeout(Duration::from_secs(1), started_rx)
5199 .await
5200 .context("REP child did not start")??;
5201 let mut join = Box::pin(owner.join());
5202 let pending = std::future::poll_fn(|cx| match join.as_mut().poll(cx) {
5203 std::task::Poll::Pending => std::task::Poll::Ready(true),
5204 std::task::Poll::Ready(_) => std::task::Poll::Ready(false),
5205 })
5206 .await;
5207 assert!(pending, "REP owner join unexpectedly completed");
5208 drop(join);
5209 tokio::time::timeout(Duration::from_secs(1), async {
5210 while !dropped.load(Ordering::Acquire) {
5211 tokio::task::yield_now().await;
5212 }
5213 })
5214 .await
5215 .context("cancellation did not abort owned REP task")?;
5216 Ok(())
5217 }
5218
5219 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5220 async fn rep_startup_ack_failure_preserves_task_error() -> Result<()> {
5221 let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>();
5222 let task =
5223 tokio::spawn(async { Err::<(), anyhow::Error>(anyhow::anyhow!("REP bind sentinel")) });
5224 let owner = RepTaskOwner::new(task);
5225 drop(armed_tx);
5226 assert!(armed_rx.await.is_err(), "startup arm channel must close");
5227 let error = owner
5228 .join()
5229 .await
5230 .expect_err("REP startup failure must remain visible");
5231 assert!(error.to_string().contains("REP bind sentinel"));
5232 Ok(())
5233 }
5234}
5235
5236#[cfg(test)]
5237#[allow(clippy::unwrap_used, clippy::expect_used)]
5238mod tests {
5239 use super::*;
5253 use crate::runtime::GenerationStats;
5254 use std::future::Future;
5255 use std::pin::Pin;
5256 use std::task::{Context, Poll};
5257
5258 #[test]
5259 fn standalone_ready_follows_quic_drain_arm() {
5260 let source = include_str!("inference.rs");
5261 let drain_arm = source
5262 .find("let _ = drain_armed_rx.await;")
5263 .expect("standalone inference must await the drain arm barrier");
5264 let outer = &source[drain_arm..];
5265 let ready = outer
5266 .find("hyprstream_rpc::notify::ready()")
5267 .expect("outer inference must emit systemd readiness");
5268 let oneshot = outer
5269 .find("if let Some(on_ready) = on_ready")
5270 .expect("outer readiness oneshot must remain explicit");
5271 assert!(ready < oneshot, "systemd READY must follow drain arming");
5272 assert!(
5273 source.contains("serve_bridged_with_shutdown_armed_silent"),
5274 "nested REP helper must suppress its own readiness"
5275 );
5276 }
5277
5278 #[tokio::test]
5279 async fn lifecycle_shutdown_observer_arms_before_owner_readiness() {
5280 let source = include_str!("inference.rs");
5281 assert!(
5282 source.contains("lifecycle_armed_rx.await"),
5283 "inference startup must wait for the lifecycle shutdown observer to arm"
5284 );
5285
5286 let shutdown = Arc::new(tokio::sync::Notify::new());
5287 let network_ready = Arc::new(AtomicBool::new(true));
5288 let draining = Arc::new(AtomicBool::new(false));
5289 let (armed_tx, armed_rx) = tokio::sync::oneshot::channel();
5290 let observer = tokio::spawn(observe_inference_shutdown(
5291 Arc::clone(&shutdown),
5292 Arc::clone(&network_ready),
5293 Arc::clone(&draining),
5294 armed_tx,
5295 ));
5296
5297 tokio::time::timeout(std::time::Duration::from_secs(5), armed_rx)
5298 .await
5299 .expect("lifecycle observer did not arm")
5300 .expect("lifecycle observer dropped before arming");
5301 shutdown.notify_waiters();
5302 tokio::time::timeout(std::time::Duration::from_secs(5), observer)
5303 .await
5304 .expect("lifecycle observer did not finish")
5305 .expect("lifecycle observer panicked");
5306 assert!(draining.load(Ordering::Acquire));
5307 assert!(!network_ready.load(Ordering::Acquire));
5308 }
5309
5310 fn stats(prefill: usize, generated: usize) -> GenerationStats {
5312 GenerationStats {
5313 tokens_generated: generated,
5314 generation_time_ms: 0,
5315 tokens_per_second: 0.0,
5316 finish_reason: None,
5317 quality_metrics: None,
5318 prefill_tokens: prefill,
5319 prefill_time_ms: 0,
5320 prefill_tokens_per_sec: 0.0,
5321 inference_tokens: generated,
5322 inference_time_ms: 0,
5323 inference_tokens_per_sec: 0.0,
5324 inference_tokens_per_sec_ema: 0.0,
5325 speculative_accepted: 0,
5326 speculative_rejected: 0,
5327 }
5328 }
5329
5330 struct FakeStream {
5333 chunks: Vec<Result<String, String>>,
5336 pos: usize,
5337 prefill: usize,
5338 }
5339
5340 impl FakeStream {
5341 fn new(chunks: Vec<Result<String, String>>, prefill: usize) -> Self {
5342 FakeStream {
5343 chunks,
5344 pos: 0,
5345 prefill,
5346 }
5347 }
5348 fn generated(&self) -> usize {
5349 self.chunks[..self.pos].iter().filter(|c| c.is_ok()).count()
5350 }
5351 }
5352
5353 impl futures::Stream for FakeStream {
5354 type Item = anyhow::Result<String>;
5355 fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
5356 let this = self.get_mut();
5357 if this.pos < this.chunks.len() {
5358 let c = this.chunks[this.pos].clone();
5359 this.pos += 1;
5360 Poll::Ready(Some(c.map_err(anyhow::Error::msg)))
5361 } else {
5362 Poll::Ready(None)
5363 }
5364 }
5365 }
5366
5367 impl GenOutput for FakeStream {
5368 fn ema_rate(&self) -> f32 {
5369 1.0
5370 }
5371 fn completion_stats(&self) -> GenerationStats {
5372 stats(self.prefill, self.generated())
5373 }
5374 }
5375
5376 #[derive(Default)]
5378 struct RecordingSink {
5379 published: Vec<String>,
5380 }
5381
5382 impl GenSink for RecordingSink {
5383 fn publish_chunk(&mut self, data: &str, _rate: f32) -> impl Future<Output = Result<()>> {
5384 self.published.push(data.to_owned());
5385 std::future::ready(Ok(()))
5386 }
5387 }
5388
5389 struct CancelAfterSink {
5392 published: Vec<String>,
5393 cancel: tokio_util::sync::CancellationToken,
5394 cancel_after: usize,
5395 }
5396
5397 impl CancelAfterSink {
5398 fn new(cancel: tokio_util::sync::CancellationToken, cancel_after: usize) -> Self {
5399 CancelAfterSink {
5400 published: vec![],
5401 cancel,
5402 cancel_after,
5403 }
5404 }
5405 }
5406
5407 impl GenSink for CancelAfterSink {
5408 fn publish_chunk(&mut self, data: &str, _rate: f32) -> impl Future<Output = Result<()>> {
5409 self.published.push(data.to_owned());
5410 if self.published.len() >= self.cancel_after {
5411 self.cancel.cancel();
5412 }
5413 std::future::ready(Ok(()))
5414 }
5415 }
5416
5417 struct FailOnSink {
5419 published: Vec<String>,
5420 fail_on: usize,
5421 }
5422
5423 impl FailOnSink {
5424 fn new(fail_on: usize) -> Self {
5425 FailOnSink {
5426 published: vec![],
5427 fail_on,
5428 }
5429 }
5430 }
5431
5432 impl GenSink for FailOnSink {
5433 fn publish_chunk(&mut self, data: &str, _rate: f32) -> impl Future<Output = Result<()>> {
5434 let will_fail = self.published.len() + 1 == self.fail_on;
5435 self.published.push(data.to_owned());
5436 std::future::ready(if will_fail {
5437 Err(anyhow!("publish failed"))
5438 } else {
5439 Ok(())
5440 })
5441 }
5442 }
5443
5444 #[tokio::test]
5447 async fn cancel_after_one_token_captures_partial_stats() {
5448 let cancel = tokio_util::sync::CancellationToken::new();
5449 let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5450 let mut sink = CancelAfterSink::new(cancel.clone(), 1);
5451 let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5452 assert!(matches!(outcome, GenLoopOutcome::Cancelled), "{outcome:?}");
5453 assert_eq!(sink.published.len(), 1);
5455 let s = stream.completion_stats();
5457 assert_eq!(s.prefill_tokens, 5);
5458 assert_eq!(s.tokens_generated, 1);
5459 }
5460
5461 #[tokio::test]
5462 async fn publish_failure_after_partial_captures_stats() {
5463 let cancel = tokio_util::sync::CancellationToken::new();
5464 let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5465 let mut sink = FailOnSink::new(2);
5466 let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5467 assert!(
5468 matches!(outcome, GenLoopOutcome::PublishFailed(_)),
5469 "{outcome:?}"
5470 );
5471 let s = stream.completion_stats();
5473 assert_eq!(s.prefill_tokens, 5);
5474 assert_eq!(s.tokens_generated, 2);
5475 }
5476
5477 #[tokio::test]
5478 async fn stream_error_after_partial_captures_stats() {
5479 let cancel = tokio_util::sync::CancellationToken::new();
5480 let mut stream = FakeStream::new(vec![Ok("a".into()), Err("decode boom".to_owned())], 5);
5481 let mut sink = RecordingSink::default();
5482 let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5483 assert!(
5484 matches!(outcome, GenLoopOutcome::StreamError(_)),
5485 "{outcome:?}"
5486 );
5487 let s = stream.completion_stats();
5489 assert_eq!(s.prefill_tokens, 5);
5490 assert_eq!(s.tokens_generated, 1);
5491 }
5492
5493 #[tokio::test]
5494 async fn normal_completion_captures_full_stats() {
5495 let cancel = tokio_util::sync::CancellationToken::new();
5496 let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into())], 5);
5497 let mut sink = RecordingSink::default();
5498 let outcome = drive_generation_loop(&mut stream, &cancel, &mut sink).await;
5499 assert!(matches!(outcome, GenLoopOutcome::Exhausted), "{outcome:?}");
5500 let s = stream.completion_stats();
5501 assert_eq!(s.prefill_tokens, 5);
5502 assert_eq!(s.tokens_generated, 2);
5503 }
5504
5505 #[test]
5508 fn missing_identity_denies_only_when_emitter_attached() {
5509 assert!(ledger_requires_identity(true, None));
5511 assert!(!ledger_requires_identity(true, Some("did:web:alice")));
5513 assert!(!ledger_requires_identity(false, None));
5515 assert!(!ledger_requires_identity(false, Some("did:web:alice")));
5516 }
5517
5518 #[test]
5519 fn deny_frame_is_content_free() {
5520 assert!(LEDGER_DENY_NO_IDENTITY.contains("billing identity"));
5522 assert!(!LEDGER_DENY_NO_IDENTITY.contains("did:"));
5523 }
5524
5525 #[cfg(feature = "ledger")]
5527 mod spend {
5528 use super::*;
5529 use crate::services::ledger::inference_spend::tests::{available, fixture};
5530
5531 async fn drive_and_post(
5535 stream: &mut FakeStream,
5536 cancel: &tokio_util::sync::CancellationToken,
5537 sink: &mut impl GenSink,
5538 emitter: &InferenceSpendEmitter,
5539 stream_id: &str,
5540 ) -> Option<SpendResult> {
5541 drive_generation_loop(stream, cancel, sink).await;
5542 post_completion_spend(
5543 emitter,
5544 Some(&stream.completion_stats()),
5545 Some("did:web:alice"),
5546 stream_id,
5547 )
5548 .await
5549 }
5550
5551 #[tokio::test]
5552 async fn cancelled_after_one_token_posts_spend() {
5553 let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5554 let cancel = tokio_util::sync::CancellationToken::new();
5555 let mut stream =
5556 FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5557 let mut sink = CancelAfterSink::new(cancel.clone(), 1);
5558 let res =
5559 drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-cancel").await;
5560 assert!(
5561 matches!(res, Some(SpendResult::Posted { amount: 6, .. })),
5562 "{res:?}"
5563 );
5564 assert_eq!(available(&handle, "did:web:alice").await, 994);
5566 }
5567
5568 #[tokio::test]
5569 async fn publish_failure_after_partial_posts_spend() {
5570 let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5571 let cancel = tokio_util::sync::CancellationToken::new();
5572 let mut stream =
5573 FakeStream::new(vec![Ok("a".into()), Ok("b".into()), Ok("c".into())], 5);
5574 let mut sink = FailOnSink::new(2);
5575 let res =
5576 drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-pubfail").await;
5577 assert!(
5578 matches!(res, Some(SpendResult::Posted { amount: 7, .. })),
5579 "{res:?}"
5580 );
5581 assert_eq!(available(&handle, "did:web:alice").await, 993);
5583 }
5584
5585 #[tokio::test]
5586 async fn stream_error_after_partial_posts_spend() {
5587 let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5588 let cancel = tokio_util::sync::CancellationToken::new();
5589 let mut stream = FakeStream::new(vec![Ok("a".into()), Err("boom".to_owned())], 5);
5590 let mut sink = RecordingSink::default();
5591 let res = drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-err").await;
5592 assert!(
5593 matches!(res, Some(SpendResult::Posted { amount: 6, .. })),
5594 "{res:?}"
5595 );
5596 assert_eq!(available(&handle, "did:web:alice").await, 994);
5598 }
5599
5600 #[tokio::test]
5601 async fn normal_completion_posts_spend() {
5602 let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5603 let cancel = tokio_util::sync::CancellationToken::new();
5604 let mut stream = FakeStream::new(vec![Ok("a".into()), Ok("b".into())], 5);
5605 let mut sink = RecordingSink::default();
5606 let res = drive_and_post(&mut stream, &cancel, &mut sink, &emitter, "stream-ok").await;
5607 assert!(
5608 matches!(res, Some(SpendResult::Posted { amount: 7, .. })),
5609 "{res:?}"
5610 );
5611 assert_eq!(available(&handle, "did:web:alice").await, 993);
5612 }
5613
5614 #[tokio::test]
5615 async fn missing_identity_posts_no_spend() {
5616 let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5619 let s = stats(5, 1);
5620 let res = post_completion_spend(&emitter, Some(&s), None, "stream-anon").await;
5621 assert!(res.is_none(), "{res:?}");
5622 assert_eq!(available(&handle, "did:web:alice").await, 1000);
5623 }
5624
5625 #[tokio::test]
5626 async fn no_stats_means_nothing_to_account() {
5627 let (emitter, handle) = fixture(&[("did:web:alice", 1000)]).await;
5629 let res =
5630 post_completion_spend(&emitter, None, Some("did:web:alice"), "stream-nostart")
5631 .await;
5632 assert!(res.is_none(), "{res:?}");
5633 assert_eq!(available(&handle, "did:web:alice").await, 1000);
5634 }
5635 }
5636}
5637
5638#[cfg(test)]
5652#[allow(clippy::unwrap_used, clippy::expect_used)]
5653mod single_service_boundary_tests {
5654 use super::*;
5655
5656 use crate::runtime::inference_profile::InferenceInstanceId;
5657 use hyprstream_rpc::auth::internal_work::{
5658 encode_internal_work, InternalWorkClaims, INTERNAL_WORK_ISSUER,
5659 };
5660 use hyprstream_rpc::envelope::{RequestEnvelope, SignedEnvelope};
5661
5662 const CONTROLLER_KEY: [u8; 32] = [0xC7; 32];
5663 const STRANGER_KEY: [u8; 32] = [0xC9; 32];
5664 const TENANT: &str = "tenant-boundary";
5665 const MODEL: &str = "boundary-model:main";
5666
5667 struct DeadPolicyRpc;
5670
5671 #[async_trait::async_trait]
5672 impl hyprstream_rpc::RpcClient for DeadPolicyRpc {
5673 async fn call(&self, _payload: Vec<u8>) -> Result<Vec<u8>> {
5674 anyhow::bail!("policy must not be dialed on the internal work leg")
5675 }
5676 async fn call_for_service(
5677 &self,
5678 _service_domain: &str,
5679 _payload: Vec<u8>,
5680 ) -> Result<Vec<u8>> {
5681 anyhow::bail!("policy must not be dialed on the internal work leg")
5682 }
5683 async fn call_for_service_with_method(
5684 &self,
5685 _service_domain: &str,
5686 _method_discriminator: u16,
5687 _payload: Vec<u8>,
5688 ) -> Result<Vec<u8>> {
5689 anyhow::bail!("policy must not be dialed on the internal work leg")
5690 }
5691 async fn call_with_options(
5692 &self,
5693 _payload: Vec<u8>,
5694 _options: hyprstream_rpc::CallOptions,
5695 ) -> Result<Vec<u8>> {
5696 anyhow::bail!("policy must not be dialed on the internal work leg")
5697 }
5698 async fn call_with_options_for_service(
5699 &self,
5700 _service_domain: &str,
5701 _payload: Vec<u8>,
5702 _options: hyprstream_rpc::CallOptions,
5703 ) -> Result<Vec<u8>> {
5704 anyhow::bail!("policy must not be dialed on the internal work leg")
5705 }
5706 async fn call_streaming(
5707 &self,
5708 _payload: Vec<u8>,
5709 _ephemeral_pubkey: [u8; 32],
5710 ) -> Result<Vec<u8>> {
5711 anyhow::bail!("policy must not be dialed on the internal work leg")
5712 }
5713 async fn call_streaming_for_service(
5714 &self,
5715 _service_domain: &str,
5716 _payload: Vec<u8>,
5717 _ephemeral_pubkey: [u8; 32],
5718 ) -> Result<Vec<u8>> {
5719 anyhow::bail!("policy must not be dialed on the internal work leg")
5720 }
5721 async fn call_streaming_for_service_with_method(
5722 &self,
5723 _service_domain: &str,
5724 _method_discriminator: u16,
5725 _payload: Vec<u8>,
5726 _ephemeral_pubkey: [u8; 32],
5727 ) -> Result<Vec<u8>> {
5728 anyhow::bail!("policy must not be dialed on the internal work leg")
5729 }
5730 async fn open_stream(
5731 &self,
5732 _payload: Vec<u8>,
5733 ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
5734 anyhow::bail!("policy must not be dialed on the internal work leg")
5735 }
5736 async fn open_stream_from_info(
5737 &self,
5738 _stream_info: hyprstream_rpc::stream_info::StreamInfo,
5739 _client_secret: [u8; 32],
5740 _client_pubkey: [u8; 32],
5741 ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
5742 anyhow::bail!("policy must not be dialed on the internal work leg")
5743 }
5744 fn next_id(&self) -> u64 {
5745 1
5746 }
5747 }
5748
5749 const TEST_INCARNATION: &str = "test-incarnation-0000";
5753
5754 fn boundary_adapter(
5755 controller: &SigningKey,
5756 instance_service_name: &str,
5757 tenant: &str,
5758 model: &str,
5759 with_model_binding: bool,
5760 ) -> InferenceZmqAdapter {
5761 boundary_adapter_with_pool(
5762 controller,
5763 instance_service_name,
5764 tenant,
5765 model,
5766 with_model_binding,
5767 None,
5768 TEST_INCARNATION,
5769 )
5770 }
5771
5772 fn boundary_adapter_with_pool(
5773 controller: &SigningKey,
5774 instance_service_name: &str,
5775 tenant: &str,
5776 model: &str,
5777 with_model_binding: bool,
5778 delta_pool: Option<Arc<DeltaPool>>,
5779 incarnation: &str,
5780 ) -> InferenceZmqAdapter {
5781 let service = InferenceService {
5782 inner: Arc::new(InferenceServiceInner {
5783 engine: parking_lot::RwLock::new(
5784 create_engine(&RuntimeConfig::default())
5785 .expect("engine construction (no weights)"),
5786 ),
5787 model_path: PathBuf::from("/nonexistent"),
5788 state_path: PathBuf::from("/nonexistent"),
5789 _pinned_artifact: None,
5790 session_id: parking_lot::RwLock::new(None),
5791 runtime_handle: Handle::current(),
5792 stream_channel: None,
5793 server_pubkey: controller.verifying_key(),
5794 signing_key: controller.clone(),
5795 nonce_cache: Arc::new(InMemoryNonceCache::new()),
5796 policy_client: PolicyClient::new(Arc::new(DeadPolicyRpc)),
5797 ttt_trainer: None,
5798 tokenizer: None,
5799 delta_pool,
5800 base_delta: Mutex::new(None),
5801 fs: None,
5802 transport: hyprstream_rpc::transport::TransportConfig::inproc("boundary-test"),
5803 lora_generation: Arc::new(AtomicU64::new(0)),
5804 tenant_domain: tenant.to_owned(),
5805 controller_pubkey: controller.verifying_key(),
5806 network_ready: Arc::new(AtomicBool::new(true)),
5807 draining: Arc::new(AtomicBool::new(false)),
5808 shutdown: Arc::new(tokio::sync::Notify::new()),
5809 #[cfg(feature = "ledger")]
5810 ledger: parking_lot::RwLock::new(None),
5811 object_label: inference_object_label(),
5812 }),
5813 };
5814 InferenceZmqAdapter {
5815 service,
5816 transport: hyprstream_rpc::transport::TransportConfig::inproc("boundary-test"),
5817 signing_key: controller.clone(),
5818 expected_audience: None,
5819 jwt_key_source: None,
5820 instance_model_ref: with_model_binding.then(|| model.to_owned()),
5821 instance_audience: internal_work_audience(instance_service_name, incarnation),
5822 }
5823 }
5824
5825 fn fixture_work_claims(
5826 controller: &SigningKey,
5827 instance_service_name: &str,
5828 now: i64,
5829 ) -> InternalWorkClaims {
5830 fixture_work_claims_for_subject(controller, instance_service_name, now, "alice")
5831 }
5832
5833 fn fixture_work_claims_for_subject(
5834 controller: &SigningKey,
5835 instance_service_name: &str,
5836 now: i64,
5837 subject_string: &str,
5838 ) -> InternalWorkClaims {
5839 InternalWorkClaims {
5840 iss: INTERNAL_WORK_ISSUER.to_owned(),
5841 sub: subject_string.to_owned(),
5842 aud: internal_work_audience(instance_service_name, TEST_INCARNATION),
5843 tenant: TENANT.to_owned(),
5844 model: MODEL.to_owned(),
5845 resource: "inference:GenerateStream".to_owned(),
5846 operation: "infer".to_owned(),
5847 iat: now,
5848 exp: now + 60,
5849 jti: format!(
5850 "jti-{}-{}",
5851 hex::encode(&controller.verifying_key().as_bytes()[..4]),
5852 std::time::SystemTime::now()
5853 .duration_since(std::time::UNIX_EPOCH)
5854 .expect("clock")
5855 .as_nanos()
5856 ),
5857 cnf: InternalWorkClaims::controller_cnf(&controller.verifying_key()),
5858 owner_did: Some("did:key:z6Mkboundary".to_owned()),
5859 caller: hyprstream_rpc::auth::Claims::new(subject_string.to_owned(), now, now + 3600)
5860 .with_tenant(TENANT.to_owned())
5861 .with_clearance(inference_object_label()),
5864 }
5865 }
5866
5867 fn ctx_with_work(token: String, controller: &SigningKey) -> EnvelopeContext {
5871 let envelope = SignedEnvelope::new_signed(
5872 RequestEnvelope::anonymous(Vec::new()).with_jwt_token(token),
5873 controller,
5874 );
5875 EnvelopeContext::from_verified_as_system(&envelope)
5876 }
5877
5878 #[tokio::test]
5879 async fn internal_work_admits_through_real_verify_claims_and_authorize() {
5880 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
5881 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
5882 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5883 let now = chrono::Utc::now().timestamp();
5884 let token = encode_internal_work(
5885 &fixture_work_claims(&controller, &instance.service_name(), now),
5886 &controller,
5887 );
5888 let mut ctx = ctx_with_work(token, &controller);
5889
5890 adapter
5892 .verify_claims(&mut ctx)
5893 .await
5894 .expect("controller work order must pass the real verify_claims routing");
5895 let work = ctx.internal_work().expect("work order installed");
5896 assert_eq!(work.resource, "inference:GenerateStream");
5897 assert_eq!(work.operation, "infer");
5898 assert_eq!(ctx.verified_tenant(), Some(TENANT));
5899 assert_eq!(ctx.claims().map(|c| c.sub.as_str()), Some("alice"));
5900
5901 InferenceHandler::authorize(&adapter.service, &ctx, "inference:GenerateStream", "infer")
5904 .await
5905 .expect("internal work authorizes exactly its bound operation");
5906 assert_eq!(
5910 ctx.internal_work().expect("work order").subject.name(),
5911 Some("alice")
5912 );
5913 }
5914
5915 #[tokio::test]
5916 async fn internal_work_rejects_wrong_instance_tenant_model_and_standalone() {
5917 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
5918 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
5919 let now = chrono::Utc::now().timestamp();
5920
5921 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5923 let mut ctx = ctx_with_work(
5924 encode_internal_work(
5925 &fixture_work_claims(&controller, "inference-other-instance", now),
5926 &controller,
5927 ),
5928 &controller,
5929 );
5930 let err = adapter
5931 .verify_claims(&mut ctx)
5932 .await
5933 .expect_err("a work order for another instance must deny");
5934 assert!(err.to_string().contains("audience"), "{err:#}");
5935
5936 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5939 let mut claims = fixture_work_claims(&controller, &instance.service_name(), now);
5940 claims.tenant = "tenant-other".to_owned();
5941 let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
5942 let err = adapter
5943 .verify_claims(&mut ctx)
5944 .await
5945 .expect_err("a work order for another tenant must deny");
5946 assert!(err.to_string().contains("tenant"), "{err:#}");
5947
5948 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5951 let mut claims = fixture_work_claims(&controller, &instance.service_name(), now);
5952 claims.model = "model-other:main".to_owned();
5953 let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
5954 let err = adapter
5955 .verify_claims(&mut ctx)
5956 .await
5957 .expect_err("a work order for another model must deny");
5958 assert!(err.to_string().contains("model"), "{err:#}");
5959
5960 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, false);
5962 let claims = fixture_work_claims(&controller, &instance.service_name(), now);
5963 let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
5964 let err = adapter
5965 .verify_claims(&mut ctx)
5966 .await
5967 .expect_err("a standalone instance accepts no internal work");
5968 assert!(err.to_string().contains("standalone"), "{err:#}");
5969 }
5970
5971 #[tokio::test]
5972 async fn internal_work_rejects_foreign_controller_expired_and_replayed() {
5973 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
5974 let stranger = SigningKey::from_bytes(&STRANGER_KEY);
5975 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
5976 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
5977 let now = chrono::Utc::now().timestamp();
5978
5979 let mut ctx = ctx_with_work(
5982 encode_internal_work(
5983 &fixture_work_claims(&stranger, &instance.service_name(), now),
5984 &stranger,
5985 ),
5986 &stranger,
5987 );
5988 let err = adapter
5989 .verify_claims(&mut ctx)
5990 .await
5991 .expect_err("a foreign controller cannot mint for a pinned instance");
5992 assert!(err.to_string().contains("pinned controller"), "{err:#}");
5993
5994 let mut expired = fixture_work_claims(&controller, &instance.service_name(), now);
5996 expired.exp = now - 1;
5997 let mut ctx = ctx_with_work(encode_internal_work(&expired, &controller), &controller);
5998 let err = adapter
5999 .verify_claims(&mut ctx)
6000 .await
6001 .expect_err("expired work orders deny");
6002 assert!(err.to_string().contains("expired"), "{err:#}");
6003
6004 let mut long = fixture_work_claims(&controller, &instance.service_name(), now);
6006 long.exp = now + 60 * 60;
6007 let mut ctx = ctx_with_work(encode_internal_work(&long, &controller), &controller);
6008 let err = adapter
6009 .verify_claims(&mut ctx)
6010 .await
6011 .expect_err("oversized lifetimes deny");
6012 assert!(
6013 err.to_string().contains("maximum internal lifetime"),
6014 "{err:#}"
6015 );
6016
6017 let claims = fixture_work_claims(&controller, &instance.service_name(), now);
6019 let token = encode_internal_work(&claims, &controller);
6020 let mut first = ctx_with_work(token.clone(), &controller);
6021 adapter
6022 .verify_claims(&mut first)
6023 .await
6024 .expect("first admission succeeds");
6025 let mut second = ctx_with_work(token, &controller);
6026 let err = adapter
6027 .verify_claims(&mut second)
6028 .await
6029 .expect_err("the same work order cannot be admitted twice");
6030 assert!(err.to_string().contains("replay"), "{err:#}");
6031 }
6032
6033 #[tokio::test]
6034 async fn internal_work_authorize_denies_unbound_operations() {
6035 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6036 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6037 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6038 let now = chrono::Utc::now().timestamp();
6039 let token = encode_internal_work(
6040 &fixture_work_claims(&controller, &instance.service_name(), now),
6041 &controller,
6042 );
6043 let mut ctx = ctx_with_work(token, &controller);
6044 adapter.verify_claims(&mut ctx).await.expect("admitted");
6045
6046 let err =
6049 InferenceHandler::authorize(&adapter.service, &ctx, "inference:tttZero", "manage")
6050 .await
6051 .expect_err("work order does not authorize other operations");
6052 assert!(err.to_string().contains("does not authorize"), "{err:#}");
6053
6054 let err = InferenceHandler::authorize(&adapter.service, &ctx, "inference:embed", "infer")
6056 .await
6057 .expect_err("work order does not authorize other resources");
6058 assert!(err.to_string().contains("does not authorize"), "{err:#}");
6059 }
6060
6061 #[tokio::test]
6062 async fn direct_caller_path_still_requires_policy_on_dead_transport() {
6063 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6067 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6068 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6069 let mut label_claims = hyprstream_rpc::auth::Claims::new(
6072 "alice".to_owned(),
6073 chrono::Utc::now().timestamp(),
6074 chrono::Utc::now().timestamp() + 3600,
6075 )
6076 .with_tenant(TENANT.to_owned());
6077 label_claims.clearance = Some(
6078 crate::mac::genesis::SitePolicy::conservative()
6079 .label_for("/srv/inference")
6080 .into(),
6081 );
6082 let ctx = EnvelopeContext::for_test_authenticated_subject_with_claims(
6083 Subject::new("alice"),
6084 TENANT,
6085 controller.verifying_key(),
6086 label_claims,
6087 );
6088 let err = InferenceHandler::authorize(
6089 &adapter.service,
6090 &ctx,
6091 "inference:generateStream",
6092 "infer",
6093 )
6094 .await
6095 .expect_err("direct callers still require the Policy decision");
6096 assert!(
6097 err.to_string().contains("policy must not be dialed")
6098 || err.to_string().contains("Unauthorized")
6099 || err.to_string().contains("denying"),
6100 "expected the dead-policy denial, got: {err:#}"
6101 );
6102 }
6103
6104 use crate::services::generated::inference_client::dispatch_inference;
6117
6118 fn encoded_request(
6120 build: impl FnOnce(&mut hyprstream_rpc_std::inference_capnp::inference_request::Builder<'_>),
6121 ) -> Result<Vec<u8>> {
6122 let mut message = capnp::message::Builder::new_default();
6123 {
6124 let mut request = message
6125 .init_root::<hyprstream_rpc_std::inference_capnp::inference_request::Builder>(
6126 );
6127 request.set_id(42);
6128 build(&mut request);
6129 }
6130 let mut bytes = Vec::new();
6131 capnp::serialize::write_message(&mut bytes, &message)?;
6132 Ok(bytes)
6133 }
6134
6135 async fn admitted_ctx(
6138 adapter: &InferenceZmqAdapter,
6139 controller: &SigningKey,
6140 instance_service_name: &str,
6141 resource: String,
6142 operation: &str,
6143 ) -> Result<EnvelopeContext> {
6144 let now = chrono::Utc::now().timestamp();
6145 let mut claims = fixture_work_claims(controller, instance_service_name, now);
6146 claims.resource = resource;
6147 claims.operation = operation.to_owned();
6148 let mut ctx = ctx_with_work(encode_internal_work(&claims, controller), controller);
6149 adapter.verify_claims(&mut ctx).await?;
6150 Ok(ctx)
6151 }
6152
6153 #[tokio::test]
6154 async fn real_dispatch_void_variant_has_lora_succeeds_through_full_chain() {
6155 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6156 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6157 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6158 let resource = crate::services::model::ModelService::inference_dispatch_resource("hasLora");
6161 assert_eq!(resource, "inference:HasLora");
6162 let ctx = admitted_ctx(
6163 &adapter,
6164 &controller,
6165 &instance.service_name(),
6166 resource,
6167 "query",
6168 )
6169 .await
6170 .expect("work order must pass the real verify_claims routing");
6171
6172 let payload = encoded_request(|request| {
6173 request.set_has_lora(());
6174 })
6175 .expect("encode hasLora");
6176 let decoded =
6177 crate::services::generated::inference_client::decode_inference_request_body(&payload)
6178 .expect("decode hasLora");
6179 let (response, continuation) =
6181 match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6182 Ok(pair) => pair,
6183 Err(error) => panic!(
6184 "real dispatch of a correctly-scoped work order must reach the handler: {error:#}"
6185 ),
6186 };
6187 assert!(continuation.is_none());
6188 let parsed =
6189 hyprstream_rpc_std::inference_client::InferenceClient::parse_response(&response)
6190 .expect("parse hasLora response");
6191 match parsed {
6192 InferenceResponseVariant::HasLoraResult(has) => {
6193 assert!(!has, "unloaded engine reports no adapter");
6194 }
6195 other => panic!("expected HasLoraResult, got {other:?}"),
6196 }
6197 }
6198
6199 #[tokio::test]
6200 async fn real_dispatch_text_variant_load_lora_binds_exact_payload_coordinate() {
6201 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6202 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6203 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6204 let path = "probe/adapters/k3";
6208 let ctx = admitted_ctx(
6209 &adapter,
6210 &controller,
6211 &instance.service_name(),
6212 format!("inference:{path}"),
6213 "write",
6214 )
6215 .await
6216 .expect("payload-bound work order must pass the real verify_claims routing");
6217
6218 let payload = encoded_request(|request| {
6219 request.set_load_lora(path);
6220 })
6221 .expect("encode loadLora");
6222 let decoded =
6223 crate::services::generated::inference_client::decode_inference_request_body(&payload)
6224 .expect("decode loadLora");
6225 let (response, _continuation) =
6226 match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6227 Ok(pair) => pair,
6228 Err(error) => panic!(
6229 "real dispatch must pass the PEP for the payload-bound coordinate: {error:#}"
6230 ),
6231 };
6232 let parsed =
6235 hyprstream_rpc_std::inference_client::InferenceClient::parse_response(&response)
6236 .expect("parse loadLora response");
6237 match parsed {
6238 InferenceResponseVariant::Error(info) => {
6239 assert!(
6240 !info.message.contains("does not authorize")
6241 && !info.message.contains("UNAUTHORIZED"),
6242 "handler error must be post-PEP, got: {}",
6243 info.message
6244 );
6245 assert!(
6246 info.message.contains("FsOps not available"),
6247 "expected the fixture's path-containment handler error, got: {}",
6248 info.message
6249 );
6250 }
6251 other => panic!("expected the handler Error variant, got {other:?}"),
6252 }
6253 }
6254
6255 #[tokio::test]
6256 async fn real_dispatch_struct_variant_generate_stream_reaches_handler_after_pep() {
6257 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6258 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6259 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6260 let ctx = admitted_ctx(
6261 &adapter,
6262 &controller,
6263 &instance.service_name(),
6264 crate::services::model::ModelService::inference_dispatch_resource("generateStream"),
6265 "infer",
6266 )
6267 .await
6268 .expect("work order must pass the real verify_claims routing");
6269
6270 let payload = encoded_request(|request| {
6271 request.reborrow().init_generate_stream();
6272 })
6273 .expect("encode generateStream");
6274 let decoded =
6275 crate::services::generated::inference_client::decode_inference_request_body(&payload)
6276 .expect("decode generateStream");
6277 let error = match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6282 Ok(_) => panic!("struct variant must reach the handler after the PEP"),
6283 Err(error) => error,
6284 };
6285 assert!(
6286 error.to_string().contains("ephemeral pubkey"),
6287 "expected the post-PEP handler error, got: {error:#}"
6288 );
6289 assert!(
6290 !error.to_string().contains("does not authorize"),
6291 "the PEP must not deny a correctly-scoped work order"
6292 );
6293 }
6294
6295 #[tokio::test]
6296 async fn real_dispatch_denies_when_mint_grammar_and_dispatch_disagree() {
6297 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6300 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6301 let adapter = boundary_adapter(&controller, &instance.service_name(), TENANT, MODEL, true);
6302 let ctx = admitted_ctx(
6303 &adapter,
6304 &controller,
6305 &instance.service_name(),
6306 "inference:hasLora".to_owned(),
6307 "query",
6308 )
6309 .await
6310 .expect("admission is coordinate-agnostic; the PEP compares");
6311 let payload = encoded_request(|request| {
6312 request.set_has_lora(());
6313 })
6314 .expect("encode hasLora");
6315 let decoded =
6316 crate::services::generated::inference_client::decode_inference_request_body(&payload)
6317 .expect("decode hasLora");
6318 let error = match dispatch_inference(&adapter.service, &ctx, &decoded).await {
6319 Ok(_) => panic!("the stale camelCase grammar must keep denying"),
6320 Err(error) => error,
6321 };
6322 assert!(
6323 error.to_string().contains("does not authorize"),
6324 "expected the exact-match PEP denial, got: {error:#}"
6325 );
6326 }
6327
6328 #[tokio::test]
6337 async fn incarnation_a_token_denied_at_worker_incarnation_b() {
6338 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6339 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6340 let adapter_a = boundary_adapter_with_pool(
6341 &controller,
6342 &instance.service_name(),
6343 TENANT,
6344 MODEL,
6345 true,
6346 None,
6347 "incarnation-aaaa",
6348 );
6349 let adapter_b = boundary_adapter_with_pool(
6350 &controller,
6351 &instance.service_name(),
6352 TENANT,
6353 MODEL,
6354 true,
6355 None,
6356 "incarnation-bbbb",
6357 );
6358 let now = chrono::Utc::now().timestamp();
6359 let mut claims =
6360 fixture_work_claims_for_subject(&controller, &instance.service_name(), now, "alice");
6361 claims.aud = internal_work_audience(&instance.service_name(), "incarnation-aaaa");
6363 let mut ctx = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
6364 adapter_a
6365 .verify_claims(&mut ctx)
6366 .await
6367 .expect("A token admits at A");
6368 let mut ctx_b = ctx_with_work(encode_internal_work(&claims, &controller), &controller);
6371 let error = adapter_b
6372 .verify_claims(&mut ctx_b)
6373 .await
6374 .expect_err("A token must fail at worker incarnation B");
6375 assert!(error.to_string().contains("audience"), "{error:#}");
6376 assert!(ctx_b.internal_work().is_none());
6377
6378 let mut b_claims =
6380 fixture_work_claims_for_subject(&controller, &instance.service_name(), now, "alice");
6381 b_claims.aud = internal_work_audience(&instance.service_name(), "incarnation-bbbb");
6382 b_claims.jti = format!("{}-b", b_claims.jti);
6383 let mut ctx_b2 = ctx_with_work(encode_internal_work(&b_claims, &controller), &controller);
6384 adapter_b
6385 .verify_claims(&mut ctx_b2)
6386 .await
6387 .expect("B token admits at B");
6388 }
6389
6390 #[tokio::test]
6391 async fn duplicate_incarnation_receipt_denies_through_jti_gate() {
6392 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6393 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6394 let adapter = boundary_adapter_with_pool(
6395 &controller,
6396 &instance.service_name(),
6397 TENANT,
6398 MODEL,
6399 true,
6400 None,
6401 "incarnation-cccc",
6402 );
6403 let now = chrono::Utc::now().timestamp();
6404 let mut claims =
6405 fixture_work_claims_for_subject(&controller, &instance.service_name(), now, "alice");
6406 claims.aud = internal_work_audience(&instance.service_name(), "incarnation-cccc");
6407 let token = encode_internal_work(&claims, &controller);
6408 let mut first = ctx_with_work(token.clone(), &controller);
6411 adapter
6412 .verify_claims(&mut first)
6413 .await
6414 .expect("first submission admitted");
6415 let mut second = ctx_with_work(token, &controller);
6416 let error = adapter
6417 .verify_claims(&mut second)
6418 .await
6419 .expect_err("second submission of one receipt must deny");
6420 assert!(error.to_string().contains("replay"), "{error:#}");
6421 }
6422
6423 #[tokio::test]
6433 async fn sol_s1_distinct_callers_keep_subject_keyed_ttt_state_separate() {
6434 use hyprstream_rpc::envelope::Subject;
6435
6436 let controller = SigningKey::from_bytes(&CONTROLLER_KEY);
6437 let instance = InferenceInstanceId::new(TENANT, MODEL, 0).expect("fixture instance");
6438 let device = create_engine(&RuntimeConfig::default())
6441 .expect("engine construction (no weights)")
6442 .device();
6443 let snapshots_dir = std::env::temp_dir().join(format!(
6444 "sol-s1-delta-pool-{}",
6445 std::time::SystemTime::now()
6446 .duration_since(std::time::UNIX_EPOCH)
6447 .expect("clock")
6448 .as_nanos()
6449 ));
6450 let module_dims = std::collections::HashMap::from([
6453 ("q_proj".to_owned(), (16usize, 16usize)),
6454 ("v_proj".to_owned(), (16usize, 16usize)),
6455 ]);
6456 let pool = Arc::new(DeltaPool::new(
6457 crate::training::TenantDeltaConfig::default(),
6458 module_dims,
6459 device,
6460 None,
6461 snapshots_dir,
6462 None,
6463 2,
6464 ));
6465 let adapter = boundary_adapter_with_pool(
6466 &controller,
6467 &instance.service_name(),
6468 TENANT,
6469 MODEL,
6470 true,
6471 Some(Arc::clone(&pool)),
6472 "test-incarnation-0000",
6473 );
6474
6475 let caller_a = "user:alice";
6476 let caller_b = "alice";
6477
6478 let subject_a = Subject::new(caller_a);
6481 let delta_a = pool.get_or_create(&subject_a).expect("seed caller A delta");
6482 delta_a.lock().accumulated_steps = 7;
6483
6484 async fn admit(
6487 adapter: &InferenceZmqAdapter,
6488 controller: &SigningKey,
6489 instance_service_name: &str,
6490 subject_string: &str,
6491 ) -> EnvelopeContext {
6492 let now = chrono::Utc::now().timestamp();
6493 let mut claims = fixture_work_claims_for_subject(
6494 controller,
6495 instance_service_name,
6496 now,
6497 subject_string,
6498 );
6499 claims.resource = "inference:GetDeltaStatus".to_owned();
6500 claims.operation = "query".to_owned();
6501 let mut ctx = ctx_with_work(encode_internal_work(&claims, controller), controller);
6502 adapter
6503 .verify_claims(&mut ctx)
6504 .await
6505 .unwrap_or_else(|error| panic!("{subject_string} must be admitted: {error:#}"));
6506 ctx
6507 }
6508 let ctx_a = admit(&adapter, &controller, &instance.service_name(), caller_a).await;
6509 let ctx_b = admit(&adapter, &controller, &instance.service_name(), caller_b).await;
6510 assert_eq!(
6511 ctx_a.internal_work().expect("A work order").subject,
6512 Subject::new(caller_a),
6513 "caller A identity must install verbatim"
6514 );
6515 assert_eq!(
6516 ctx_b.internal_work().expect("B work order").subject,
6517 Subject::new(caller_b),
6518 "caller B identity must install verbatim"
6519 );
6520
6521 let handler_ctx = |subject_string: &str| {
6528 let now = chrono::Utc::now().timestamp();
6529 let caller_claims =
6530 hyprstream_rpc::auth::Claims::new(subject_string.to_owned(), now, now + 3600)
6531 .with_tenant(TENANT.to_owned());
6532 EnvelopeContext::for_test_authenticated_subject_with_claims(
6533 Subject::new(subject_string),
6534 TENANT,
6535 controller.verifying_key(),
6536 caller_claims,
6537 )
6538 };
6539 let handler_ctx_a = handler_ctx(caller_a);
6540 let handler_ctx_b = handler_ctx(caller_b);
6541
6542 let status_b =
6545 InferenceHandler::handle_get_delta_status(&adapter.service, &handler_ctx_b, 1)
6546 .await
6547 .unwrap_or_else(|error| panic!("B delta status failed: {error:#}"));
6548 match status_b {
6549 InferenceResponseVariant::GetDeltaStatusResult(status) => {
6550 assert!(
6551 !status.exists,
6552 "caller B must not see caller A's subject-keyed delta"
6553 );
6554 }
6555 other => panic!("expected GetDeltaStatusResult, got {other:?}"),
6556 }
6557
6558 InferenceHandler::handle_ttt_zero(&adapter.service, &handler_ctx_b, 2)
6560 .await
6561 .unwrap_or_else(|error| panic!("B tttZero failed: {error:#}"));
6562 let delta_a_after = pool.get(&subject_a).expect("A delta survives B's reset");
6563 assert_eq!(
6564 delta_a_after.lock().accumulated_steps,
6565 7,
6566 "A's accumulated TTT steps must be untouched by caller B's operations"
6567 );
6568
6569 let status_a =
6571 InferenceHandler::handle_get_delta_status(&adapter.service, &handler_ctx_a, 3)
6572 .await
6573 .unwrap_or_else(|error| panic!("A delta status failed: {error:#}"));
6574 match status_a {
6575 InferenceResponseVariant::GetDeltaStatusResult(status) => {
6576 assert!(status.exists, "A observes its own delta");
6577 assert_eq!(status.accumulated_steps, 7);
6578 }
6579 other => panic!("expected GetDeltaStatusResult, got {other:?}"),
6580 }
6581 }
6582}