1use async_trait::async_trait;
31use crate::runtime::inference_profile::{
33 InferenceCompute, InferenceDeploymentProfile, InferenceInstanceId, InferenceIsolationProfile,
34};
35use crate::runtime::RuntimeConfig;
36use crate::services::WorktreeClientExt;
37use crate::services::{EnvelopeContext, PolicyClient};
38use crate::storage::ModelRef;
39use anyhow::{anyhow, Context, Result};
40use hyprstream_rpc::events::EventPublisher;
41use hyprstream_rpc::latch::{Terminal, TerminalStore};
42use hyprstream_rpc::prelude::*;
43use hyprstream_rpc::stream_info::TransportConfig as WireTransportConfig;
44use hyprstream_rpc::transport::{EndpointType, TransportConfig};
45use hyprstream_rpc_std::inference_client::InferenceClient;
46use hyprstream_rpc_std::model_client::KVQuantType;
47use hyprstream_rpc_std::policy_client::PolicyCheck;
48use hyprstream_rpc_std::registry_client::RegistryClient;
49use hyprstream_rpc_std::registry_client::{CommitWithAuthorRequest, StageFilesRequest};
50use lru::LruCache;
51use std::collections::{HashMap, HashSet};
52use std::num::NonZeroUsize;
53use std::sync::atomic::{AtomicU64, Ordering};
54use std::sync::Arc;
55use std::time::Instant;
56use tokio::sync::{Mutex, RwLock};
57use tracing::{debug, info, warn};
58#[cfg(test)]
59use zeroize::Zeroizing;
60
61pub const MODEL_ENDPOINT: &str = "inproc://hyprstream/model";
63
64#[derive(Clone, Debug, PartialEq)]
78pub enum ModelLoadTerminal {
79 Loaded { endpoint: String },
81 LoadFailed { error: String },
83}
84
85#[derive(Clone, Debug, PartialEq, Eq, Hash)]
93pub struct ModelLoadKey {
94 pub tenant: String,
96 pub model_ref: String,
98 pub epoch: u64,
100}
101
102pub struct LoadedModel {
108 pub instance: InferenceInstanceId,
110 pub model_ref: String,
112 pub pinned_commit: Option<git2::Oid>,
114 pub transport: TransportConfig,
118 pub network_transport: TransportConfig,
121 pub service_handle: hyprstream_service::SpawnedService,
123 pub client: InferenceClient,
126 pub incarnation: String,
131 pub work_audience: String,
135 pub load_generation: u64,
139 pub router: crate::services::router::CellRouter,
146 pub load_state: Vec<crate::services::router::InferenceServerInfo>,
150 pub loaded_at: Instant,
152 pub last_used: Instant,
154 pub ttt_config: Option<crate::training::ttt::TTTConfig>,
156 pub generation_defaults: crate::config::SamplingParams,
158}
159
160pub struct ModelServiceConfig {
162 pub max_models: usize,
164 pub max_context: Option<u32>,
166 pub kv_quant: KVQuantType,
168 pub inference_deployment: InferenceDeploymentProfile,
170 pub staging_model_pin: Option<StagingModelPin>,
172}
173
174#[derive(Clone, Debug, PartialEq, Eq)]
175pub struct StagingModelPin {
176 pub model_ref: String,
177 pub commit: git2::Oid,
178 projection_policy: git2db::pinned_disk::ProjectionPolicy,
179 #[cfg(test)]
180 projection_parent: Option<std::path::PathBuf>,
181}
182
183const REVIEWED_STAGING_MODEL_REF: &str = "qwen2.5-0.5b-instruct:main";
184const REVIEWED_STAGING_MODEL_OID: &str = "18c562db6830c2ef6636b8eaf42fb479272052f7";
185
186impl StagingModelPin {
187 pub(crate) fn reviewed() -> Result<Self> {
188 Self::new(
189 REVIEWED_STAGING_MODEL_REF,
190 git2::Oid::from_str(REVIEWED_STAGING_MODEL_OID)?,
191 )
192 }
193
194 #[cfg(feature = "postgres")]
195 pub(crate) fn reviewed_repo_name() -> &'static str {
196 "qwen2.5-0.5b-instruct"
197 }
198
199 pub fn new(model_ref: impl Into<String>, commit: git2::Oid) -> Result<Self> {
200 let model_ref = model_ref.into();
201 let parsed = ModelRef::parse(&model_ref)
202 .with_context(|| format!("invalid bounded staging modelRef {model_ref:?}"))?;
203 anyhow::ensure!(
204 matches!(parsed.git_ref, crate::storage::GitRef::Branch(_))
205 && parsed.to_string() == model_ref,
206 "bounded staging modelRef must be a canonical local branch ref"
207 );
208 Ok(Self {
209 model_ref,
210 commit,
211 projection_policy: Default::default(),
212 #[cfg(test)]
213 projection_parent: None,
214 })
215 }
216
217 pub fn reviewed_clef_flash(
221 local_lfs: std::collections::BTreeMap<String, std::path::PathBuf>,
222 ) -> Result<Self> {
223 let mut pin = Self::new(
224 "clef-flash-fp8:main",
225 git2::Oid::from_str("7b43c2d799e7b5e9fef96b6137840088d75af0d6")?,
226 )?;
227 pin.projection_policy = git2db::pinned_disk::ProjectionPolicy::from_trusted_contract(
229 12_174_008_714,
230 local_lfs,
231 )?;
232 Ok(pin)
233 }
234
235 fn capture_policy(&self) -> Result<git2db::pinned_disk::ProjectionPolicy> {
236 if self.projection_policy != Default::default() {
237 anyhow::ensure!(
238 self.model_ref == "clef-flash-fp8:main"
239 && self.commit
240 == git2::Oid::from_str("7b43c2d799e7b5e9fef96b6137840088d75af0d6")?,
241 "Clef projection policy pin identity changed"
242 );
243 }
244 Ok(self.projection_policy.clone())
245 }
246
247 pub fn from_env() -> Result<Option<Self>> {
250 const PROFILE: &str = "HYPRSTREAM_MODEL_ADMISSION_PROFILE";
251 const REF: &str = "HYPRSTREAM_STAGING_MODEL_REF";
252 const OID: &str = "HYPRSTREAM_STAGING_MODEL_OID";
253 let profile = match std::env::var(PROFILE) {
254 Ok(value) => value,
255 Err(std::env::VarError::NotPresent) => "default".to_owned(),
256 Err(error) => return Err(error.into()),
257 };
258 let read_optional = |name| match std::env::var(name) {
259 Ok(value) => Ok(Some(value)),
260 Err(std::env::VarError::NotPresent) => Ok(None),
261 Err(error) => Err(anyhow!("read {name}: {error}")),
262 };
263 let model_ref = read_optional(REF)?;
264 let oid = read_optional(OID)?;
265 Self::from_settings(&profile, model_ref.as_deref(), oid.as_deref())
266 }
267
268 fn from_settings(
269 profile: &str,
270 model_ref: Option<&str>,
271 oid: Option<&str>,
272 ) -> Result<Option<Self>> {
273 const PROFILE: &str = "HYPRSTREAM_MODEL_ADMISSION_PROFILE";
274 const REF: &str = "HYPRSTREAM_STAGING_MODEL_REF";
275 const OID: &str = "HYPRSTREAM_STAGING_MODEL_OID";
276 if profile == "staging" {
277 match (model_ref, oid) {
278 (None, None) => return Self::reviewed().map(Some),
279 (Some(model_ref), Some(oid)) => anyhow::ensure!(
280 model_ref == REVIEWED_STAGING_MODEL_REF && oid == REVIEWED_STAGING_MODEL_OID,
281 "staging override must match the complete reviewed model pin"
282 ),
283 _ => anyhow::bail!(
284 "{REF} and {OID} must either both be absent or both match the reviewed pin"
285 ),
286 }
287 return Self::reviewed().map(Some);
288 }
289 anyhow::ensure!(
290 profile == "default",
291 "{PROFILE} must be 'default' or 'staging'"
292 );
293 match (model_ref, oid) {
294 (None, None) => Ok(None),
295 (Some(model_ref), Some(oid)) => {
296 anyhow::ensure!(
297 oid.len() == 40 && oid.bytes().all(|byte| byte.is_ascii_hexdigit()),
298 "{OID} must be a full 40-character hexadecimal Git OID"
299 );
300 Self::new(model_ref, git2::Oid::from_str(oid)?).map(Some)
301 }
302 _ => anyhow::bail!("{REF} and {OID} must be configured together"),
303 }
304 }
305
306 fn admits(&self, model_ref: &str) -> bool {
307 self.model_ref == model_ref
308 }
309}
310
311impl Default for ModelServiceConfig {
312 fn default() -> Self {
313 Self {
314 max_models: 5,
315 max_context: None,
316 kv_quant: KVQuantType::None,
317 inference_deployment: InferenceDeploymentProfile::default(),
318 staging_model_pin: None,
319 }
320 }
321}
322
323pub struct ModelServiceInner {
325 loaded_models: RwLock<LruCache<InferenceInstanceId, LoadedModel>>,
328 pending_loads: parking_lot::Mutex<HashMap<InferenceInstanceId, u64>>,
330 staging_pinned_artifacts: crate::storage::pinned_model::StagingPinnedArtifactCache,
333 unloading_models: Mutex<HashSet<InferenceInstanceId>>,
337 load_unload_gate: Mutex<()>,
341 lifecycle_event_publish_gate: Mutex<()>,
347 config: ModelServiceConfig,
349 signing_key: SigningKey,
351 policy_client: PolicyClient,
353 #[cfg(feature = "postgres")]
355 federate_dispatch: Option<Arc<crate::services::oauth::federate_proof::DispatchAdapter>>,
356 event_publisher: EventPublisher,
359 registry: RegistryClient,
361 transport: TransportConfig,
363 policy_transport: TransportConfig,
364 expected_audience: Option<String>,
366 delegated_oai_audience: Option<String>,
368 delegated_mcp_audience: Option<String>,
369 jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
371 discovery_client: Option<Arc<hyprstream_rpc_std::discovery_client::DiscoveryClient>>,
375 fs_trees: dashmap::DashMap<String, Arc<crate::services::fs::SyntheticTree>>,
377 load_terminals: TerminalStore<ModelLoadKey, ModelLoadTerminal>,
384 load_epoch: AtomicU64,
388 incarnation_generation: AtomicU64,
393 in_process_tenant: std::sync::OnceLock<String>,
396 producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
401 moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
403 #[cfg(test)]
408 test_work_order_capture: Mutex<Option<Zeroizing<String>>>,
409 #[cfg(test)]
412 before_unloaded_event: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
413}
414
415pub struct ModelService {
425 inner: Arc<ModelServiceInner>,
426}
427
428struct SpawnPublicationGuard(Option<hyprstream_service::SpawnedService>);
429
430impl SpawnPublicationGuard {
431 fn new(handle: hyprstream_service::SpawnedService) -> Self {
432 Self(Some(handle))
433 }
434 fn publish(mut self) -> Result<hyprstream_service::SpawnedService> {
435 self.0
436 .take()
437 .ok_or_else(|| anyhow!("spawn publication guard was already disarmed"))
438 }
439}
440
441impl Drop for SpawnPublicationGuard {
442 fn drop(&mut self) {
443 if let Some(mut handle) = self.0.take() {
444 match tokio::runtime::Handle::try_current() {
445 Ok(runtime) => {
446 runtime.spawn(async move {
447 if let Err(error) = handle.stop().await {
448 tracing::error!(%error, "failed to reclaim unpublished inference worker");
449 }
450 });
451 }
452 Err(error) => {
453 tracing::error!(%error, "cannot reclaim unpublished inference worker outside Tokio runtime");
454 }
455 }
456 }
457 }
458}
459
460enum InterceptedLoadAdmission {
466 Loaded { reach: Vec<WireTransportConfig> },
467 Pending,
468 Accepted(InterceptedLoadReservation),
469}
470
471struct InterceptedLoadReservation {
476 inner: Arc<ModelServiceInner>,
477 instance: InferenceInstanceId,
478 attempt: u64,
479 active: bool,
480}
481
482impl InterceptedLoadReservation {
483 fn new(inner: Arc<ModelServiceInner>, instance: InferenceInstanceId, attempt: u64) -> Self {
484 Self {
485 inner,
486 instance,
487 attempt,
488 active: true,
489 }
490 }
491
492 fn release(&mut self) {
493 if self.active {
494 let mut pending = self.inner.pending_loads.lock();
495 if pending.get(&self.instance) == Some(&self.attempt) {
496 pending.remove(&self.instance);
497 }
498 self.active = false;
499 }
500 }
501}
502
503impl Drop for InterceptedLoadReservation {
504 fn drop(&mut self) {
505 self.release();
506 }
507}
508
509fn admit_in_process_tenant(
510 admitted_tenant: &std::sync::OnceLock<String>,
511 verified_tenant: &str,
512) -> Result<()> {
513 if let Some(admitted) = admitted_tenant.get() {
514 anyhow::ensure!(
515 admitted == verified_tenant,
516 "in-process inference is bound to tenant {admitted:?}; refusing cross-tenant engine sharing"
517 );
518 return Ok(());
519 }
520
521 if admitted_tenant.set(verified_tenant.to_owned()).is_err() {
522 let admitted = admitted_tenant
523 .get()
524 .ok_or_else(|| anyhow!("in-process tenant admission raced without a winner"))?;
525 anyhow::ensure!(
526 admitted == verified_tenant,
527 "in-process inference is bound to tenant {admitted:?}; refusing cross-tenant engine sharing"
528 );
529 }
530 Ok(())
531}
532
533impl Clone for ModelService {
534 fn clone(&self) -> Self {
535 Self {
536 inner: Arc::clone(&self.inner),
537 }
538 }
539}
540
541impl std::ops::Deref for ModelService {
542 type Target = ModelServiceInner;
543 fn deref(&self) -> &Self::Target {
544 &self.inner
545 }
546}
547
548#[derive(Debug, Clone, PartialEq, Eq)]
563pub enum ModelRefDispatch<'a> {
564 Federated {
566 scheme: &'static str,
568 rest: &'a str,
570 },
571 Local,
573}
574
575pub fn model_ref_dispatch(s: &str) -> ModelRefDispatch<'_> {
580 if let Some(rest) = s.strip_prefix("at://") {
581 ModelRefDispatch::Federated {
582 scheme: "at://",
583 rest,
584 }
585 } else if let Some(rest) = s.strip_prefix("did:") {
586 ModelRefDispatch::Federated {
587 scheme: "did:",
588 rest,
589 }
590 } else {
591 ModelRefDispatch::Local
592 }
593}
594
595#[derive(Debug, Clone, Copy, PartialEq, Eq)]
600enum RouteDecision {
601 CoLocated,
605 Remote(crate::services::router::ReplicaId),
610 NoHealthyCandidate,
613}
614
615#[derive(Debug, Clone, Copy, PartialEq, Eq)]
616enum ReplicaLocality {
617 CoLocated,
618 Remote,
619}
620
621struct RoutedReplica<T> {
622 value: T,
623 replica_id: crate::services::router::ReplicaId,
624 locality: ReplicaLocality,
625}
626
627async fn route_and_dial_replica<T, F, Fut>(
633 router: &mut crate::services::router::CellRouter,
634 load_state: &[crate::services::router::InferenceServerInfo],
635 co_located: crate::services::router::ReplicaId,
636 session_id: &str,
637 local_value: T,
638 mut dial_remote: F,
639) -> Result<RoutedReplica<T>>
640where
641 F: FnMut(crate::services::router::InferenceServerInfo) -> Fut,
642 Fut: std::future::Future<Output = Result<T>>,
643{
644 let mut local_value = Some(local_value);
645 let mut failures = Vec::new();
646 for _ in 0..load_state.len() {
647 let now = Instant::now();
648 match ModelService::route_decision(router, load_state, co_located, session_id, now) {
649 RouteDecision::CoLocated => {
650 let value = local_value
651 .take()
652 .ok_or_else(|| anyhow!("co-located inference client was already consumed"))?;
653 return Ok(RoutedReplica {
654 value,
655 replica_id: co_located,
656 locality: ReplicaLocality::CoLocated,
657 });
658 }
659 RouteDecision::Remote(replica_id) => {
660 let candidate = load_state
661 .iter()
662 .find(|candidate| candidate.replica_id == replica_id)
663 .cloned()
664 .ok_or_else(|| anyhow!("router selected an absent replica"))?;
665 match dial_remote(candidate).await {
666 Ok(value) => {
667 return Ok(RoutedReplica {
668 value,
669 replica_id,
670 locality: ReplicaLocality::Remote,
671 });
672 }
673 Err(error) => {
674 router.report_dial_fail(replica_id, now);
675 failures.push(format!("{}: {error}", hex::encode(replica_id.as_bytes())));
676 }
677 }
678 }
679 RouteDecision::NoHealthyCandidate => break,
680 }
681 }
682 if failures.is_empty() {
683 anyhow::bail!("no healthy inference replica candidates");
684 }
685 anyhow::bail!(
686 "inference replica candidates exhausted after dial/authorization failures: {}",
687 failures.join("; ")
688 )
689}
690
691impl ModelService {
692 pub async fn new(
694 config: ModelServiceConfig,
695 signing_key: SigningKey,
696 policy_client: PolicyClient,
697 registry: RegistryClient,
698 transport: TransportConfig,
699 policy_transport: TransportConfig,
700 ) -> Result<Self> {
701 config.inference_deployment.validate()?;
702 const DEFAULT_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(5) {
704 Some(n) => n,
705 None => unreachable!(),
706 };
707 let cache_size = NonZeroUsize::new(config.max_models).unwrap_or(DEFAULT_CACHE_SIZE);
708
709 let event_publisher = EventPublisher::new("model")?;
712
713 Ok(Self {
714 inner: Arc::new(ModelServiceInner {
715 loaded_models: RwLock::new(LruCache::new(cache_size)),
716 pending_loads: parking_lot::Mutex::new(HashMap::new()),
717 staging_pinned_artifacts:
718 crate::storage::pinned_model::StagingPinnedArtifactCache::default(),
719 unloading_models: Mutex::new(HashSet::new()),
720 load_unload_gate: Mutex::new(()),
721 lifecycle_event_publish_gate: Mutex::new(()),
722 config,
723 signing_key,
724 policy_client,
725 #[cfg(feature = "postgres")]
726 federate_dispatch: None,
727 event_publisher,
728 registry,
729 transport,
730 policy_transport,
731 expected_audience: None,
732 delegated_oai_audience: None,
733 delegated_mcp_audience: None,
734 jwt_key_source: None,
735 discovery_client: None,
736 fs_trees: dashmap::DashMap::new(),
737 load_terminals: TerminalStore::new(),
738 load_epoch: AtomicU64::new(0),
739 incarnation_generation: AtomicU64::new(0),
740 in_process_tenant: std::sync::OnceLock::new(),
741 producer_reach_config: std::sync::Arc::new(parking_lot::RwLock::new(
742 hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
743 )),
744 moq_origin: std::sync::Arc::new(parking_lot::RwLock::new(None)),
745 #[cfg(test)]
746 test_work_order_capture: Mutex::new(None),
747 #[cfg(test)]
748 before_unloaded_event: Mutex::new(None),
749 }),
750 })
751 }
752
753 #[cfg(test)]
757 async fn capture_test_work_order(&self, work_order: &str) {
758 let mut capture = self.test_work_order_capture.lock().await;
759 *capture = Some(Zeroizing::new(work_order.to_owned()));
760 }
761
762 #[cfg(test)]
766 async fn take_test_work_order(&self) -> Option<Zeroizing<String>> {
767 self.test_work_order_capture.lock().await.take()
768 }
769
770 #[allow(clippy::expect_used)]
776 pub fn with_expected_audience(mut self, audience: String) -> Self {
777 Arc::get_mut(&mut self.inner)
778 .expect("with_expected_audience must be called before service is shared")
779 .expected_audience = Some(audience);
780 self
781 }
782
783 #[allow(clippy::expect_used)]
786 pub fn with_delegated_resource_audiences(mut self, oai: String, mcp: String) -> Self {
787 let inner = Arc::get_mut(&mut self.inner)
788 .expect("delegated resources must be configured before service sharing");
789 inner.delegated_oai_audience = Some(oai);
790 inner.delegated_mcp_audience = Some(mcp);
791 self
792 }
793
794 #[allow(clippy::expect_used)]
799 pub fn with_jwt_key_source(
800 mut self,
801 src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
802 ) -> Self {
803 Arc::get_mut(&mut self.inner)
804 .expect("with_jwt_key_source must be called before service is shared")
805 .jwt_key_source = Some(src);
806 self
807 }
808
809 #[cfg(feature = "postgres")]
810 #[allow(clippy::expect_used)]
811 pub(crate) fn with_federate_dispatch(
812 mut self,
813 adapter: Arc<crate::services::oauth::federate_proof::DispatchAdapter>,
814 ) -> Self {
815 Arc::get_mut(&mut self.inner)
816 .expect("with_federate_dispatch must be called before service is shared")
817 .federate_dispatch = Some(adapter);
818 self
819 }
820
821 #[allow(clippy::expect_used)]
826 pub fn with_discovery_client(
827 mut self,
828 client: Arc<hyprstream_rpc_std::discovery_client::DiscoveryClient>,
829 ) -> Self {
830 Arc::get_mut(&mut self.inner)
831 .expect("with_discovery_client must be called before service is shared")
832 .discovery_client = Some(client);
833 self
834 }
835
836 fn allocate_load_epoch(&self) -> u64 {
846 self.load_epoch.fetch_add(1, Ordering::Relaxed) + 1
847 }
848
849 fn latch_load_terminal(
856 &self,
857 instance: &InferenceInstanceId,
858 epoch: u64,
859 result: &Result<String>,
860 ) {
861 let terminal = match result {
862 Ok(endpoint) => Terminal {
863 value: ModelLoadTerminal::Loaded {
864 endpoint: endpoint.clone(),
865 },
866 latched_by: "model-service".to_owned(),
867 },
868 Err(e) => Terminal {
869 value: ModelLoadTerminal::LoadFailed {
870 error: e.to_string(),
871 },
872 latched_by: "model-service".to_owned(),
873 },
874 };
875 let key = ModelLoadKey {
876 tenant: instance.tenant().to_owned(),
877 model_ref: instance.model_ref().to_owned(),
878 epoch,
879 };
880 let _ = self.load_terminals.latch(key, terminal);
881 }
882
883 async fn reserve_intercepted_load(
888 &self,
889 instance: &InferenceInstanceId,
890 ) -> Result<InterceptedLoadAdmission> {
891 self.validate_staging_model_checkout(instance).await?;
892 self.admit_instance(instance)?;
893 let _load_unload_gate = self.load_unload_gate.lock().await;
894 self.ensure_not_unloading(instance).await?;
895
896 {
897 let mut cache = self.loaded_models.write().await;
898 if let Some(model) = cache.get_mut(instance) {
899 model.last_used = Instant::now();
900 return Ok(InterceptedLoadAdmission::Loaded {
901 reach: Self::model_reach(&model.network_transport),
902 });
903 }
904 }
905
906 let mut pending = self.pending_loads.lock();
907 if pending.contains_key(instance) {
908 return Ok(InterceptedLoadAdmission::Pending);
909 }
910
911 let attempt = self.allocate_load_epoch();
912 pending.insert(instance.clone(), attempt);
913 Ok(InterceptedLoadAdmission::Accepted(
914 InterceptedLoadReservation::new(Arc::clone(&self.inner), instance.clone(), attempt),
915 ))
916 }
917
918 fn validate_model_load_ref(&self, model_ref: &str) -> Result<()> {
919 if let Some(pin) = &self.config.staging_model_pin {
920 anyhow::ensure!(
921 pin.admits(model_ref),
922 "Model.load is restricted to the configured staging modelRef"
923 );
924 }
925 Ok(())
926 }
927
928 async fn validate_staging_model_checkout(&self, instance: &InferenceInstanceId) -> Result<()> {
929 let Some(pin) = &self.config.staging_model_pin else {
930 return Ok(());
931 };
932 self.validate_model_load_ref(instance.model_ref())?;
933 let model_ref = ModelRef::parse(instance.model_ref())?;
934 let tracked = self
935 .registry
936 .get_by_name(model_ref.name())
937 .await
938 .map_err(|e| {
939 anyhow!(
940 "staging model '{}' not found in registry: {e}",
941 model_ref.name()
942 )
943 })?;
944 let repo_client = self.registry.repo(&tracked.id);
945 let worktrees = repo_client.list_worktrees().await?;
946 let branch = match &model_ref.git_ref {
947 crate::storage::GitRef::Branch(branch) => branch,
948 _ => anyhow::bail!("staging modelRef must select its configured local branch"),
949 };
950 anyhow::ensure!(
951 worktrees
952 .iter()
953 .any(|worktree| worktree.branch_name == branch.as_str()),
954 "staging worktree for {}:{} not found",
955 model_ref.name(),
956 branch
957 );
958 let path = crate::storage::StoragePaths::new()?.worktree_path(model_ref.name(), branch)?;
959 crate::storage::pinned_model::verify_selected_checkout(&path, pin.commit)
960 .with_context(|| format!("admit staging worktree for {}", pin.model_ref))
961 }
962
963 async fn run_intercepted_load(
967 &self,
968 instance: &InferenceInstanceId,
969 max_context: Option<u32>,
970 kv_quant: Option<KVQuantType>,
971 mut reservation: InterceptedLoadReservation,
972 ) -> Result<String> {
973 let epoch = reservation.attempt;
974 let result = self.load_model_inner(instance, max_context, kv_quant).await;
975 reservation.release();
976 self.latch_load_terminal(instance, epoch, &result);
977 result
978 }
979
980 fn inference_instance(
982 ctx: &EnvelopeContext,
983 model_ref_str: &str,
984 ) -> Result<InferenceInstanceId> {
985 let object_label = crate::services::inference::inference_object_label();
986 crate::services::inference::enforce_inference_mac(ctx, &object_label)?;
987 InferenceInstanceId::new(&ctx.domain()?, model_ref_str, 0)
988 }
989
990 fn admit_instance(&self, instance: &InferenceInstanceId) -> Result<()> {
992 self.config.inference_deployment.validate()?;
993 match &self.config.inference_deployment.isolation {
994 InferenceIsolationProfile::InProcess => {
995 admit_in_process_tenant(&self.in_process_tenant, instance.tenant())
996 }
997 InferenceIsolationProfile::PerTenantSubprocess
998 | InferenceIsolationProfile::PerTenantMicroVmTask { .. } => {
999 anyhow::bail!(
1000 "inference isolation profile {:?} requires the external tenant launcher; \
1001 refusing to downgrade to an in-process engine",
1002 self.config.inference_deployment.isolation
1003 )
1004 }
1005 }
1006 }
1007
1008 fn inference_transport(instance: &InferenceInstanceId) -> TransportConfig {
1009 TransportConfig::inproc(format!("hyprstream/{}", instance.service_name()))
1013 }
1014
1015 fn inference_endpoint(instance: &InferenceInstanceId) -> String {
1020 Self::inference_transport(instance).endpoint_string()
1021 }
1022
1023 fn model_reach(transport: &TransportConfig) -> Vec<WireTransportConfig> {
1028 hyprstream_rpc::moq_stream::dial_transport_to_wire(transport)
1029 .into_iter()
1030 .collect()
1031 }
1032
1033 fn route_decision(
1039 router: &mut crate::services::router::CellRouter,
1040 load_state: &[crate::services::router::InferenceServerInfo],
1041 co_located: crate::services::router::ReplicaId,
1042 session_id: &str,
1043 now: Instant,
1044 ) -> RouteDecision {
1045 match router.place(session_id, load_state, now) {
1046 Some(placement) if placement.replica_id == co_located => RouteDecision::CoLocated,
1047 Some(placement) => RouteDecision::Remote(placement.replica_id),
1048 None => RouteDecision::NoHealthyCandidate,
1049 }
1050 }
1051
1052 async fn select_inference_routed(
1071 &self,
1072 model: &mut LoadedModel,
1073 session_id: &str,
1074 bearer: Option<&str>,
1075 ) -> Result<RoutedReplica<InferenceClient>> {
1076 let co_located = Self::co_located_replica_id(model);
1077 let resolution_service_name = model.instance.service_name();
1078 let signing_key = self.signing_key.clone();
1079 let bearer = bearer.map(ToOwned::to_owned);
1080 let routed = route_and_dial_replica(
1081 &mut model.router,
1082 &model.load_state,
1083 co_located,
1084 session_id,
1085 model.client.clone(),
1086 move |candidate| {
1087 let resolution_service_name = resolution_service_name.clone();
1088 let signing_key = signing_key.clone();
1089 let bearer = bearer.clone();
1090 async move {
1091 anyhow::ensure!(
1092 matches!(
1093 &candidate.transport.endpoint,
1094 EndpointType::Quic { .. } | EndpointType::Iroh { .. }
1095 ),
1096 "remote inference replica advertised a non-network transport"
1097 );
1098 let relay_bearer = bearer.clone().ok_or_else(|| {
1103 anyhow!(
1104 "remote inference replica selection requires a relayable caller bearer"
1105 )
1106 })?;
1107 let rpc = hyprstream_discovery::production_inference_rpc_client_at_transport(
1108 &resolution_service_name,
1109 &candidate.transport,
1110 signing_key,
1111 None,
1112 )?;
1113 let client = InferenceClient::new(rpc).with_delegated_bearer(relay_bearer);
1114 anyhow::ensure!(
1115 client.is_ready().await?,
1116 "remote inference replica is not ready"
1117 );
1118 Ok(client)
1119 }
1120 },
1121 )
1122 .await?;
1123 info!(
1124 replica_id = %hex::encode(routed.replica_id.as_bytes()),
1125 locality = ?routed.locality,
1126 session_id,
1127 tenant = model.instance.tenant(),
1128 model_ref = model.instance.model_ref(),
1129 "selected inference replica"
1130 );
1131 Ok(routed)
1132 }
1133
1134 fn co_located_replica_id(model: &LoadedModel) -> crate::services::router::ReplicaId {
1136 model
1139 .load_state
1140 .first()
1141 .map(|info| info.replica_id)
1142 .unwrap_or(crate::services::router::ReplicaId::from_bytes([0u8; 32]))
1143 }
1144
1145 async fn resolve_model_ref(&self, model_ref_str: &str) -> Result<ModelRef> {
1165 match model_ref_dispatch(model_ref_str) {
1166 ModelRefDispatch::Federated { scheme, rest } => {
1167 if scheme == "at://" {
1172 match self.resolve_federated_at_uri(model_ref_str).await {
1173 Ok(Some(model_ref)) => return Ok(model_ref),
1174 Ok(None) => {
1175 if self.inner.discovery_client.is_none() {
1184 anyhow::bail!(
1185 "federation not enabled: cannot resolve federated ref '{model_ref_str}' \
1186 (no DiscoveryClient configured on this node)"
1187 );
1188 }
1189 anyhow::bail!(
1190 "federated ref not resolvable: '{model_ref_str}' is not a full \
1191 at://<did>/<collection>/<rkey>"
1192 );
1193 }
1194 Err(e) => {
1195 return Err(e);
1198 }
1199 }
1200 }
1201 anyhow::bail!(
1204 "federated ModelRef scheme '{scheme}' for '{rest}' is not resolvable on this node \
1205 (bare did: resolution not implemented; use a full at:// record ref)"
1206 )
1207 }
1208 ModelRefDispatch::Local => ModelRef::parse(model_ref_str),
1209 }
1210 }
1211
1212 async fn resolve_federated_at_uri(&self, at_uri: &str) -> Result<Option<ModelRef>> {
1235 let Some(dc) = self.inner.discovery_client.as_ref() else {
1236 return Ok(None);
1237 };
1238
1239 let rest = match at_uri.strip_prefix("at://") {
1242 Some(r) => r,
1243 None => return Ok(None),
1244 };
1245 let mut parts = rest.splitn(3, '/');
1246 let (did, collection, rkey) = match (parts.next(), parts.next(), parts.next()) {
1247 (Some(d), Some(c), Some(r)) if !d.is_empty() && !c.is_empty() && !r.is_empty() => {
1248 (d.to_owned(), c.to_owned(), r.to_owned())
1249 }
1250 _ => return Ok(None),
1252 };
1253
1254 let car_resp = dc
1255 .get_record(&hyprstream_rpc_std::discovery_client::GetRecordRequest {
1256 uri: at_uri.to_owned(),
1257 did: did.clone(),
1258 collection: collection.clone(),
1259 rkey: rkey.clone(),
1260 })
1261 .await
1262 .map_err(|e| anyhow!("DiscoveryService.getRecord failed for {at_uri}: {e}"))?;
1263
1264 let (roots, blocks) = hyprstream_pds::car::parse_car_v1(&car_resp.car)
1266 .map_err(|e| anyhow!("getRecord returned an unparseable CAR for {at_uri}: {e}"))?;
1267 let commit_cid = roots
1268 .first()
1269 .copied()
1270 .ok_or_else(|| anyhow!("getRecord CAR for {at_uri} has no root commit CID"))?;
1271
1272 let _commit = blocks
1276 .iter()
1277 .find(|(c, _)| *c == commit_cid)
1278 .map(|(_, b)| hyprstream_pds::commit::Commit::from_dag_cbor(b))
1279 .transpose()
1280 .map_err(|e| anyhow!("getRecord CAR commit block did not decode: {e}"))?
1281 .ok_or_else(|| anyhow!("getRecord CAR for {at_uri} is missing its commit block"))?;
1282
1283 let record = blocks
1286 .iter()
1287 .filter(|(c, _)| *c != commit_cid)
1288 .find_map(|(_, b)| hyprstream_pds::record::ModelRecord::from_dag_cbor(b).ok())
1289 .ok_or_else(|| {
1290 anyhow!("getRecord CAR for {at_uri} contained no ai.hyprstream.model record")
1291 })?;
1292
1293 let cid = hyprstream_rpc::cid::decode_cid(&record.current_oid)
1295 .map_err(|e| anyhow!("record currentOid is not a valid CID: {e}"))?;
1296 let oid_hex = cid
1297 .multihash
1298 .digest
1299 .iter()
1300 .map(|b| format!("{b:02x}"))
1301 .collect::<String>();
1302
1303 info!(
1304 %at_uri,
1305 did = %did,
1306 current_oid = %record.current_oid,
1307 git_oid = %oid_hex,
1308 "resolved federated at-uri via DiscoveryService.getRecord (#431)"
1309 );
1310
1311 let local_ref = format!("{did}:{oid_hex}");
1314 match ModelRef::parse(&local_ref) {
1315 Ok(model_ref) => Ok(Some(model_ref)),
1316 Err(e) => Err(anyhow!(
1320 "resolved {at_uri} → git OID {oid_hex}, but local ModelRef::parse rejected {local_ref:?}: {e}"
1321 )),
1322 }
1323 }
1324
1325 async fn load_model(
1327 &self,
1328 instance: &InferenceInstanceId,
1329 max_context: Option<u32>,
1330 kv_quant: Option<KVQuantType>,
1331 ) -> Result<String> {
1332 self.validate_staging_model_checkout(instance).await?;
1333 self.admit_instance(instance)?;
1334 let model_ref_str = instance.model_ref();
1335 let load_unload_gate = self.load_unload_gate.lock().await;
1339 self.ensure_not_unloading(instance).await?;
1340 {
1342 let mut cache = self.loaded_models.write().await;
1343 if let Some(model) = cache.get_mut(instance) {
1344 model.last_used = Instant::now();
1345 debug!("Model {} already loaded", model_ref_str);
1346 return Ok(model.network_transport.endpoint_string());
1347 }
1348 }
1349
1350 let epoch;
1355 {
1356 let mut pending = self.pending_loads.lock();
1357 if pending.contains_key(instance) {
1358 anyhow::bail!(
1359 "Model {} is already being loaded — please retry shortly",
1360 model_ref_str
1361 );
1362 }
1363 epoch = self.allocate_load_epoch();
1364 pending.insert(instance.clone(), epoch);
1365 }
1366 drop(load_unload_gate);
1367
1368 let result = self.load_model_inner(instance, max_context, kv_quant).await;
1371
1372 self.pending_loads.lock().remove(instance);
1374
1375 self.latch_load_terminal(instance, epoch, &result);
1378
1379 if result.is_ok() {
1380 info!("Model {} loaded successfully", model_ref_str);
1381 }
1382 result
1383 }
1384
1385 async fn load_model_inner(
1387 &self,
1388 instance: &InferenceInstanceId,
1389 max_context: Option<u32>,
1390 kv_quant: Option<KVQuantType>,
1391 ) -> Result<String> {
1392 let model_ref_str = instance.model_ref();
1393 self.validate_model_load_ref(model_ref_str)?;
1394 let model_ref = self.resolve_model_ref(model_ref_str).await?;
1396
1397 #[cfg(test)]
1399 crate::services::restart_diag::phase("registry.get_by_name.enter");
1400 let tracked = self
1401 .registry
1402 .get_by_name(model_ref.name())
1403 .await
1404 .map_err(|e| anyhow!("Model '{}' not found in registry: {}", model_ref.name(), e))?;
1405 #[cfg(test)]
1406 crate::services::restart_diag::phase("registry.get_by_name.done");
1407 let repo_client = self.registry.repo(&tracked.id);
1408
1409 let branch_name = match &model_ref.git_ref {
1410 crate::storage::GitRef::Branch(name) => name.clone(),
1411 crate::storage::GitRef::Commit(_) => repo_client
1412 .get_head()
1413 .await
1414 .context("resolve selected worktree branch for pinned commit")?,
1415 crate::storage::GitRef::Tag(_) | crate::storage::GitRef::Revspec(_) => {
1416 anyhow::bail!("tag and revspec model loads are unsupported; use a full commit OID")
1417 }
1418 crate::storage::GitRef::DefaultBranch => repo_client
1419 .get_head()
1420 .await
1421 .unwrap_or_else(|_| "main".to_owned()),
1422 };
1423 #[cfg(test)]
1424 crate::services::restart_diag::phase("registry.list_worktrees.enter");
1425 let worktrees = repo_client.list_worktrees().await?;
1426 #[cfg(test)]
1427 crate::services::restart_diag::phase("registry.list_worktrees.done");
1428 if !worktrees.iter().any(|wt| wt.branch_name == branch_name) {
1429 return Err(anyhow!(
1430 "worktree for {}:{} not found",
1431 model_ref.name(),
1432 branch_name
1433 ));
1434 }
1435 let storage_paths = crate::storage::StoragePaths::new()?;
1437 let model_path = storage_paths.worktree_path(model_ref.name(), &branch_name)?;
1438
1439 if !model_path.exists() {
1440 return Err(anyhow!(
1441 "Model worktree not found for {}. Please clone the model first.",
1442 model_ref_str
1443 ));
1444 }
1445
1446 let pinned_artifact = if let Some(pin) = &self.config.staging_model_pin {
1447 anyhow::ensure!(
1448 model_ref_str == pin.model_ref,
1449 "staging modelRef changed after admission"
1450 );
1451 let policy = pin.capture_policy()?;
1452 #[cfg(test)]
1453 let artifact = if let Some(parent) = pin.projection_parent.as_deref() {
1454 self.inner
1455 .staging_pinned_artifacts
1456 .acquire_with_policy_in(&model_path, pin.commit, parent, policy)
1457 .await?
1458 } else {
1459 self.inner
1460 .staging_pinned_artifacts
1461 .acquire_with_policy(&model_path, pin.commit, policy)
1462 .await?
1463 };
1464 #[cfg(not(test))]
1465 let artifact = self
1466 .inner
1467 .staging_pinned_artifacts
1468 .acquire_with_policy(&model_path, pin.commit, policy)
1469 .await?;
1470 Some(artifact)
1471 } else if let crate::storage::GitRef::Commit(oid) = &model_ref.git_ref {
1472 Some(crate::storage::pinned_model::acquire_pinned_model(&model_path, *oid).await?)
1473 } else {
1474 None
1475 };
1476 let model_input_path = pinned_artifact
1477 .as_ref()
1478 .map_or(model_path.as_path(), |artifact| artifact.root());
1479
1480 let endpoint = Self::inference_endpoint(instance);
1481
1482 info!("Loading model {} at endpoint {}", model_ref_str, endpoint);
1483
1484 let runtime_config = RuntimeConfig {
1486 max_context: max_context.or(self.config.max_context),
1487 kv_quant_type: kv_quant.unwrap_or(self.config.kv_quant),
1488 use_gpu: self.config.inference_deployment.compute != InferenceCompute::Cpu,
1489 gpu_device_id: if self.config.inference_deployment.compute == InferenceCompute::Cpu {
1490 None
1491 } else {
1492 RuntimeConfig::default().gpu_device_id
1493 },
1494 devices: if self.config.inference_deployment.compute == InferenceCompute::Cpu {
1495 Vec::new()
1496 } else {
1497 RuntimeConfig::default().devices
1498 },
1499 ..Default::default()
1500 };
1501
1502 let fs: Option<hyprstream_rpc_std::registry_client::WorktreeClient> =
1504 Some(repo_client.worktree(&branch_name));
1505
1506 let spawner = hyprstream_service::ServiceSpawner::threaded();
1508
1509 let transport = Self::inference_transport(instance);
1516 let mut service_config = crate::services::InferenceServiceConfig::new(
1517 &model_path,
1518 runtime_config,
1519 self.signing_key.verifying_key(),
1520 self.signing_key.clone(),
1521 transport.clone(),
1522 self.policy_transport.clone(),
1523 fs,
1524 );
1525 if let Some(ref artifact) = pinned_artifact {
1526 service_config = service_config.with_pinned_artifact(Arc::clone(artifact));
1527 }
1528 let mut service_config = service_config
1529 .with_instance_identity(
1530 instance.service_name(),
1531 instance.tenant().to_owned(),
1532 self.signing_key.verifying_key(),
1533 )
1534 .with_model_ref(instance.model_ref().to_owned())
1535 .with_stream_plane(
1536 Arc::clone(&self.producer_reach_config),
1537 Arc::clone(&self.moq_origin),
1538 );
1539 if let Some(ref aud) = self.expected_audience {
1540 service_config = service_config.with_expected_audience(aud.clone());
1541 }
1542 if let Some(ref src) = self.jwt_key_source {
1543 service_config = service_config.with_jwt_key_source(src.clone());
1544 }
1545 let generation = self
1553 .inner
1554 .incarnation_generation
1555 .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
1556 + 1;
1557 let (incarnation_tx, incarnation_rx) = tokio::sync::oneshot::channel();
1558 service_config = service_config.with_incarnation_handoff(incarnation_tx, generation);
1559
1560 let network_reach = service_config.network_reach_handle();
1561 #[cfg(test)]
1562 crate::services::restart_diag::phase("inference.spawn.enter");
1563 let service_handle = spawner
1564 .spawn(service_config)
1565 .await
1566 .map_err(|e| anyhow!("Failed to spawn inference service: {}", e))?;
1567 let spawn_guard = SpawnPublicationGuard::new(service_handle);
1568 #[cfg(test)]
1569 crate::services::restart_diag::phase("inference.spawn.done");
1570 let network_transport = network_reach
1571 .read()
1572 .clone()
1573 .ok_or_else(|| anyhow!("InferenceService became ready without network reach"))?;
1574 let endpoint = network_transport.endpoint_string();
1575
1576 let expected_controller = self.signing_key.verifying_key().to_bytes();
1580 const HANDOFF_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(
1581 crate::services::inference::INCARNATION_HANDOFF_TIMEOUT_SECS,
1582 );
1583 #[cfg(test)]
1584 crate::services::restart_diag::phase("incarnation.handoff.enter");
1585 let incarnation_binding = match tokio::time::timeout(HANDOFF_TIMEOUT, incarnation_rx).await
1586 {
1587 Err(_) => anyhow::bail!(
1588 "pinned worker incarnation handoff timed out after {}s without a ready worker",
1589 crate::services::inference::INCARNATION_HANDOFF_TIMEOUT_SECS
1590 ),
1591 Ok(Err(_)) => {
1592 anyhow::bail!("pinned worker died before reporting its incarnation handoff")
1593 }
1594 Ok(Ok(handoff)) => handoff,
1595 };
1596 #[cfg(test)]
1597 crate::services::restart_diag::phase("incarnation.handoff.done");
1598 anyhow::ensure!(
1599 incarnation_binding.generation == generation,
1600 "stale incarnation handoff: worker reported generation {} for attempt {generation}",
1601 incarnation_binding.generation
1602 );
1603 anyhow::ensure!(
1604 incarnation_binding.instance_service_name == instance.service_name(),
1605 "incarnation handoff names the wrong deterministic instance"
1606 );
1607 anyhow::ensure!(
1608 incarnation_binding.controller_pubkey == expected_controller,
1609 "incarnation handoff names the wrong pinned controller"
1610 );
1611
1612 let client = InferenceClient::for_local_transport_bootstrap(
1616 &transport,
1617 self.signing_key.clone(),
1618 self.signing_key.verifying_key(),
1619 None,
1620 )?;
1621
1622 let ttt_config =
1624 crate::runtime::model_config::ModelConfig::load_training_config(model_input_path)
1625 .and_then(|tc| {
1626 if tc.is_enabled() && tc.mode == crate::config::TrainingMode::TestTimeTraining {
1627 Some(crate::training::ttt::TTTConfig {
1628 learning_rate: tc.ttt.learning_rate,
1629 gradient_steps: tc.ttt.gradient_steps,
1630 max_grad_norm: tc.ttt.max_grad_norm,
1631 min_input_length: tc.ttt.min_input_length,
1632 max_ttt_context: tc.ttt.max_ttt_context,
1633 enabled: true,
1634 ..crate::training::ttt::TTTConfig::default()
1635 })
1636 } else {
1637 None
1638 }
1639 });
1640
1641 let generation_defaults = crate::config::SamplingParams::from_model_path(model_input_path)
1643 .await
1644 .unwrap_or_default();
1645
1646 {
1648 let mut cache = self.loaded_models.write().await;
1649 if cache.len() >= self.config.max_models {
1650 if let Some((evicted_ref, mut evicted)) = cache.pop_lru() {
1651 info!(
1652 "Evicting model {} to load {}",
1653 evicted_ref.model_ref(),
1654 model_ref_str
1655 );
1656 #[allow(clippy::let_underscore_future)]
1658 let _ = tokio::spawn(async move {
1659 let _ = evicted.service_handle.stop().await;
1660 });
1661 }
1662 }
1663
1664 let service_handle = spawn_guard.publish()?;
1666 cache.put(
1667 instance.clone(),
1668 LoadedModel {
1669 instance: instance.clone(),
1670 model_ref: model_ref_str.to_owned(),
1671 pinned_commit: pinned_artifact.as_ref().map(|artifact| artifact.commit()),
1672 transport: transport.clone(),
1673 network_transport,
1674 service_handle,
1675 client,
1676 incarnation: incarnation_binding.incarnation,
1679 work_audience: incarnation_binding.audience,
1680 load_generation: incarnation_binding.generation,
1681 router: crate::services::router::CellRouter::default(),
1688 load_state: vec![crate::services::router::InferenceServerInfo {
1689 replica_id: crate::services::router::ReplicaId::from_bytes(
1690 hyprstream_rpc::node_identity::derive_purpose_key(
1691 &self.signing_key,
1692 "hyprstream-inference-replica-placement-v1",
1693 )
1694 .verifying_key()
1695 .to_bytes(),
1696 ),
1697 transport: transport.clone(),
1698 gpu_memory_free: 0,
1699 active_sessions: 0,
1700 last_heartbeat: Instant::now(),
1701 }],
1702 loaded_at: Instant::now(),
1703 last_used: Instant::now(),
1704 ttt_config,
1705 generation_defaults,
1706 },
1707 );
1708 }
1709
1710 if let Some(ref artifact) = pinned_artifact {
1711 info!(model = %model_ref_str, commit = %artifact.commit(), files = artifact.file_count(),
1712 "loaded model from private disk exact-tree artifact");
1713 }
1714
1715 Ok(endpoint)
1716 }
1717
1718 async fn ensure_not_unloading(&self, instance: &InferenceInstanceId) -> Result<()> {
1719 anyhow::ensure!(
1720 !self.unloading_models.lock().await.contains(instance),
1721 "Model {} is unloading — please retry after teardown completes",
1722 instance.model_ref(),
1723 );
1724 Ok(())
1725 }
1726
1727 async fn unload_model(&self, instance: &InferenceInstanceId) -> Result<()> {
1736 let (completion_tx, completion_rx) = tokio::sync::oneshot::channel();
1737 let inner = Arc::clone(&self.inner);
1738 let instance = instance.clone();
1739
1740 tokio::spawn(async move {
1741 let result = Self::unload_model_owned(Arc::clone(&inner), &instance).await;
1742 if let Err(Err(error)) = completion_tx.send(result) {
1743 warn!(
1744 model = %instance.model_ref(),
1745 error = %error,
1746 "model unload caller cancelled; owned teardown completed with an error"
1747 );
1748 }
1749 });
1750
1751 completion_rx
1752 .await
1753 .map_err(|_| anyhow!("model unload task exited before reporting completion"))?
1754 }
1755
1756 async fn unload_model_owned(
1757 inner: Arc<ModelServiceInner>,
1758 instance: &InferenceInstanceId,
1759 ) -> Result<()> {
1760 let model_ref_str = instance.model_ref().to_owned();
1761 let (model, pending_load) = {
1762 let load_unload_gate = inner.load_unload_gate.lock().await;
1763 let mut unloading = inner.unloading_models.lock().await;
1764 let inserted = unloading.insert(instance.clone());
1765 drop(unloading);
1766 anyhow::ensure!(inserted, "Model {} is already unloading", model_ref_str,);
1767 let model = {
1768 let mut cache = inner.loaded_models.write().await;
1769 cache.pop_entry(instance).map(|(_, model)| model)
1770 };
1771 let pending_load = if model.is_none() {
1772 inner.pending_loads.lock().contains_key(instance)
1773 } else {
1774 false
1775 };
1776 drop(load_unload_gate);
1777 (model, pending_load)
1778 };
1779
1780 let Some(mut model) = model else {
1781 if pending_load {
1782 inner.unloading_models.lock().await.remove(instance);
1783 anyhow::bail!("Model {} is loading", model_ref_str);
1784 }
1785 inner.unloading_models.lock().await.remove(instance);
1786 anyhow::bail!("Model {} is not loaded", model_ref_str);
1787 };
1788
1789 info!("Unloading model {}", model_ref_str);
1790 if let Err(error) = model.service_handle.stop().await {
1791 warn!(
1797 model = %model_ref_str,
1798 error = %error,
1799 "model unload stop failed; retaining unload reservation"
1800 );
1801 return Err(anyhow!(
1802 "failed to stop model {} before unload: {error}",
1803 model_ref_str
1804 ));
1805 }
1806 let _event_publish_gate = inner.lifecycle_event_publish_gate.lock().await;
1811 inner.unloading_models.lock().await.remove(instance);
1812 #[cfg(test)]
1813 if let Some(before_publish) = inner.before_unloaded_event.lock().await.take() {
1814 let _ = before_publish.send(());
1815 }
1816 let model_name = model_ref_str.split(':').next().unwrap_or(&model_ref_str);
1817 let scope = format!("serve:model:{}", model_name);
1818 let event = crate::events::EventEnvelope::new(
1819 crate::events::EventSource::Model,
1820 scope.clone(),
1821 crate::events::EventPayload::ModelUnloaded {
1822 model_ref: model_ref_str.clone(),
1823 },
1824 );
1825 if let Ok(payload) = serde_json::to_vec(&event) {
1826 let _ = inner
1827 .event_publisher
1828 .publish("lifecycle", "unloaded", &payload)
1829 .await;
1830 }
1831 Ok(())
1832 }
1833
1834 fn ttt_config_to_wire(cfg: &crate::training::ttt::TTTConfig) -> GenOnlineTrainingConfig {
1836 GenOnlineTrainingConfig {
1837 enabled: cfg.enabled,
1838 learning_rate: cfg.learning_rate,
1839 gradient_steps: cfg.gradient_steps,
1840 max_grad_norm: cfg.max_grad_norm,
1841 min_input_length: cfg.min_input_length,
1842 max_ttt_context: cfg.max_ttt_context,
1843 }
1844 }
1845
1846 fn sampling_params_to_wire(params: &crate::config::SamplingParams) -> GenGenerationDefaults {
1848 GenGenerationDefaults {
1849 temperature: params.temperature,
1850 top_p: params.top_p,
1851 top_k: params.top_k.map(|v| v as u32),
1852 max_tokens: params.max_tokens.map(|v| v as u32),
1853 repeat_penalty: params.repeat_penalty,
1854 stop_tokens: params.stop_tokens.clone().unwrap_or_default(),
1855 do_sample: params.do_sample,
1856 }
1857 }
1858
1859 async fn model_status_all(&self, verified_tenant: &str) -> Vec<GenModelStatusEntry> {
1862 let _load_unload_gate = self.load_unload_gate.lock().await;
1867 let unloading: HashSet<_> = self
1868 .unloading_models
1869 .lock()
1870 .await
1871 .iter()
1872 .filter(|instance| instance.tenant() == verified_tenant)
1873 .cloned()
1874 .collect();
1875 let (mut entries, loaded): (Vec<_>, HashSet<_>) = {
1876 let cache = self.loaded_models.read().await;
1877 (
1878 cache
1879 .iter()
1880 .filter(|(instance, _)| {
1881 instance.tenant() == verified_tenant && !unloading.contains(*instance)
1882 })
1883 .map(|(_, model)| GenModelStatusEntry {
1884 model_ref: model.model_ref.clone(),
1885 status: "loaded".to_owned(),
1886 reach: Self::model_reach(&model.network_transport),
1887 loaded_at: model.loaded_at.elapsed().as_millis() as i64,
1888 last_used: model.last_used.elapsed().as_millis() as i64,
1889 online_training_config: model
1890 .ttt_config
1891 .as_ref()
1892 .map(Self::ttt_config_to_wire)
1893 .unwrap_or_default(),
1894 generation_defaults: Self::sampling_params_to_wire(
1895 &model.generation_defaults,
1896 ),
1897 })
1898 .collect(),
1899 cache
1900 .iter()
1901 .filter(|(instance, _)| {
1902 instance.tenant() == verified_tenant && !unloading.contains(*instance)
1903 })
1904 .map(|(instance, _)| instance.clone())
1905 .collect(),
1906 )
1907 };
1908 for instance in &unloading {
1909 entries.push(GenModelStatusEntry {
1910 model_ref: instance.model_ref().to_owned(),
1911 status: "unloading".to_owned(),
1912 reach: Vec::new(),
1913 loaded_at: 0,
1914 last_used: 0,
1915 online_training_config: GenOnlineTrainingConfig::default(),
1916 generation_defaults: GenGenerationDefaults::default(),
1917 });
1918 }
1919 let pending: HashSet<_> = {
1920 self.pending_loads
1921 .lock()
1922 .keys()
1923 .filter(|instance| instance.tenant() == verified_tenant)
1924 .cloned()
1925 .collect()
1926 };
1927 for instance in &pending {
1928 if !loaded.contains(instance) && !unloading.contains(instance) {
1929 entries.push(GenModelStatusEntry {
1930 model_ref: instance.model_ref().to_owned(),
1931 status: "loading".to_owned(),
1932 reach: Vec::new(),
1933 loaded_at: 0,
1934 last_used: 0,
1935 online_training_config: GenOnlineTrainingConfig::default(),
1936 generation_defaults: GenGenerationDefaults::default(),
1937 });
1938 }
1939 }
1940 entries
1941 }
1942
1943 async fn model_status_single(
1945 &self,
1946 instance: &InferenceInstanceId,
1947 ) -> Vec<GenModelStatusEntry> {
1948 let _load_unload_gate = self.load_unload_gate.lock().await;
1949 if self.unloading_models.lock().await.contains(instance) {
1950 return vec![GenModelStatusEntry {
1951 model_ref: instance.model_ref().to_owned(),
1952 status: "unloading".to_owned(),
1953 reach: Vec::new(),
1954 loaded_at: 0,
1955 last_used: 0,
1956 online_training_config: GenOnlineTrainingConfig::default(),
1957 generation_defaults: GenGenerationDefaults::default(),
1958 }];
1959 }
1960 {
1961 let cache = self.loaded_models.read().await;
1962 if let Some(model) = cache.peek(instance) {
1963 return vec![GenModelStatusEntry {
1964 model_ref: instance.model_ref().to_owned(),
1965 status: "loaded".to_owned(),
1966 reach: Self::model_reach(&model.network_transport),
1967 loaded_at: model.loaded_at.elapsed().as_millis() as i64,
1968 last_used: model.last_used.elapsed().as_millis() as i64,
1969 online_training_config: model
1970 .ttt_config
1971 .as_ref()
1972 .map(Self::ttt_config_to_wire)
1973 .unwrap_or_default(),
1974 generation_defaults: Self::sampling_params_to_wire(&model.generation_defaults),
1975 }];
1976 }
1977 }
1978 let pending = self.pending_loads.lock();
1979 if pending.contains_key(instance) {
1980 vec![GenModelStatusEntry {
1981 model_ref: instance.model_ref().to_owned(),
1982 status: "loading".to_owned(),
1983 reach: Vec::new(),
1984 loaded_at: 0,
1985 last_used: 0,
1986 online_training_config: GenOnlineTrainingConfig::default(),
1987 generation_defaults: GenGenerationDefaults::default(),
1988 }]
1989 } else {
1990 vec![]
1991 }
1992 }
1993
1994 async fn model_status(&self, instance: &InferenceInstanceId) -> ModelStatusResponse {
1996 let cache = self.loaded_models.read().await;
1997 if let Some(model) = cache.peek(instance) {
1998 ModelStatusResponse {
1999 loaded: true,
2000 reach: Self::model_reach(&model.network_transport),
2001 online_training_config: model
2002 .ttt_config
2003 .as_ref()
2004 .map(Self::ttt_config_to_wire)
2005 .unwrap_or_default(),
2006 ..Default::default()
2007 }
2008 } else {
2009 ModelStatusResponse {
2010 loaded: false,
2011 ..Default::default()
2012 }
2013 }
2014 }
2015 fn mint_internal_work_token(
2025 &self,
2026 instance: &InferenceInstanceId,
2027 ctx: &EnvelopeContext,
2028 scope: &hyprstream_rpc::auth::internal_work::InternalWorkScope,
2029 live_work_audience: &str,
2030 ) -> Result<String> {
2031 use hyprstream_rpc::auth::internal_work::{
2032 encode_internal_work, InternalWorkClaims, INTERNAL_WORK_ISSUER, MAX_LIFETIME_SECS,
2033 };
2034 let caller = ctx.claims().ok_or_else(|| {
2035 anyhow!("internal work requires a verified caller identity at Model ingress")
2036 })?;
2037 let subject = ctx.subject();
2038 anyhow::ensure!(
2039 !subject.is_anonymous(),
2040 "internal work requires an authenticated original subject"
2041 );
2042 let now = chrono::Utc::now().timestamp();
2043 let exp = caller.exp.min(now + MAX_LIFETIME_SECS);
2044 anyhow::ensure!(
2045 exp > now,
2046 "caller credential is expired; refusing to mint internal work"
2047 );
2048 let subject_string = subject.to_string();
2053 let mut caller_snapshot = caller.clone();
2054 caller_snapshot.sub = subject_string.clone();
2055 let claims = InternalWorkClaims {
2056 iss: INTERNAL_WORK_ISSUER.to_owned(),
2057 sub: subject_string,
2058 aud: live_work_audience.to_owned(),
2062 tenant: instance.tenant().to_owned(),
2063 model: instance.model_ref().to_owned(),
2064 resource: scope.resource.clone(),
2065 operation: scope.operation.clone(),
2066 iat: now,
2067 exp,
2068 jti: hex::encode(hyprstream_rpc::envelope::generate_nonce()),
2069 cnf: InternalWorkClaims::controller_cnf(&self.signing_key.verifying_key()),
2070 owner_did: ctx
2071 .authenticated_pairwise_did()
2072 .map(|d| d.as_str().to_owned()),
2073 caller: caller_snapshot,
2074 };
2075 Ok(encode_internal_work(&claims, &self.signing_key))
2076 }
2077
2078 pub(crate) fn inference_dispatch_resource(variant: &str) -> String {
2095 let mut chars = variant.chars();
2096 match chars.next() {
2097 Some(first) => {
2098 format!("inference:{}{}", first.to_ascii_uppercase(), chars.as_str())
2099 }
2100 None => "inference:".to_owned(),
2101 }
2102 }
2103
2104 fn inference_scope(
2107 variant: &str,
2108 operation: &str,
2109 ) -> hyprstream_rpc::auth::internal_work::InternalWorkScope {
2110 hyprstream_rpc::auth::internal_work::InternalWorkScope::new(
2111 Self::inference_dispatch_resource(variant),
2112 operation,
2113 )
2114 }
2115
2116 async fn get_inference_client(
2117 &self,
2118 model_ref_str: &str,
2119 ctx: &EnvelopeContext,
2120 work: hyprstream_rpc::auth::internal_work::InternalWorkScope,
2121 ) -> Result<InferenceClient> {
2122 let instance = Self::inference_instance(ctx, model_ref_str)?;
2123 let _endpoint = self.load_model(&instance, None, None).await?;
2124 let mut cache = self.loaded_models.write().await;
2125 let model = cache
2126 .get_mut(&instance)
2127 .ok_or_else(|| anyhow!("Model {} not found after loading", model_ref_str))?;
2128 if !model.service_handle.is_running() {
2133 cache.pop(&instance);
2134 anyhow::bail!(
2135 "pinned worker for model {model_ref_str} is not running; \
2136 the model must be loaded again before internal work"
2137 );
2138 }
2139 model.last_used = Instant::now();
2140 let live_work_audience = model.work_audience.clone();
2143 let live_generation = model.load_generation;
2144 let placement_key = ctx.subject().to_string();
2151 let relay_bearer = Self::remote_relay_bearer(ctx).map(ToOwned::to_owned);
2160 let routed = self
2161 .select_inference_routed(model, &placement_key, relay_bearer.as_deref())
2162 .await?;
2163 let work_token =
2164 self.mint_internal_work_token(&instance, ctx, &work, &live_work_audience)?;
2165 #[cfg(test)]
2166 self.capture_test_work_order(&work_token).await;
2167 tracing::debug!(
2168 generation = live_generation,
2169 "minted internal work order against live worker incarnation"
2170 );
2171 Ok(Self::attach_work_order(routed, work_token))
2172 }
2173
2174 fn attach_work_order(
2182 routed: RoutedReplica<InferenceClient>,
2183 work_token: String,
2184 ) -> InferenceClient {
2185 match routed.locality {
2186 ReplicaLocality::CoLocated => routed.value.with_bearer(work_token),
2187 ReplicaLocality::Remote => routed.value,
2188 }
2189 }
2190
2191 fn remote_relay_bearer(ctx: &EnvelopeContext) -> Option<&str> {
2192 if ctx.is_federate_admitted() {
2196 return None;
2197 }
2198 Self::remote_relay_bearer_for_claims(ctx.claims(), ctx.jwt_token())
2199 }
2200
2201 fn remote_relay_bearer_for_claims<'a>(
2202 claims: Option<&hyprstream_rpc::auth::Claims>,
2203 bearer: Option<&'a str>,
2204 ) -> Option<&'a str> {
2205 if claims.is_some_and(|claims| claims.sub.starts_with("service:")) {
2206 None
2207 } else {
2208 bearer
2209 }
2210 }
2211
2212 async fn load_lora(
2214 &self,
2215 model_ref_str: &str,
2216 ctx: &EnvelopeContext,
2217 path: &str,
2218 ) -> Result<()> {
2219 let client = self
2223 .get_inference_client(
2224 model_ref_str,
2225 ctx,
2226 hyprstream_rpc::auth::internal_work::InternalWorkScope::new(
2227 format!("inference:{path}"),
2228 "write",
2229 ),
2230 )
2231 .await?;
2232 client.load_lora(path).await
2233 }
2234
2235 async fn unload_lora(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2237 let client = self
2238 .get_inference_client(
2239 model_ref_str,
2240 ctx,
2241 Self::inference_scope("unloadLora", "write"),
2242 )
2243 .await?;
2244 client.unload_lora().await
2245 }
2246
2247 async fn has_lora(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<bool> {
2249 let client = self
2250 .get_inference_client(
2251 model_ref_str,
2252 ctx,
2253 Self::inference_scope("hasLora", "query"),
2254 )
2255 .await?;
2256 client.has_lora().await
2257 }
2258
2259 async fn writeback_adaptation(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2261 let client = self
2262 .get_inference_client(
2263 model_ref_str,
2264 ctx,
2265 Self::inference_scope("tttWriteback", "train"),
2266 )
2267 .await?;
2268 client.ttt_writeback().await
2269 }
2270
2271 async fn evict_adaptation(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2272 let client = self
2273 .get_inference_client(
2274 model_ref_str,
2275 ctx,
2276 Self::inference_scope("tttEvict", "train"),
2277 )
2278 .await?;
2279 client.ttt_evict().await
2280 }
2281
2282 async fn zero_delta(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2283 let client = self
2284 .get_inference_client(
2285 model_ref_str,
2286 ctx,
2287 Self::inference_scope("tttZero", "manage"),
2288 )
2289 .await?;
2290 client.ttt_zero().await
2291 }
2292
2293 async fn get_delta_status_forward(
2294 &self,
2295 model_ref_str: &str,
2296 ctx: &EnvelopeContext,
2297 ) -> Result<hyprstream_rpc_std::inference_client::DeltaStatusResult> {
2298 let client = self
2299 .get_inference_client(
2300 model_ref_str,
2301 ctx,
2302 Self::inference_scope("getDeltaStatus", "query"),
2303 )
2304 .await?;
2305 client.get_delta_status().await
2306 }
2307
2308 async fn snapshot_delta_forward(
2309 &self,
2310 model_ref_str: &str,
2311 ctx: &EnvelopeContext,
2312 ) -> Result<hyprstream_rpc_std::inference_client::SnapshotDeltaResult> {
2313 let client = self
2314 .get_inference_client(
2315 model_ref_str,
2316 ctx,
2317 Self::inference_scope("snapshotDelta", "write"),
2318 )
2319 .await?;
2320 client.snapshot_delta().await
2321 }
2322
2323 async fn export_peft_adapter_forward(
2324 &self,
2325 model_ref_str: &str,
2326 ctx: &EnvelopeContext,
2327 data: &ExportPeftRequest,
2328 ) -> Result<ExportPeftResult> {
2329 let client = self
2330 .get_inference_client(
2331 model_ref_str,
2332 ctx,
2333 Self::inference_scope("exportPeftAdapter", "write"),
2334 )
2335 .await?;
2336 client.export_peft_adapter(data).await
2337 }
2338}
2339
2340use crate::services::generated::model_client::{
2345 dispatch_model, serialize_response, AdapterHandler, InferHandler, ModelHandler, TttHandler,
2346};
2347use hyprstream_rpc_std::model_client::{
2348 AdapterInfo,
2350 ChatTemplateRequest,
2351 DeltaStatusResult,
2352 EmbedImagesRequest,
2353 EmbedImagesResponse,
2354 ErrorInfo,
2355 ExportPeftRequest,
2356 ExportPeftResult,
2357 GenerationDefaults as GenGenerationDefaults,
2358 GenerationRequest,
2360 LoadModelRequest,
2362 LoadedModelResponse,
2363 LoraConfig,
2365 MergeLoraRequest,
2366 ModelHealthStatus,
2367 ModelResponseVariant,
2368 ModelStatusEntry as GenModelStatusEntry,
2369 ModelStatusResponse,
2370 OnlineTrainingConfig as GenOnlineTrainingConfig,
2371 SaveAdaptationRequest,
2372 SaveAdaptationResult,
2373 SnapshotDeltaResult,
2374 StatusRequest,
2375 TrainStepRequest,
2376 TrainStepResult,
2377 UnloadModelRequest,
2378 WriteTttConfigRequest,
2379};
2380#[async_trait::async_trait(?Send)]
2384impl TttHandler for ModelService {
2385 async fn handle_init(
2386 &self,
2387 ctx: &EnvelopeContext,
2388 _request_id: u64,
2389 model_ref: &str,
2390 data: &LoraConfig,
2391 ) -> Result<()> {
2392 let client = self
2393 .get_inference_client(model_ref, ctx, Self::inference_scope("createLora", "write"))
2394 .await?;
2395 client.create_lora(data).await
2396 }
2397
2398 async fn handle_train(
2399 &self,
2400 ctx: &EnvelopeContext,
2401 _request_id: u64,
2402 model_ref: &str,
2403 data: &TrainStepRequest,
2404 ) -> Result<TrainStepResult> {
2405 let client = self
2406 .get_inference_client(model_ref, ctx, Self::inference_scope("trainStep", "train"))
2407 .await?;
2408 client.train_step(data).await
2409 }
2410
2411 async fn handle_train_stream(
2412 &self,
2413 ctx: &EnvelopeContext,
2414 _request_id: u64,
2415 model_ref: &str,
2416 data: &TrainStepRequest,
2417 ) -> Result<(
2418 hyprstream_rpc_std::model_client::StreamInfo,
2419 hyprstream_rpc::service::Continuation,
2420 )> {
2421 let client = self
2422 .get_inference_client(
2423 model_ref,
2424 ctx,
2425 Self::inference_scope("trainStepStream", "train"),
2426 )
2427 .await?;
2428 let ephemeral_pubkey = ctx.ephemeral_pubkey().ok_or_else(|| {
2429 anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
2430 })?;
2431 let stream_info = client.train_step_stream(data, ephemeral_pubkey).await?;
2432 Ok((stream_info, Box::pin(async {})))
2433 }
2434
2435 async fn handle_writeback(
2436 &self,
2437 ctx: &EnvelopeContext,
2438 _request_id: u64,
2439 model_ref: &str,
2440 ) -> Result<()> {
2441 self.writeback_adaptation(model_ref, ctx).await
2442 }
2443
2444 async fn handle_evict(
2445 &self,
2446 ctx: &EnvelopeContext,
2447 _request_id: u64,
2448 model_ref: &str,
2449 ) -> Result<()> {
2450 self.evict_adaptation(model_ref, ctx).await
2451 }
2452
2453 async fn handle_zero(
2454 &self,
2455 ctx: &EnvelopeContext,
2456 _request_id: u64,
2457 model_ref: &str,
2458 ) -> Result<()> {
2459 self.zero_delta(model_ref, ctx).await
2460 }
2461
2462 async fn handle_status(
2463 &self,
2464 ctx: &EnvelopeContext,
2465 _request_id: u64,
2466 model_ref: &str,
2467 ) -> Result<DeltaStatusResult> {
2468 self.get_delta_status_forward(model_ref, ctx).await
2469 }
2470
2471 async fn handle_save(
2472 &self,
2473 ctx: &EnvelopeContext,
2474 _request_id: u64,
2475 model_ref: &str,
2476 data: &SaveAdaptationRequest,
2477 ) -> Result<SaveAdaptationResult> {
2478 let client = self
2479 .get_inference_client(
2480 model_ref,
2481 ctx,
2482 Self::inference_scope("saveAdaptation", "write"),
2483 )
2484 .await?;
2485 client.save_adaptation(data).await
2486 }
2487
2488 async fn handle_snapshot(
2489 &self,
2490 ctx: &EnvelopeContext,
2491 _request_id: u64,
2492 model_ref: &str,
2493 ) -> Result<SnapshotDeltaResult> {
2494 self.snapshot_delta_forward(model_ref, ctx).await
2495 }
2496
2497 async fn handle_export(
2498 &self,
2499 ctx: &EnvelopeContext,
2500 _request_id: u64,
2501 model_ref: &str,
2502 data: &ExportPeftRequest,
2503 ) -> Result<ExportPeftResult> {
2504 self.export_peft_adapter_forward(model_ref, ctx, data).await
2505 }
2506
2507 async fn handle_write_ttt_config(
2508 &self,
2509 ctx: &EnvelopeContext,
2510 _request_id: u64,
2511 model_ref: &str,
2512 data: &WriteTttConfigRequest,
2513 ) -> Result<()> {
2514 let parsed = self.resolve_model_ref(model_ref).await?;
2516 let tracked = self
2517 .registry
2518 .get_by_name(parsed.name())
2519 .await
2520 .map_err(|e| anyhow!("Model '{}' not found in registry: {}", parsed.name(), e))?;
2521 let repo_client = self.registry.repo(&tracked.id);
2522
2523 let branch_name = match &parsed.git_ref {
2524 crate::storage::GitRef::Branch(name) => name.clone(),
2525 _ => repo_client
2526 .get_head()
2527 .await
2528 .unwrap_or_else(|_| "main".to_owned()),
2529 };
2530
2531 let storage_paths = crate::storage::StoragePaths::new()?;
2532 let model_path = storage_paths.worktree_path(parsed.name(), &branch_name)?;
2533
2534 if !model_path.exists() {
2535 return Err(anyhow!("Model worktree not found for {}", model_ref));
2536 }
2537
2538 let ttt_config = crate::config::TTTTrainingConfig {
2540 learning_rate: if data.learning_rate > 0.0 {
2541 data.learning_rate
2542 } else {
2543 3e-4
2544 },
2545 gradient_steps: if data.gradient_steps > 0 {
2546 data.gradient_steps
2547 } else {
2548 3
2549 },
2550 max_grad_norm: if data.max_grad_norm > 0.0 {
2551 data.max_grad_norm
2552 } else {
2553 1.0
2554 },
2555 min_input_length: if data.min_input_length > 0 {
2556 data.min_input_length
2557 } else {
2558 32
2559 },
2560 max_ttt_context: if data.max_ttt_context > 0 {
2561 data.max_ttt_context
2562 } else {
2563 512
2564 },
2565 rank_oracle: None,
2566 gradient_gating: None,
2567 };
2568
2569 let training_config = crate::config::HyprstreamTrainingConfig {
2570 mode: crate::config::TrainingMode::TestTimeTraining,
2571 ttt: ttt_config,
2572 lora_rank: if data.lora_rank > 0 {
2573 data.lora_rank as usize
2574 } else {
2575 crate::config::default_lora_rank()
2576 },
2577 lora_alpha: if data.lora_alpha > 0.0 {
2578 Some(data.lora_alpha)
2579 } else {
2580 None
2581 },
2582 target_modules: if data.target_modules.is_empty() {
2583 crate::config::default_target_modules()
2584 } else {
2585 data.target_modules.clone()
2586 },
2587 ..Default::default()
2588 };
2589
2590 crate::runtime::model_config::ModelConfig::save_training_config(
2592 &model_path,
2593 &training_config,
2594 )?;
2595
2596 let wt = repo_client.worktree(&branch_name);
2598 wt.stage_files(&StageFilesRequest {
2599 files: vec!["config.json".to_owned()],
2600 })
2601 .await?;
2602 wt.commit_with_author(&CommitWithAuthorRequest {
2603 message: "Update hyprstream_training config via RPC".to_owned(),
2604 author_name: "hyprstream".to_owned(),
2605 author_email: "noreply@hyprstream.dev".to_owned(),
2606 })
2607 .await?;
2608
2609 info!("TTT config written for {}", model_ref);
2610
2611 if data.auto_reload {
2613 let instance = Self::inference_instance(ctx, model_ref)?;
2614 let is_loaded = {
2615 let cache = self.loaded_models.read().await;
2616 cache.contains(&instance)
2617 };
2618 if is_loaded {
2619 info!("Auto-reloading {} after TTT config change", model_ref);
2620 self.unload_model(&instance).await?;
2621 self.load_model(&instance, None, None).await?;
2622 }
2623 }
2624
2625 Ok(())
2626 }
2627}
2628
2629#[async_trait::async_trait(?Send)]
2630impl AdapterHandler for ModelService {
2631 async fn handle_load(
2632 &self,
2633 ctx: &EnvelopeContext,
2634 _request_id: u64,
2635 model_ref: &str,
2636 value: &str,
2637 ) -> Result<()> {
2638 self.load_lora(model_ref, ctx, value).await
2639 }
2640
2641 async fn handle_unload(
2642 &self,
2643 ctx: &EnvelopeContext,
2644 _request_id: u64,
2645 model_ref: &str,
2646 ) -> Result<()> {
2647 self.unload_lora(model_ref, ctx).await
2648 }
2649
2650 async fn handle_status(
2651 &self,
2652 ctx: &EnvelopeContext,
2653 _request_id: u64,
2654 model_ref: &str,
2655 ) -> Result<bool> {
2656 self.has_lora(model_ref, ctx).await
2657 }
2658
2659 async fn handle_inspect(
2660 &self,
2661 _ctx: &EnvelopeContext,
2662 _request_id: u64,
2663 model_ref: &str,
2664 value: &str,
2665 ) -> Result<AdapterInfo> {
2666 let parsed = self.resolve_model_ref(model_ref).await?;
2668 let tracked = self
2669 .registry
2670 .get_by_name(parsed.name())
2671 .await
2672 .map_err(|e| anyhow!("Model '{}' not found in registry: {}", parsed.name(), e))?;
2673 let repo_client = self.registry.repo(&tracked.id);
2674 let branch_name = match &parsed.git_ref {
2675 crate::storage::GitRef::Branch(name) => name.clone(),
2676 _ => repo_client
2677 .get_head()
2678 .await
2679 .unwrap_or_else(|_| "main".to_owned()),
2680 };
2681 let fs = repo_client.worktree(&branch_name);
2682
2683 let config_path = format!("{}/adapter_config.json", value);
2685 let config_bytes = fs
2686 .read_file_chunked(&config_path)
2687 .await
2688 .map_err(|e| anyhow!("Failed to read {}: {}", config_path, e))?;
2689 let config_json: serde_json::Value = serde_json::from_slice(&config_bytes)
2690 .map_err(|e| anyhow!("Failed to parse adapter_config.json: {}", e))?;
2691
2692 let model_path = format!("{}/adapter_model.safetensors", value);
2694 let stat = fs
2695 .stat_path(&model_path)
2696 .await
2697 .map_err(|e| anyhow!("Failed to stat {}: {}", model_path, e))?;
2698 if !stat.exists {
2699 anyhow::bail!("adapter_model.safetensors not found in {}", value);
2700 }
2701
2702 let rank = config_json
2704 .get("r")
2705 .and_then(serde_json::Value::as_u64)
2706 .unwrap_or(0) as u32;
2707 let lora_alpha = config_json
2708 .get("lora_alpha")
2709 .and_then(serde_json::Value::as_f64)
2710 .unwrap_or(0.0) as f32;
2711 let target_modules = config_json
2712 .get("target_modules")
2713 .and_then(|v| v.as_array())
2714 .map(|arr| {
2715 arr.iter()
2716 .filter_map(|v| v.as_str().map(String::from))
2717 .collect()
2718 })
2719 .unwrap_or_default();
2720 let base_model = config_json
2721 .get("base_model_name_or_path")
2722 .and_then(|v| v.as_str())
2723 .unwrap_or("")
2724 .to_owned();
2725
2726 let name = value.rsplit('/').next().unwrap_or(value).to_owned();
2728
2729 Ok(AdapterInfo {
2730 name,
2731 path: value.to_owned(),
2732 rank,
2733 lora_alpha,
2734 target_modules,
2735 base_model,
2736 })
2737 }
2738
2739 async fn handle_merge(
2740 &self,
2741 ctx: &EnvelopeContext,
2742 _request_id: u64,
2743 model_ref: &str,
2744 data: &MergeLoraRequest,
2745 ) -> Result<()> {
2746 let client = self
2747 .get_inference_client(model_ref, ctx, Self::inference_scope("mergeLora", "write"))
2748 .await?;
2749 client.merge_lora(data).await
2750 }
2751}
2752
2753#[async_trait::async_trait(?Send)]
2754impl InferHandler for ModelService {
2755 async fn handle_generate_stream(
2756 &self,
2757 ctx: &EnvelopeContext,
2758 _request_id: u64,
2759 model_ref: &str,
2760 data: &GenerationRequest,
2761 ) -> Result<(
2762 hyprstream_rpc_std::model_client::StreamInfo,
2763 hyprstream_rpc::service::Continuation,
2764 )> {
2765 let client = self
2766 .get_inference_client(
2767 model_ref,
2768 ctx,
2769 Self::inference_scope("generateStream", "infer"),
2770 )
2771 .await?;
2772 let ephemeral_pubkey = ctx.ephemeral_pubkey().ok_or_else(|| {
2773 anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
2774 })?;
2775 let stream_info = client.generate_stream(data, ephemeral_pubkey).await?;
2776 Ok((stream_info, Box::pin(async {})))
2777 }
2778
2779 async fn handle_apply_chat_template(
2780 &self,
2781 ctx: &EnvelopeContext,
2782 _request_id: u64,
2783 model_ref: &str,
2784 data: &ChatTemplateRequest,
2785 ) -> Result<String> {
2786 let client = self
2787 .get_inference_client(
2788 model_ref,
2789 ctx,
2790 Self::inference_scope("applyChatTemplate", "query"),
2791 )
2792 .await?;
2793 client.apply_chat_template(data).await
2794 }
2795
2796 async fn handle_embed(
2797 &self,
2798 ctx: &EnvelopeContext,
2799 _request_id: u64,
2800 model_ref: &str,
2801 data: &EmbedImagesRequest,
2802 ) -> Result<EmbedImagesResponse> {
2803 let client = self
2804 .get_inference_client(model_ref, ctx, Self::inference_scope("embed", "infer"))
2805 .await?;
2806 client.embed(data).await
2807 }
2808
2809 async fn handle_status(
2810 &self,
2811 ctx: &EnvelopeContext,
2812 _request_id: u64,
2813 model_ref: &str,
2814 ) -> Result<ModelStatusResponse> {
2815 let instance = Self::inference_instance(ctx, model_ref)?;
2816 Ok(self.model_status(&instance).await)
2817 }
2818}
2819
2820#[async_trait::async_trait(?Send)]
2821impl ModelHandler for ModelService {
2822 async fn authorize(
2823 &self,
2824 ctx: &EnvelopeContext,
2825 resource: &str,
2826 operation: &str,
2827 ) -> Result<()> {
2828 let subject = ctx.subject();
2829 let domain = ctx.domain()?;
2830 let object_label = crate::services::inference::inference_object_label();
2831 crate::services::inference::enforce_inference_mac(ctx, &object_label)?;
2832 let request = PolicyCheck {
2833 subject: subject.to_string(),
2834 domain,
2835 resource: resource.to_owned(),
2836 operation: operation.to_owned(),
2837 };
2838 if ctx.is_federate_admitted() {
2839 anyhow::ensure!(
2840 ctx.admitted_federate_operation(resource, operation),
2841 "Federate request-local operation mismatch"
2842 );
2843 return Ok(());
2848 }
2849 let allowed = crate::services::policy::check_with_holder_evidence(
2850 &self.policy_client,
2851 &request,
2852 ctx.jwt_token(),
2853 &ctx.subject(),
2854 ctx.original_holder_evidence(),
2855 )
2856 .await
2857 .unwrap_or_else(|e| {
2858 warn!(
2859 "Policy check failed for {} on {}: {} - denying access",
2860 subject, resource, e
2861 );
2862 false
2863 });
2864 if allowed {
2865 Ok(())
2866 } else {
2867 anyhow::bail!(
2868 "Unauthorized: {} cannot {} on {}",
2869 subject,
2870 operation,
2871 resource
2872 )
2873 }
2874 }
2875
2876 async fn handle_load(
2877 &self,
2878 ctx: &EnvelopeContext,
2879 _request_id: u64,
2880 data: &LoadModelRequest,
2881 ) -> Result<ModelResponseVariant> {
2882 let max_ctx = data.max_context.filter(|&n| n != 0);
2883 let kv_q = data.kv_quant.filter(|q| *q != KVQuantType::None);
2884 let model_ref = &data.model_ref;
2885 let instance = Self::inference_instance(ctx, model_ref)?;
2886 match self.load_model(&instance, max_ctx, kv_q).await {
2887 Ok(_endpoint) => {
2888 let cache = self.loaded_models.read().await;
2889 let model = cache
2890 .peek(&instance)
2891 .ok_or_else(|| anyhow!("model missing after successful load"))?;
2892 Ok(ModelResponseVariant::LoadResult(LoadedModelResponse {
2893 model_ref: model_ref.to_owned(),
2894 reach: Self::model_reach(&model.network_transport),
2895 }))
2896 }
2897 Err(e) => Ok(ModelResponseVariant::Error(ErrorInfo {
2898 message: format!("Failed to load model: {e}"),
2899 code: "LOAD_FAILED".into(),
2900 details: String::new(),
2901 })),
2902 }
2903 }
2904
2905 async fn handle_unload(
2906 &self,
2907 ctx: &EnvelopeContext,
2908 _request_id: u64,
2909 data: &UnloadModelRequest,
2910 ) -> Result<ModelResponseVariant> {
2911 let model_ref = &data.model_ref;
2912 let instance = Self::inference_instance(ctx, model_ref)?;
2913 match self.unload_model(&instance).await {
2914 Ok(()) => Ok(ModelResponseVariant::UnloadResult),
2915 Err(e) => Ok(ModelResponseVariant::Error(ErrorInfo {
2916 message: format!("Failed to unload model: {e}"),
2917 code: "UNLOAD_FAILED".into(),
2918 details: String::new(),
2919 })),
2920 }
2921 }
2922
2923 async fn handle_status(
2924 &self,
2925 ctx: &EnvelopeContext,
2926 _request_id: u64,
2927 data: &StatusRequest,
2928 ) -> Result<ModelResponseVariant> {
2929 let tenant = ctx.domain()?;
2930 let entries = if data.model_ref.is_empty() {
2931 self.model_status_all(&tenant).await
2932 } else {
2933 let instance = InferenceInstanceId::new(&tenant, &data.model_ref, 0)?;
2934 self.model_status_single(&instance).await
2935 };
2936 Ok(ModelResponseVariant::StatusResult(entries))
2937 }
2938
2939 async fn handle_health_check(
2940 &self,
2941 ctx: &EnvelopeContext,
2942 _request_id: u64,
2943 ) -> Result<ModelResponseVariant> {
2944 let tenant = ctx.domain()?;
2945 let cache = self.loaded_models.read().await;
2946 let loaded_count = cache
2947 .iter()
2948 .filter(|(instance, _)| instance.tenant() == tenant.as_str())
2949 .count() as u32;
2950 let max_models = self.config.max_models as u32;
2951 drop(cache);
2952 Ok(ModelResponseVariant::HealthCheckResult(ModelHealthStatus {
2953 status: "healthy".into(),
2954 loaded_model_count: loaded_count,
2955 max_models,
2956 total_memory_bytes: 0,
2957 }))
2958 }
2959}
2960
2961use crate::services::fs::{SyntheticNode, SyntheticQid, SyntheticTree};
2966use crate::services::generated::model_client::FsHandler;
2967use hyprstream_rpc_std::model_client::{
2968 NpClunk, NpCreate, NpOpen, NpRead, NpRemove, NpStat as GenNpStat, NpStatReq, NpWalk, NpWrite,
2969 Qid as GenQid, ROpen, RRead, RStat, RWalk, RWrite,
2970};
2971use hyprstream_vfs::DirEntry;
2972
2973impl ModelService {
2974 fn fs_tree(&self, verified_tenant: &str) -> Arc<SyntheticTree> {
2976 if let Some(tree) = self.inner.fs_trees.get(verified_tenant) {
2977 return Arc::clone(tree.value());
2978 }
2979 let tree = Arc::new(self.build_fs_tree(verified_tenant.to_owned()));
2980 let entry = self
2981 .inner
2982 .fs_trees
2983 .entry(verified_tenant.to_owned())
2984 .or_insert(tree);
2985 Arc::clone(entry.value())
2986 }
2987
2988 fn build_fs_tree(&self, verified_tenant: String) -> SyntheticTree {
2990 let inner_list = Arc::clone(&self.inner);
2991 let inner_resolve = Arc::clone(&self.inner);
2992 let list_tenant = verified_tenant.clone();
2993 let resolve_tenant = verified_tenant;
2994
2995 SyntheticTree::new(SyntheticNode::DynamicDir {
2996 list: Box::new(move || {
2997 let cache = inner_list.loaded_models.blocking_read();
2998 let pending = inner_list.pending_loads.lock();
2999 let mut entries: Vec<DirEntry> = cache
3000 .iter()
3001 .filter(|(instance, _)| instance.tenant() == list_tenant)
3002 .map(|(instance, _)| DirEntry {
3003 name: instance.service_name(),
3004 is_dir: true,
3005 size: 0,
3006 stat: None,
3007 })
3008 .collect();
3009 for instance in pending
3010 .keys()
3011 .filter(|instance| instance.tenant() == list_tenant)
3012 {
3013 if !cache.contains(instance) {
3014 entries.push(DirEntry {
3015 name: instance.service_name(),
3016 is_dir: true,
3017 size: 0,
3018 stat: None,
3019 });
3020 }
3021 }
3022 entries
3023 }),
3024 resolve: Box::new(move |ref_name| {
3025 let cache = inner_resolve.loaded_models.blocking_read();
3026 let model = cache
3027 .iter()
3028 .find(|(instance, _)| {
3029 instance.tenant() == resolve_tenant && instance.service_name() == ref_name
3030 })
3031 .map(|(_, model)| model);
3032 let is_loaded = model.is_some();
3033 let defaults_json = if let Some(model) = model {
3034 serde_json::to_vec_pretty(&model.generation_defaults).unwrap_or_default()
3035 } else {
3036 b"{}".to_vec()
3037 };
3038 let status_str = if is_loaded { "loaded" } else { "unloaded" };
3039
3040 let mut children = std::collections::HashMap::new();
3041 let status_owned = status_str.to_owned();
3042 children.insert(
3043 "status".to_owned(),
3044 SyntheticNode::ReadFile(Box::new(move || {
3045 format!("{status_owned}\n").into_bytes()
3046 })),
3047 );
3048 children.insert(
3049 "defaults".to_owned(),
3050 SyntheticNode::ReadFile(Box::new(move || defaults_json.clone())),
3051 );
3052 children.insert(
3053 "ctl".to_owned(),
3054 SyntheticNode::CtlFile {
3055 handler: Box::new(|data, _subject| {
3056 let cmd = String::from_utf8_lossy(data).trim().to_owned();
3057 Ok(format!("ctl: {cmd}\n").into_bytes())
3058 }),
3059 },
3060 );
3061 Some(SyntheticNode::Dir { children })
3062 }),
3063 })
3064 }
3065
3066 fn qid_to_gen(qid: &SyntheticQid) -> GenQid {
3067 GenQid {
3068 qtype: qid.qtype,
3069 version: qid.version,
3070 path: qid.path,
3071 }
3072 }
3073}
3074
3075#[async_trait::async_trait(?Send)]
3076impl FsHandler for ModelService {
3077 async fn handle_walk(
3078 &self,
3079 ctx: &EnvelopeContext,
3080 _request_id: u64,
3081 _model_ref: &str,
3082 data: &NpWalk,
3083 ) -> Result<RWalk> {
3084 let tree = self.fs_tree(&ctx.domain()?);
3085 let owner = ctx.subject().to_string();
3086 let (fid, qid) = tree
3087 .walk(&data.wnames, &owner, Some(data.newfid))
3088 .map_err(|e| anyhow::anyhow!(e))?;
3089 Ok(RWalk {
3090 qid: Self::qid_to_gen(&qid),
3091 fid,
3092 })
3093 }
3094
3095 async fn handle_open(
3096 &self,
3097 ctx: &EnvelopeContext,
3098 _request_id: u64,
3099 _model_ref: &str,
3100 data: &NpOpen,
3101 ) -> Result<ROpen> {
3102 let tree = self.fs_tree(&ctx.domain()?);
3103 let owner = ctx.subject().to_string();
3104 let (qid, iounit) = tree
3106 .open(data.fid, data.mode, &owner)
3107 .map_err(|e| anyhow::anyhow!(e))?;
3108 Ok(ROpen {
3109 qid: Self::qid_to_gen(&qid),
3110 iounit,
3111 })
3112 }
3113
3114 async fn handle_read(
3115 &self,
3116 ctx: &EnvelopeContext,
3117 _request_id: u64,
3118 _model_ref: &str,
3119 data: &NpRead,
3120 ) -> Result<RRead> {
3121 let tree = self.fs_tree(&ctx.domain()?);
3122 let owner = ctx.subject().to_string();
3123 let bytes = tree
3124 .read(data.fid, data.offset, data.count, &owner)
3125 .map_err(|e| anyhow::anyhow!(e))?;
3126 Ok(RRead { data: bytes })
3127 }
3128
3129 async fn handle_write(
3130 &self,
3131 ctx: &EnvelopeContext,
3132 _request_id: u64,
3133 _model_ref: &str,
3134 data: &NpWrite,
3135 ) -> Result<RWrite> {
3136 let tree = self.fs_tree(&ctx.domain()?);
3137 let subject = ctx.subject();
3138 let owner = subject.to_string();
3139 let count = tree
3140 .write(data.fid, data.offset, &data.data, &owner, &subject)
3141 .map_err(|e| anyhow::anyhow!(e))?;
3142 Ok(RWrite { count })
3143 }
3144
3145 async fn handle_clunk(
3146 &self,
3147 ctx: &EnvelopeContext,
3148 _request_id: u64,
3149 _model_ref: &str,
3150 data: &NpClunk,
3151 ) -> Result<()> {
3152 let tree = self.fs_tree(&ctx.domain()?);
3153 let owner = ctx.subject().to_string();
3154 tree.clunk(data.fid, &owner);
3155 Ok(())
3156 }
3157
3158 async fn handle_stat(
3159 &self,
3160 ctx: &EnvelopeContext,
3161 _request_id: u64,
3162 _model_ref: &str,
3163 data: &NpStatReq,
3164 ) -> Result<RStat> {
3165 let tree = self.fs_tree(&ctx.domain()?);
3166 let owner = ctx.subject().to_string();
3167 let (qid, name) = tree
3168 .stat(data.fid, &owner)
3169 .map_err(|e| anyhow::anyhow!(e))?;
3170 Ok(RStat {
3171 stat: GenNpStat {
3172 qid: Self::qid_to_gen(&qid),
3173 mode: if qid.qtype & 0x80 != 0 {
3174 0o040755
3175 } else {
3176 0o100644
3177 },
3178 atime: 0,
3179 mtime: 0,
3180 length: 0,
3181 name,
3182 uid: String::new(),
3183 gid: String::new(),
3184 muid: String::new(),
3185 },
3186 })
3187 }
3188
3189 async fn handle_create(
3190 &self,
3191 _ctx: &EnvelopeContext,
3192 _request_id: u64,
3193 _model_ref: &str,
3194 _data: &NpCreate,
3195 ) -> Result<ROpen> {
3196 anyhow::bail!("create not supported on model fs")
3197 }
3198
3199 async fn handle_remove(
3200 &self,
3201 _ctx: &EnvelopeContext,
3202 _request_id: u64,
3203 _model_ref: &str,
3204 _data: &NpRemove,
3205 ) -> Result<()> {
3206 anyhow::bail!("remove not supported on model fs")
3207 }
3208}
3209
3210struct ParsedLoadRequest {
3216 model_ref: String,
3217 max_context: u32,
3218 kv_quant: hyprstream_rpc_std::model_capnp::KVQuantType,
3219}
3220
3221impl ParsedLoadRequest {
3222 fn to_load_params(&self) -> (Option<u32>, Option<KVQuantType>) {
3223 use hyprstream_rpc_std::model_capnp::KVQuantType as CKV;
3224 let max_ctx = match self.max_context {
3225 0 => None,
3226 n => Some(n),
3227 };
3228 let kv_q = match self.kv_quant {
3229 CKV::Int8 => Some(KVQuantType::Int8),
3230 CKV::Nf4 => Some(KVQuantType::Nf4),
3231 CKV::Fp4 => Some(KVQuantType::Fp4),
3232 CKV::None => None,
3233 };
3234 (max_ctx, kv_q)
3235 }
3236}
3237
3238impl ModelService {
3239 fn try_parse_load_request(
3245 body: &hyprstream_rpc::service::DecodedRequestBody,
3246 ) -> Option<(u64, ParsedLoadRequest)> {
3247 use hyprstream_rpc::optional_capnp::option_uint32;
3248 use hyprstream_rpc_std::model_capnp::model_request;
3249 use hyprstream_rpc_std::model_capnp::option_k_v_quant_type;
3250 use hyprstream_rpc_std::model_capnp::KVQuantType as CKV;
3251 let req = body.root::<model_request::Reader>().ok()?;
3252 let request_id = req.get_id();
3253 match req.which().ok()? {
3254 model_request::Which::Load(data) => {
3255 let data = data.ok()?;
3256 let model_ref = data.get_model_ref().ok()?.to_str().ok()?.to_owned();
3257 let max_context = match data.get_max_context().ok()?.which().ok()? {
3258 option_uint32::Which::None(()) => 0u32,
3259 option_uint32::Which::Some(v) => v,
3260 };
3261 let kv_quant = match data.get_kv_quant().ok()?.which().ok()? {
3262 option_k_v_quant_type::Which::None(()) => CKV::None,
3263 option_k_v_quant_type::Which::Some(v) => v.unwrap_or(CKV::None),
3264 };
3265 Some((
3266 request_id,
3267 ParsedLoadRequest {
3268 model_ref,
3269 max_context,
3270 kv_quant,
3271 },
3272 ))
3273 }
3274 _ => None,
3275 }
3276 }
3277}
3278
3279#[cfg(feature = "postgres")]
3288pub(crate) const FEDERATE_MODEL_REQUEST_ROOT_TYPE_ID: u64 =
3289 <hyprstream_rpc_std::model_capnp::model_request::Reader<'static> as capnp::traits::HasTypeId>::TYPE_ID;
3290
3291#[cfg(feature = "postgres")]
3292fn federate_model_operation(
3293 body: &hyprstream_rpc::service::DecodedRequestBody,
3294 pin: &StagingModelPin,
3295) -> Result<(String, String)> {
3296 use hyprstream_rpc_std::model_capnp::{infer_request, model_request};
3297 let request = body.root::<model_request::Reader>()?;
3298 let (model_ref, resource, operation) = match request.which()? {
3299 model_request::Which::Load(inner) => (
3300 inner?.get_model_ref()?.to_str()?.to_owned(),
3301 "model:Load".to_owned(),
3302 "write",
3303 ),
3304 model_request::Which::Status(inner) => (
3305 inner?.get_model_ref()?.to_str()?.to_owned(),
3306 "model:Status".to_owned(),
3307 "query",
3308 ),
3309 model_request::Which::Infer(inner) => {
3310 let infer = inner?;
3311 anyhow::ensure!(
3312 matches!(infer.which()?, infer_request::Which::GenerateStream(_)),
3313 "unsupported Federate inference method"
3314 );
3315 let model_ref = infer.get_model_ref()?.to_str()?.to_owned();
3316 (model_ref.clone(), format!("model:{model_ref}"), "infer")
3317 }
3318 _ => anyhow::bail!("unsupported Federate Model method"),
3319 };
3320 anyhow::ensure!(
3321 pin == &StagingModelPin::reviewed()? && pin.admits(&model_ref),
3322 "Federate Model request is not the reviewed pinned staging model"
3323 );
3324 Ok((resource, operation.to_owned()))
3325}
3326
3327#[async_trait(?Send)]
3328impl crate::services::RequestService for ModelService {
3329 fn accept_deferred_federate_credential(&self) -> bool {
3330 #[cfg(feature = "postgres")]
3331 {
3332 self.federate_dispatch.is_some()
3333 }
3334 #[cfg(not(feature = "postgres"))]
3335 {
3336 false
3337 }
3338 }
3339
3340 async fn admit_deferred_federate_request(
3341 &self,
3342 ctx: &EnvelopeContext,
3343 body: &hyprstream_rpc::service::DecodedRequestBody,
3344 proof: &hyprstream_rpc::proof::parser::ParsedProof,
3345 ) -> anyhow::Result<(
3346 hyprstream_rpc::proof::verify::VerifiedProof,
3347 [u8; 32],
3348 String,
3349 String,
3350 hyprstream_rpc::proof::admission::ProofReplayKey,
3351 u64,
3352 )> {
3353 #[cfg(feature = "postgres")]
3354 {
3355 let adapter = self
3356 .federate_dispatch
3357 .as_ref()
3358 .ok_or_else(|| anyhow!("Federate dispatch is disabled"))?;
3359 let pin =
3360 self.config.staging_model_pin.as_ref().ok_or_else(|| {
3361 anyhow!("Federate Model requires an immutable staging model pin")
3362 })?;
3363 let (resource, operation) = federate_model_operation(body, pin)?;
3364 let (verified, holder, replay_key, replay_expiry) = adapter
3365 .admit(
3366 ctx,
3367 proof,
3368 "model",
3369 FEDERATE_MODEL_REQUEST_ROOT_TYPE_ID,
3370 body.bytes(),
3371 )
3372 .await?;
3373 Ok((
3374 verified,
3375 holder,
3376 resource,
3377 operation,
3378 replay_key,
3379 replay_expiry,
3380 ))
3381 }
3382 #[cfg(not(feature = "postgres"))]
3383 {
3384 let _ = (ctx, body, proof);
3385 anyhow::bail!("Federate dispatch is disabled")
3386 }
3387 }
3388
3389 async fn commit_deferred_federate_request(
3390 &self,
3391 ctx: &EnvelopeContext,
3392 resource: &str,
3393 operation: &str,
3394 replay_key: &hyprstream_rpc::proof::admission::ProofReplayKey,
3395 replay_expiry: u64,
3396 ) -> anyhow::Result<()> {
3397 #[cfg(feature = "postgres")]
3398 {
3399 let adapter = self
3400 .federate_dispatch
3401 .as_ref()
3402 .ok_or_else(|| anyhow!("Federate dispatch is disabled"))?;
3403 adapter
3404 .commit_request_use(ctx, resource, operation, replay_key, replay_expiry)
3405 .await
3406 .map_err(anyhow::Error::new)
3407 }
3408 #[cfg(not(feature = "postgres"))]
3409 {
3410 let _ = (ctx, resource, operation, replay_key, replay_expiry);
3411 anyhow::bail!("Federate dispatch is disabled")
3412 }
3413 }
3414
3415 fn decode_request_body(
3416 &self,
3417 signed_body: &[u8],
3418 ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
3419 crate::services::generated::model_client::decode_model_request_body(signed_body)
3423 }
3424
3425 async fn handle_request(
3426 &self,
3427 ctx: &EnvelopeContext,
3428 body: &hyprstream_rpc::service::DecodedRequestBody,
3429 ) -> Result<(Vec<u8>, Option<crate::services::Continuation>)> {
3430 debug!(
3431 "Model request from {} (id={})",
3432 ctx.subject(),
3433 ctx.request_id
3434 );
3435
3436 if let Some((request_id, load_data)) = Self::try_parse_load_request(body) {
3442 let audit_resource = "model:Load";
3446 let authorization =
3447 <Self as ModelHandler>::authorize(self, ctx, audit_resource, "write").await;
3448 ctx.audit_authz(audit_resource, "write", authorization.is_ok());
3449 authorization?;
3450
3451 self.validate_model_load_ref(&load_data.model_ref)?;
3452 let instance = Self::inference_instance(ctx, &load_data.model_ref)?;
3453 self.validate_staging_model_checkout(&instance).await?;
3454
3455 let model_ref = load_data.model_ref.clone();
3458 let reservation = match self.reserve_intercepted_load(&instance).await? {
3459 InterceptedLoadAdmission::Loaded { reach } => {
3460 let response = serialize_response(
3461 request_id,
3462 &ModelResponseVariant::LoadResult(LoadedModelResponse { model_ref, reach }),
3463 )?;
3464 return Ok((response, None));
3465 }
3466 InterceptedLoadAdmission::Pending => {
3467 debug!("Model {} already being loaded, deduplicating", model_ref);
3468 let response = serialize_response(
3469 request_id,
3470 &ModelResponseVariant::LoadResult(LoadedModelResponse {
3471 model_ref,
3472 reach: Vec::new(),
3473 }),
3474 )?;
3475 return Ok((response, None));
3476 }
3477 InterceptedLoadAdmission::Accepted(reservation) => reservation,
3478 };
3479
3480 info!("Load request accepted for {} (async)", model_ref);
3486 let response_model_ref = model_ref.clone();
3487 let service = self.clone(); let (load_max_context, load_kv_quant) = load_data.to_load_params();
3489 let continuation: crate::services::Continuation = Box::pin(async move {
3490 let model_name = model_ref.split(':').next().unwrap_or(&model_ref);
3491 let scope = format!("serve:model:{}", model_name);
3492 match service
3493 .run_intercepted_load(&instance, load_max_context, load_kv_quant, reservation)
3494 .await
3495 {
3496 Ok(endpoint) => {
3497 info!("Model {} loaded successfully at {}", model_ref, endpoint);
3498 let event = crate::events::EventEnvelope::new(
3499 crate::events::EventSource::Model,
3500 scope.clone(),
3501 crate::events::EventPayload::ModelLoaded {
3502 model_ref: model_ref.clone(),
3503 endpoint,
3504 },
3505 );
3506 if let Ok(payload) = serde_json::to_vec(&event) {
3507 let _event_publish_gate =
3508 service.lifecycle_event_publish_gate.lock().await;
3509 let _ = service
3510 .event_publisher
3511 .publish("lifecycle", "loaded", &payload)
3512 .await;
3513 debug!("Published model.loaded event");
3514 }
3515 }
3516 Err(e) => {
3517 warn!("Model {} failed to load: {}", model_ref, e);
3518 let event = crate::events::EventEnvelope::new(
3519 crate::events::EventSource::Model,
3520 scope.clone(),
3521 crate::events::EventPayload::ModelFailed {
3522 model_ref: model_ref.clone(),
3523 error: e.to_string(),
3524 },
3525 );
3526 if let Ok(payload) = serde_json::to_vec(&event) {
3527 let _event_publish_gate =
3528 service.lifecycle_event_publish_gate.lock().await;
3529 let _ = service
3530 .event_publisher
3531 .publish("lifecycle", "failed", &payload)
3532 .await;
3533 }
3534 }
3535 }
3536 });
3537
3538 let response = serialize_response(
3539 request_id,
3540 &ModelResponseVariant::LoadResult(LoadedModelResponse {
3541 model_ref: response_model_ref,
3542 reach: Vec::new(),
3543 }),
3544 )?;
3545
3546 return Ok((response, Some(continuation)));
3547 }
3548
3549 dispatch_model(self, ctx, body).await
3550 }
3551
3552 fn name(&self) -> &str {
3553 "model"
3554 }
3555
3556 fn transport(&self) -> &TransportConfig {
3557 &self.transport
3558 }
3559
3560 fn signing_key(&self) -> SigningKey {
3561 self.signing_key.clone()
3562 }
3563
3564 fn expected_audience(&self) -> Option<&str> {
3565 self.expected_audience.as_deref()
3566 }
3567
3568 fn accept_delegated_bearer(&self, signer: &[u8; 32]) -> bool {
3569 let Ok(key) = hyprstream_rpc::prelude::VerifyingKey::from_bytes(signer) else {
3570 return false;
3571 };
3572 let trust = hyprstream_service::global_trust_store();
3573 (self.delegated_oai_audience.is_some() && trust.is_authorized(&key, "oai"))
3574 || (self.delegated_mcp_audience.is_some() && trust.is_authorized(&key, "mcp"))
3575 }
3576
3577 fn delegated_user_audience(
3578 &self,
3579 signer: &[u8; 32],
3580 untrusted_aud: Option<&str>,
3581 ) -> Option<&str> {
3582 let key = hyprstream_rpc::prelude::VerifyingKey::from_bytes(signer).ok()?;
3583 let trust = hyprstream_service::global_trust_store();
3584 if let Some(aud) = self.delegated_oai_audience.as_deref() {
3585 if untrusted_aud == Some(aud) && trust.is_authorized(&key, "oai") {
3586 return Some(aud);
3587 }
3588 }
3589 if let Some(aud) = self.delegated_mcp_audience.as_deref() {
3590 if untrusted_aud == Some(aud) && trust.is_authorized(&key, "mcp") {
3591 return Some(aud);
3592 }
3593 }
3594 self.expected_audience()
3595 }
3596
3597 fn accept_delegated_resource_leaf(
3598 &self,
3599 ctx: &EnvelopeContext,
3600 body: &hyprstream_rpc::service::DecodedRequestBody,
3601 ) -> bool {
3602 use hyprstream_rpc_std::model_capnp::{infer_request, model_request};
3603
3604 let Some(claims) = ctx.claims() else {
3605 return false;
3606 };
3607 let audience = claims.aud.as_deref();
3608 let mcp = audience.is_some() && self.delegated_mcp_audience.as_deref() == audience;
3609 let oai = audience.is_some() && self.delegated_oai_audience.as_deref() == audience;
3610 if !mcp && !oai {
3611 return false;
3612 }
3613 let Ok(request) = body.root::<model_request::Reader>() else {
3614 return false;
3615 };
3616 match (body.leaf_path(), request.which()) {
3617 (Some([2]), Ok(model_request::Which::Status(Ok(_)))) if mcp || oai => {
3618 claims.has_scope("read:model:Status") || claims.has_scope("read:model:*")
3619 }
3620 (
3621 Some([6, 0] | [6, 1] | [6, 2] | [6, 3]),
3622 Ok(model_request::Which::Infer(Ok(infer))),
3623 ) if oai => {
3624 let Ok(model_ref) = infer.get_model_ref() else {
3625 return false;
3626 };
3627 let Ok(model_ref) = model_ref.to_str() else {
3628 return false;
3629 };
3630 if model_ref.is_empty() {
3631 return false;
3632 }
3633 let has_scope = |operation| {
3634 claims.has_scope(&format!("{operation}:model:{model_ref}"))
3635 || claims.has_scope(&format!("{operation}:model:*"))
3636 };
3637 match (body.leaf_path(), infer.which()) {
3638 (Some([6, 0]), Ok(infer_request::Which::GenerateStream(_)))
3639 | (Some([6, 3]), Ok(infer_request::Which::Embed(_))) => has_scope("infer"),
3640 (Some([6, 1]), Ok(infer_request::Which::ApplyChatTemplate(_))) => {
3641 has_scope("read") || has_scope("infer")
3644 }
3645 (Some([6, 2]), Ok(infer_request::Which::Status(_))) => has_scope("read"),
3646 _ => false,
3647 }
3648 }
3649 _ => false,
3650 }
3651 }
3652
3653 fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
3654 self.inner.jwt_key_source.clone()
3655 }
3656
3657 fn producer_reach_config_handle(
3658 &self,
3659 ) -> Option<hyprstream_rpc::moq_stream::ProducerReachConfigHandle> {
3660 Some(Arc::clone(&self.producer_reach_config))
3661 }
3662
3663 fn moq_origin_handle(&self) -> Option<hyprstream_rpc::moq_stream::MoqStreamOriginHandle> {
3664 Some(Arc::clone(&self.moq_origin))
3665 }
3666
3667 fn build_error_payload(&self, request_id: u64, error: &str) -> Vec<u8> {
3668 let variant = ModelResponseVariant::Error(ErrorInfo {
3669 message: error.to_owned(),
3670 code: "INTERNAL".to_owned(),
3671 details: String::new(),
3672 });
3673 serialize_response(request_id, &variant).unwrap_or_default()
3674 }
3675}
3676
3677#[cfg(test)]
3690mod tests {
3691 #[test]
3692 fn clef_projection_contract_keeps_a0_default_and_rejects_rebinding() -> anyhow::Result<()> {
3693 let a0 = super::StagingModelPin::reviewed()?;
3694 assert_eq!(
3695 a0.capture_policy()?,
3696 git2db::pinned_disk::ProjectionPolicy::default()
3697 );
3698 let mut clef = super::StagingModelPin::reviewed_clef_flash(Default::default())?;
3699 assert_ne!(
3700 clef.capture_policy()?,
3701 git2db::pinned_disk::ProjectionPolicy::default()
3702 );
3703 clef.commit = a0.commit;
3704 assert!(clef.capture_policy().is_err());
3705 let mut clef = super::StagingModelPin::reviewed_clef_flash(Default::default())?;
3706 clef.model_ref = a0.model_ref;
3707 assert!(clef.capture_policy().is_err());
3708 Ok(())
3709 }
3710
3711 use super::*;
3712 use hyprstream_rpc::registry::{global as registry, SocketKind};
3713
3714 #[cfg(feature = "postgres")]
3715 #[test]
3716 fn federate_model_route_is_exact_load_status_stream_and_pinned_ref() -> Result<()> {
3717 let pin = StagingModelPin::from_settings("staging", None, None)?
3718 .ok_or_else(|| anyhow!("staging pin missing"))?;
3719 let model_ref = pin.model_ref.as_str();
3720 assert_eq!(
3721 model_ref.split_once(':').map(|(name, _)| name),
3722 Some(StagingModelPin::reviewed_repo_name())
3723 );
3724 let bytes = hyprstream_rpc::serialize_message(|message| {
3725 let mut request =
3726 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3727 request.set_id(1);
3728 let mut infer = request.init_infer();
3729 infer.set_model_ref(model_ref);
3730 infer.init_generate_stream();
3731 })?;
3732 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3733 assert_eq!(
3734 federate_model_operation(&body, &pin)?,
3735 (format!("model:{model_ref}"), "infer".into())
3736 );
3737 let wrong_oid = StagingModelPin::new(
3738 model_ref,
3739 git2::Oid::from_str("0000000000000000000000000000000000000001")?,
3740 )?;
3741 assert!(federate_model_operation(&body, &wrong_oid).is_err());
3742
3743 let bytes = hyprstream_rpc::serialize_message(|message| {
3744 let mut request =
3745 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3746 request.set_id(2);
3747 let mut infer = request.init_infer();
3748 infer.set_model_ref(model_ref);
3749 infer.set_status(());
3750 })?;
3751 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3752 assert!(federate_model_operation(&body, &pin).is_err());
3753
3754 let bytes = hyprstream_rpc::serialize_message(|message| {
3755 let mut request =
3756 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3757 request.set_id(3);
3758 request.init_load().set_model_ref(model_ref);
3759 })?;
3760 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3761 assert_eq!(
3762 federate_model_operation(&body, &pin)?,
3763 ("model:Load".into(), "write".into())
3764 );
3765
3766 let bytes = hyprstream_rpc::serialize_message(|message| {
3767 let mut request =
3768 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3769 request.set_id(4);
3770 request.init_status().set_model_ref(model_ref);
3771 })?;
3772 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3773 assert_eq!(
3774 federate_model_operation(&body, &pin)?,
3775 ("model:Status".into(), "query".into())
3776 );
3777
3778 for rejected in ["", "qwen2.5-0.5b-instruct:other", "attacker:main"] {
3779 let bytes = hyprstream_rpc::serialize_message(|message| {
3780 let mut request =
3781 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3782 request.set_id(5);
3783 request.init_status().set_model_ref(rejected);
3784 })?;
3785 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3786 assert!(
3787 federate_model_operation(&body, &pin).is_err(),
3788 "status accepted {rejected:?}"
3789 );
3790 }
3791 let bytes = hyprstream_rpc::serialize_message(|message| {
3792 let mut request =
3793 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3794 request.set_id(6);
3795 request.init_load().set_model_ref("attacker:main");
3796 })?;
3797 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3798 assert!(federate_model_operation(&body, &pin).is_err());
3799
3800 let bytes = hyprstream_rpc::serialize_message(|message| {
3801 let mut request =
3802 message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3803 request.set_id(7);
3804 let mut infer = request.init_infer();
3805 infer.set_model_ref("attacker:main");
3806 infer.init_generate_stream();
3807 })?;
3808 let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3809 assert!(federate_model_operation(&body, &pin).is_err());
3810 Ok(())
3811 }
3812
3813 #[tokio::test]
3814 async fn delegated_resource_model_leaves_intersect_signed_oauth_scope() -> Result<()> {
3815 use hyprstream_rpc::auth::{jwt, Claims};
3816 use hyprstream_rpc::service::RequestService;
3817 use hyprstream_rpc_std::model_capnp::model_request;
3818
3819 let oai = "https://oai.example";
3820 let mcp = "https://mcp.example";
3821 let service = selector_model_service()
3822 .await
3823 .with_delegated_resource_audiences(oai.to_owned(), mcp.to_owned());
3824 let relay = SigningKey::from_bytes(&[0x7f; 32]);
3825 hyprstream_service::global_trust_store().insert(
3826 relay.verifying_key(),
3827 hyprstream_service::Attestation {
3828 scopes: ["oai".to_owned(), "mcp".to_owned()].into_iter().collect(),
3829 subject: Some("service:oai".to_owned()),
3830 jwt: None,
3831 expires_at: 0,
3832 attested_by: None,
3833 },
3834 );
3835 assert!(service.accept_delegated_bearer(relay.verifying_key().as_bytes()));
3836 let signer = SigningKey::from_bytes(&[0x7e; 32]);
3837 let model_ref = "fixture-model:main";
3838 let body_for = |method: &str| -> Result<hyprstream_rpc::service::DecodedRequestBody> {
3839 let bytes = hyprstream_rpc::serialize_message(|message| {
3840 let mut request = message.init_root::<model_request::Builder>();
3841 request.set_id(1);
3842 match method {
3843 "status" => request.init_status().set_model_ref(model_ref),
3844 "generate" => {
3845 let mut infer = request.init_infer();
3846 infer.set_model_ref(model_ref);
3847 infer.init_generate_stream();
3848 }
3849 "template" => {
3850 let mut infer = request.init_infer();
3851 infer.set_model_ref(model_ref);
3852 infer.init_apply_chat_template();
3853 }
3854 _ => unreachable!(),
3855 }
3856 })?;
3857 crate::services::generated::model_client::decode_model_request_body(&bytes)
3858 };
3859 let status = body_for("status")?;
3860 let generate = body_for("generate")?;
3861 let template = body_for("template")?;
3862 assert_eq!(status.leaf_path(), Some([2].as_slice()));
3863 assert_eq!(generate.leaf_path(), Some([6, 0].as_slice()));
3864 assert_eq!(template.leaf_path(), Some([6, 1].as_slice()));
3865 let now = chrono::Utc::now().timestamp();
3866 for (audience, scope, body, allowed) in [
3867 (oai, "infer:model:fixture-model:main", &generate, true),
3868 (oai, "infer:model:*", &generate, true),
3869 (oai, "openid", &generate, false),
3870 (oai, "read:model:fixture-model:main", &generate, false),
3871 (oai, "infer:model:other", &generate, false),
3872 (oai, "infer:model:fixture-model:main", &template, true),
3873 (oai, "read:model:fixture-model:main", &template, true),
3874 (oai, "infer:model:fixture-model:main", &status, false),
3875 (oai, "read:model:Status", &status, true),
3876 (oai, "read:model:*", &status, true),
3877 (mcp, "read:model:Status", &status, true),
3878 (mcp, "read:model:*", &status, true),
3879 (mcp, "query:model:*", &status, false),
3880 (mcp, "openid", &status, false),
3881 (mcp, "infer:model:fixture-model:main", &status, false),
3882 (mcp, "read:model:other", &status, false),
3883 (mcp, "read:model:Status", &generate, false),
3884 ] {
3885 let claims = Claims::new("alice".to_owned(), now, now + 3600)
3886 .with_audience(Some(audience.to_owned()))
3887 .with_scope(Some(scope.to_owned()))
3888 .with_tenant("fixture-tenant".to_owned())
3889 .with_clearance(crate::services::inference::inference_object_label());
3890 let token = jwt::encode(&claims, &signer);
3891 let signed = hyprstream_rpc::auth::decode_with_key(
3892 &token,
3893 &signer.verifying_key(),
3894 Some(audience),
3895 )?;
3896 let ctx = EnvelopeContext::for_test_authenticated_subject_with_claims(
3897 hyprstream_rpc::envelope::Subject::new("alice"),
3898 "fixture-tenant",
3899 relay.verifying_key(),
3900 signed,
3901 );
3902 assert_eq!(ctx.domain()?, "fixture-tenant");
3903 assert!(crate::services::inference::enforce_inference_mac(
3904 &ctx,
3905 &crate::services::inference::inference_object_label(),
3906 )
3907 .is_ok());
3908 assert_eq!(
3909 service.accept_delegated_resource_leaf(&ctx, body),
3910 allowed,
3911 "audience={audience} scope={scope} leaf={:?}",
3912 body.leaf_path()
3913 );
3914 }
3915 for audience in [oai, mcp] {
3916 let claims = Claims::new("alice".to_owned(), now, now + 3600)
3917 .with_audience(Some(audience.to_owned()))
3918 .with_tenant("fixture-tenant".to_owned())
3919 .with_clearance(crate::services::inference::inference_object_label());
3920 let token = jwt::encode(&claims, &signer);
3921 let signed = hyprstream_rpc::auth::decode_with_key(
3922 &token,
3923 &signer.verifying_key(),
3924 Some(audience),
3925 )?;
3926 let ctx = EnvelopeContext::for_test_authenticated_subject_with_claims(
3927 hyprstream_rpc::envelope::Subject::new("alice"),
3928 "fixture-tenant",
3929 relay.verifying_key(),
3930 signed,
3931 );
3932 assert!(!service.accept_delegated_resource_leaf(&ctx, &status));
3933 assert!(!service.accept_delegated_resource_leaf(&ctx, &generate));
3934 }
3935 Ok(())
3936 }
3937
3938 #[test]
3939 fn test_config_defaults() {
3940 let config = ModelServiceConfig::default();
3941 assert_eq!(config.max_models, 5);
3942 assert_eq!(config.max_context, None);
3943 assert_eq!(config.kv_quant, KVQuantType::None);
3944 assert_eq!(
3945 config.inference_deployment.isolation,
3946 InferenceIsolationProfile::InProcess
3947 );
3948 }
3949
3950 #[test]
3951 fn staging_pin_rejects_wrong_name_and_ref() -> Result<()> {
3952 let oid = git2::Oid::from_str("18c562db6830c2ef6636b8eaf42fb479272052f7")?;
3953 let pin = StagingModelPin::new("qwen2.5-0.5b-instruct:main", oid)?;
3954 assert!(pin.admits("qwen2.5-0.5b-instruct:main"));
3955 for wrong in [
3956 "other-model:main",
3957 "qwen2.5-0.5b-instruct:other",
3958 "qwen2.5-0.5b-instruct",
3959 ] {
3960 assert!(!pin.admits(wrong));
3961 }
3962 assert!(StagingModelPin::new("qwen2.5-0.5b-instruct:refs/heads/main", oid).is_err());
3963 Ok(())
3964 }
3965
3966 #[tokio::test]
3967 async fn denied_model_load_is_audited_before_pin_or_registry_io() -> Result<()> {
3968 use tracing::field::{Field, Visit};
3969 use tracing_subscriber::layer::{Context, Layer, SubscriberExt};
3970
3971 #[derive(Clone, Default, Debug)]
3972 struct AuditObservation {
3973 target: String,
3974 resource: String,
3975 action: String,
3976 decision: String,
3977 }
3978
3979 #[derive(Default)]
3980 struct AuditVisitor(AuditObservation);
3981
3982 impl Visit for AuditVisitor {
3983 fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
3984 self.record_str(field, format!("{value:?}").trim_matches('"'));
3985 }
3986
3987 fn record_str(&mut self, field: &Field, value: &str) {
3988 match field.name() {
3989 "resource" => self.0.resource = value.to_owned(),
3990 "action" => self.0.action = value.to_owned(),
3991 "decision" => self.0.decision = value.to_owned(),
3992 _ => {}
3993 }
3994 }
3995 }
3996
3997 #[derive(Clone)]
3998 struct AuditLayer(Arc<parking_lot::Mutex<Vec<AuditObservation>>>);
3999
4000 impl<S: tracing::Subscriber> Layer<S> for AuditLayer {
4001 fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
4002 if event.metadata().target() != "audit" {
4003 return;
4004 }
4005 let mut visitor = AuditVisitor::default();
4006 event.record(&mut visitor);
4007 visitor.0.target = event.metadata().target().to_owned();
4008 self.0.lock().push(visitor.0);
4009 }
4010 }
4011
4012 let registry_calls = Arc::new(BoundaryDialState::default());
4013 let policy_calls = Arc::new(BoundaryDialState::default());
4014 let registry_rpc = Arc::new(BoundaryRpcClient::new(
4015 TransportConfig::inproc("pin-admission-registry"),
4016 Arc::clone(®istry_calls),
4017 ));
4018 let policy_rpc = Arc::new(BoundaryRpcClient::new(
4019 TransportConfig::inproc("pin-admission-policy"),
4020 Arc::clone(&policy_calls),
4021 ));
4022 let mut config = ModelServiceConfig::default();
4023 config.staging_model_pin = Some(StagingModelPin::new(
4024 "qwen2.5-0.5b-instruct:main",
4025 git2::Oid::from_str("18c562db6830c2ef6636b8eaf42fb479272052f7")?,
4026 )?);
4027 let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
4028 hyprstream_rpc::moq_event::MoqEventOrigin::new(),
4029 );
4030 let service = ModelService::new(
4031 config,
4032 SigningKey::from_bytes(&[0x45; 32]),
4033 PolicyClient::new(policy_rpc),
4034 RegistryClient::new(registry_rpc),
4035 TransportConfig::inproc("pin-admission-model"),
4036 TransportConfig::inproc("pin-admission-policy-transport"),
4037 )
4038 .await?;
4039
4040 let context = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4041 hyprstream_rpc::envelope::Subject::new("unauthorized-caller"),
4042 "pin-admission-tenant",
4043 SigningKey::from_bytes(&[0x46; 32]).verifying_key(),
4044 );
4045 let records = Arc::new(parking_lot::Mutex::new(Vec::new()));
4046 let subscriber =
4047 tracing_subscriber::Registry::default().with(AuditLayer(Arc::clone(&records)));
4048 tracing::subscriber::with_default(subscriber, || -> Result<()> {
4049 for model_ref in ["other-model:main", "qwen2.5-0.5b-instruct:main"] {
4050 let bytes = hyprstream_rpc::serialize_message(|message| {
4051 let mut request = message
4052 .init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>(
4053 );
4054 request.set_id(91);
4055 request.init_load().set_model_ref(model_ref);
4056 })?;
4057 let body =
4058 crate::services::generated::model_client::decode_model_request_body(&bytes)?;
4059 let error =
4060 match futures::executor::block_on(service.handle_request(&context, &body)) {
4061 Err(error) => error,
4062 Ok(_) => anyhow::bail!(
4063 "missing MAC clearance must deny the load before pin/ref status handling"
4064 ),
4065 };
4066 anyhow::ensure!(
4067 error.to_string().contains(
4068 "authorization denied: inference caller has no verified MAC clearance"
4069 ),
4070 "denial must be authorization-specific, not a pin or registry error: {error}"
4071 );
4072 anyhow::ensure!(
4073 !error.to_string().contains("configured staging modelRef"),
4074 "unauthorized callers must not receive pin-specific diagnostics"
4075 );
4076 }
4077 Ok(())
4078 })?;
4079
4080 let observations = records.lock().clone();
4081 anyhow::ensure!(
4082 observations.len() == 2,
4083 "each denied load must emit one authz audit event: {observations:?}"
4084 );
4085 for observation in observations {
4086 anyhow::ensure!(
4087 observation.target == "audit",
4088 "authorization denial must use the audit target"
4089 );
4090 anyhow::ensure!(
4091 observation.resource == "model:Load",
4092 "unexpected audited resource: {observation:?}"
4093 );
4094 anyhow::ensure!(
4095 observation.action == "write",
4096 "unexpected audited action: {observation:?}"
4097 );
4098 anyhow::ensure!(
4099 observation.decision == "deny",
4100 "denied caller must be audited as denied: {observation:?}"
4101 );
4102 }
4103 Ok(())
4104 }
4105
4106 #[test]
4107 fn staging_pin_settings_fail_closed_on_missing_or_partial_override() -> Result<()> {
4108 let pin = StagingModelPin::from_settings("staging", None, None)?
4109 .ok_or_else(|| anyhow!("staging profile must have reviewed pin"))?;
4110 assert!(pin.admits("qwen2.5-0.5b-instruct:main"));
4111 assert!(StagingModelPin::from_settings(
4112 "staging",
4113 Some("qwen2.5-0.5b-instruct:main"),
4114 None
4115 )
4116 .is_err());
4117 assert!(StagingModelPin::from_settings(
4118 "staging",
4119 None,
4120 Some("18c562db6830c2ef6636b8eaf42fb479272052f7")
4121 )
4122 .is_err());
4123 assert!(StagingModelPin::from_settings(
4124 "staging",
4125 Some("other:main"),
4126 Some("18c562db6830c2ef6636b8eaf42fb479272052f7")
4127 )
4128 .is_err());
4129 assert!(StagingModelPin::from_settings("default", None, None)?.is_none());
4130 assert!(StagingModelPin::from_settings("default", Some("x:main"), None).is_err());
4131 assert!(
4132 StagingModelPin::from_settings("default", Some("x:main"), Some("deadbeef")).is_err()
4133 );
4134 Ok(())
4135 }
4136
4137 #[tokio::test]
4138 async fn unpublished_spawn_guard_stops_worker_after_forced_failure() -> Result<()> {
4139 let temp = tempfile::tempdir()?;
4140 let stopped = temp.path().join("worker-stopped");
4141 let witness = stopped.clone();
4142 let shutdown = Arc::new(tokio::sync::Notify::new());
4143 let worker_shutdown = Arc::clone(&shutdown);
4144 let worker = std::thread::spawn(move || {
4145 let Ok(runtime) = tokio::runtime::Builder::new_current_thread()
4146 .enable_all()
4147 .build()
4148 else {
4149 return;
4150 };
4151 runtime.block_on(worker_shutdown.notified());
4152 let _ = std::fs::write(witness, b"stopped");
4153 });
4154 let handle = hyprstream_service::SpawnedService::thread(
4155 "model-pin-guard-test".to_owned(),
4156 Some(worker),
4157 shutdown,
4158 None,
4159 );
4160 let result: Result<()> = async {
4161 let _guard = SpawnPublicationGuard::new(handle);
4162 anyhow::bail!("forced post-spawn pre-publication failure")
4163 }
4164 .await;
4165 assert!(result.is_err());
4166 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4167 while !stopped.exists() {
4168 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4169 }
4170 })
4171 .await?;
4172 assert_eq!(std::fs::read(stopped)?, b"stopped");
4173 Ok(())
4174 }
4175
4176 #[test]
4177 fn inference_instance_denies_unlabeled_tenant_before_load() {
4178 let signer = SigningKey::from_bytes(&[44u8; 32]).verifying_key();
4179 let ctx = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4180 hyprstream_rpc::envelope::Subject::new("alice"),
4181 "did-hosted-account",
4182 signer,
4183 );
4184 assert!(
4185 ModelService::inference_instance(&ctx, "tiny-llama:main").is_err(),
4186 "an authority-bound tenant without MAC clearance must not reach engine loading"
4187 );
4188 }
4189
4190 fn loaded_term(endpoint: &str) -> Terminal<ModelLoadTerminal> {
4202 Terminal {
4203 value: ModelLoadTerminal::Loaded {
4204 endpoint: endpoint.to_owned(),
4205 },
4206 latched_by: "model-service".to_owned(),
4207 }
4208 }
4209
4210 fn failed_term(err: &str) -> Terminal<ModelLoadTerminal> {
4211 Terminal {
4212 value: ModelLoadTerminal::LoadFailed {
4213 error: err.to_owned(),
4214 },
4215 latched_by: "model-service".to_owned(),
4216 }
4217 }
4218
4219 fn latched_value(
4221 store: &TerminalStore<ModelLoadKey, ModelLoadTerminal>,
4222 key: &ModelLoadKey,
4223 ) -> ModelLoadTerminal {
4224 match store.get(key) {
4225 Some(t) => t.value,
4226 None => panic!("expected a terminal latched for {key:?}"),
4227 }
4228 }
4229
4230 #[test]
4233 fn load_terminal_is_monotonic_write_once_per_epoch() {
4234 let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4235 let key = ModelLoadKey {
4236 tenant: "tenant-a".to_owned(),
4237 model_ref: "qwen3:main".to_owned(),
4238 epoch: 1,
4239 };
4240 assert!(store.latch(key.clone(), loaded_term("ep1")));
4241 assert!(!store.latch(key.clone(), failed_term("race")));
4243 assert_eq!(
4244 latched_value(&store, &key),
4245 ModelLoadTerminal::Loaded {
4246 endpoint: "ep1".to_owned()
4247 }
4248 );
4249 }
4250
4251 #[test]
4255 fn reload_latches_new_terminal_under_new_epoch() {
4256 let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4257 let k1 = ModelLoadKey {
4259 tenant: "tenant-a".to_owned(),
4260 model_ref: "qwen3:main".to_owned(),
4261 epoch: 1,
4262 };
4263 assert!(store.latch(k1.clone(), loaded_term("ep1")));
4264 let k2 = ModelLoadKey {
4266 tenant: "tenant-a".to_owned(),
4267 model_ref: "qwen3:main".to_owned(),
4268 epoch: 2,
4269 };
4270 assert!(store.latch(k2.clone(), loaded_term("ep2")));
4271 assert_eq!(
4273 latched_value(&store, &k1),
4274 ModelLoadTerminal::Loaded {
4275 endpoint: "ep1".to_owned()
4276 }
4277 );
4278 assert_eq!(
4279 latched_value(&store, &k2),
4280 ModelLoadTerminal::Loaded {
4281 endpoint: "ep2".to_owned()
4282 }
4283 );
4284 }
4285
4286 #[test]
4290 fn failed_then_reload_latches_distinct_terminals() {
4291 let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4292 let kf = ModelLoadKey {
4293 tenant: "tenant-a".to_owned(),
4294 model_ref: "qwen3:main".to_owned(),
4295 epoch: 1,
4296 };
4297 assert!(store.latch(kf.clone(), failed_term("oom")));
4298 let ks = ModelLoadKey {
4299 tenant: "tenant-a".to_owned(),
4300 model_ref: "qwen3:main".to_owned(),
4301 epoch: 2,
4302 };
4303 assert!(store.latch(ks.clone(), loaded_term("ep2")));
4304 assert!(matches!(
4305 latched_value(&store, &kf),
4306 ModelLoadTerminal::LoadFailed { .. }
4307 ));
4308 assert!(matches!(
4309 latched_value(&store, &ks),
4310 ModelLoadTerminal::Loaded { .. }
4311 ));
4312 }
4313
4314 #[test]
4315 fn load_terminals_are_tenant_scoped() {
4316 let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4317 let tenant_a = ModelLoadKey {
4318 tenant: "tenant-a".to_owned(),
4319 model_ref: "qwen3:main".to_owned(),
4320 epoch: 1,
4321 };
4322 let tenant_b = ModelLoadKey {
4323 tenant: "tenant-b".to_owned(),
4324 model_ref: "qwen3:main".to_owned(),
4325 epoch: 1,
4326 };
4327 assert!(store.latch(tenant_a.clone(), loaded_term("a")));
4328 assert!(store.latch(tenant_b.clone(), loaded_term("b")));
4329 assert_ne!(
4330 latched_value(&store, &tenant_a),
4331 latched_value(&store, &tenant_b)
4332 );
4333 }
4334
4335 #[test]
4336 fn in_process_profile_never_admits_two_tenants() {
4337 let admitted = std::sync::OnceLock::new();
4338 assert!(admit_in_process_tenant(&admitted, "tenant-a").is_ok());
4339 assert!(admit_in_process_tenant(&admitted, "tenant-a").is_ok());
4340 assert!(admit_in_process_tenant(&admitted, "tenant-b").is_err());
4341 }
4342
4343 #[test]
4346 fn inference_transport_is_inproc_co_located() -> Result<()> {
4347 const CHILD: &str = "HYPRSTREAM_TEST_INFERENCE_IPC_PROFILE_CHILD";
4348 if std::env::var_os(CHILD).is_none() {
4349 let output = std::process::Command::new(std::env::current_exe()?)
4350 .args([
4351 "--exact",
4352 "services::model::tests::inference_transport_is_inproc_co_located",
4353 "--nocapture",
4354 ])
4355 .env(CHILD, "1")
4356 .output()?;
4357 assert!(
4358 output.status.success(),
4359 "{}{}",
4360 String::from_utf8_lossy(&output.stdout),
4361 String::from_utf8_lossy(&output.stderr)
4362 );
4363 return Ok(());
4364 }
4365 hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Ipc, None);
4366 let instance = InferenceInstanceId::new("tenant-a", "qwen3-small:main", 0)
4367 .unwrap_or_else(|e| panic!("valid instance: {e}"));
4368 assert!(matches!(
4369 registry()
4370 .endpoint(&instance.service_name(), SocketKind::Rep)
4371 .endpoint,
4372 hyprstream_rpc::transport::EndpointType::Ipc { .. }
4373 ));
4374 let t = ModelService::inference_transport(&instance);
4375 match &t.endpoint {
4376 hyprstream_rpc::transport::EndpointType::Inproc { endpoint } => {
4377 assert!(
4378 endpoint.contains("inference-"),
4379 "opaque deterministic per-tenant/model inproc name, got {endpoint}"
4380 );
4381 assert_eq!(endpoint, &format!("hyprstream/{}", instance.service_name()));
4382 }
4383 other => panic!("expected Inproc co-located transport, got {other:?}"),
4384 }
4385 let other = InferenceInstanceId::new("tenant-b", "qwen3-small:main", 0)?;
4386 assert_ne!(
4387 t.endpoint_string(),
4388 ModelService::inference_endpoint(&other)
4389 );
4390 assert_eq!(
4391 t.endpoint_string(),
4392 ModelService::inference_endpoint(&instance)
4393 );
4394 Ok(())
4395 }
4396
4397 #[test]
4402 fn model_reach_co_located_is_empty() {
4403 let inproc = TransportConfig::inproc("hyprstream/inference-x");
4404 assert!(
4405 ModelService::model_reach(&inproc).is_empty(),
4406 "co-located Inproc reach must not be wire-advertised"
4407 );
4408 }
4409
4410 #[test]
4415 fn model_reach_networked_iroh_single_select() {
4416 let node_id = [0xEEu8; 32];
4417 let iroh = TransportConfig::iroh(
4418 node_id,
4419 Vec::new(),
4420 Some("https://relay.example".to_owned()),
4421 );
4422 let reach = ModelService::model_reach(&iroh);
4423 assert_eq!(
4424 reach.len(),
4425 1,
4426 "single-select: exactly one reach per service"
4427 );
4428 match &reach[0] {
4429 WireTransportConfig::Iroh(i) => assert_eq!(i.node_id, node_id),
4430 other => panic!("expected wire Iroh reach, got {other:?}"),
4431 }
4432 }
4433
4434 #[test]
4437 fn model_ref_dispatch_at_uri_is_federated() {
4438 assert_eq!(
4440 model_ref_dispatch("at://did:plc:x123/hyprstream.models/qwen3/v1"),
4441 ModelRefDispatch::Federated {
4442 scheme: "at://",
4443 rest: "did:plc:x123/hyprstream.models/qwen3/v1",
4444 }
4445 );
4446 }
4447
4448 #[test]
4449 fn model_ref_dispatch_bare_did_is_federated() {
4450 assert_eq!(
4454 model_ref_dispatch("did:plc:abcdef"),
4455 ModelRefDispatch::Federated {
4456 scheme: "did:",
4457 rest: "plc:abcdef",
4458 }
4459 );
4460 assert_eq!(
4462 model_ref_dispatch("did:web:hyprstream.example.com"),
4463 ModelRefDispatch::Federated {
4464 scheme: "did:",
4465 rest: "web:hyprstream.example.com",
4466 }
4467 );
4468 }
4469
4470 #[test]
4471 fn model_ref_dispatch_local_name_is_local() {
4472 assert_eq!(model_ref_dispatch("qwen3"), ModelRefDispatch::Local);
4474 assert_eq!(model_ref_dispatch("qwen3:main"), ModelRefDispatch::Local);
4476 assert_eq!(
4477 model_ref_dispatch("Qwen3-0.6B:tags/v1.0"),
4478 ModelRefDispatch::Local
4479 );
4480 assert_eq!(
4482 model_ref_dispatch("org/model-name"),
4483 ModelRefDispatch::Local
4484 );
4485 }
4486
4487 #[test]
4488 fn model_ref_dispatch_uuid_is_local() {
4489 assert_eq!(
4491 model_ref_dispatch("550e8400-e29b-41d4-a716-446655440000"),
4492 ModelRefDispatch::Local
4493 );
4494 }
4495
4496 #[test]
4497 fn model_ref_dispatch_local_arms_parse_unchanged() {
4498 for s in &["qwen3", "qwen3:main", "Qwen3-0.6B:tags/v1.0"] {
4501 assert_eq!(model_ref_dispatch(s), ModelRefDispatch::Local);
4502 assert!(
4503 ModelRef::parse(s).is_ok(),
4504 "local ModelRef '{s}' must still parse after #395"
4505 );
4506 }
4507 }
4508
4509 use crate::services::router::{CellRouter, InferenceServerInfo};
4522
4523 fn server(id: u8, mem_free: u64, active: u64) -> InferenceServerInfo {
4524 InferenceServerInfo {
4525 replica_id: crate::services::router::ReplicaId::from_bytes([id; 32]),
4526 transport: TransportConfig::inproc("test"),
4527 gpu_memory_free: mem_free,
4528 active_sessions: active,
4529 last_heartbeat: Instant::now(),
4530 }
4531 }
4532
4533 const SELECTOR_BEARER: &str = "fixture-delegated-bearer";
4534 static SELECTOR_BOUNDARY_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
4535 static SELECTOR_DIAL_STATE: std::sync::OnceLock<Arc<BoundaryDialState>> =
4536 std::sync::OnceLock::new();
4537 static SELECTOR_FIXTURE: std::sync::OnceLock<hyprstream_discovery::ProductionInferenceFixture> =
4538 std::sync::OnceLock::new();
4539
4540 #[test]
4541 fn holder_bound_service_credentials_are_not_relayed_to_remote_replicas() {
4542 let service =
4543 hyprstream_rpc::auth::Claims::new("service:registry".to_owned(), 0, 9_999_999_999);
4544 let user = hyprstream_rpc::auth::Claims::new("alice".to_owned(), 0, 9_999_999_999);
4545
4546 assert_eq!(
4547 ModelService::remote_relay_bearer_for_claims(Some(&service), Some("service-jwt")),
4548 None,
4549 "a service credential is holder-bound and must not be delegated to another controller"
4550 );
4551 assert_eq!(
4552 ModelService::remote_relay_bearer_for_claims(Some(&user), Some("user-jwt")),
4553 Some("user-jwt"),
4554 "an authenticated end-user bearer remains relayable"
4555 );
4556 assert_eq!(
4557 ModelService::remote_relay_bearer_for_claims(None, None),
4558 None,
4559 "an unauthenticated caller cannot select a remote relay path"
4560 );
4561 }
4562
4563 #[derive(Clone, Debug)]
4564 struct ReadinessObservation {
4565 transport: TransportConfig,
4566 service: String,
4567 method_discriminator: u16,
4568 delegated_bearer: Option<String>,
4569 jwt: Option<String>,
4570 }
4571
4572 #[derive(Default)]
4573 struct BoundaryDialBehavior {
4574 failures_remaining: usize,
4575 fail_all: bool,
4576 }
4577
4578 #[derive(Default)]
4579 struct BoundaryDialState {
4580 dials: parking_lot::Mutex<Vec<TransportConfig>>,
4581 readiness: parking_lot::Mutex<Vec<ReadinessObservation>>,
4582 expected_bearer: parking_lot::Mutex<Option<String>>,
4583 expected_jwt: parking_lot::Mutex<Option<String>>,
4584 behavior: parking_lot::Mutex<BoundaryDialBehavior>,
4585 }
4586
4587 impl BoundaryDialState {
4588 fn reset(&self, expected_bearer: Option<&str>) {
4589 self.dials.lock().clear();
4590 self.readiness.lock().clear();
4591 *self.expected_bearer.lock() = expected_bearer.map(ToOwned::to_owned);
4592 *self.behavior.lock() = BoundaryDialBehavior::default();
4593 }
4594
4595 fn fail_next(&self, count: usize) {
4596 self.behavior.lock().failures_remaining = count;
4597 }
4598
4599 fn fail_all(&self) {
4600 self.behavior.lock().fail_all = true;
4601 }
4602 }
4603
4604 struct BoundaryRpcClient {
4605 transport: TransportConfig,
4606 state: Arc<BoundaryDialState>,
4607 request_id: std::sync::atomic::AtomicU64,
4608 }
4609
4610 impl BoundaryRpcClient {
4611 fn new(transport: TransportConfig, state: Arc<BoundaryDialState>) -> Self {
4612 Self {
4613 transport,
4614 state,
4615 request_id: std::sync::atomic::AtomicU64::new(1),
4616 }
4617 }
4618
4619 fn unsupported() -> Result<Vec<u8>> {
4620 anyhow::bail!("selector fixture received an unexpected RPC method")
4621 }
4622 }
4623
4624 #[async_trait]
4625 impl hyprstream_rpc::RpcClient for BoundaryRpcClient {
4626 async fn call(&self, _payload: Vec<u8>) -> Result<Vec<u8>> {
4627 Self::unsupported()
4628 }
4629
4630 async fn call_for_service(
4631 &self,
4632 _service_domain: &str,
4633 _payload: Vec<u8>,
4634 ) -> Result<Vec<u8>> {
4635 Self::unsupported()
4636 }
4637
4638 async fn call_for_service_with_method(
4639 &self,
4640 _service_domain: &str,
4641 _method_discriminator: u16,
4642 _payload: Vec<u8>,
4643 ) -> Result<Vec<u8>> {
4644 Self::unsupported()
4645 }
4646
4647 async fn call_with_options(
4648 &self,
4649 _payload: Vec<u8>,
4650 _options: hyprstream_rpc::CallOptions,
4651 ) -> Result<Vec<u8>> {
4652 Self::unsupported()
4653 }
4654
4655 async fn call_with_options_for_service(
4656 &self,
4657 _service_domain: &str,
4658 _payload: Vec<u8>,
4659 _options: hyprstream_rpc::CallOptions,
4660 ) -> Result<Vec<u8>> {
4661 Self::unsupported()
4662 }
4663
4664 async fn call_with_options_for_service_with_method(
4665 &self,
4666 service_domain: &str,
4667 method_discriminator: u16,
4668 payload: Vec<u8>,
4669 options: hyprstream_rpc::CallOptions,
4670 ) -> Result<Vec<u8>> {
4671 let reader = capnp::serialize::read_message(
4672 &mut std::io::Cursor::new(&payload),
4673 capnp::message::ReaderOptions::new(),
4674 )?;
4675 let request = reader
4676 .get_root::<hyprstream_rpc_std::inference_capnp::inference_request::Reader>()?;
4677 anyhow::ensure!(
4678 matches!(
4679 request.which()?,
4680 hyprstream_rpc_std::inference_capnp::inference_request::Which::IsReady(())
4681 ),
4682 "selector fixture received a non-readiness request"
4683 );
4684 let expected_bearer = self.state.expected_bearer.lock().clone();
4685 let expected_jwt = self.state.expected_jwt.lock().clone();
4686 self.state.readiness.lock().push(ReadinessObservation {
4687 transport: self.transport.clone(),
4688 service: service_domain.to_owned(),
4689 method_discriminator,
4690 delegated_bearer: options.delegated_bearer.clone(),
4691 jwt: options.jwt.clone(),
4692 });
4693 anyhow::ensure!(
4694 service_domain == InferenceClient::SERVICE_NAME,
4695 "readiness used the wrong generated service authority"
4696 );
4697 anyhow::ensure!(
4698 method_discriminator == 2,
4699 "readiness used the wrong schema method discriminator"
4700 );
4701 anyhow::ensure!(
4702 options.jwt == expected_jwt && options.delegated_bearer == expected_bearer,
4703 "readiness did not carry the expected bearer shape"
4704 );
4705 let should_fail = {
4706 let mut behavior = self.state.behavior.lock();
4707 if behavior.fail_all {
4708 true
4709 } else if behavior.failures_remaining > 0 {
4710 behavior.failures_remaining -= 1;
4711 true
4712 } else {
4713 false
4714 }
4715 };
4716 if should_fail {
4717 anyhow::bail!("fixture readiness failure");
4718 }
4719 crate::services::generated::inference_client::serialize_response(
4720 request.get_id(),
4721 &hyprstream_rpc_std::inference_client::InferenceResponseVariant::IsReadyResult(
4722 true,
4723 ),
4724 )
4725 }
4726
4727 async fn call_streaming(
4728 &self,
4729 _payload: Vec<u8>,
4730 _ephemeral_pubkey: [u8; 32],
4731 ) -> Result<Vec<u8>> {
4732 Self::unsupported()
4733 }
4734
4735 async fn call_streaming_for_service(
4736 &self,
4737 _service_domain: &str,
4738 _payload: Vec<u8>,
4739 _ephemeral_pubkey: [u8; 32],
4740 ) -> Result<Vec<u8>> {
4741 Self::unsupported()
4742 }
4743
4744 async fn call_streaming_for_service_with_method(
4745 &self,
4746 _service_domain: &str,
4747 _method_discriminator: u16,
4748 _payload: Vec<u8>,
4749 _ephemeral_pubkey: [u8; 32],
4750 ) -> Result<Vec<u8>> {
4751 Self::unsupported()
4752 }
4753
4754 async fn open_stream(
4755 &self,
4756 _payload: Vec<u8>,
4757 ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
4758 anyhow::bail!("selector fixture received an unexpected streaming RPC")
4759 }
4760
4761 async fn open_stream_from_info(
4762 &self,
4763 _stream_info: hyprstream_rpc::stream_info::StreamInfo,
4764 _client_secret: [u8; 32],
4765 _client_pubkey: [u8; 32],
4766 ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
4767 anyhow::bail!("selector fixture received an unexpected stream-open RPC")
4768 }
4769
4770 fn next_id(&self) -> u64 {
4771 self.request_id.fetch_add(1, Ordering::Relaxed)
4772 }
4773 }
4774
4775 fn selector_instance() -> InferenceInstanceId {
4776 InferenceInstanceId::new("fixture-tenant", "fixture-model:main", 0)
4777 .unwrap_or_else(|error| panic!("fixture inference identity failed: {error}"))
4778 }
4779
4780 fn remote_transport(id: u8) -> TransportConfig {
4781 TransportConfig::iroh([id; 32], Vec::new(), None)
4782 }
4783
4784 fn selector_fixture() -> (
4785 &'static hyprstream_discovery::ProductionInferenceFixture,
4786 &'static Arc<BoundaryDialState>,
4787 ) {
4788 let state = SELECTOR_DIAL_STATE.get_or_init(|| Arc::new(BoundaryDialState::default()));
4789 let fixture = SELECTOR_FIXTURE.get_or_init(|| {
4790 let dial_state = Arc::clone(state);
4791 let dial = Arc::new(move |transport: &TransportConfig| {
4792 dial_state.dials.lock().push(transport.clone());
4793 Ok(Arc::new(BoundaryRpcClient::new(
4794 transport.clone(),
4795 Arc::clone(&dial_state),
4796 )) as Arc<dyn hyprstream_rpc::RpcClient>)
4797 });
4798 hyprstream_discovery::install_production_inference_fixture(
4799 &selector_instance().service_name(),
4800 &[remote_transport(0x01)],
4801 dial,
4802 )
4803 .unwrap_or_else(|error| panic!("install selector fixture failed: {error}"))
4804 });
4805 (fixture, state)
4806 }
4807
4808 async fn selector_model_service() -> ModelService {
4809 let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
4810 hyprstream_rpc::moq_event::MoqEventOrigin::new(),
4811 );
4812 let infrastructure_state = Arc::new(BoundaryDialState::default());
4813 let infrastructure_rpc = Arc::new(BoundaryRpcClient::new(
4814 TransportConfig::inproc("selector-infrastructure"),
4815 infrastructure_state,
4816 )) as Arc<dyn hyprstream_rpc::RpcClient>;
4817 ModelService::new(
4818 ModelServiceConfig::default(),
4819 SigningKey::from_bytes(&[0x41; 32]),
4820 PolicyClient::new(Arc::clone(&infrastructure_rpc)),
4821 RegistryClient::new(infrastructure_rpc),
4822 TransportConfig::inproc("selector-model-service"),
4823 TransportConfig::inproc("policy"),
4824 )
4825 .await
4826 .unwrap_or_else(|error| panic!("construct model-free selector service failed: {error}"))
4827 }
4828
4829 fn selector_loaded_model(
4830 load_state: Vec<InferenceServerInfo>,
4831 local_state: Arc<BoundaryDialState>,
4832 ) -> LoadedModel {
4833 let local_transport = TransportConfig::inproc("selector-local-inference");
4834 let network_transport = load_state
4835 .iter()
4836 .find(|candidate| {
4837 matches!(
4838 &candidate.transport.endpoint,
4839 EndpointType::Quic { .. } | EndpointType::Iroh { .. }
4840 )
4841 })
4842 .map_or_else(
4843 || local_transport.clone(),
4844 |candidate| candidate.transport.clone(),
4845 );
4846 LoadedModel {
4847 instance: selector_instance(),
4848 model_ref: "fixture-model:main".to_owned(),
4849 pinned_commit: None,
4850 incarnation: "fixture-incarnation-0000".to_owned(),
4851 work_audience: crate::services::inference::internal_work_audience(
4852 &selector_instance().service_name(),
4853 "fixture-incarnation-0000",
4854 ),
4855 load_generation: 1,
4856 transport: local_transport.clone(),
4857 network_transport,
4858 service_handle: hyprstream_service::SpawnedService::dummy(),
4859 client: InferenceClient::new(Arc::new(BoundaryRpcClient::new(
4860 local_transport,
4861 local_state,
4862 ))),
4863 router: CellRouter::default(),
4864 load_state,
4865 loaded_at: Instant::now(),
4866 last_used: Instant::now(),
4867 ttt_config: None,
4868 generation_defaults: crate::config::SamplingParams::default(),
4869 }
4870 }
4871
4872 fn server_at(id: u8, transport: TransportConfig) -> InferenceServerInfo {
4873 InferenceServerInfo {
4874 replica_id: crate::services::router::ReplicaId::from_bytes([id; 32]),
4875 transport,
4876 gpu_memory_free: 1024,
4877 active_sessions: 0,
4878 last_heartbeat: Instant::now(),
4879 }
4880 }
4881
4882 fn exclude_local(model: &mut LoadedModel) -> crate::services::router::ReplicaId {
4883 let local = ModelService::co_located_replica_id(model);
4884 model.router.report_dial_fail(local, Instant::now());
4885 local
4886 }
4887
4888 #[tokio::test]
4889 async fn selector_boundary_fixed_authority_exact_reach_and_delegated_readiness() {
4890 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4891 let (fixture, dial_state) = selector_fixture();
4892 let advertised = remote_transport(0x11);
4893 fixture
4894 .reset(std::slice::from_ref(&advertised))
4895 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4896 dial_state.reset(Some(SELECTOR_BEARER));
4897 let local_state = Arc::new(BoundaryDialState::default());
4898 let local = server_at(0x10, TransportConfig::inproc("selector-local"));
4899 let remote = server_at(0x20, advertised.clone());
4900 let mut model = selector_loaded_model(vec![local, remote], local_state);
4901 exclude_local(&mut model);
4902 let service = selector_model_service().await;
4903
4904 service
4905 .select_inference_routed(&mut model, "selector-positive", Some(SELECTOR_BEARER))
4906 .await
4907 .unwrap_or_else(|error| panic!("production selector rejected exact reach: {error}"));
4908
4909 let dials = dial_state.dials.lock().clone();
4910 assert_eq!(dials, vec![advertised.clone()]);
4911 let readiness = dial_state.readiness.lock().clone();
4912 assert_eq!(readiness.len(), 1);
4913 assert_eq!(readiness[0].transport, advertised);
4914 assert_eq!(readiness[0].service, "inference");
4915 assert_eq!(readiness[0].method_discriminator, 2);
4916 assert_eq!(
4917 readiness[0].delegated_bearer.as_deref(),
4918 Some(SELECTOR_BEARER)
4919 );
4920 assert!(readiness[0].jwt.is_none());
4921 assert_ne!(selector_instance().service_name(), "inference");
4922 }
4923
4924 #[tokio::test]
4925 async fn selector_boundary_rejects_unadvertised_reach_before_bearer_disclosure() {
4926 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4927 let (fixture, dial_state) = selector_fixture();
4928 fixture
4929 .reset(&[remote_transport(0x21)])
4930 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4931 dial_state.reset(Some(SELECTOR_BEARER));
4932 let local_state = Arc::new(BoundaryDialState::default());
4933 let selected_id = crate::services::router::ReplicaId::from_bytes([0x22; 32]);
4934 let mut model = selector_loaded_model(
4935 vec![
4936 server_at(0x10, TransportConfig::inproc("selector-local")),
4937 server_at(0x22, remote_transport(0x22)),
4938 ],
4939 local_state,
4940 );
4941 exclude_local(&mut model);
4942 let error = selector_model_service()
4943 .await
4944 .select_inference_routed(&mut model, "selector-unadvertised", Some(SELECTOR_BEARER))
4945 .await
4946 .err()
4947 .unwrap_or_else(|| panic!("unadvertised reach was accepted"));
4948 assert!(error
4949 .to_string()
4950 .contains("not a current authorized candidate"));
4951 assert!(dial_state.dials.lock().is_empty());
4952 assert!(dial_state.readiness.lock().is_empty());
4953 assert!(model
4954 .router
4955 .is_excluded(selected_id, Instant::now())
4956 .is_some());
4957 }
4958
4959 #[tokio::test]
4960 async fn selector_boundary_rejects_stale_reach_before_bearer_disclosure() {
4961 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4962 let (fixture, dial_state) = selector_fixture();
4963 let advertised = remote_transport(0x31);
4964 fixture
4965 .reset(std::slice::from_ref(&advertised))
4966 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4967 fixture.mark_stale();
4968 dial_state.reset(Some(SELECTOR_BEARER));
4969 let selected_id = crate::services::router::ReplicaId::from_bytes([0x23; 32]);
4970 let mut model = selector_loaded_model(
4971 vec![
4972 server_at(0x10, TransportConfig::inproc("selector-local")),
4973 server_at(0x23, advertised),
4974 ],
4975 Arc::new(BoundaryDialState::default()),
4976 );
4977 exclude_local(&mut model);
4978 assert!(selector_model_service()
4979 .await
4980 .select_inference_routed(&mut model, "selector-stale", Some(SELECTOR_BEARER))
4981 .await
4982 .is_err());
4983 assert!(dial_state.dials.lock().is_empty());
4984 assert!(dial_state.readiness.lock().is_empty());
4985 assert!(model
4986 .router
4987 .is_excluded(selected_id, Instant::now())
4988 .is_some());
4989 }
4990
4991 #[tokio::test]
4992 async fn selector_boundary_rejects_cross_authority_retry_set_before_bearer_disclosure() {
4993 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4994 let (fixture, dial_state) = selector_fixture();
4995 let selected = remote_transport(0x41);
4996 fixture
4997 .reset(std::slice::from_ref(&selected))
4998 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4999 fixture
5000 .add_foreign_authority(&remote_transport(0x42))
5001 .unwrap_or_else(|error| panic!("add foreign fixture authority failed: {error}"));
5002 dial_state.reset(Some(SELECTOR_BEARER));
5003 let selected_id = crate::services::router::ReplicaId::from_bytes([0x24; 32]);
5004 let mut model = selector_loaded_model(
5005 vec![
5006 server_at(0x10, TransportConfig::inproc("selector-local")),
5007 server_at(0x24, selected),
5008 ],
5009 Arc::new(BoundaryDialState::default()),
5010 );
5011 exclude_local(&mut model);
5012 let error = selector_model_service()
5013 .await
5014 .select_inference_routed(&mut model, "selector-authority", Some(SELECTOR_BEARER))
5015 .await
5016 .err()
5017 .unwrap_or_else(|| panic!("cross-authority retry set was accepted"));
5018 assert!(error
5019 .to_string()
5020 .contains("ambiguous validated service authority"));
5021 assert!(dial_state.dials.lock().is_empty());
5022 assert!(dial_state.readiness.lock().is_empty());
5023 assert!(model
5024 .router
5025 .is_excluded(selected_id, Instant::now())
5026 .is_some());
5027 }
5028
5029 #[tokio::test]
5030 async fn selector_boundary_rejects_non_network_transport_before_bearer_disclosure() {
5031 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5032 let (fixture, dial_state) = selector_fixture();
5033 fixture
5034 .reset(&[remote_transport(0x51)])
5035 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5036 dial_state.reset(Some(SELECTOR_BEARER));
5037 let selected_id = crate::services::router::ReplicaId::from_bytes([0x25; 32]);
5038 let mut model = selector_loaded_model(
5039 vec![
5040 server_at(0x10, TransportConfig::inproc("selector-local")),
5041 server_at(0x25, TransportConfig::ipc("/run/hyprstream/attacker.sock")),
5042 ],
5043 Arc::new(BoundaryDialState::default()),
5044 );
5045 exclude_local(&mut model);
5046 let error = selector_model_service()
5047 .await
5048 .select_inference_routed(&mut model, "selector-transport", Some(SELECTOR_BEARER))
5049 .await
5050 .err()
5051 .unwrap_or_else(|| panic!("non-network remote transport was accepted"));
5052 assert!(error.to_string().contains("non-network transport"));
5053 assert!(dial_state.dials.lock().is_empty());
5054 assert!(dial_state.readiness.lock().is_empty());
5055 assert!(model
5056 .router
5057 .is_excluded(selected_id, Instant::now())
5058 .is_some());
5059 }
5060
5061 #[tokio::test]
5062 async fn selector_boundary_failure_marks_exact_selected_replica_and_reselects_once() {
5063 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5064 let (fixture, dial_state) = selector_fixture();
5065 let first = remote_transport(0x61);
5066 let second = remote_transport(0x62);
5067 fixture
5068 .reset(&[first.clone(), second.clone()])
5069 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5070 dial_state.reset(Some(SELECTOR_BEARER));
5071 dial_state.fail_next(1);
5072 let first_id = crate::services::router::ReplicaId::from_bytes([0x26; 32]);
5073 let second_id = crate::services::router::ReplicaId::from_bytes([0x27; 32]);
5074 let mut model = selector_loaded_model(
5075 vec![
5076 server_at(0x10, TransportConfig::inproc("selector-local")),
5077 server_at(0x26, first.clone()),
5078 server_at(0x27, second.clone()),
5079 ],
5080 Arc::new(BoundaryDialState::default()),
5081 );
5082 exclude_local(&mut model);
5083 selector_model_service()
5084 .await
5085 .select_inference_routed(&mut model, "selector-reselect", Some(SELECTOR_BEARER))
5086 .await
5087 .unwrap_or_else(|error| panic!("bounded reselection did not recover: {error}"));
5088 let dials = dial_state.dials.lock().clone();
5089 assert_eq!(dials.len(), 2);
5090 assert_ne!(dials[0], dials[1]);
5091 let (failed_id, surviving_id) = if dials[0] == first {
5092 (first_id, second_id)
5093 } else {
5094 assert_eq!(dials[0], second);
5095 (second_id, first_id)
5096 };
5097 assert!(model
5098 .router
5099 .is_excluded(failed_id, Instant::now())
5100 .is_some());
5101 assert!(model
5102 .router
5103 .is_excluded(surviving_id, Instant::now())
5104 .is_none());
5105 assert_eq!(dial_state.readiness.lock().len(), 2);
5106 }
5107
5108 #[tokio::test]
5109 async fn selector_boundary_explicit_colocation_returns_local_client_without_remote_dial() {
5110 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5111 let (fixture, dial_state) = selector_fixture();
5112 fixture
5113 .reset(&[remote_transport(0x71)])
5114 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5115 dial_state.reset(Some(SELECTOR_BEARER));
5116 let local_state = Arc::new(BoundaryDialState::default());
5117 local_state.reset(None);
5118 let mut model = selector_loaded_model(
5119 vec![server_at(0x10, TransportConfig::inproc("selector-local"))],
5120 Arc::clone(&local_state),
5121 );
5122 let client = selector_model_service()
5123 .await
5124 .select_inference_routed(&mut model, "selector-local", Some(SELECTOR_BEARER))
5125 .await
5126 .unwrap_or_else(|error| panic!("explicit co-location failed: {error}"))
5127 .value;
5128 assert!(dial_state.dials.lock().is_empty());
5129 assert!(dial_state.readiness.lock().is_empty());
5130 assert!(client
5131 .is_ready()
5132 .await
5133 .unwrap_or_else(|error| panic!("returned local client failed: {error}")));
5134 assert_eq!(local_state.readiness.lock().len(), 1);
5135 }
5136
5137 #[tokio::test]
5138 async fn selector_boundary_exhaustion_fails_without_local_fallback() {
5139 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5140 let (fixture, dial_state) = selector_fixture();
5141 let first = remote_transport(0x81);
5142 let second = remote_transport(0x82);
5143 fixture
5144 .reset(&[first.clone(), second.clone()])
5145 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5146 dial_state.reset(Some(SELECTOR_BEARER));
5147 dial_state.fail_all();
5148 let first_id = crate::services::router::ReplicaId::from_bytes([0x28; 32]);
5149 let second_id = crate::services::router::ReplicaId::from_bytes([0x29; 32]);
5150 let local_state = Arc::new(BoundaryDialState::default());
5151 local_state.reset(None);
5152 let mut model = selector_loaded_model(
5153 vec![
5154 server_at(0x10, TransportConfig::inproc("selector-local")),
5155 server_at(0x28, first),
5156 server_at(0x29, second),
5157 ],
5158 Arc::clone(&local_state),
5159 );
5160 exclude_local(&mut model);
5161 let error = selector_model_service()
5162 .await
5163 .select_inference_routed(&mut model, "selector-exhausted", Some(SELECTOR_BEARER))
5164 .await
5165 .err()
5166 .unwrap_or_else(|| panic!("exhausted remote candidates fell back locally"));
5167 assert!(error.to_string().contains("candidates exhausted"));
5168 assert_eq!(dial_state.dials.lock().len(), 2);
5169 assert_eq!(dial_state.readiness.lock().len(), 2);
5170 assert!(local_state.readiness.lock().is_empty());
5171 assert!(model.router.is_excluded(first_id, Instant::now()).is_some());
5172 assert!(model
5173 .router
5174 .is_excluded(second_id, Instant::now())
5175 .is_some());
5176 }
5177
5178 #[test]
5182 fn route_decision_single_replica_is_co_located() {
5183 let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5184 let load_state = vec![server(0x10, 1024, 0)];
5185 let mut router = CellRouter::default();
5186 let decision = ModelService::route_decision(
5187 &mut router,
5188 &load_state,
5189 co_located,
5190 "sess-A",
5191 Instant::now(),
5192 );
5193 assert_eq!(decision, RouteDecision::CoLocated);
5194 }
5195
5196 #[test]
5202 fn route_decision_multi_replica_distributes_and_detects_remote() {
5203 let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5204 let load_state = vec![
5205 server(0x10, 16 * 1024 * 1024 * 1024, 0), server(0x20, 16 * 1024 * 1024 * 1024, 0), server(0x30, 16 * 1024 * 1024 * 1024, 0), ];
5209 let mut saw_co_located = false;
5210 let mut saw_remote = false;
5211 for i in 0..150u32 {
5212 let mut router = CellRouter::default();
5215 match ModelService::route_decision(
5216 &mut router,
5217 &load_state,
5218 co_located,
5219 &format!("sess-{i}"),
5220 Instant::now(),
5221 ) {
5222 RouteDecision::CoLocated => saw_co_located = true,
5223 RouteDecision::Remote(_) => saw_remote = true,
5224 RouteDecision::NoHealthyCandidate => panic!("healthy replica set must place"),
5225 }
5226 }
5227 assert!(
5228 saw_co_located,
5229 "some sessions must land on the co-located replica"
5230 );
5231 assert!(
5232 saw_remote,
5233 "some sessions must land on a remote replica (not always co-located)"
5234 );
5235 }
5236
5237 #[test]
5241 fn route_decision_session_affinity_holds() {
5242 let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5243 let load_state = vec![
5244 server(0x10, 16 * 1024 * 1024 * 1024, 0),
5245 server(0x20, 16 * 1024 * 1024 * 1024, 0),
5246 server(0x30, 16 * 1024 * 1024 * 1024, 0),
5247 ];
5248 let mut router = CellRouter::default();
5249 let now = Instant::now();
5250 let first =
5251 ModelService::route_decision(&mut router, &load_state, co_located, "sess-sticky", now);
5252 for _ in 0..5 {
5253 let again = ModelService::route_decision(
5254 &mut router,
5255 &load_state,
5256 co_located,
5257 "sess-sticky",
5258 now,
5259 );
5260 assert_eq!(
5261 first, again,
5262 "same session must stick to the same replica within its lease"
5263 );
5264 }
5265 }
5266
5267 #[test]
5271 fn route_decision_fails_over_on_replica_loss() {
5272 let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5273 let load_state = vec![
5274 server(0x10, 16 * 1024 * 1024 * 1024, 0),
5275 server(0x20, 16 * 1024 * 1024 * 1024, 0),
5276 server(0x30, 16 * 1024 * 1024 * 1024, 0),
5277 ];
5278 let mut router = CellRouter::default();
5279 let now = Instant::now();
5280 let first =
5281 ModelService::route_decision(&mut router, &load_state, co_located, "sess-B", now);
5282 let first_node = match first {
5283 RouteDecision::CoLocated => co_located,
5284 RouteDecision::Remote(n) => n,
5285 RouteDecision::NoHealthyCandidate => panic!("healthy replica set must place"),
5286 };
5287 router.report_dial_fail(first_node, now);
5288 let second =
5289 ModelService::route_decision(&mut router, &load_state, co_located, "sess-B", now);
5290 let second_node = match second {
5291 RouteDecision::CoLocated => co_located,
5292 RouteDecision::Remote(n) => n,
5293 RouteDecision::NoHealthyCandidate => {
5294 panic!("two healthy replicas remain after one failure")
5295 }
5296 };
5297 assert_ne!(
5298 second_node, first_node,
5299 "must fail over off the down replica"
5300 );
5301 }
5302
5303 #[test]
5306 fn route_decision_empty_replica_set_is_no_healthy_candidate() {
5307 let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5308 let load_state: Vec<InferenceServerInfo> = vec![];
5309 let mut router = CellRouter::default();
5310 let decision = ModelService::route_decision(
5311 &mut router,
5312 &load_state,
5313 co_located,
5314 "sess-A",
5315 Instant::now(),
5316 );
5317 assert_eq!(decision, RouteDecision::NoHealthyCandidate);
5318 }
5319
5320 #[tokio::test]
5321 async fn selector_boundary_colocated_client_carries_internal_work_bearer_not_relay() {
5322 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5323 let local_state = Arc::new(BoundaryDialState::default());
5324 let work_token = "iw-minted-work-order-fixture".to_owned();
5327 *local_state.expected_jwt.lock() = Some(work_token.clone());
5328 local_state.reset(None);
5329 let mut model = selector_loaded_model(
5330 vec![server_at(0x10, TransportConfig::inproc("selector-local"))],
5331 Arc::clone(&local_state),
5332 );
5333 let service = selector_model_service().await;
5334 let routed = service
5335 .select_inference_routed(&mut model, "selector-internal", None)
5336 .await
5337 .unwrap_or_else(|error| panic!("co-location without a relay bearer failed: {error}"));
5338 let bearer_client = ModelService::attach_work_order(routed, work_token.clone());
5341 assert!(
5342 bearer_client
5343 .is_ready()
5344 .await
5345 .unwrap_or_else(|error| panic!("returned local client failed: {error}")),
5346 "readiness with the internal work bearer must succeed"
5347 );
5348 let readiness = local_state.readiness.lock().clone();
5349 assert_eq!(readiness.len(), 1);
5350 assert_eq!(readiness[0].jwt.as_deref(), Some(work_token.as_str()));
5351 assert!(
5352 readiness[0].delegated_bearer.is_none(),
5353 "the internal leg must never relay a caller credential as delegated bearer"
5354 );
5355 }
5356
5357 #[tokio::test]
5363 async fn selector_boundary_remote_arm_never_overlays_internal_work_order() {
5364 let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5365 let (fixture, dial_state) = selector_fixture();
5366 let advertised = remote_transport(0x91);
5367 fixture
5368 .reset(std::slice::from_ref(&advertised))
5369 .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5370 dial_state.reset(Some(SELECTOR_BEARER));
5371 *dial_state.expected_jwt.lock() = None;
5374 let local_state = Arc::new(BoundaryDialState::default());
5375 local_state.reset(None);
5376 let mut model = selector_loaded_model(
5377 vec![
5378 server_at(0x10, TransportConfig::inproc("selector-local")),
5379 server_at(0x91, advertised.clone()),
5380 ],
5381 Arc::clone(&local_state),
5382 );
5383 exclude_local(&mut model);
5384 let service = selector_model_service().await;
5385 let routed = service
5386 .select_inference_routed(
5387 &mut model,
5388 "selector-remote-internal",
5389 Some(SELECTOR_BEARER),
5390 )
5391 .await
5392 .unwrap_or_else(|error| panic!("remote selection failed: {error}"));
5393 let client =
5396 ModelService::attach_work_order(routed, "iw-foreign-controller-order".to_owned());
5397 assert!(
5398 client
5399 .is_ready()
5400 .await
5401 .unwrap_or_else(|error| panic!("remote client failed: {error}")),
5402 "remote relay readiness must succeed with the delegated bearer only"
5403 );
5404 let readiness = dial_state.readiness.lock().clone();
5408 assert_eq!(readiness.len(), 2);
5409 for observation in &readiness {
5410 assert_eq!(
5411 observation.delegated_bearer.as_deref(),
5412 Some(SELECTOR_BEARER),
5413 "remote relay must carry the delegated bearer"
5414 );
5415 assert!(
5416 observation.jwt.is_none(),
5417 "the remote arm must not carry the internal work order as a direct jwt"
5418 );
5419 }
5420 }
5421
5422 use base64::Engine as _;
5433 use hyprstream_rpc::auth::internal_work::{
5434 verify_internal_work, InternalWorkScope, MAX_LIFETIME_SECS,
5435 };
5436
5437 const BOUNDARY_CALLER_KEY: [u8; 32] = [0xCB; 32];
5438
5439 fn audience_fixture() -> String {
5442 crate::services::inference::internal_work_audience(
5443 &selector_instance().service_name(),
5444 "test-incarnation-0000",
5445 )
5446 }
5447
5448 fn running_worker() -> hyprstream_service::SpawnedService {
5451 let handle = std::thread::spawn(|| loop {
5454 std::thread::park_timeout(std::time::Duration::from_secs(3600));
5455 });
5456 hyprstream_service::SpawnedService::thread(
5457 "s1-alive-worker".to_owned(),
5458 Some(handle),
5459 Arc::new(tokio::sync::Notify::new()),
5460 None,
5461 )
5462 }
5463
5464 fn dead_worker() -> hyprstream_service::SpawnedService {
5465 hyprstream_service::SpawnedService::thread(
5466 "s1-dead-worker".to_owned(),
5467 None,
5468 Arc::new(tokio::sync::Notify::new()),
5469 None,
5470 )
5471 }
5472
5473 fn blocking_unload_worker(
5474 armed: std::sync::mpsc::Sender<()>,
5475 shutdown_observed: tokio::sync::oneshot::Sender<()>,
5476 release: Arc<tokio::sync::Notify>,
5477 ) -> hyprstream_service::SpawnedService {
5478 let shutdown = Arc::new(tokio::sync::Notify::new());
5479 let worker_shutdown = Arc::clone(&shutdown);
5480 let handle = std::thread::spawn(move || {
5481 let runtime = tokio::runtime::Builder::new_current_thread()
5482 .enable_all()
5483 .build()
5484 .unwrap_or_else(|error| panic!("build blocking-unload runtime: {error}"));
5485 runtime.block_on(async move {
5486 let notified = worker_shutdown.notified();
5487 tokio::pin!(notified);
5488 notified.as_mut().enable();
5489 armed
5490 .send(())
5491 .unwrap_or_else(|error| panic!("signal blocking worker readiness: {error}"));
5492 notified.await;
5493 let _ = shutdown_observed.send(());
5494 release.notified().await;
5495 });
5496 });
5497 hyprstream_service::SpawnedService::thread(
5498 "s1-blocking-unload-worker".to_owned(),
5499 Some(handle),
5500 shutdown,
5501 None,
5502 )
5503 }
5504
5505 fn panicking_unload_worker() -> hyprstream_service::SpawnedService {
5506 let handle = std::thread::spawn(|| panic!("test worker panics before join"));
5507 hyprstream_service::SpawnedService::thread(
5508 "s1-panicking-unload-worker".to_owned(),
5509 Some(handle),
5510 Arc::new(tokio::sync::Notify::new()),
5511 None,
5512 )
5513 }
5514
5515 #[tokio::test]
5516 async fn cancelled_unload_keeps_instance_reserved_until_thread_join() {
5517 let service = selector_model_service().await;
5518 let instance = selector_instance();
5519 let local_state = Arc::new(BoundaryDialState::default());
5520 let (armed_tx, armed_rx) = std::sync::mpsc::channel();
5521 let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
5522 let release = Arc::new(tokio::sync::Notify::new());
5523 let mut model = selector_loaded_model(Vec::new(), local_state);
5524 model.service_handle =
5525 blocking_unload_worker(armed_tx, shutdown_observed_tx, Arc::clone(&release));
5526 service
5527 .loaded_models
5528 .write()
5529 .await
5530 .put(instance.clone(), model);
5531 service.pending_loads.lock().insert(instance.clone(), 1);
5536
5537 tokio::task::spawn_blocking(move || {
5538 armed_rx
5539 .recv_timeout(std::time::Duration::from_secs(5))
5540 .map_err(|error| anyhow!("blocking worker did not arm: {error}"))
5541 })
5542 .await
5543 .unwrap_or_else(|error| panic!("wait for blocking worker readiness task: {error}"))
5544 .unwrap_or_else(|error| panic!("wait for blocking worker readiness: {error}"));
5545
5546 let unload_service = service.clone();
5547 let unload_instance = instance.clone();
5548 let caller =
5549 tokio::spawn(async move { unload_service.unload_model(&unload_instance).await });
5550
5551 tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_observed_rx)
5552 .await
5553 .unwrap_or_else(|_| panic!("unload did not notify the blocking worker"))
5554 .unwrap_or_else(|error| {
5555 panic!("blocking worker dropped shutdown observation: {error}")
5556 });
5557
5558 caller.abort();
5559 assert!(
5560 matches!(caller.await, Err(error) if error.is_cancelled()),
5561 "the caller task must be cancelled while owned teardown continues"
5562 );
5563 assert!(
5564 service.unloading_models.lock().await.contains(&instance),
5565 "cancellation must retain the unloading reservation until join completes"
5566 );
5567 assert!(
5568 !service.loaded_models.read().await.contains(&instance),
5569 "the cache must not expose a worker after its shutdown begins"
5570 );
5571 let (all_status, single_status) = tokio::join!(
5572 service.model_status_all(instance.tenant()),
5573 service.model_status_single(&instance),
5574 );
5575 assert_eq!(
5576 all_status
5577 .iter()
5578 .filter(|entry| entry.model_ref == instance.model_ref())
5579 .map(|entry| entry.status.as_str())
5580 .collect::<Vec<_>>(),
5581 vec!["unloading"],
5582 "all-status must retain an unloading worker after its caller is cancelled"
5583 );
5584 assert_eq!(
5585 single_status
5586 .iter()
5587 .map(|entry| entry.status.as_str())
5588 .collect::<Vec<_>>(),
5589 vec!["unloading"],
5590 "single-status must not report the draining worker as absent"
5591 );
5592 let error = match service.load_model(&instance, None, None).await {
5593 Err(error) => error,
5594 Ok(_) => panic!("a replacement load must not race the draining worker"),
5595 };
5596 assert!(
5597 error.to_string().contains("is unloading"),
5598 "replacement denial must disclose only lifecycle state: {error:#}"
5599 );
5600
5601 service.pending_loads.lock().remove(&instance);
5605 release.notify_one();
5606 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5607 loop {
5608 if !service.unloading_models.lock().await.contains(&instance) {
5609 break;
5610 }
5611 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5612 }
5613 })
5614 .await
5615 .unwrap_or_else(|_| panic!("owned teardown did not clear its reservation after join"));
5616 let (all_status, single_status) = tokio::join!(
5617 service.model_status_all(instance.tenant()),
5618 service.model_status_single(&instance),
5619 );
5620 assert!(
5621 all_status.is_empty() && single_status.is_empty(),
5622 "status absence is permitted only after teardown releases its reservation"
5623 );
5624 }
5625
5626 #[tokio::test]
5627 async fn unloading_reservation_precedes_loaded_and_pending_status_snapshot() {
5628 let service = selector_model_service().await;
5629 let instance = selector_instance();
5630
5631 service.loaded_models.write().await.put(
5636 instance.clone(),
5637 selector_loaded_model(Vec::new(), Arc::new(BoundaryDialState::default())),
5638 );
5639 service.pending_loads.lock().insert(instance.clone(), 1);
5640 service
5641 .unloading_models
5642 .lock()
5643 .await
5644 .insert(instance.clone());
5645
5646 let (all_status, single_status) = tokio::join!(
5647 service.model_status_all(instance.tenant()),
5648 service.model_status_single(&instance),
5649 );
5650 assert_eq!(
5651 all_status
5652 .iter()
5653 .filter(|entry| entry.model_ref == instance.model_ref())
5654 .map(|entry| entry.status.as_str())
5655 .collect::<Vec<_>>(),
5656 vec!["unloading"],
5657 "list status must not let pending_loads conceal an active teardown"
5658 );
5659 assert_eq!(
5660 single_status
5661 .iter()
5662 .map(|entry| entry.status.as_str())
5663 .collect::<Vec<_>>(),
5664 vec!["unloading"],
5665 "single status must not let pending_loads conceal an active teardown"
5666 );
5667 }
5668
5669 #[tokio::test]
5670 async fn intercepted_load_reservation_rejects_unload_before_continuation_runs() {
5671 let service = selector_model_service().await;
5672 let instance = selector_instance();
5673
5674 let reservation = match service.reserve_intercepted_load(&instance).await {
5679 Ok(InterceptedLoadAdmission::Accepted(reservation)) => reservation,
5680 Ok(_) => panic!("fixture load must receive a new accepted reservation"),
5681 Err(error) => panic!("fixture load admission failed: {error:#}"),
5682 };
5683 let unload_error = match service.unload_model(&instance).await {
5684 Err(error) => error,
5685 Ok(()) => panic!("unload must not cancel an accepted pending load"),
5686 };
5687 assert!(
5688 unload_error.to_string().contains("is loading"),
5689 "unload must reject against the accepted pending reservation: {unload_error:#}"
5690 );
5691 assert!(
5692 service.pending_loads.lock().contains_key(&instance),
5693 "the continuation must retain ownership of the accepted reservation"
5694 );
5695 assert!(
5696 !service.unloading_models.lock().await.contains(&instance),
5697 "a rejected unload must not leave a false teardown reservation"
5698 );
5699
5700 assert!(
5704 service
5705 .run_intercepted_load(&instance, None, None, reservation)
5706 .await
5707 .is_err(),
5708 "fixture resolution must reach the continuation failure path"
5709 );
5710 assert!(
5711 !service.pending_loads.lock().contains_key(&instance),
5712 "continuation completion must release its accepted reservation"
5713 );
5714 }
5715
5716 #[tokio::test]
5717 async fn dropped_intercepted_load_reservation_rolls_back_pending_marker() {
5718 let service = selector_model_service().await;
5719 let instance = selector_instance();
5720
5721 let reservation = match service.reserve_intercepted_load(&instance).await {
5727 Ok(InterceptedLoadAdmission::Accepted(reservation)) => reservation,
5728 Ok(_) => panic!("fixture load must receive a new accepted reservation"),
5729 Err(error) => panic!("fixture load admission failed: {error:#}"),
5730 };
5731 assert!(service.pending_loads.lock().contains_key(&instance));
5732 let unpolled: crate::services::Continuation = Box::pin(async move {
5733 let _reservation = reservation;
5734 std::future::pending::<()>().await;
5735 });
5736 drop(unpolled);
5737 assert!(
5738 !service.pending_loads.lock().contains_key(&instance),
5739 "dropping an unpolled accepted continuation must roll back pending"
5740 );
5741
5742 let stale =
5746 InterceptedLoadReservation::new(Arc::clone(&service.inner), instance.clone(), 7);
5747 service.pending_loads.lock().insert(instance.clone(), 8);
5748 drop(stale);
5749 assert_eq!(
5750 service.pending_loads.lock().get(&instance),
5751 Some(&8),
5752 "a stale guard must release only its own attempt marker"
5753 );
5754 service.pending_loads.lock().remove(&instance);
5755 let retry = service.reserve_intercepted_load(&instance).await;
5756 assert!(
5757 matches!(retry, Ok(InterceptedLoadAdmission::Accepted(_))),
5758 "a later load must not deduplicate against a dropped reservation"
5759 );
5760 }
5761
5762 #[tokio::test]
5763 async fn successful_unload_releases_admission_before_unloaded_event() {
5764 let service = selector_model_service().await;
5765 let instance = selector_instance();
5766 service.loaded_models.write().await.put(
5767 instance.clone(),
5768 selector_loaded_model(Vec::new(), Arc::new(BoundaryDialState::default())),
5769 );
5770 let (before_publish_tx, before_publish_rx) = tokio::sync::oneshot::channel();
5771 *service.before_unloaded_event.lock().await = Some(before_publish_tx);
5772
5773 let unload_service = service.clone();
5774 let unload_instance = instance.clone();
5775 let unload =
5776 tokio::spawn(async move { unload_service.unload_model(&unload_instance).await });
5777 before_publish_rx
5778 .await
5779 .unwrap_or_else(|error| panic!("unload did not reach event boundary: {error}"));
5780
5781 assert!(
5782 !service.unloading_models.lock().await.contains(&instance),
5783 "successful join must release admission before model.unloaded is published"
5784 );
5785 let reload = service.reserve_intercepted_load(&instance).await;
5786 assert!(
5787 matches!(reload, Ok(InterceptedLoadAdmission::Accepted(_))),
5788 "an event-driven reload must be admitted after a successful join"
5789 );
5790 unload
5791 .await
5792 .unwrap_or_else(|error| panic!("unload task panicked: {error}"))
5793 .unwrap_or_else(|error| panic!("dummy worker unload failed: {error:#}"));
5794 }
5795
5796 #[tokio::test]
5797 async fn unload_stop_error_retains_instance_reservation() {
5798 let service = selector_model_service().await;
5799 let instance = selector_instance();
5800 let local_state = Arc::new(BoundaryDialState::default());
5801 let mut model = selector_loaded_model(Vec::new(), local_state);
5802 model.service_handle = panicking_unload_worker();
5803 service
5804 .loaded_models
5805 .write()
5806 .await
5807 .put(instance.clone(), model);
5808
5809 let error = match service.unload_model(&instance).await {
5810 Err(error) => error,
5811 Ok(()) => panic!("a thread join panic must make unload fail"),
5812 };
5813 assert!(
5814 error.to_string().contains("failed to stop model"),
5815 "unload must return the stop failure: {error:#}"
5816 );
5817 assert!(
5818 service.unloading_models.lock().await.contains(&instance),
5819 "an ambiguous stop failure must retain the replacement-load reservation"
5820 );
5821 assert!(
5822 !service.loaded_models.read().await.contains(&instance),
5823 "a failed stop must not expose the popped worker as live"
5824 );
5825 let error = match service.load_model(&instance, None, None).await {
5826 Err(error) => error,
5827 Ok(_) => panic!("a failed stop must not admit a replacement worker"),
5828 };
5829 assert!(
5830 error.to_string().contains("is unloading"),
5831 "replacement denial must remain a lifecycle-only message: {error:#}"
5832 );
5833 }
5834
5835 #[tokio::test]
5836 async fn dead_worker_cache_entry_cannot_mint_and_is_evicted() {
5837 let service = selector_model_service().await;
5838 let instance = selector_instance();
5839 let mut cache = service.loaded_models.write().await;
5840 cache.put(
5841 instance.clone(),
5842 LoadedModel {
5843 instance: instance.clone(),
5844 model_ref: instance.model_ref().to_owned(),
5845 pinned_commit: None,
5846 transport: TransportConfig::inproc("dead-worker-test"),
5847 network_transport: TransportConfig::inproc("dead-worker-test"),
5848 service_handle: dead_worker(),
5849 client: InferenceClient::new(Arc::new(BoundaryRpcClient::new(
5850 TransportConfig::inproc("dead-worker-test"),
5851 Arc::new(BoundaryDialState::default()),
5852 ))),
5853 incarnation: "dead-incarnation".to_owned(),
5854 work_audience: "iw1/dead/dead".to_owned(),
5855 load_generation: 1,
5856 router: CellRouter::default(),
5857 load_state: Vec::new(),
5858 loaded_at: Instant::now(),
5859 last_used: Instant::now(),
5860 ttt_config: None,
5861 generation_defaults: crate::config::SamplingParams::default(),
5862 },
5863 );
5864 drop(cache);
5865
5866 let ctx = boundary_caller_ctx(chrono::Utc::now().timestamp() + 3600);
5867 let scope = InternalWorkScope::new("inference:GenerateStream", "infer");
5868 let error = match service
5869 .get_inference_client(instance.model_ref(), &ctx, scope)
5870 .await
5871 {
5872 Ok(_) => panic!("a dead worker cache entry must not mint work orders"),
5873 Err(error) => error,
5874 };
5875 assert!(
5876 error.to_string().contains("not running"),
5877 "dead-worker denial must name the liveness guard: {error:#}"
5878 );
5879 assert!(!service.loaded_models.read().await.contains(&instance));
5881 }
5882
5883 #[test]
5884 fn running_worker_liveness_guard_passes() {
5885 let worker = running_worker();
5888 assert!(
5889 worker.is_running(),
5890 "live thread fixture must report running"
5891 );
5892 let dead = dead_worker();
5893 assert!(!dead.is_running(), "dead fixture must report not-running");
5894 drop(worker);
5895 }
5896
5897 #[test]
5898 fn incarnation_handoff_validation_rejects_stale_or_mismatched() {
5899 let instance = selector_instance();
5904 let expected_generation: u64 = 3;
5905 let expected_controller = SigningKey::from_bytes(&[0x41; 32])
5906 .verifying_key()
5907 .to_bytes();
5908
5909 let handoff = crate::services::inference::IncarnationHandoff {
5910 instance_service_name: instance.service_name(),
5911 controller_pubkey: expected_controller,
5912 generation: expected_generation,
5913 incarnation: "aa11".to_owned(),
5914 audience: "iw1/x/aa11".to_owned(),
5915 };
5916 assert_eq!(handoff.generation, expected_generation);
5917 assert_eq!(handoff.instance_service_name, instance.service_name());
5918 assert_eq!(handoff.controller_pubkey, expected_controller);
5919
5920 let stale = crate::services::inference::IncarnationHandoff {
5921 generation: expected_generation - 1,
5922 ..handoff.clone()
5923 };
5924 assert_ne!(stale.generation, expected_generation, "stale handoff");
5925 }
5926
5927 fn boundary_caller_ctx(caller_exp: i64) -> EnvelopeContext {
5928 let caller_key = SigningKey::from_bytes(&BOUNDARY_CALLER_KEY);
5929 let now = chrono::Utc::now().timestamp();
5930 let claims = hyprstream_rpc::auth::Claims::new("alice".to_owned(), now, caller_exp)
5931 .with_tenant("fixture-tenant".to_owned())
5932 .with_cnf_jwk(caller_key.verifying_key().as_bytes())
5933 .with_clearance(crate::services::inference::inference_object_label());
5938 EnvelopeContext::for_test_authenticated_subject_with_claims(
5939 hyprstream_rpc::envelope::Subject::new("alice"),
5940 "fixture-tenant",
5941 caller_key.verifying_key(),
5942 claims,
5943 )
5944 }
5945
5946 fn restart_acceptance_caller_ctx(
5947 subject: &str,
5948 caller_key: &SigningKey,
5949 tenant: &str,
5950 ) -> EnvelopeContext {
5951 let now = chrono::Utc::now().timestamp();
5952 let claims = hyprstream_rpc::auth::Claims::new(subject.to_owned(), now, now + 300)
5953 .with_tenant(tenant.to_owned())
5954 .with_cnf_jwk(caller_key.verifying_key().as_bytes())
5955 .with_clearance(crate::services::inference::inference_object_label());
5956 EnvelopeContext::for_test_authenticated_subject_with_claims(
5957 hyprstream_rpc::envelope::Subject::new(subject),
5958 tenant,
5959 caller_key.verifying_key(),
5960 claims,
5961 )
5962 }
5963
5964 fn resolve_gitdir_reference(
5965 reference_file: &std::path::Path,
5966 gitfile_format: bool,
5967 ) -> Result<std::path::PathBuf> {
5968 let contents = std::fs::read_to_string(reference_file)?;
5969 let reference = if gitfile_format {
5970 contents
5971 .strip_prefix("gitdir: ")
5972 .ok_or_else(|| anyhow!("fixture Git file is malformed"))?
5973 } else {
5974 contents.as_str()
5975 }
5976 .trim();
5977 anyhow::ensure!(
5978 !reference.is_empty() && !reference.contains(['\n', '\r']),
5979 "fixture Git link is malformed"
5980 );
5981 let path = std::path::PathBuf::from(reference);
5982 let resolved = if path.is_absolute() {
5983 path
5984 } else {
5985 reference_file
5986 .parent()
5987 .ok_or_else(|| anyhow!("fixture Git link has no parent"))?
5988 .join(path)
5989 };
5990 Ok(resolved.canonicalize()?)
5991 }
5992
5993 fn validate_restart_fixture_worktree_link(
5994 repo: &std::path::Path,
5995 worktree: &std::path::Path,
5996 ) -> Result<()> {
5997 let admin_worktree = repo.join("worktrees").join("main");
5998 let expected_gitfile = worktree.join(".git");
5999 anyhow::ensure!(
6000 expected_gitfile.is_file(),
6001 "fixture main worktree has no Git file"
6002 );
6003 anyhow::ensure!(
6004 resolve_gitdir_reference(&admin_worktree.join("gitdir"), false)?
6005 == expected_gitfile.canonicalize()?,
6006 "fixture repository main worktree does not name the isolated StoragePaths worktree"
6007 );
6008 anyhow::ensure!(
6009 resolve_gitdir_reference(&expected_gitfile, true)? == admin_worktree.canonicalize()?,
6010 "isolated StoragePaths worktree does not link back to fixture repository main"
6011 );
6012 Ok(())
6013 }
6014
6015 struct RestartAcceptanceFixture {
6020 model: ModelService,
6021 model_key: SigningKey,
6022 model_ref: String,
6023 instance: InferenceInstanceId,
6024 _auth: RestartAuthServices,
6025 }
6026
6027 const RESTART_FIXTURE_ISSUER: &str = "http://127.0.0.1:6791";
6028
6029 struct RestartAuthServices {
6032 model_key: SigningKey,
6033 policy_key: SigningKey,
6034 registry_key: SigningKey,
6035 model_token: String,
6036 policy_endpoint: String,
6037 registry_endpoint: String,
6038 policy_transport: TransportConfig,
6039 jwt_source: Arc<hyprstream_rpc::auth::ClusterKeySource>,
6040 _registry_handle: hyprstream_service::SpawnedService,
6041 _policy_handle: hyprstream_service::SpawnedService,
6042 _registry_base: tempfile::TempDir,
6043 _policy_base: tempfile::TempDir,
6044 _credentials: tempfile::TempDir,
6045 }
6046
6047 fn restart_acceptance_inproc_endpoint(kind: &str, tag: &str) -> (TransportConfig, String) {
6048 let transport = TransportConfig::inproc(format!("restart-acceptance-{kind}-{tag}"));
6049 let uri = transport.endpoint_string();
6050 (transport, uri)
6051 }
6052
6053 fn install_restart_fixture_auth(keys: &[SigningKey]) -> Result<()> {
6054 use hyprstream_rpc::crypto::CryptoPolicy;
6055 use hyprstream_rpc::envelope::{
6056 EnvelopeVerifyConfig, KeyedPqTrustStore, ResponseVerifyConfig,
6057 };
6058
6059 let _ = hyprstream_rpc::proof::admission::set_global_proof_replay_store(Box::new(
6060 hyprstream_rpc::proof::admission::InMemoryProofReplayStore::single_verifier_instance(
6061 10_000,
6062 ),
6063 ));
6064 if hyprstream_rpc::auth::global_credential_revocation_store().is_none() {
6065 let _ = hyprstream_rpc::auth::set_global_credential_revocation_store(Arc::new(
6066 hyprstream_rpc::auth::InMemoryCredentialRevocationStore::new(),
6067 ));
6068 }
6069 let mut pq_store = KeyedPqTrustStore::new();
6070 for key in keys {
6071 let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(key);
6072 let pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
6073 &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(&pq),
6074 )?;
6075 pq_store.bind(key.verifying_key().to_bytes(), &pq_vk);
6076 }
6077 let pq_store = Arc::new(pq_store);
6078 hyprstream_rpc::envelope::install_verify_config(EnvelopeVerifyConfig {
6079 policy: CryptoPolicy::Hybrid,
6080 pq_store: Some(pq_store.clone()),
6081 })?;
6082 hyprstream_rpc::envelope::install_response_verify_config(ResponseVerifyConfig {
6083 policy: CryptoPolicy::Hybrid,
6084 pq_store: Some(pq_store),
6085 })?;
6086 crate::mac::install_production_rpc_dispatch_pep()?;
6087 anyhow::ensure!(
6088 hyprstream_rpc::auth::mac::global_mac_dispatch_pep().is_some(),
6089 "restart fixture has no production dispatch PEP"
6090 );
6091 Ok(())
6092 }
6093
6094 fn restart_fixture_service_jwt(
6095 credentials: &tempfile::TempDir,
6096 name: &str,
6097 ca: &SigningKey,
6098 key: &SigningKey,
6099 ) -> Result<String> {
6100 let bootstrap = crate::auth::identity_store::BootstrapPubkey::for_service_key(key)?;
6101 crate::auth::service_jwt::issue_or_load_service_jwt(
6102 credentials.path(),
6103 name,
6104 ca,
6105 &bootstrap,
6106 RESTART_FIXTURE_ISSUER,
6107 chrono::Utc::now().timestamp(),
6108 Some(&crate::mac::dispatch_labels::BOOTSTRAP_SERVICE_CLEARANCE),
6109 )
6110 }
6111
6112 impl RestartAuthServices {
6113 async fn new(repo: &std::path::Path, model_name: &str) -> Result<Self> {
6114 use hyprstream_service::ServiceManager as _;
6115
6116 let tag = hex::encode(hyprstream_rpc::envelope::generate_nonce());
6117 let policy_key = SigningKey::from_bytes(&[0x91; 32]);
6118 let registry_key = SigningKey::from_bytes(&[0x92; 32]);
6119 let model_key = SigningKey::from_bytes(&[0x93; 32]);
6120 let wrong_holder = SigningKey::from_bytes(&[0x94; 32]);
6123 install_restart_fixture_auth(&[
6124 policy_key.clone(),
6125 registry_key.clone(),
6126 model_key.clone(),
6127 wrong_holder,
6128 ])?;
6129 let ca =
6130 hyprstream_rpc::node_identity::derive_purpose_key(&policy_key, "hyprstream-jwt-v1");
6131 let ca_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&ca);
6132 let jwt_source = Arc::new(
6133 hyprstream_rpc::auth::ClusterKeySource::new(
6134 ca.verifying_key(),
6135 RESTART_FIXTURE_ISSUER.to_owned(),
6136 )
6137 .with_ca_composite_key(hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk(&ca_pq)),
6138 );
6139 let credentials = tempfile::TempDir::new()?;
6140 let model_token = restart_fixture_service_jwt(&credentials, "model", &ca, &model_key)?;
6141 let registry_token =
6142 restart_fixture_service_jwt(&credentials, "registry", &ca, ®istry_key)?;
6143 for (key, name) in [(&policy_key, "policy"), (®istry_key, "registry")] {
6144 hyprstream_service::global_trust_store().insert(
6145 key.verifying_key(),
6146 hyprstream_service::Attestation {
6147 scopes: std::iter::once(name.to_owned()).collect(),
6148 subject: Some(format!("service:{name}")),
6149 jwt: None,
6150 expires_at: 0,
6151 attested_by: None,
6152 },
6153 );
6154 }
6155
6156 let policy_base = tempfile::TempDir::new()?;
6157 let policy_manager = Arc::new(crate::auth::PolicyManager::permissive().await?);
6158 let (policy_transport, policy_endpoint) =
6159 restart_acceptance_inproc_endpoint("policy", &tag);
6160 let policy_service = crate::services::PolicyService::new(
6161 policy_manager,
6162 Arc::new(policy_key.clone()),
6163 crate::config::TokenConfig::default(),
6164 Arc::new(RwLock::new(git2db::Git2DB::open(policy_base.path()).await?)),
6165 policy_transport.clone(),
6166 )
6167 .with_jwt_key_source(jwt_source.clone());
6168 crate::services::restart_diag::phase("policy.spawn.enter");
6169 let manager = hyprstream_service::InprocManager::new();
6170 let policy_handle = manager.spawn(Box::new(policy_service)).await?;
6171 crate::services::restart_diag::phase("policy.spawn.done");
6172
6173 let registry_base = tempfile::TempDir::new()?;
6174 {
6175 let mut registry_store = git2db::Git2DB::open(registry_base.path()).await?;
6176 registry_store
6177 .register(git2db::RepoId::new())
6178 .name(model_name)
6179 .worktree_path(repo)
6180 .url(String::new())
6181 .exec()
6182 .await?;
6183 }
6184 let (registry_transport, registry_endpoint) =
6185 restart_acceptance_inproc_endpoint("registry", &tag);
6186 let registry_policy = PolicyClient::for_local_endpoint_bootstrap(
6187 &policy_endpoint,
6188 registry_key.clone(),
6189 policy_key.verifying_key(),
6190 Some(registry_token),
6191 )?;
6192 let registry_service = crate::services::RegistryService::new(
6193 registry_base.path(),
6194 registry_policy,
6195 registry_transport,
6196 registry_key.clone(),
6197 )
6198 .await?
6199 .with_expected_audience(RESTART_FIXTURE_ISSUER.to_owned())
6200 .with_jwt_key_source(jwt_source.clone());
6201 crate::services::restart_diag::phase("registry.spawn.enter");
6202 let registry_handle = manager.spawn(Box::new(registry_service)).await?;
6203 crate::services::restart_diag::phase("registry.spawn.done");
6204 Ok(Self {
6205 model_key,
6206 policy_key,
6207 registry_key,
6208 model_token,
6209 policy_endpoint,
6210 registry_endpoint,
6211 policy_transport,
6212 jwt_source,
6213 _registry_handle: registry_handle,
6214 _policy_handle: policy_handle,
6215 _registry_base: registry_base,
6216 _policy_base: policy_base,
6217 _credentials: credentials,
6218 })
6219 }
6220
6221 fn model_registry_client(&self) -> Result<RegistryClient> {
6222 RegistryClient::for_local_endpoint_bootstrap(
6223 &self.registry_endpoint,
6224 self.model_key.clone(),
6225 self.registry_key.verifying_key(),
6226 Some(self.model_token.clone()),
6227 )
6228 }
6229
6230 fn model_policy_client(&self) -> Result<PolicyClient> {
6231 PolicyClient::for_local_endpoint_bootstrap(
6232 &self.policy_endpoint,
6233 self.model_key.clone(),
6234 self.policy_key.verifying_key(),
6235 Some(self.model_token.clone()),
6236 )
6237 }
6238 }
6239
6240 #[test]
6241 fn restart_acceptance_inproc_endpoints_match_client_dial_names() {
6242 for kind in ["policy", "registry"] {
6243 let (server, uri) = restart_acceptance_inproc_endpoint(kind, "fixture");
6244 assert_eq!(
6245 server.endpoint,
6246 TransportConfig::from_endpoint(&uri).endpoint,
6247 "{kind} server registration and client dial must use the same name"
6248 );
6249 assert_eq!(uri, format!("inproc://restart-acceptance-{kind}-fixture"));
6250 }
6251 }
6252
6253 struct ClassicalRestartSigner(SigningKey);
6254
6255 #[async_trait::async_trait]
6256 impl hyprstream_rpc::transport_traits::Signer for ClassicalRestartSigner {
6257 fn pubkey(&self) -> [u8; 32] {
6258 self.0.verifying_key().to_bytes()
6259 }
6260
6261 async fn sign(&self, bytes: &[u8]) -> Result<[u8; 64]> {
6262 use ed25519_dalek::Signer as _;
6263 Ok(self.0.sign(bytes).to_bytes())
6264 }
6265 }
6266
6267 fn assert_restart_dispatch_denied<T>(result: Result<T>, case: &str) -> Result<()> {
6268 let error = result
6269 .err()
6270 .ok_or_else(|| anyhow!("{case} reached the Registry handler"))?;
6271 anyhow::ensure!(
6272 error
6273 .to_string()
6274 .contains(hyprstream_rpc::service::dispatch::DISPATCH_DENIED),
6275 "{case} failed outside the uniform dispatch boundary: {error}"
6276 );
6277 Ok(())
6278 }
6279
6280 #[tokio::test]
6283 #[ignore = "requires first-write verify-config globals; run with --exact --test-threads=1"]
6284 async fn restart_fixture_authenticated_hybrid_get_by_name_requires_model_holder() -> Result<()>
6285 {
6286 use hyprstream_rpc::rpc_client::RpcClientImpl;
6287 use hyprstream_rpc::transport::in_memory::InMemoryTransport;
6288
6289 let repo = tempfile::TempDir::new()?;
6290 let model_name = "restart-auth-regression";
6291 let auth = RestartAuthServices::new(repo.path(), model_name).await?;
6292 let model = auth.model_registry_client()?;
6293 let policy_check = hyprstream_rpc_std::policy_client::PolicyCheck {
6294 subject: "service:model".to_owned(),
6295 domain: "*".to_owned(),
6296 resource: "registry:*".to_owned(),
6297 operation: "query".to_owned(),
6298 };
6299 anyhow::ensure!(
6300 auth.model_policy_client()?.check(&policy_check).await?,
6301 "model holder must reach real Policy dispatch"
6302 );
6303 let found = model.get_by_name(model_name).await?;
6304 anyhow::ensure!(
6305 found.name == model_name,
6306 "the authenticated model lookup returned the wrong repository"
6307 );
6308 anyhow::ensure!(
6309 model
6310 .list()
6311 .await?
6312 .iter()
6313 .any(|entry| entry.name == model_name),
6314 "Registry-to-Policy mediation lost the authenticated model caller"
6315 );
6316
6317 let client = |key: SigningKey, token: Option<String>| {
6318 RegistryClient::for_local_endpoint_bootstrap(
6319 &auth.registry_endpoint,
6320 key,
6321 auth.registry_key.verifying_key(),
6322 token,
6323 )
6324 };
6325 let missing = client(auth.model_key.clone(), None)?
6326 .get_by_name(model_name)
6327 .await;
6328 assert_restart_dispatch_denied(missing, "tokenless model lookup")?;
6329 let wrong_holder = client(
6330 SigningKey::from_bytes(&[0x94; 32]),
6331 Some(auth.model_token.clone()),
6332 )?
6333 .get_by_name(model_name)
6334 .await;
6335 assert_restart_dispatch_denied(wrong_holder, "wrong holder with model token")?;
6336
6337 let processor = hyprstream_rpc::dial::lookup_inproc(
6338 auth.registry_endpoint
6339 .strip_prefix("inproc://")
6340 .ok_or_else(|| anyhow!("fixture Registry endpoint is not in-process"))?,
6341 )
6342 .ok_or_else(|| anyhow!("fixture Registry handle was not retained"))?;
6343 let classical = RpcClientImpl::new(
6344 ClassicalRestartSigner(auth.model_key.clone()),
6345 InMemoryTransport::new(processor),
6346 Some(auth.registry_key.verifying_key()),
6347 )
6348 .with_default_jwt(auth.model_token.clone());
6349 let classical = RegistryClient::new(Arc::new(classical));
6350 let classical_error = classical
6351 .get_by_name(model_name)
6352 .await
6353 .err()
6354 .ok_or_else(|| anyhow!("classical signer reached Registry"))?;
6355 anyhow::ensure!(
6356 classical_error
6357 .to_string()
6358 .contains("mandatory Hybrid suite requires an ML-DSA-65 signer key"),
6359 "classical signer failed for an unrelated reason: {classical_error}"
6360 );
6361
6362 let unanchored_key = SigningKey::from_bytes(&[0x95; 32]);
6365 let ca = hyprstream_rpc::node_identity::derive_purpose_key(
6366 &auth.policy_key,
6367 "hyprstream-jwt-v1",
6368 );
6369 let unanchored_credentials = tempfile::TempDir::new()?;
6370 let unanchored_token =
6371 restart_fixture_service_jwt(&unanchored_credentials, "model", &ca, &unanchored_key)?;
6372 let unanchored = client(unanchored_key, Some(unanchored_token))?
6373 .get_by_name(model_name)
6374 .await;
6375 let unanchored_error = unanchored
6376 .err()
6377 .ok_or_else(|| anyhow!("unanchored PQ signer reached Registry"))?;
6378 anyhow::ensure!(
6379 unanchored_error.to_string() == "registry envelope admission failed",
6380 "unanchored PQ signer failed outside envelope admission: {unanchored_error:#}"
6381 );
6382 anyhow::ensure!(
6383 unanchored_error.chain().any(|cause| matches!(
6384 cause.downcast_ref::<hyprstream_rpc::EnvelopeError>(),
6385 Some(hyprstream_rpc::EnvelopeError::PqSignatureInvalid(reason))
6386 if reason == "mandatory Hybrid suite requires an anchored ML-DSA-65 signer key"
6387 )),
6388 "unanchored PQ signer failed for an unrelated admission reason: {unanchored_error:#}"
6389 );
6390 Ok(())
6391 }
6392
6393 impl RestartAcceptanceFixture {
6394 async fn from_guest_env() -> Result<Self> {
6395 let repo = std::path::PathBuf::from(std::env::var(
6396 "HYPRSTREAM_RESTART_ACCEPTANCE_MODEL_REPO",
6397 ).map_err(|_| anyhow!(
6398 "HYPRSTREAM_RESTART_ACCEPTANCE_MODEL_REPO must name the sealed local bare model repository"
6399 ))?);
6400 anyhow::ensure!(
6401 repo.is_dir(),
6402 "restart acceptance model repository is not a directory"
6403 );
6404 let model_name = std::env::var("HYPRSTREAM_RESTART_ACCEPTANCE_MODEL_NAME")
6405 .unwrap_or_else(|_| "restart-acceptance-fixture".to_owned());
6406 let instance_name = std::env::var("HYPRSTREAM_INSTANCE").map_err(|_| {
6407 anyhow!("HYPRSTREAM_INSTANCE must be set to an isolated synthetic-guest namespace")
6408 })?;
6409 anyhow::ensure!(
6410 instance_name.starts_with("restart-acceptance-"),
6411 "HYPRSTREAM_INSTANCE must use the restart-acceptance-* isolated namespace"
6412 );
6413
6414 let storage = crate::storage::StoragePaths::new()?;
6415 let worktree = storage.worktree_path(&model_name, "main")?;
6416 anyhow::ensure!(worktree.is_dir(), "fixture main worktree is absent");
6417 anyhow::ensure!(
6418 worktree.join("config.json").is_file()
6419 && worktree.join("tokenizer.json").is_file()
6420 && std::fs::read_dir(&worktree)?.flatten().any(|entry| {
6421 entry
6422 .file_name()
6423 .to_string_lossy()
6424 .ends_with(".safetensors")
6425 }),
6426 "fixture must provide config.json, tokenizer.json, and safetensors weights"
6427 );
6428 validate_restart_fixture_worktree_link(&repo, &worktree)?;
6429
6430 let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
6431 hyprstream_rpc::moq_event::MoqEventOrigin::new(),
6432 );
6433 hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6434 anyhow::ensure!(
6435 hyprstream_rpc::registry::global().mode()
6436 == hyprstream_rpc::registry::EndpointMode::Inproc,
6437 "restart fixture endpoint registry must use Inproc mode"
6438 );
6439 let auth = RestartAuthServices::new(&repo, &model_name).await?;
6440 let mut config = ModelServiceConfig::default();
6441 config.inference_deployment.compute = InferenceCompute::Cpu;
6442 let model = ModelService::new(
6443 config,
6444 auth.model_key.clone(),
6445 auth.model_policy_client()?,
6446 auth.model_registry_client()?,
6447 TransportConfig::inproc(format!(
6448 "restart-acceptance-model-{}",
6449 hex::encode(hyprstream_rpc::envelope::generate_nonce())
6450 )),
6451 auth.policy_transport.clone(),
6452 )
6453 .await?
6454 .with_expected_audience(RESTART_FIXTURE_ISSUER.to_owned())
6455 .with_jwt_key_source(auth.jwt_source.clone());
6456 let model_ref = format!("{model_name}:main");
6457 let instance = InferenceInstanceId::new("restart-acceptance-tenant", &model_ref, 0)?;
6458 Ok(Self {
6459 model,
6460 model_key: auth.model_key.clone(),
6461 model_ref,
6462 instance,
6463 _auth: auth,
6464 })
6465 }
6466 }
6467
6468 #[test]
6469 fn restart_fixture_gitdir_backlink_requires_expected_checkout() -> Result<()> {
6470 fn git(args: &[&str]) -> Result<()> {
6471 let status = std::process::Command::new("git")
6472 .args(args)
6473 .status()
6474 .map_err(|error| anyhow!("temporary Git fixture command did not start: {error}"))?;
6475 anyhow::ensure!(
6476 status.success(),
6477 "temporary Git fixture command failed: git {args:?}"
6478 );
6479 Ok(())
6480 }
6481
6482 let temp = tempfile::tempdir()?;
6483 let source = temp.path().join("source");
6484 let repo = temp.path().join("fixture.git");
6485 let expected_checkout = temp.path().join("isolated/models/worktrees/main");
6486 let other_checkout = temp.path().join("other-worktree");
6487 let source_arg = source
6488 .to_str()
6489 .ok_or_else(|| anyhow!("temporary source path is not UTF-8"))?;
6490 let repo_arg = repo
6491 .to_str()
6492 .ok_or_else(|| anyhow!("temporary bare repository path is not UTF-8"))?;
6493 let expected_arg = expected_checkout
6494 .to_str()
6495 .ok_or_else(|| anyhow!("temporary expected checkout path is not UTF-8"))?;
6496 let other_arg = other_checkout
6497 .to_str()
6498 .ok_or_else(|| anyhow!("temporary mismatched checkout path is not UTF-8"))?;
6499
6500 git(&["init", "-b", "main", source_arg])?;
6501 git(&[
6502 "-C",
6503 source_arg,
6504 "config",
6505 "user.name",
6506 "restart-fixture-test",
6507 ])?;
6508 git(&[
6509 "-C",
6510 source_arg,
6511 "config",
6512 "user.email",
6513 "restart-fixture-test@example.invalid",
6514 ])?;
6515 std::fs::write(source.join("fixture.txt"), "fixture\n")?;
6516 git(&["-C", source_arg, "add", "fixture.txt"])?;
6517 git(&["-C", source_arg, "commit", "-m", "temporary fixture"])?;
6518 git(&["clone", "--bare", source_arg, repo_arg])?;
6519 std::fs::create_dir_all(
6520 expected_checkout
6521 .parent()
6522 .ok_or_else(|| anyhow!("temporary expected checkout has no parent"))?,
6523 )?;
6524 git(&[
6525 "--git-dir",
6526 repo_arg,
6527 "worktree",
6528 "add",
6529 expected_arg,
6530 "main",
6531 ])?;
6532 git(&[
6533 "--git-dir",
6534 repo_arg,
6535 "worktree",
6536 "add",
6537 "-b",
6538 "other",
6539 other_arg,
6540 "main",
6541 ])?;
6542
6543 validate_restart_fixture_worktree_link(&repo, &expected_checkout)?;
6544 let mismatch = match validate_restart_fixture_worktree_link(&repo, &other_checkout) {
6545 Ok(()) => anyhow::bail!("a different linked checkout must be rejected"),
6546 Err(error) => error,
6547 };
6548 anyhow::ensure!(
6549 mismatch
6550 .to_string()
6551 .contains("does not name the isolated StoragePaths worktree"),
6552 "mismatched checkout error must identify the administrative backlink"
6553 );
6554 Ok(())
6555 }
6556
6557 fn assert_restart_order_holder(
6558 order: &str,
6559 controller: &SigningKey,
6560 audience: &str,
6561 subject: &str,
6562 caller_key: &SigningKey,
6563 ) -> Result<()> {
6564 let claims = verify_internal_work(
6565 order,
6566 &controller.verifying_key(),
6567 audience,
6568 chrono::Utc::now().timestamp(),
6569 )?;
6570 anyhow::ensure!(claims.sub == subject && claims.caller.sub == subject);
6571 anyhow::ensure!(claims.tenant == "restart-acceptance-tenant");
6572 anyhow::ensure!(claims.resource == "inference:HasLora" && claims.operation == "query");
6573 anyhow::ensure!(
6574 claims.owner_did.as_deref()
6575 == Some(
6576 hyprstream_rpc::identity::Did::from_ed25519(
6577 &caller_key.verifying_key().to_bytes(),
6578 )
6579 .as_str()
6580 ),
6581 "work order owner DID must remain bound to the verified caller"
6582 );
6583 Ok(())
6584 }
6585
6586 #[derive(Clone)]
6590 struct RestartAudienceDenialProbe {
6591 seen: Arc<std::sync::atomic::AtomicBool>,
6592 }
6593
6594 impl RestartAudienceDenialProbe {
6595 fn install() -> Result<Arc<std::sync::atomic::AtomicBool>> {
6596 use tracing_subscriber::prelude::*;
6597
6598 let seen = Arc::new(std::sync::atomic::AtomicBool::new(false));
6599 let subscriber = tracing_subscriber::registry().with(Self {
6600 seen: Arc::clone(&seen),
6601 });
6602 tracing::subscriber::set_global_default(subscriber).map_err(|_| {
6607 anyhow!(
6608 "restart acceptance requires an isolated test process without a preinstalled tracing subscriber"
6609 )
6610 })?;
6611 Ok(seen)
6612 }
6613 }
6614
6615 struct RestartAudienceDenialVisitor<'a> {
6616 seen: &'a std::sync::atomic::AtomicBool,
6617 }
6618
6619 impl RestartAudienceDenialVisitor<'_> {
6620 fn record_message(&self, field: &tracing::field::Field, message: &str) {
6621 if field.name() == "message"
6622 && message.contains("claims verification failed")
6623 && message.contains("audience")
6624 {
6625 self.seen.store(true, std::sync::atomic::Ordering::SeqCst);
6626 }
6627 }
6628 }
6629
6630 impl tracing::field::Visit for RestartAudienceDenialVisitor<'_> {
6631 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
6632 self.record_message(field, &format!("{value:?}"));
6633 }
6634
6635 fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
6636 self.record_message(field, value);
6637 }
6638 }
6639
6640 impl<S> tracing_subscriber::Layer<S> for RestartAudienceDenialProbe
6641 where
6642 S: tracing::Subscriber + for<'lookup> tracing_subscriber::registry::LookupSpan<'lookup>,
6643 {
6644 fn on_event(
6645 &self,
6646 event: &tracing::Event<'_>,
6647 _context: tracing_subscriber::layer::Context<'_, S>,
6648 ) {
6649 if event.metadata().target() == "hyprstream_rpc::service::dispatch" {
6650 event.record(&mut RestartAudienceDenialVisitor { seen: &self.seen });
6651 }
6652 }
6653 }
6654
6655 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6656 #[ignore = "requires the sealed local CPU-model repository and isolated XDG guest fixture"]
6657 async fn real_cpu_worker_restart_rejects_old_order_and_preserves_holders() -> Result<()> {
6658 crate::services::restart_diag::enable();
6659 crate::services::restart_diag::phase("test.enter");
6660 let audience_denial = RestartAudienceDenialProbe::install()?;
6661 crate::services::restart_diag::phase("fixture.enter");
6662 let fixture = tokio::time::timeout(
6663 std::time::Duration::from_secs(180),
6664 RestartAcceptanceFixture::from_guest_env(),
6665 )
6666 .await
6667 .map_err(|_| anyhow!("RESTART_DIAG_PHASE_TIMEOUT=fixture"))??;
6668 crate::services::restart_diag::phase("fixture.done");
6669 let caller_a_key = SigningKey::from_bytes(&[0xA1; 32]);
6670 let caller_b_key = SigningKey::from_bytes(&[0xB1; 32]);
6671 let caller_a = restart_acceptance_caller_ctx(
6672 "restart-caller-a",
6673 &caller_a_key,
6674 "restart-acceptance-tenant",
6675 );
6676 let caller_b = restart_acceptance_caller_ctx(
6677 "restart-caller-b",
6678 &caller_b_key,
6679 "restart-acceptance-tenant",
6680 );
6681 let scope = ModelService::inference_scope("hasLora", "query");
6682
6683 crate::services::restart_diag::phase("worker_a.load.enter");
6685 tokio::time::timeout(
6686 std::time::Duration::from_secs(600),
6687 fixture.model.load_model(&fixture.instance, None, None),
6688 )
6689 .await
6690 .map_err(|_| anyhow!("RESTART_DIAG_PHASE_TIMEOUT=worker_a.load"))??;
6691 crate::services::restart_diag::phase("worker_a.load.done");
6692 let (audience_a, incarnation_a) = {
6693 let cache = fixture.model.loaded_models.read().await;
6694 let loaded = cache
6695 .peek(&fixture.instance)
6696 .ok_or_else(|| anyhow!("worker A missing after successful load"))?;
6697 (loaded.work_audience.clone(), loaded.incarnation.clone())
6698 };
6699 let a_client = fixture
6700 .model
6701 .get_inference_client(&fixture.model_ref, &caller_a, scope.clone())
6702 .await?;
6703 let mut old_order = fixture
6704 .model
6705 .take_test_work_order()
6706 .await
6707 .ok_or_else(|| anyhow!("test-only capture missed worker A work order"))?;
6708 assert_restart_order_holder(
6709 &old_order,
6710 &fixture.model_key,
6711 &audience_a,
6712 "restart-caller-a",
6713 &caller_a_key,
6714 )?;
6715 crate::services::restart_diag::phase("worker_a.call.enter");
6716 let _ = a_client.has_lora().await?;
6717 crate::services::restart_diag::phase("worker_a.call.done");
6718
6719 crate::services::restart_diag::phase("worker_a.unload.enter");
6722 fixture.model.unload_model(&fixture.instance).await?;
6723 crate::services::restart_diag::phase("worker_a.unload.done");
6724 anyhow::ensure!(
6725 !fixture
6726 .model
6727 .loaded_models
6728 .read()
6729 .await
6730 .contains(&fixture.instance),
6731 "unload must remove worker A before B is created"
6732 );
6733 crate::services::restart_diag::phase("worker_b.load.enter");
6734 tokio::time::timeout(
6735 std::time::Duration::from_secs(600),
6736 fixture.model.load_model(&fixture.instance, None, None),
6737 )
6738 .await
6739 .map_err(|_| anyhow!("RESTART_DIAG_PHASE_TIMEOUT=worker_b.load"))??;
6740 crate::services::restart_diag::phase("worker_b.load.done");
6741 let (audience_b, incarnation_b, worker_b) = {
6742 let cache = fixture.model.loaded_models.read().await;
6743 let loaded = cache
6744 .peek(&fixture.instance)
6745 .ok_or_else(|| anyhow!("worker B missing after successful reload"))?;
6746 (
6747 loaded.work_audience.clone(),
6748 loaded.incarnation.clone(),
6749 loaded.client.clone(),
6750 )
6751 };
6752 anyhow::ensure!(
6753 audience_a != audience_b && incarnation_a != incarnation_b,
6754 "reload must install a fresh worker-generated incarnation"
6755 );
6756
6757 crate::services::restart_diag::phase("old_order.denial.enter");
6762 let replay_error = match worker_b
6763 .with_bearer(std::mem::take(&mut *old_order))
6764 .has_lora()
6765 .await
6766 {
6767 Ok(_) => anyhow::bail!("worker B accepted worker A's order"),
6768 Err(error) => error,
6769 };
6770 anyhow::ensure!(
6771 replay_error.to_string() == hyprstream_rpc::service::dispatch::DISPATCH_DENIED,
6772 "old order must receive the uniform dispatch denial"
6773 );
6774 anyhow::ensure!(
6775 audience_denial.load(std::sync::atomic::Ordering::SeqCst),
6776 "replacement worker must classify the old order as an audience mismatch"
6777 );
6778 crate::services::restart_diag::phase("old_order.denial.done");
6779
6780 let b_client_for_a = fixture
6782 .model
6783 .get_inference_client(&fixture.model_ref, &caller_a, scope.clone())
6784 .await?;
6785 let fresh_a_order = fixture
6786 .model
6787 .take_test_work_order()
6788 .await
6789 .ok_or_else(|| anyhow!("test-only capture missed worker B fresh order"))?;
6790 assert_restart_order_holder(
6791 &fresh_a_order,
6792 &fixture.model_key,
6793 &audience_b,
6794 "restart-caller-a",
6795 &caller_a_key,
6796 )?;
6797 crate::services::restart_diag::phase("fresh_a.call.enter");
6798 let _ = b_client_for_a.has_lora().await?;
6799 crate::services::restart_diag::phase("fresh_a.call.done");
6800
6801 let b_client_for_b = fixture
6804 .model
6805 .get_inference_client(&fixture.model_ref, &caller_b, scope)
6806 .await?;
6807 let fresh_b_order = fixture
6808 .model
6809 .take_test_work_order()
6810 .await
6811 .ok_or_else(|| anyhow!("test-only capture missed caller B order"))?;
6812 assert_restart_order_holder(
6813 &fresh_b_order,
6814 &fixture.model_key,
6815 &audience_b,
6816 "restart-caller-b",
6817 &caller_b_key,
6818 )?;
6819 anyhow::ensure!(
6820 fresh_a_order != fresh_b_order,
6821 "distinct caller requests require distinct work orders"
6822 );
6823 crate::services::restart_diag::phase("fresh_b.call.enter");
6824 let _ = b_client_for_b.has_lora().await?;
6825 crate::services::restart_diag::phase("fresh_b.call.done");
6826 crate::services::restart_diag::phase("holders.done");
6827
6828 crate::services::restart_diag::phase("worker_b.unload.enter");
6829 fixture.model.unload_model(&fixture.instance).await?;
6830 crate::services::restart_diag::phase("worker_b.unload.done");
6831 Ok(())
6832 }
6833
6834 #[tokio::test]
6835 async fn internal_work_mint_binds_instance_tenant_model_operation_and_caller() {
6836 let service = selector_model_service().await;
6837 let instance = selector_instance();
6838 let scope = ModelService::inference_scope("generateStream", "infer");
6839 let caller_exp = chrono::Utc::now().timestamp() + 3600;
6840 let ctx = boundary_caller_ctx(caller_exp);
6841 let token = service
6842 .mint_internal_work_token(&instance, &ctx, &scope, &audience_fixture())
6843 .unwrap_or_else(|e| panic!("mint failed: {e}"));
6844
6845 let model_key = SigningKey::from_bytes(&[0x41; 32]);
6846 let now = chrono::Utc::now().timestamp();
6847 let claims =
6848 verify_internal_work(&token, &model_key.verifying_key(), &audience_fixture(), now)
6849 .unwrap_or_else(|e| {
6850 panic!("minted order must verify against the pinned controller: {e}")
6851 });
6852 assert_eq!(claims.sub, "alice");
6853 assert_eq!(claims.tenant, "fixture-tenant");
6854 assert_eq!(claims.model, "fixture-model:main");
6855 assert_eq!(claims.resource, "inference:GenerateStream");
6856 assert_eq!(claims.operation, "infer");
6857 assert!(
6858 claims.exp <= caller_exp && claims.exp <= now + MAX_LIFETIME_SECS,
6859 "exp must be capped by the caller credential and the hard bound"
6860 );
6861 assert!(
6862 claims.exp > now,
6863 "a live caller credential must mint a live work order"
6864 );
6865 assert!(!claims.jti.is_empty());
6866 let expected_x = base64::engine::general_purpose::URL_SAFE_NO_PAD
6867 .encode(model_key.verifying_key().as_bytes());
6868 assert_eq!(
6869 claims.cnf.jwk.as_ref().map(|j| j.x.as_str()),
6870 Some(expected_x.as_str()),
6871 "cnf must bind the pinned controller key"
6872 );
6873 let caller_key = SigningKey::from_bytes(&BOUNDARY_CALLER_KEY);
6876 assert_eq!(
6877 claims.owner_did.as_deref(),
6878 Some(
6879 hyprstream_rpc::identity::Did::from_ed25519(&caller_key.verifying_key().to_bytes())
6880 .as_str()
6881 )
6882 );
6883 assert_eq!(claims.caller.sub, "alice");
6886 assert!(claims.caller.token.is_none());
6887 }
6888
6889 #[tokio::test]
6890 async fn internal_work_mint_requires_verified_caller_identity_and_live_expiry() {
6891 let service = selector_model_service().await;
6892 let instance = selector_instance();
6893 let scope = InternalWorkScope::new("inference:probe/adapters/k3", "write");
6895
6896 let bare = EnvelopeContext::for_test_authenticated_subject(
6899 hyprstream_rpc::envelope::Subject::anonymous(),
6900 SigningKey::from_bytes(&BOUNDARY_CALLER_KEY).verifying_key(),
6901 );
6902 assert!(
6903 service
6904 .mint_internal_work_token(&instance, &bare, &scope, &audience_fixture())
6905 .is_err(),
6906 "internal work without a verified caller identity must not mint"
6907 );
6908
6909 let expired = boundary_caller_ctx(chrono::Utc::now().timestamp() - 10);
6911 assert!(
6912 service
6913 .mint_internal_work_token(&instance, &expired, &scope, &audience_fixture())
6914 .is_err(),
6915 "internal work must not outlive an expired caller credential"
6916 );
6917 }
6918}