Skip to main content

hyprstream_core/services/
model.rs

1//! Model service for managing InferenceService instances over ZMQ
2//!
3//! This service manages the lifecycle of InferenceService instances.
4//! It handles model loading, unloading, and routes inference requests
5//! to the appropriate InferenceService based on model reference.
6//!
7//! # Architecture
8//!
9//! ```text
10//! REST API / CLI
11//!       │
12//!       │ ModelClient (async ZMQ I/O)
13//!       ▼
14//! ModelService (multi-threaded runtime)
15//!       │
16//!       ├── LRU cache of loaded models
17//!       ├── Spawns InferenceService per model
18//!       └── Routes requests to InferenceService
19//!             │
20//!             │ InferenceClient (async ZMQ I/O)
21//!             ▼
22//!       InferenceService (dedicated thread per model)
23//! ```
24//!
25//! # Endpoint
26//!
27//! Uses `registry().endpoint("model", SocketKind::Rep)` for the REP endpoint.
28//! Default fallback: `inproc://hyprstream/model`
29
30use async_trait::async_trait;
31// GenerationRequest import removed — was only used by deleted ModelZmqClient
32use crate::runtime::inference_profile::{
33    InferenceCompute, InferenceDeploymentProfile, InferenceInstanceId, InferenceIsolationProfile,
34};
35use crate::runtime::RuntimeConfig;
36use crate::services::WorktreeClientExt;
37use crate::services::{EnvelopeContext, PolicyClient};
38use crate::storage::ModelRef;
39use anyhow::{anyhow, Context, Result};
40use hyprstream_rpc::events::EventPublisher;
41use hyprstream_rpc::latch::{Terminal, TerminalStore};
42use hyprstream_rpc::prelude::*;
43use hyprstream_rpc::stream_info::TransportConfig as WireTransportConfig;
44use hyprstream_rpc::transport::{EndpointType, TransportConfig};
45use hyprstream_rpc_std::inference_client::InferenceClient;
46use hyprstream_rpc_std::model_client::KVQuantType;
47use hyprstream_rpc_std::policy_client::PolicyCheck;
48use hyprstream_rpc_std::registry_client::RegistryClient;
49use hyprstream_rpc_std::registry_client::{CommitWithAuthorRequest, StageFilesRequest};
50use lru::LruCache;
51use std::collections::{HashMap, HashSet};
52use std::num::NonZeroUsize;
53use std::sync::atomic::{AtomicU64, Ordering};
54use std::sync::Arc;
55use std::time::Instant;
56use tokio::sync::{Mutex, RwLock};
57use tracing::{debug, info, warn};
58#[cfg(test)]
59use zeroize::Zeroizing;
60
61/// Default endpoint for the model service
62pub const MODEL_ENDPOINT: &str = "inproc://hyprstream/model";
63
64// ============================================================================
65// Latched load terminal (EV7/#649) — host-side retained load outcome
66// ============================================================================
67
68/// Terminal payload for a model load attempt (EV7/#649). Latched host-side by
69/// [`ModelService`] when a load completes — [`Loaded`](Self::Loaded) on success,
70/// [`LoadFailed`](Self::LoadFailed) otherwise — so a late `load --wait` (or a
71/// future P9 `/model/<ref>/loaded` file) is served the retained value instead
72/// of missing the live `model.lifecycle` edge it subscribed to too late.
73///
74/// Rides the same plaintext `model` EventService source as the live edge (EV5
75/// /#605; encrypted-default flip deferred to #555), so retaining it host-side
76/// releases no plaintext the live edge would not.
77#[derive(Clone, Debug, PartialEq)]
78pub enum ModelLoadTerminal {
79    /// Load succeeded; `endpoint` is the inference endpoint string.
80    Loaded { endpoint: String },
81    /// Load failed; `error` is the failure message.
82    LoadFailed { error: String },
83}
84
85/// Per-load-attempt key into the model-service [`TerminalStore`] (EV7/#649).
86///
87/// Combines the model ref with a monotonic per-process epoch (see
88/// [`ModelServiceInner::load_epoch`]) so a reload allocates a fresh key and
89/// latches a NEW terminal — monotonic write-once then holds *per epoch*, the
90/// EV7 "reload = new latch, write-once within an attempt" contract. Without
91/// per-epoch keying the first `Loaded` latch would shadow every reload.
92#[derive(Clone, Debug, PartialEq, Eq, Hash)]
93pub struct ModelLoadKey {
94    /// Authority-verified tenant that owns this load attempt.
95    pub tenant: String,
96    /// The model reference string (e.g. "qwen3-small:main").
97    pub model_ref: String,
98    /// Monotonic load-attempt number for this process.
99    pub epoch: u64,
100}
101
102// ============================================================================
103// ModelService (server-side)
104// ============================================================================
105
106/// Information about a loaded model
107pub struct LoadedModel {
108    /// Tenant/model/replica identity of this inference instance.
109    pub instance: InferenceInstanceId,
110    /// Model reference string (e.g., "qwen3-small:main")
111    pub model_ref: String,
112    /// Exact Git commit whose owned base inputs were used, if pinned.
113    pub pinned_commit: Option<git2::Oid>,
114    /// Local transport for this model's InferenceService (#320). This remains
115    /// the `Inproc` arm registered by the spawner and is never advertised as a
116    /// remotely dialable reach.
117    pub transport: TransportConfig,
118    /// Advertised network reach for remote callers. Local dispatch continues
119    /// to use `transport` as the in-process fast path.
120    pub network_transport: TransportConfig,
121    /// Handle to stop the InferenceService
122    pub service_handle: hyprstream_service::SpawnedService,
123    /// Client for communicating with the InferenceService (built from
124    /// `transport` via `dial()` — the co-located fast path).
125    pub client: InferenceClient,
126    /// Worker-generated 256-bit incarnation of THIS running pinned worker,
127    /// reported through the private readiness handoff for this spawn attempt.
128    /// Fresh per run: pre-restart work orders fail audience verification at
129    /// the replacement worker (Sol bounded incarnation plan).
130    pub incarnation: String,
131    /// Exact versioned internal-work audience of this incarnation
132    /// (`iw1/{deterministic service name}/{incarnation}`). Minting uses THIS
133    /// binding — never a recomputed deterministic name.
134    pub work_audience: String,
135    /// Spawn-attempt generation this binding came from. A delayed result from
136    /// an older attempt cannot overwrite a newer binding (generation checked
137    /// at handoff validation).
138    pub load_generation: u64,
139    /// #322 leaf cell-router state for this model. Holds the session→owner
140    /// affinity map (heartbeat-lease renewal, KV-cache stickiness) and the
141    /// per-node load/health counters used by HRW placement. In v1 this is a
142    /// single co-located node (the router fast-paths to `client`); the
143    /// `load_state` set grows to multiple nodes when cross-host replicas are
144    /// resolved via the Resolver.
145    pub router: crate::services::router::CellRouter,
146    /// Replica-set snapshot for the router (one entry per known inference
147    /// server serving this model's OID). Updated as the Resolver yields new
148    /// reaches; consumed by HRW placement.
149    pub load_state: Vec<crate::services::router::InferenceServerInfo>,
150    /// When the model was loaded
151    pub loaded_at: Instant,
152    /// When the model was last used
153    pub last_used: Instant,
154    /// Online training (TTT) configuration (if enabled)
155    pub ttt_config: Option<crate::training::ttt::TTTConfig>,
156    /// Generation parameter defaults from model's generation_config.json
157    pub generation_defaults: crate::config::SamplingParams,
158}
159
160/// Model service configuration
161pub struct ModelServiceConfig {
162    /// Maximum number of models to keep loaded
163    pub max_models: usize,
164    /// Maximum context length for KV cache allocation
165    pub max_context: Option<u32>,
166    /// KV cache quantization type
167    pub kv_quant: KVQuantType,
168    /// Isolation, tenancy, compute, and resource contract for spawned engines.
169    pub inference_deployment: InferenceDeploymentProfile,
170    /// Optional exact-ref admission for the bounded staging deployment.
171    pub staging_model_pin: Option<StagingModelPin>,
172}
173
174#[derive(Clone, Debug, PartialEq, Eq)]
175pub struct StagingModelPin {
176    pub model_ref: String,
177    pub commit: git2::Oid,
178    projection_policy: git2db::pinned_disk::ProjectionPolicy,
179    #[cfg(test)]
180    projection_parent: Option<std::path::PathBuf>,
181}
182
183const REVIEWED_STAGING_MODEL_REF: &str = "qwen2.5-0.5b-instruct:main";
184const REVIEWED_STAGING_MODEL_OID: &str = "18c562db6830c2ef6636b8eaf42fb479272052f7";
185
186impl StagingModelPin {
187    pub(crate) fn reviewed() -> Result<Self> {
188        Self::new(
189            REVIEWED_STAGING_MODEL_REF,
190            git2::Oid::from_str(REVIEWED_STAGING_MODEL_OID)?,
191        )
192    }
193
194    #[cfg(feature = "postgres")]
195    pub(crate) fn reviewed_repo_name() -> &'static str {
196        "qwen2.5-0.5b-instruct"
197    }
198
199    pub fn new(model_ref: impl Into<String>, commit: git2::Oid) -> Result<Self> {
200        let model_ref = model_ref.into();
201        let parsed = ModelRef::parse(&model_ref)
202            .with_context(|| format!("invalid bounded staging modelRef {model_ref:?}"))?;
203        anyhow::ensure!(
204            matches!(parsed.git_ref, crate::storage::GitRef::Branch(_))
205                && parsed.to_string() == model_ref,
206            "bounded staging modelRef must be a canonical local branch ref"
207        );
208        Ok(Self {
209            model_ref,
210            commit,
211            projection_policy: Default::default(),
212            #[cfg(test)]
213            projection_parent: None,
214        })
215    }
216
217    /// Reviewed Clef complete-tree contract; preserves the ordinary 2 GiB
218    /// default and A0 allowlist. This is trusted startup wiring, never RPC data.
219    /// Local objects are independently verified against captured LFS oid/size.
220    pub fn reviewed_clef_flash(
221        local_lfs: std::collections::BTreeMap<String, std::path::PathBuf>,
222    ) -> Result<Self> {
223        let mut pin = Self::new(
224            "clef-flash-fp8:main",
225            git2::Oid::from_str("7b43c2d799e7b5e9fef96b6137840088d75af0d6")?,
226        )?;
227        // R0 normalized complete 21-file pinned tree (head + backbone + metadata).
228        pin.projection_policy = git2db::pinned_disk::ProjectionPolicy::from_trusted_contract(
229            12_174_008_714,
230            local_lfs,
231        )?;
232        Ok(pin)
233    }
234
235    fn capture_policy(&self) -> Result<git2db::pinned_disk::ProjectionPolicy> {
236        if self.projection_policy != Default::default() {
237            anyhow::ensure!(
238                self.model_ref == "clef-flash-fp8:main"
239                    && self.commit
240                        == git2::Oid::from_str("7b43c2d799e7b5e9fef96b6137840088d75af0d6")?,
241                "Clef projection policy pin identity changed"
242            );
243        }
244        Ok(self.projection_policy.clone())
245    }
246
247    /// `staging` selects the code-reviewed A0 identity. Optional explicit values
248    /// must match it; incomplete/invalid profiles fail closed at startup.
249    pub fn from_env() -> Result<Option<Self>> {
250        const PROFILE: &str = "HYPRSTREAM_MODEL_ADMISSION_PROFILE";
251        const REF: &str = "HYPRSTREAM_STAGING_MODEL_REF";
252        const OID: &str = "HYPRSTREAM_STAGING_MODEL_OID";
253        let profile = match std::env::var(PROFILE) {
254            Ok(value) => value,
255            Err(std::env::VarError::NotPresent) => "default".to_owned(),
256            Err(error) => return Err(error.into()),
257        };
258        let read_optional = |name| match std::env::var(name) {
259            Ok(value) => Ok(Some(value)),
260            Err(std::env::VarError::NotPresent) => Ok(None),
261            Err(error) => Err(anyhow!("read {name}: {error}")),
262        };
263        let model_ref = read_optional(REF)?;
264        let oid = read_optional(OID)?;
265        Self::from_settings(&profile, model_ref.as_deref(), oid.as_deref())
266    }
267
268    fn from_settings(
269        profile: &str,
270        model_ref: Option<&str>,
271        oid: Option<&str>,
272    ) -> Result<Option<Self>> {
273        const PROFILE: &str = "HYPRSTREAM_MODEL_ADMISSION_PROFILE";
274        const REF: &str = "HYPRSTREAM_STAGING_MODEL_REF";
275        const OID: &str = "HYPRSTREAM_STAGING_MODEL_OID";
276        if profile == "staging" {
277            match (model_ref, oid) {
278                (None, None) => return Self::reviewed().map(Some),
279                (Some(model_ref), Some(oid)) => anyhow::ensure!(
280                    model_ref == REVIEWED_STAGING_MODEL_REF && oid == REVIEWED_STAGING_MODEL_OID,
281                    "staging override must match the complete reviewed model pin"
282                ),
283                _ => anyhow::bail!(
284                    "{REF} and {OID} must either both be absent or both match the reviewed pin"
285                ),
286            }
287            return Self::reviewed().map(Some);
288        }
289        anyhow::ensure!(
290            profile == "default",
291            "{PROFILE} must be 'default' or 'staging'"
292        );
293        match (model_ref, oid) {
294            (None, None) => Ok(None),
295            (Some(model_ref), Some(oid)) => {
296                anyhow::ensure!(
297                    oid.len() == 40 && oid.bytes().all(|byte| byte.is_ascii_hexdigit()),
298                    "{OID} must be a full 40-character hexadecimal Git OID"
299                );
300                Self::new(model_ref, git2::Oid::from_str(oid)?).map(Some)
301            }
302            _ => anyhow::bail!("{REF} and {OID} must be configured together"),
303        }
304    }
305
306    fn admits(&self, model_ref: &str) -> bool {
307        self.model_ref == model_ref
308    }
309}
310
311impl Default for ModelServiceConfig {
312    fn default() -> Self {
313        Self {
314            max_models: 5,
315            max_context: None,
316            kv_quant: KVQuantType::None,
317            inference_deployment: InferenceDeploymentProfile::default(),
318            staging_model_pin: None,
319        }
320    }
321}
322
323/// Inner state for ModelService, behind Arc for continuation capture.
324pub struct ModelServiceInner {
325    // Business logic
326    /// LRU cache of loaded models
327    loaded_models: RwLock<LruCache<InferenceInstanceId, LoadedModel>>,
328    /// Models currently being loaded (accepted but not yet in LRU cache)
329    pending_loads: parking_lot::Mutex<HashMap<InferenceInstanceId, u64>>,
330    /// Shared read-only staging projection for tenant-specific workers using
331    /// the same reviewed commit pin.
332    staging_pinned_artifacts: crate::storage::pinned_model::StagingPinnedArtifactCache,
333    /// Models whose cache entry has been removed and whose worker teardown is
334    /// still in progress. This survives cancellation of the caller awaiting
335    /// `unload_model`, so a replacement load cannot race a draining worker.
336    unloading_models: Mutex<HashSet<InferenceInstanceId>>,
337    /// Serializes admission of a new load with the transition that reserves an
338    /// existing worker for teardown. The guard covers the unloading check,
339    /// cache observation, and pending-load insertion as one decision.
340    load_unload_gate: Mutex<()>,
341    /// Serializes lifecycle event delivery. It is deliberately separate from
342    /// lifecycle admission: a successful unload acquires this order before
343    /// releasing its slot, preventing a later loaded event from overtaking the
344    /// listener-visible unloaded completion without holding admission locks
345    /// across network/event delivery.
346    lifecycle_event_publish_gate: Mutex<()>,
347    /// Service configuration
348    config: ModelServiceConfig,
349    /// Ed25519 signing key for creating InferenceClients
350    signing_key: SigningKey,
351    /// Policy client for authorization checks in InferenceService
352    policy_client: PolicyClient,
353    /// No runtime installer in this slice; reserved Federate JWTs still deny.
354    #[cfg(feature = "postgres")]
355    federate_dispatch: Option<Arc<crate::services::oauth::federate_proof::DispatchAdapter>>,
356    /// Event publisher for model lifecycle events (replaces NotificationService, EV5/#605).
357    /// Plaintext (Public) path; the encrypted-default flip is deferred to #555.
358    event_publisher: EventPublisher,
359    /// Registry client for resolving model paths
360    registry: RegistryClient,
361    // Infrastructure (for Spawnable)
362    transport: TransportConfig,
363    policy_transport: TransportConfig,
364    /// Expected JWT audience for token validation (RFC 8707).
365    expected_audience: Option<String>,
366    /// Exact HTTP resource audiences admitted only for current OAI/MCP relays.
367    delegated_oai_audience: Option<String>,
368    delegated_mcp_audience: Option<String>,
369    /// Unified JWT key source for verifying JWTs (local and federated).
370    jwt_key_source: Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>>,
371    /// Discovery client for federated record resolution (#431). None = no
372    /// federation; `resolve_model_ref`'s at:// branch then falls through to
373    /// local resolution.
374    discovery_client: Option<Arc<hyprstream_rpc_std::discovery_client::DiscoveryClient>>,
375    /// Persistent 9P synthetic trees, partitioned by authority-verified tenant.
376    fs_trees: dashmap::DashMap<String, Arc<crate::services::fs::SyntheticTree>>,
377    /// Retained terminal for each model load attempt (EV7/#649) — the
378    /// host-side "file holds the truth" retain. A late `load --wait` reads the
379    /// retained [`ModelLoadTerminal`] from here instead of missing the live
380    /// `model.lifecycle` edge. Distinct from the live EventService edge (which
381    /// never holds the retained value) and from the `loaded_models` cache
382    /// (which tracks residency, not the per-attempt terminal outcome).
383    load_terminals: TerminalStore<ModelLoadKey, ModelLoadTerminal>,
384    /// Monotonic per-process epoch counter; each genuine load attempt
385    /// allocates the next value, keying a fresh [`TerminalStore`] entry so a
386    /// reload latches under a new key (reload ⇒ new terminal, EV7/#649).
387    load_epoch: AtomicU64,
388    /// Monotonic spawn-attempt generation for pinned worker incarnation
389    /// handoffs. Each `InferenceService` spawn attempt allocates the next
390    /// value; the worker echoes it in its private readiness handoff and a
391    /// delayed/stale handoff from an older attempt is discarded.
392    incarnation_generation: AtomicU64,
393    /// The sole tenant admitted by an in-process deployment. A second tenant
394    /// must never share the FFI engine fault radius.
395    in_process_tenant: std::sync::OnceLock<String>,
396    /// QUIC/MoQ reach populated by the outer service spawner after bind.
397    ///
398    /// Dynamically loaded inference engines share this handle so their stream
399    /// tokens advertise the ModelService's browser-addressable `/moq` relay.
400    producer_reach_config: hyprstream_rpc::moq_stream::ProducerReachConfigHandle,
401    /// Service-scoped MoQ origin populated by the outer service spawner.
402    moq_origin: hyprstream_rpc::moq_stream::MoqStreamOriginHandle,
403    /// Test-only, in-memory capture of the most recently minted direct work
404    /// order. This deliberately does not exist in production builds: a work
405    /// order must never become an observable RPC value, log field, file, or
406    /// environment value merely to support restart acceptance coverage.
407    #[cfg(test)]
408    test_work_order_capture: Mutex<Option<Zeroizing<String>>>,
409    /// Test-only ordering witness: successful unload clears admission before
410    /// making its `model.unloaded` event visible.
411    #[cfg(test)]
412    before_unloaded_event: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
413}
414
415/// Model service that manages InferenceService lifecycle.
416///
417/// Wraps `ModelServiceInner` in `Arc` so continuations can capture a cheap
418/// clone. All field access is transparent via `Deref`.
419///
420/// Load requests are handled asynchronously: the request loop returns an
421/// immediate "accepted" response and spawns the actual model loading as a
422/// `Continuation` (via `spawn_local`), keeping the service responsive for
423/// list, health, info, and other requests during long GPU weight transfers.
424pub struct ModelService {
425    inner: Arc<ModelServiceInner>,
426}
427
428struct SpawnPublicationGuard(Option<hyprstream_service::SpawnedService>);
429
430impl SpawnPublicationGuard {
431    fn new(handle: hyprstream_service::SpawnedService) -> Self {
432        Self(Some(handle))
433    }
434    fn publish(mut self) -> Result<hyprstream_service::SpawnedService> {
435        self.0
436            .take()
437            .ok_or_else(|| anyhow!("spawn publication guard was already disarmed"))
438    }
439}
440
441impl Drop for SpawnPublicationGuard {
442    fn drop(&mut self) {
443        if let Some(mut handle) = self.0.take() {
444            match tokio::runtime::Handle::try_current() {
445                Ok(runtime) => {
446                    runtime.spawn(async move {
447                    if let Err(error) = handle.stop().await {
448                        tracing::error!(%error, "failed to reclaim unpublished inference worker");
449                    }
450                });
451                }
452                Err(error) => {
453                    tracing::error!(%error, "cannot reclaim unpublished inference worker outside Tokio runtime");
454                }
455            }
456        }
457    }
458}
459
460/// Result of the gated admission used by the non-blocking load RPC path.
461///
462/// Only `Accepted` owns a continuation and a pending reservation. An
463/// already-loaded or already-pending request is a read/deduplication fast path
464/// and must not create a second load attempt.
465enum InterceptedLoadAdmission {
466    Loaded { reach: Vec<WireTransportConfig> },
467    Pending,
468    Accepted(InterceptedLoadReservation),
469}
470
471/// Owns an intercepted load's pending marker from admission through either
472/// continuation completion or cancellation. The marker uses a tiny synchronous
473/// mutex deliberately: `Drop` must be able to roll it back if serialization or
474/// the continuation handoff is cancelled before the future is ever polled.
475struct InterceptedLoadReservation {
476    inner: Arc<ModelServiceInner>,
477    instance: InferenceInstanceId,
478    attempt: u64,
479    active: bool,
480}
481
482impl InterceptedLoadReservation {
483    fn new(inner: Arc<ModelServiceInner>, instance: InferenceInstanceId, attempt: u64) -> Self {
484        Self {
485            inner,
486            instance,
487            attempt,
488            active: true,
489        }
490    }
491
492    fn release(&mut self) {
493        if self.active {
494            let mut pending = self.inner.pending_loads.lock();
495            if pending.get(&self.instance) == Some(&self.attempt) {
496                pending.remove(&self.instance);
497            }
498            self.active = false;
499        }
500    }
501}
502
503impl Drop for InterceptedLoadReservation {
504    fn drop(&mut self) {
505        self.release();
506    }
507}
508
509fn admit_in_process_tenant(
510    admitted_tenant: &std::sync::OnceLock<String>,
511    verified_tenant: &str,
512) -> Result<()> {
513    if let Some(admitted) = admitted_tenant.get() {
514        anyhow::ensure!(
515            admitted == verified_tenant,
516            "in-process inference is bound to tenant {admitted:?}; refusing cross-tenant engine sharing"
517        );
518        return Ok(());
519    }
520
521    if admitted_tenant.set(verified_tenant.to_owned()).is_err() {
522        let admitted = admitted_tenant
523            .get()
524            .ok_or_else(|| anyhow!("in-process tenant admission raced without a winner"))?;
525        anyhow::ensure!(
526            admitted == verified_tenant,
527            "in-process inference is bound to tenant {admitted:?}; refusing cross-tenant engine sharing"
528        );
529    }
530    Ok(())
531}
532
533impl Clone for ModelService {
534    fn clone(&self) -> Self {
535        Self {
536            inner: Arc::clone(&self.inner),
537        }
538    }
539}
540
541impl std::ops::Deref for ModelService {
542    type Target = ModelServiceInner;
543    fn deref(&self) -> &Self::Target {
544        &self.inner
545    }
546}
547
548/// Prefix-dispatch arm for the `modelRef` grammar (#395).
549///
550/// `modelRef :Text` stays `Text` in the capnp schema; the Rust-side grammar is:
551///
552/// ```text
553/// modelRef ::= "at://" <at-uri>   # federated (resolve via atproto NAME → git OID)
554///            | "did:" <did>       # federated, bare-DID form
555///            | <name> [":" <gitref>]  # local ModelRef (unchanged, backward-compatible)
556/// ```
557///
558/// `at://` and `did:` are federated; everything else is the legacy local
559/// [`ModelRef::parse`] path. This enum is split out from
560/// [`ModelService::resolve_model_ref`] so the grammar is unit-testable without
561/// constructing a full [`ModelService`].
562#[derive(Debug, Clone, PartialEq, Eq)]
563pub enum ModelRefDispatch<'a> {
564    /// `at://…` or `did:…` — federated resolution (atproto record store, #392).
565    Federated {
566        /// The matched scheme prefix (`"at://"` or `"did:"`).
567        scheme: &'static str,
568        /// The ModelRef string with the scheme prefix stripped.
569        rest: &'a str,
570    },
571    /// No federated prefix — fall through to the local [`ModelRef::parse`].
572    Local,
573}
574
575/// Classify a `modelRef` string into its grammar arm (#395 prefix dispatch).
576///
577/// This is the pure prefix-detection core of [`ModelService::resolve_model_ref`];
578/// it performs no I/O and no validation, so it can be unit-tested in isolation.
579pub fn model_ref_dispatch(s: &str) -> ModelRefDispatch<'_> {
580    if let Some(rest) = s.strip_prefix("at://") {
581        ModelRefDispatch::Federated {
582            scheme: "at://",
583            rest,
584        }
585    } else if let Some(rest) = s.strip_prefix("did:") {
586        ModelRefDispatch::Federated {
587            scheme: "did:",
588            rest,
589        }
590    } else {
591        ModelRefDispatch::Local
592    }
593}
594
595/// Outcome of [`ModelService::route_decision`] — which arm of the replica
596/// set HRW/least-loaded selected, kept separate from `InferenceClient`
597/// construction so the pure ranking decision is unit-testable without a
598/// live [`LoadedModel`] (no `InferenceClient`/`SpawnedService` required).
599#[derive(Debug, Clone, Copy, PartialEq, Eq)]
600enum RouteDecision {
601    /// The router selected the co-located replica (`load_state`'s
602    /// co-located entry) — the in-process fast path, no dial, no mesh
603    /// policy fires (same trust domain).
604    CoLocated,
605    /// The router selected a DIFFERENT (non-co-located) replica from the
606    /// resolved candidate set. #282 has not wired a cross-host dial yet,
607    /// so callers still fall back to the co-located client. Placement metadata
608    /// supplies neither network reach nor authority.
609    Remote(crate::services::router::ReplicaId),
610    /// No healthy candidate at all (every entry excluded as down/stalled,
611    /// or the replica set is empty).
612    NoHealthyCandidate,
613}
614
615#[derive(Debug, Clone, Copy, PartialEq, Eq)]
616enum ReplicaLocality {
617    CoLocated,
618    Remote,
619}
620
621struct RoutedReplica<T> {
622    value: T,
623    replica_id: crate::services::router::ReplicaId,
624    locality: ReplicaLocality,
625}
626
627/// Apply affinity-preserving selection and bounded dial-failure reselection.
628///
629/// A remote failure is marked down before the next placement. The co-located
630/// value is returned only when the router explicitly selects that replica; an
631/// exhausted/empty set is an error, never an implicit local fallback.
632async fn route_and_dial_replica<T, F, Fut>(
633    router: &mut crate::services::router::CellRouter,
634    load_state: &[crate::services::router::InferenceServerInfo],
635    co_located: crate::services::router::ReplicaId,
636    session_id: &str,
637    local_value: T,
638    mut dial_remote: F,
639) -> Result<RoutedReplica<T>>
640where
641    F: FnMut(crate::services::router::InferenceServerInfo) -> Fut,
642    Fut: std::future::Future<Output = Result<T>>,
643{
644    let mut local_value = Some(local_value);
645    let mut failures = Vec::new();
646    for _ in 0..load_state.len() {
647        let now = Instant::now();
648        match ModelService::route_decision(router, load_state, co_located, session_id, now) {
649            RouteDecision::CoLocated => {
650                let value = local_value
651                    .take()
652                    .ok_or_else(|| anyhow!("co-located inference client was already consumed"))?;
653                return Ok(RoutedReplica {
654                    value,
655                    replica_id: co_located,
656                    locality: ReplicaLocality::CoLocated,
657                });
658            }
659            RouteDecision::Remote(replica_id) => {
660                let candidate = load_state
661                    .iter()
662                    .find(|candidate| candidate.replica_id == replica_id)
663                    .cloned()
664                    .ok_or_else(|| anyhow!("router selected an absent replica"))?;
665                match dial_remote(candidate).await {
666                    Ok(value) => {
667                        return Ok(RoutedReplica {
668                            value,
669                            replica_id,
670                            locality: ReplicaLocality::Remote,
671                        });
672                    }
673                    Err(error) => {
674                        router.report_dial_fail(replica_id, now);
675                        failures.push(format!("{}: {error}", hex::encode(replica_id.as_bytes())));
676                    }
677                }
678            }
679            RouteDecision::NoHealthyCandidate => break,
680        }
681    }
682    if failures.is_empty() {
683        anyhow::bail!("no healthy inference replica candidates");
684    }
685    anyhow::bail!(
686        "inference replica candidates exhausted after dial/authorization failures: {}",
687        failures.join("; ")
688    )
689}
690
691impl ModelService {
692    /// Create a new model service with infrastructure
693    pub async fn new(
694        config: ModelServiceConfig,
695        signing_key: SigningKey,
696        policy_client: PolicyClient,
697        registry: RegistryClient,
698        transport: TransportConfig,
699        policy_transport: TransportConfig,
700    ) -> Result<Self> {
701        config.inference_deployment.validate()?;
702        // SAFETY: 5 is a valid non-zero value
703        const DEFAULT_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(5) {
704            Some(n) => n,
705            None => unreachable!(),
706        };
707        let cache_size = NonZeroUsize::new(config.max_models).unwrap_or(DEFAULT_CACHE_SIZE);
708
709        // EV5/#605: model lifecycle events now ride the unified EventService
710        // (plaintext Public path); the parallel NotificationService is removed.
711        let event_publisher = EventPublisher::new("model")?;
712
713        Ok(Self {
714            inner: Arc::new(ModelServiceInner {
715                loaded_models: RwLock::new(LruCache::new(cache_size)),
716                pending_loads: parking_lot::Mutex::new(HashMap::new()),
717                staging_pinned_artifacts:
718                    crate::storage::pinned_model::StagingPinnedArtifactCache::default(),
719                unloading_models: Mutex::new(HashSet::new()),
720                load_unload_gate: Mutex::new(()),
721                lifecycle_event_publish_gate: Mutex::new(()),
722                config,
723                signing_key,
724                policy_client,
725                #[cfg(feature = "postgres")]
726                federate_dispatch: None,
727                event_publisher,
728                registry,
729                transport,
730                policy_transport,
731                expected_audience: None,
732                delegated_oai_audience: None,
733                delegated_mcp_audience: None,
734                jwt_key_source: None,
735                discovery_client: None,
736                fs_trees: dashmap::DashMap::new(),
737                load_terminals: TerminalStore::new(),
738                load_epoch: AtomicU64::new(0),
739                incarnation_generation: AtomicU64::new(0),
740                in_process_tenant: std::sync::OnceLock::new(),
741                producer_reach_config: std::sync::Arc::new(parking_lot::RwLock::new(
742                    hyprstream_rpc::moq_stream::ProducerReachConfig::default(),
743                )),
744                moq_origin: std::sync::Arc::new(parking_lot::RwLock::new(None)),
745                #[cfg(test)]
746                test_work_order_capture: Mutex::new(None),
747                #[cfg(test)]
748                before_unloaded_event: Mutex::new(None),
749            }),
750        })
751    }
752
753    /// Retain a direct work order only for an in-crate test. The production
754    /// Model→Inference boundary still attaches the bearer directly to the
755    /// generated client and never exposes it to a caller.
756    #[cfg(test)]
757    async fn capture_test_work_order(&self, work_order: &str) {
758        let mut capture = self.test_work_order_capture.lock().await;
759        *capture = Some(Zeroizing::new(work_order.to_owned()));
760    }
761
762    /// Consume the in-memory test capture. Keeping the value move-only avoids
763    /// a durable test transcript and makes each assertion explicitly own the
764    /// short-lived secret it needs to replay against the replacement worker.
765    #[cfg(test)]
766    async fn take_test_work_order(&self) -> Option<Zeroizing<String>> {
767        self.test_work_order_capture.lock().await.take()
768    }
769
770    /// Set the expected JWT audience for token validation.
771    ///
772    /// # Panics
773    /// Panics if called after the service has been cloned (Arc refcount > 1).
774    /// Must be called during construction, before the service is shared.
775    #[allow(clippy::expect_used)]
776    pub fn with_expected_audience(mut self, audience: String) -> Self {
777        Arc::get_mut(&mut self.inner)
778            .expect("with_expected_audience must be called before service is shared")
779            .expected_audience = Some(audience);
780        self
781    }
782
783    /// Admit only current OAI/MCP service keys for the configured HTTP resources.
784    /// Without this factory configuration external resource tokens stay denied.
785    #[allow(clippy::expect_used)]
786    pub fn with_delegated_resource_audiences(mut self, oai: String, mcp: String) -> Self {
787        let inner = Arc::get_mut(&mut self.inner)
788            .expect("delegated resources must be configured before service sharing");
789        inner.delegated_oai_audience = Some(oai);
790        inner.delegated_mcp_audience = Some(mcp);
791        self
792    }
793
794    /// Set the unified JWT key source for verifying JWTs.
795    ///
796    /// # Panics
797    /// Panics if called after the service has been cloned (Arc refcount > 1).
798    #[allow(clippy::expect_used)]
799    pub fn with_jwt_key_source(
800        mut self,
801        src: std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>,
802    ) -> Self {
803        Arc::get_mut(&mut self.inner)
804            .expect("with_jwt_key_source must be called before service is shared")
805            .jwt_key_source = Some(src);
806        self
807    }
808
809    #[cfg(feature = "postgres")]
810    #[allow(clippy::expect_used)]
811    pub(crate) fn with_federate_dispatch(
812        mut self,
813        adapter: Arc<crate::services::oauth::federate_proof::DispatchAdapter>,
814    ) -> Self {
815        Arc::get_mut(&mut self.inner)
816            .expect("with_federate_dispatch must be called before service is shared")
817            .federate_dispatch = Some(adapter);
818        self
819    }
820
821    /// Set the DiscoveryClient for federated `at://` record resolution (#431).
822    ///
823    /// # Panics
824    /// Panics if called after the service has been cloned (Arc refcount > 1).
825    #[allow(clippy::expect_used)]
826    pub fn with_discovery_client(
827        mut self,
828        client: Arc<hyprstream_rpc_std::discovery_client::DiscoveryClient>,
829    ) -> Self {
830        Arc::get_mut(&mut self.inner)
831            .expect("with_discovery_client must be called before service is shared")
832            .discovery_client = Some(client);
833        self
834    }
835
836    /// Derive the deterministic InferenceService transport for a model ref (#320).
837    ///
838    /// Resolved via the registry (the local [`hyprstream_rpc::Resolver`] backend):
839    /// the `Inproc` arm for a co-located service. The spawner registers it in the
840    /// in-process dial registry and the router dials it via `dial()`.
841    /// Allocate the next monotonic load epoch (EV7/#649). Each genuine load
842    /// attempt (one that proceeds past the already-loaded / already-pending
843    /// fast paths) draws a fresh epoch so its terminal latches under a new
844    /// [`ModelLoadKey`] — reload ⇒ new terminal, monotonic write-once per epoch.
845    fn allocate_load_epoch(&self) -> u64 {
846        self.load_epoch.fetch_add(1, Ordering::Relaxed) + 1
847    }
848
849    /// Latch the retained terminal for a completed load attempt (EV7/#649) —
850    /// the host-side retain a late `load --wait` reads from. Monotonic
851    /// write-once per (`model_ref`, `epoch`): the first terminal for an attempt
852    /// wins; a real attempt always latches under a fresh epoch, so the guard is
853    /// belt-and-suspenders. Split out so the latch decision is unit-testable
854    /// independent of the full load path.
855    fn latch_load_terminal(
856        &self,
857        instance: &InferenceInstanceId,
858        epoch: u64,
859        result: &Result<String>,
860    ) {
861        let terminal = match result {
862            Ok(endpoint) => Terminal {
863                value: ModelLoadTerminal::Loaded {
864                    endpoint: endpoint.clone(),
865                },
866                latched_by: "model-service".to_owned(),
867            },
868            Err(e) => Terminal {
869                value: ModelLoadTerminal::LoadFailed {
870                    error: e.to_string(),
871                },
872                latched_by: "model-service".to_owned(),
873            },
874        };
875        let key = ModelLoadKey {
876            tenant: instance.tenant().to_owned(),
877            model_ref: instance.model_ref().to_owned(),
878            epoch,
879        };
880        let _ = self.load_terminals.latch(key, terminal);
881    }
882
883    /// Atomically admit the intercepted (non-blocking) load path with unload's
884    /// reservation and cache transition. `Accepted` reserves `pending_loads`
885    /// before the response is sent; an unload that arrives afterwards rejects
886    /// against that reservation rather than invalidating an accepted request.
887    async fn reserve_intercepted_load(
888        &self,
889        instance: &InferenceInstanceId,
890    ) -> Result<InterceptedLoadAdmission> {
891        self.validate_staging_model_checkout(instance).await?;
892        self.admit_instance(instance)?;
893        let _load_unload_gate = self.load_unload_gate.lock().await;
894        self.ensure_not_unloading(instance).await?;
895
896        {
897            let mut cache = self.loaded_models.write().await;
898            if let Some(model) = cache.get_mut(instance) {
899                model.last_used = Instant::now();
900                return Ok(InterceptedLoadAdmission::Loaded {
901                    reach: Self::model_reach(&model.network_transport),
902                });
903            }
904        }
905
906        let mut pending = self.pending_loads.lock();
907        if pending.contains_key(instance) {
908            return Ok(InterceptedLoadAdmission::Pending);
909        }
910
911        let attempt = self.allocate_load_epoch();
912        pending.insert(instance.clone(), attempt);
913        Ok(InterceptedLoadAdmission::Accepted(
914            InterceptedLoadReservation::new(Arc::clone(&self.inner), instance.clone(), attempt),
915        ))
916    }
917
918    fn validate_model_load_ref(&self, model_ref: &str) -> Result<()> {
919        if let Some(pin) = &self.config.staging_model_pin {
920            anyhow::ensure!(
921                pin.admits(model_ref),
922                "Model.load is restricted to the configured staging modelRef"
923            );
924        }
925        Ok(())
926    }
927
928    async fn validate_staging_model_checkout(&self, instance: &InferenceInstanceId) -> Result<()> {
929        let Some(pin) = &self.config.staging_model_pin else {
930            return Ok(());
931        };
932        self.validate_model_load_ref(instance.model_ref())?;
933        let model_ref = ModelRef::parse(instance.model_ref())?;
934        let tracked = self
935            .registry
936            .get_by_name(model_ref.name())
937            .await
938            .map_err(|e| {
939                anyhow!(
940                    "staging model '{}' not found in registry: {e}",
941                    model_ref.name()
942                )
943            })?;
944        let repo_client = self.registry.repo(&tracked.id);
945        let worktrees = repo_client.list_worktrees().await?;
946        let branch = match &model_ref.git_ref {
947            crate::storage::GitRef::Branch(branch) => branch,
948            _ => anyhow::bail!("staging modelRef must select its configured local branch"),
949        };
950        anyhow::ensure!(
951            worktrees
952                .iter()
953                .any(|worktree| worktree.branch_name == branch.as_str()),
954            "staging worktree for {}:{} not found",
955            model_ref.name(),
956            branch
957        );
958        let path = crate::storage::StoragePaths::new()?.worktree_path(model_ref.name(), branch)?;
959        crate::storage::pinned_model::verify_selected_checkout(&path, pin.commit)
960            .with_context(|| format!("admit staging worktree for {}", pin.model_ref))
961    }
962
963    /// Execute one previously admitted intercepted load. Its pending reservation
964    /// was created under `load_unload_gate`; unload rejects against that marker
965    /// until this continuation has finished and released it.
966    async fn run_intercepted_load(
967        &self,
968        instance: &InferenceInstanceId,
969        max_context: Option<u32>,
970        kv_quant: Option<KVQuantType>,
971        mut reservation: InterceptedLoadReservation,
972    ) -> Result<String> {
973        let epoch = reservation.attempt;
974        let result = self.load_model_inner(instance, max_context, kv_quant).await;
975        reservation.release();
976        self.latch_load_terminal(instance, epoch, &result);
977        result
978    }
979
980    /// Build an instance key only from the authority-verified tenant binding.
981    fn inference_instance(
982        ctx: &EnvelopeContext,
983        model_ref_str: &str,
984    ) -> Result<InferenceInstanceId> {
985        let object_label = crate::services::inference::inference_object_label();
986        crate::services::inference::enforce_inference_mac(ctx, &object_label)?;
987        InferenceInstanceId::new(&ctx.domain()?, model_ref_str, 0)
988    }
989
990    /// Enforce the configured fault-radius contract before touching model state.
991    fn admit_instance(&self, instance: &InferenceInstanceId) -> Result<()> {
992        self.config.inference_deployment.validate()?;
993        match &self.config.inference_deployment.isolation {
994            InferenceIsolationProfile::InProcess => {
995                admit_in_process_tenant(&self.in_process_tenant, instance.tenant())
996            }
997            InferenceIsolationProfile::PerTenantSubprocess
998            | InferenceIsolationProfile::PerTenantMicroVmTask { .. } => {
999                anyhow::bail!(
1000                    "inference isolation profile {:?} requires the external tenant launcher; \
1001                     refusing to downgrade to an in-process engine",
1002                    self.config.inference_deployment.isolation
1003                )
1004            }
1005        }
1006    }
1007
1008    fn inference_transport(instance: &InferenceInstanceId) -> TransportConfig {
1009        // This worker is always spawned on a thread in the Model process.
1010        // Standalone Model's global IPC profile is for external services, not
1011        // its co-located worker; keep both worker and client on the same typed arm.
1012        TransportConfig::inproc(format!("hyprstream/{}", instance.service_name()))
1013    }
1014
1015    /// The deterministic local InferenceService endpoint string for diagnostics.
1016    ///
1017    /// This is never advertised. Loaded status and lifecycle events use the
1018    /// separately bound Iroh network reach.
1019    fn inference_endpoint(instance: &InferenceInstanceId) -> String {
1020        Self::inference_transport(instance).endpoint_string()
1021    }
1022
1023    /// The network-routable reach list a remote caller would use to dial this
1024    /// model's InferenceService (#320). An `Inproc` transport is never
1025    /// advertised; a networked (Quic/Iroh) reach maps through the single
1026    /// dial-to-wire reach codec.
1027    fn model_reach(transport: &TransportConfig) -> Vec<WireTransportConfig> {
1028        hyprstream_rpc::moq_stream::dial_transport_to_wire(transport)
1029            .into_iter()
1030            .collect()
1031    }
1032
1033    /// Pure HRW/least-loaded + session-affinity ranking decision (#523 P3):
1034    /// which candidate in `load_state` the router picked for `session_id`,
1035    /// classified against the co-located node. Contains no I/O and no
1036    /// `InferenceClient` — reused directly by unit tests that construct a
1037    /// synthetic multi-entry `load_state` without needing a live model.
1038    fn route_decision(
1039        router: &mut crate::services::router::CellRouter,
1040        load_state: &[crate::services::router::InferenceServerInfo],
1041        co_located: crate::services::router::ReplicaId,
1042        session_id: &str,
1043        now: Instant,
1044    ) -> RouteDecision {
1045        match router.place(session_id, load_state, now) {
1046            Some(placement) if placement.replica_id == co_located => RouteDecision::CoLocated,
1047            Some(placement) => RouteDecision::Remote(placement.replica_id),
1048            None => RouteDecision::NoHealthyCandidate,
1049        }
1050    }
1051
1052    /// Router seam (#322 leaf cell-router over the #320 single-select base,
1053    /// unified with #523 P3): pick ONE InferenceService for a request from a
1054    /// loaded model's replica set, applying capacity-weighted HRW +
1055    /// least-loaded + session affinity (via [`Self::route_decision`] /
1056    /// `CellRouter::place`) over the P1 candidate set resolved into
1057    /// `load_state`.
1058    ///
1059    /// A co-located selection returns the in-process client directly. A remote
1060    /// selection treats its transport only as a selector into Discovery's
1061    /// checkpoint-current, identity-bound candidate set. Resolution,
1062    /// authorization, readiness, or dial failure marks that replica down and
1063    /// re-runs placement; exhaustion fails closed.
1064    ///
1065    /// `session_id` is the placement key (defaults to "default" when the caller
1066    /// has no session — keeps HRW stable for non-session-scoped requests like
1067    /// `apply_chat_template`).
1068    /// Locality-aware selection: the caller needs [`RoutedReplica`] to apply
1069    /// the internal work order ONLY on the co-located arm (K3 finding M1).
1070    async fn select_inference_routed(
1071        &self,
1072        model: &mut LoadedModel,
1073        session_id: &str,
1074        bearer: Option<&str>,
1075    ) -> Result<RoutedReplica<InferenceClient>> {
1076        let co_located = Self::co_located_replica_id(model);
1077        let resolution_service_name = model.instance.service_name();
1078        let signing_key = self.signing_key.clone();
1079        let bearer = bearer.map(ToOwned::to_owned);
1080        let routed = route_and_dial_replica(
1081            &mut model.router,
1082            &model.load_state,
1083            co_located,
1084            session_id,
1085            model.client.clone(),
1086            move |candidate| {
1087                let resolution_service_name = resolution_service_name.clone();
1088                let signing_key = signing_key.clone();
1089                let bearer = bearer.clone();
1090                async move {
1091                    anyhow::ensure!(
1092                        matches!(
1093                            &candidate.transport.endpoint,
1094                            EndpointType::Quic { .. } | EndpointType::Iroh { .. }
1095                        ),
1096                        "remote inference replica advertised a non-network transport"
1097                    );
1098                    // Cross-host replicas are OTHER services, not this Model's
1099                    // subprocessor: they still see the caller's credential as
1100                    // a relayed delegated bearer under their own admission
1101                    // contract. No bearer ⇒ no remote relay (fail-closed).
1102                    let relay_bearer = bearer.clone().ok_or_else(|| {
1103                        anyhow!(
1104                            "remote inference replica selection requires a relayable caller bearer"
1105                        )
1106                    })?;
1107                    let rpc = hyprstream_discovery::production_inference_rpc_client_at_transport(
1108                        &resolution_service_name,
1109                        &candidate.transport,
1110                        signing_key,
1111                        None,
1112                    )?;
1113                    let client = InferenceClient::new(rpc).with_delegated_bearer(relay_bearer);
1114                    anyhow::ensure!(
1115                        client.is_ready().await?,
1116                        "remote inference replica is not ready"
1117                    );
1118                    Ok(client)
1119                }
1120            },
1121        )
1122        .await?;
1123        info!(
1124            replica_id = %hex::encode(routed.replica_id.as_bytes()),
1125            locality = ?routed.locality,
1126            session_id,
1127            tenant = model.instance.tenant(),
1128            model_ref = model.instance.model_ref(),
1129            "selected inference replica"
1130        );
1131        Ok(routed)
1132    }
1133
1134    /// Opaque placement identifier for the co-located InferenceService.
1135    fn co_located_replica_id(model: &LoadedModel) -> crate::services::router::ReplicaId {
1136        // `load_state[0]` is the co-located entry (seeded first at load). All
1137        // future multi-replica entries carry their own opaque placement ID.
1138        model
1139            .load_state
1140            .first()
1141            .map(|info| info.replica_id)
1142            .unwrap_or(crate::services::router::ReplicaId::from_bytes([0u8; 32]))
1143    }
1144
1145    /// Resolve a model identifier string to a [`ModelRef`].
1146    ///
1147    /// Accepts three prefixes (#395 prefix-dispatch grammar, no capnp/wire change
1148    /// — `modelRef` stays `Text` in the schema):
1149    ///
1150    /// ```text
1151    /// modelRef ::= "at://" <at-uri>   # federated (resolve via atproto NAME → git OID)
1152    ///            | "did:" <did>       # federated, bare-DID form
1153    ///            | <name> [":" <gitref>]  # local ModelRef (unchanged)
1154    /// ```
1155    ///
1156    /// The federated branch (`at://`, `did:`) is resolved via the atproto record
1157    /// store from #392 (PDS record → git OID). Until that store is wired up the
1158    /// federated branch logs the attempt and falls through to the local
1159    /// [`ModelRef::parse`], preserving backward compatibility — every existing
1160    /// `name:ref` caller keeps working unchanged.
1161    ///
1162    /// Local ModelRefs (no `at://` / `did:` prefix) are parsed by
1163    /// [`ModelRef::parse`] exactly as before.
1164    async fn resolve_model_ref(&self, model_ref_str: &str) -> Result<ModelRef> {
1165        match model_ref_dispatch(model_ref_str) {
1166            ModelRefDispatch::Federated { scheme, rest } => {
1167                // Federated resolution (#431): at:// → DiscoveryService.getRecord →
1168                // CAR proof → verify offline → extract currentOid → local ModelRef.
1169                // Only the full `at://<did>/<collection>/<rkey>` form resolves here;
1170                // anything else (bare `did:`, partial at://) falls through to local.
1171                if scheme == "at://" {
1172                    match self.resolve_federated_at_uri(model_ref_str).await {
1173                        Ok(Some(model_ref)) => return Ok(model_ref),
1174                        Ok(None) => {
1175                            // Attempted-but-unresolvable federated ref. FAIL CLOSED
1176                            // with a clear error rather than falling through to
1177                            // ModelRef::parse — which would mis-split `at://did:..`
1178                            // on the first ':' into model="at", git_ref="//did:..",
1179                            // a garbage local ref that fails later with a misleading
1180                            // "not found". `Ok(None)` means either no DiscoveryClient
1181                            // is configured (federation not enabled) or the string is
1182                            // not a full at://<did>/<collection>/<rkey>.
1183                            if self.inner.discovery_client.is_none() {
1184                                anyhow::bail!(
1185                                    "federation not enabled: cannot resolve federated ref '{model_ref_str}' \
1186                                     (no DiscoveryClient configured on this node)"
1187                                );
1188                            }
1189                            anyhow::bail!(
1190                                "federated ref not resolvable: '{model_ref_str}' is not a full \
1191                                 at://<did>/<collection>/<rkey>"
1192                            );
1193                        }
1194                        Err(e) => {
1195                            // Attempted and failed (record denied/missing/proof invalid):
1196                            // surface it; never mask with a local parse.
1197                            return Err(e);
1198                        }
1199                    }
1200                }
1201                // `did:` (bare-DID) is reserved for federated resolution but has no
1202                // record-fetch path yet — fail closed, do NOT reinterpret as local.
1203                anyhow::bail!(
1204                    "federated ModelRef scheme '{scheme}' for '{rest}' is not resolvable on this node \
1205                     (bare did: resolution not implemented; use a full at:// record ref)"
1206                )
1207            }
1208            ModelRefDispatch::Local => ModelRef::parse(model_ref_str),
1209        }
1210    }
1211
1212    /// Resolve a full `at://<did>/<collection>/<rkey>` to a local [`ModelRef`]
1213    /// via DiscoveryService.getRecord (#431).
1214    ///
1215    /// Steps: call `getRecord` over the (configured) DiscoveryClient → parse the
1216    /// returned CARv1 proof → locate the record block → decode it → extract its
1217    /// `currentOid` (a git-raw CID) → decode the git OID → form a local ModelRef.
1218    ///
1219    /// Returns `Ok(None)` when no DiscoveryClient is configured or the ref is not
1220    /// a full at-uri (caller falls through to local resolution). Returns `Err`
1221    /// when resolution was attempted but failed (record denied/missing/invalid).
1222    ///
1223    /// INTEGRITY NOTE: the CAR proof's ES256 commit signature is verified offline
1224    /// against the account's published `#atproto` P-256 key via
1225    /// `hyprstream_pds::car::verify_record_proof`. Resolving that published key
1226    /// requires fetching the remote DID document (`#atproto` verification method),
1227    /// for which there is not yet an in-crate resolver — so today we verify the
1228    /// CAR's *structural* integrity (it parses, the commit is its root, the record
1229    /// block is present and decodes, and the record CID is the one the proof
1230    /// addresses) and extract the OID. Full signature verification against the
1231    /// fetched DID key is a follow-up (DID-document resolution); the untrusted-
1232    /// relay posture is preserved because the eventual key check is the same
1233    /// `verify_record_proof` the e7 harness already exercises.
1234    async fn resolve_federated_at_uri(&self, at_uri: &str) -> Result<Option<ModelRef>> {
1235        let Some(dc) = self.inner.discovery_client.as_ref() else {
1236            return Ok(None);
1237        };
1238
1239        // Parse at://<did>/<collection>/<rkey>. The DID may contain ':' but no
1240        // '/', so the post-prefix remainder splits cleanly into three parts.
1241        let rest = match at_uri.strip_prefix("at://") {
1242            Some(r) => r,
1243            None => return Ok(None),
1244        };
1245        let mut parts = rest.splitn(3, '/');
1246        let (did, collection, rkey) = match (parts.next(), parts.next(), parts.next()) {
1247            (Some(d), Some(c), Some(r)) if !d.is_empty() && !c.is_empty() && !r.is_empty() => {
1248                (d.to_owned(), c.to_owned(), r.to_owned())
1249            }
1250            // Not a full at-uri — fall through to local resolution.
1251            _ => return Ok(None),
1252        };
1253
1254        let car_resp = dc
1255            .get_record(&hyprstream_rpc_std::discovery_client::GetRecordRequest {
1256                uri: at_uri.to_owned(),
1257                did: did.clone(),
1258                collection: collection.clone(),
1259                rkey: rkey.clone(),
1260            })
1261            .await
1262            .map_err(|e| anyhow!("DiscoveryService.getRecord failed for {at_uri}: {e}"))?;
1263
1264        // Parse the CARv1 proof and pull out the record block.
1265        let (roots, blocks) = hyprstream_pds::car::parse_car_v1(&car_resp.car)
1266            .map_err(|e| anyhow!("getRecord returned an unparseable CAR for {at_uri}: {e}"))?;
1267        let commit_cid = roots
1268            .first()
1269            .copied()
1270            .ok_or_else(|| anyhow!("getRecord CAR for {at_uri} has no root commit CID"))?;
1271
1272        // Decode the commit (the proof's root) — confirms the relay returned a
1273        // structurally valid signed commit. (Signature verification against the
1274        // DID's published #atproto key is the follow-up noted above.)
1275        let _commit = blocks
1276            .iter()
1277            .find(|(c, _)| *c == commit_cid)
1278            .map(|(_, b)| hyprstream_pds::commit::Commit::from_dag_cbor(b))
1279            .transpose()
1280            .map_err(|e| anyhow!("getRecord CAR commit block did not decode: {e}"))?
1281            .ok_or_else(|| anyhow!("getRecord CAR for {at_uri} is missing its commit block"))?;
1282
1283        // Find the record block: the only block that decodes as a ModelRecord.
1284        // (The CAR also contains the commit + MST nodes, which are not records.)
1285        let record = blocks
1286            .iter()
1287            .filter(|(c, _)| *c != commit_cid)
1288            .find_map(|(_, b)| hyprstream_pds::record::ModelRecord::from_dag_cbor(b).ok())
1289            .ok_or_else(|| {
1290                anyhow!("getRecord CAR for {at_uri} contained no ai.hyprstream.model record")
1291            })?;
1292
1293        // currentOid is a git-raw CID string → decode to the git OID hex.
1294        let cid = hyprstream_rpc::cid::decode_cid(&record.current_oid)
1295            .map_err(|e| anyhow!("record currentOid is not a valid CID: {e}"))?;
1296        let oid_hex = cid
1297            .multihash
1298            .digest
1299            .iter()
1300            .map(|b| format!("{b:02x}"))
1301            .collect::<String>();
1302
1303        info!(
1304            %at_uri,
1305            did = %did,
1306            current_oid = %record.current_oid,
1307            git_oid = %oid_hex,
1308            "resolved federated at-uri via DiscoveryService.getRecord (#431)"
1309        );
1310
1311        // Form a local ModelRef pinned to the resolved git OID. The name is the
1312        // DID-derived repo handle; the git ref is the resolved commit OID.
1313        let local_ref = format!("{did}:{oid_hex}");
1314        match ModelRef::parse(&local_ref) {
1315            Ok(model_ref) => Ok(Some(model_ref)),
1316            // If the resolved (name:oid) shape isn't a valid local ModelRef yet
1317            // (the local registry mapping for federated repos is a follow-up),
1318            // surface a clear error rather than a misleading local fallthrough.
1319            Err(e) => Err(anyhow!(
1320                "resolved {at_uri} → git OID {oid_hex}, but local ModelRef::parse rejected {local_ref:?}: {e}"
1321            )),
1322        }
1323    }
1324
1325    /// Load a model by reference with optional per-model config, returns the inference endpoint
1326    async fn load_model(
1327        &self,
1328        instance: &InferenceInstanceId,
1329        max_context: Option<u32>,
1330        kv_quant: Option<KVQuantType>,
1331    ) -> Result<String> {
1332        self.validate_staging_model_checkout(instance).await?;
1333        self.admit_instance(instance)?;
1334        let model_ref_str = instance.model_ref();
1335        // A load's admission decision is atomic with unload's reservation and
1336        // cache removal. Without this guard, a load could observe "not
1337        // unloading", then begin after the old worker has been removed.
1338        let load_unload_gate = self.load_unload_gate.lock().await;
1339        self.ensure_not_unloading(instance).await?;
1340        // Check if already loaded
1341        {
1342            let mut cache = self.loaded_models.write().await;
1343            if let Some(model) = cache.get_mut(instance) {
1344                model.last_used = Instant::now();
1345                debug!("Model {} already loaded", model_ref_str);
1346                return Ok(model.network_transport.endpoint_string());
1347            }
1348        }
1349
1350        // Atomically check-and-insert into pending_loads (prevents duplicate GPU loads
1351        // when multiple requests arrive during the ~40s load window).
1352        // HashMap::insert is guarded by contains_key so a direct attempt owns
1353        // the epoch stored beside its pending marker.
1354        let epoch;
1355        {
1356            let mut pending = self.pending_loads.lock();
1357            if pending.contains_key(instance) {
1358                anyhow::bail!(
1359                    "Model {} is already being loaded — please retry shortly",
1360                    model_ref_str
1361                );
1362            }
1363            epoch = self.allocate_load_epoch();
1364            pending.insert(instance.clone(), epoch);
1365        }
1366        drop(load_unload_gate);
1367
1368        // EV7/#649: the gated pending insertion allocated this genuine load's
1369        // fresh epoch, so its terminal latches under a new key on completion.
1370        let result = self.load_model_inner(instance, max_context, kv_quant).await;
1371
1372        // Always remove from pending, whether load succeeded or failed
1373        self.pending_loads.lock().remove(instance);
1374
1375        // EV7/#649: latch the retained terminal (Loaded / LoadFailed) for this
1376        // load attempt — the host-side retain a late `load --wait` reads.
1377        self.latch_load_terminal(instance, epoch, &result);
1378
1379        if result.is_ok() {
1380            info!("Model {} loaded successfully", model_ref_str);
1381        }
1382        result
1383    }
1384
1385    /// Inner model loading logic, called by load_model() which manages pending_loads.
1386    async fn load_model_inner(
1387        &self,
1388        instance: &InferenceInstanceId,
1389        max_context: Option<u32>,
1390        kv_quant: Option<KVQuantType>,
1391    ) -> Result<String> {
1392        let model_ref_str = instance.model_ref();
1393        self.validate_model_load_ref(model_ref_str)?;
1394        // Parse/resolve model reference
1395        let model_ref = self.resolve_model_ref(model_ref_str).await?;
1396
1397        // Get model path from registry
1398        #[cfg(test)]
1399        crate::services::restart_diag::phase("registry.get_by_name.enter");
1400        let tracked = self
1401            .registry
1402            .get_by_name(model_ref.name())
1403            .await
1404            .map_err(|e| anyhow!("Model '{}' not found in registry: {}", model_ref.name(), e))?;
1405        #[cfg(test)]
1406        crate::services::restart_diag::phase("registry.get_by_name.done");
1407        let repo_client = self.registry.repo(&tracked.id);
1408
1409        let branch_name = match &model_ref.git_ref {
1410            crate::storage::GitRef::Branch(name) => name.clone(),
1411            crate::storage::GitRef::Commit(_) => repo_client
1412                .get_head()
1413                .await
1414                .context("resolve selected worktree branch for pinned commit")?,
1415            crate::storage::GitRef::Tag(_) | crate::storage::GitRef::Revspec(_) => {
1416                anyhow::bail!("tag and revspec model loads are unsupported; use a full commit OID")
1417            }
1418            crate::storage::GitRef::DefaultBranch => repo_client
1419                .get_head()
1420                .await
1421                .unwrap_or_else(|_| "main".to_owned()),
1422        };
1423        #[cfg(test)]
1424        crate::services::restart_diag::phase("registry.list_worktrees.enter");
1425        let worktrees = repo_client.list_worktrees().await?;
1426        #[cfg(test)]
1427        crate::services::restart_diag::phase("registry.list_worktrees.done");
1428        if !worktrees.iter().any(|wt| wt.branch_name == branch_name) {
1429            return Err(anyhow!(
1430                "worktree for {}:{} not found",
1431                model_ref.name(),
1432                branch_name
1433            ));
1434        }
1435        // Derive worktree path locally
1436        let storage_paths = crate::storage::StoragePaths::new()?;
1437        let model_path = storage_paths.worktree_path(model_ref.name(), &branch_name)?;
1438
1439        if !model_path.exists() {
1440            return Err(anyhow!(
1441                "Model worktree not found for {}. Please clone the model first.",
1442                model_ref_str
1443            ));
1444        }
1445
1446        let pinned_artifact = if let Some(pin) = &self.config.staging_model_pin {
1447            anyhow::ensure!(
1448                model_ref_str == pin.model_ref,
1449                "staging modelRef changed after admission"
1450            );
1451            let policy = pin.capture_policy()?;
1452            #[cfg(test)]
1453            let artifact = if let Some(parent) = pin.projection_parent.as_deref() {
1454                self.inner
1455                    .staging_pinned_artifacts
1456                    .acquire_with_policy_in(&model_path, pin.commit, parent, policy)
1457                    .await?
1458            } else {
1459                self.inner
1460                    .staging_pinned_artifacts
1461                    .acquire_with_policy(&model_path, pin.commit, policy)
1462                    .await?
1463            };
1464            #[cfg(not(test))]
1465            let artifact = self
1466                .inner
1467                .staging_pinned_artifacts
1468                .acquire_with_policy(&model_path, pin.commit, policy)
1469                .await?;
1470            Some(artifact)
1471        } else if let crate::storage::GitRef::Commit(oid) = &model_ref.git_ref {
1472            Some(crate::storage::pinned_model::acquire_pinned_model(&model_path, *oid).await?)
1473        } else {
1474            None
1475        };
1476        let model_input_path = pinned_artifact
1477            .as_ref()
1478            .map_or(model_path.as_path(), |artifact| artifact.root());
1479
1480        let endpoint = Self::inference_endpoint(instance);
1481
1482        info!("Loading model {} at endpoint {}", model_ref_str, endpoint);
1483
1484        // Create runtime config - use per-model config if provided, otherwise service defaults
1485        let runtime_config = RuntimeConfig {
1486            max_context: max_context.or(self.config.max_context),
1487            kv_quant_type: kv_quant.unwrap_or(self.config.kv_quant),
1488            use_gpu: self.config.inference_deployment.compute != InferenceCompute::Cpu,
1489            gpu_device_id: if self.config.inference_deployment.compute == InferenceCompute::Cpu {
1490                None
1491            } else {
1492                RuntimeConfig::default().gpu_device_id
1493            },
1494            devices: if self.config.inference_deployment.compute == InferenceCompute::Cpu {
1495                Vec::new()
1496            } else {
1497                RuntimeConfig::default().devices
1498            },
1499            ..Default::default()
1500        };
1501
1502        // Obtain FsOps from the registry for path-contained adapter I/O
1503        let fs: Option<hyprstream_rpc_std::registry_client::WorktreeClient> =
1504            Some(repo_client.worktree(&branch_name));
1505
1506        // Start InferenceService for this model via standard Spawnable infrastructure
1507        let spawner = hyprstream_service::ServiceSpawner::threaded();
1508
1509        // Resolve the InferenceService transport via the typed `TransportConfig`
1510        // (#320). For a co-located service this is the `Inproc` arm; the spawner
1511        // registers it in the in-process dial registry (`register_inproc`) and
1512        // the same typed transport builds the client below — no string parsing on
1513        // the inference client path. The service also binds an Iroh endpoint
1514        // before readiness and publishes that separate network reach below.
1515        let transport = Self::inference_transport(instance);
1516        let mut service_config = crate::services::InferenceServiceConfig::new(
1517            &model_path,
1518            runtime_config,
1519            self.signing_key.verifying_key(),
1520            self.signing_key.clone(),
1521            transport.clone(),
1522            self.policy_transport.clone(),
1523            fs,
1524        );
1525        if let Some(ref artifact) = pinned_artifact {
1526            service_config = service_config.with_pinned_artifact(Arc::clone(artifact));
1527        }
1528        let mut service_config = service_config
1529            .with_instance_identity(
1530                instance.service_name(),
1531                instance.tenant().to_owned(),
1532                self.signing_key.verifying_key(),
1533            )
1534            .with_model_ref(instance.model_ref().to_owned())
1535            .with_stream_plane(
1536                Arc::clone(&self.producer_reach_config),
1537                Arc::clone(&self.moq_origin),
1538            );
1539        if let Some(ref aud) = self.expected_audience {
1540            service_config = service_config.with_expected_audience(aud.clone());
1541        }
1542        if let Some(ref src) = self.jwt_key_source {
1543            service_config = service_config.with_jwt_key_source(src.clone());
1544        }
1545        // Bounded worker incarnation binding (Sol plan): this spawn attempt
1546        // allocates a private handoff channel and the next generation. The
1547        // worker generates its own 256-bit incarnation inside its service
1548        // thread and reports `(generation, instance, controller, audience)`
1549        // through the handoff after its engine is initialized; Model accepts
1550        // the handoff only for THIS attempt's expected values. A worker that
1551        // dies before reporting (or a missing/failed handoff) fails the load.
1552        let generation = self
1553            .inner
1554            .incarnation_generation
1555            .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
1556            + 1;
1557        let (incarnation_tx, incarnation_rx) = tokio::sync::oneshot::channel();
1558        service_config = service_config.with_incarnation_handoff(incarnation_tx, generation);
1559
1560        let network_reach = service_config.network_reach_handle();
1561        #[cfg(test)]
1562        crate::services::restart_diag::phase("inference.spawn.enter");
1563        let service_handle = spawner
1564            .spawn(service_config)
1565            .await
1566            .map_err(|e| anyhow!("Failed to spawn inference service: {}", e))?;
1567        let spawn_guard = SpawnPublicationGuard::new(service_handle);
1568        #[cfg(test)]
1569        crate::services::restart_diag::phase("inference.spawn.done");
1570        let network_transport = network_reach
1571            .read()
1572            .clone()
1573            .ok_or_else(|| anyhow!("InferenceService became ready without network reach"))?;
1574        let endpoint = network_transport.endpoint_string();
1575
1576        // Await the worker's private incarnation handoff. Missing (worker died
1577        // before init), timed out, or mismatched (instance/controller/generation)
1578        // all FAIL THE LOAD closed — no binding, no minting from this attempt.
1579        let expected_controller = self.signing_key.verifying_key().to_bytes();
1580        const HANDOFF_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(
1581            crate::services::inference::INCARNATION_HANDOFF_TIMEOUT_SECS,
1582        );
1583        #[cfg(test)]
1584        crate::services::restart_diag::phase("incarnation.handoff.enter");
1585        let incarnation_binding = match tokio::time::timeout(HANDOFF_TIMEOUT, incarnation_rx).await
1586        {
1587            Err(_) => anyhow::bail!(
1588                "pinned worker incarnation handoff timed out after {}s without a ready worker",
1589                crate::services::inference::INCARNATION_HANDOFF_TIMEOUT_SECS
1590            ),
1591            Ok(Err(_)) => {
1592                anyhow::bail!("pinned worker died before reporting its incarnation handoff")
1593            }
1594            Ok(Ok(handoff)) => handoff,
1595        };
1596        #[cfg(test)]
1597        crate::services::restart_diag::phase("incarnation.handoff.done");
1598        anyhow::ensure!(
1599            incarnation_binding.generation == generation,
1600            "stale incarnation handoff: worker reported generation {} for attempt {generation}",
1601            incarnation_binding.generation
1602        );
1603        anyhow::ensure!(
1604            incarnation_binding.instance_service_name == instance.service_name(),
1605            "incarnation handoff names the wrong deterministic instance"
1606        );
1607        anyhow::ensure!(
1608            incarnation_binding.controller_pubkey == expected_controller,
1609            "incarnation handoff names the wrong pinned controller"
1610        );
1611
1612        // Create client for this service from the typed transport (#320).
1613        // Inference services share the model service's signing key, so use our
1614        // own verifying key directly — no PolicyService lookup needed.
1615        let client = InferenceClient::for_local_transport_bootstrap(
1616            &transport,
1617            self.signing_key.clone(),
1618            self.signing_key.verifying_key(),
1619            None,
1620        )?;
1621
1622        // Load TTT config from model's config.json (if TTT is enabled)
1623        let ttt_config =
1624            crate::runtime::model_config::ModelConfig::load_training_config(model_input_path)
1625                .and_then(|tc| {
1626                    if tc.is_enabled() && tc.mode == crate::config::TrainingMode::TestTimeTraining {
1627                        Some(crate::training::ttt::TTTConfig {
1628                            learning_rate: tc.ttt.learning_rate,
1629                            gradient_steps: tc.ttt.gradient_steps,
1630                            max_grad_norm: tc.ttt.max_grad_norm,
1631                            min_input_length: tc.ttt.min_input_length,
1632                            max_ttt_context: tc.ttt.max_ttt_context,
1633                            enabled: true,
1634                            ..crate::training::ttt::TTTConfig::default()
1635                        })
1636                    } else {
1637                        None
1638                    }
1639                });
1640
1641        // Load generation parameter defaults from model's generation_config.json
1642        let generation_defaults = crate::config::SamplingParams::from_model_path(model_input_path)
1643            .await
1644            .unwrap_or_default();
1645
1646        // Check if we need to evict
1647        {
1648            let mut cache = self.loaded_models.write().await;
1649            if cache.len() >= self.config.max_models {
1650                if let Some((evicted_ref, mut evicted)) = cache.pop_lru() {
1651                    info!(
1652                        "Evicting model {} to load {}",
1653                        evicted_ref.model_ref(),
1654                        model_ref_str
1655                    );
1656                    // Stop the evicted service in background (fire-and-forget)
1657                    #[allow(clippy::let_underscore_future)]
1658                    let _ = tokio::spawn(async move {
1659                        let _ = evicted.service_handle.stop().await;
1660                    });
1661                }
1662            }
1663
1664            // Add to cache
1665            let service_handle = spawn_guard.publish()?;
1666            cache.put(
1667                instance.clone(),
1668                LoadedModel {
1669                    instance: instance.clone(),
1670                    model_ref: model_ref_str.to_owned(),
1671                    pinned_commit: pinned_artifact.as_ref().map(|artifact| artifact.commit()),
1672                    transport: transport.clone(),
1673                    network_transport,
1674                    service_handle,
1675                    client,
1676                    // Worker incarnation binding for THIS run (Sol bounded
1677                    // incarnation plan): minting uses exactly this audience.
1678                    incarnation: incarnation_binding.incarnation,
1679                    work_audience: incarnation_binding.audience,
1680                    load_generation: incarnation_binding.generation,
1681                    // #322 leaf cell-router. v1: single co-located replica. Its
1682                    // placement ID is purpose-separated from every transport,
1683                    // application-signing, and subject identity. The
1684                    // router fast-paths to `client` when HRW picks the
1685                    // co-located node; the replica set grows when cross-host
1686                    // reaches are resolved via the Resolver.
1687                    router: crate::services::router::CellRouter::default(),
1688                    load_state: vec![crate::services::router::InferenceServerInfo {
1689                        replica_id: crate::services::router::ReplicaId::from_bytes(
1690                            hyprstream_rpc::node_identity::derive_purpose_key(
1691                                &self.signing_key,
1692                                "hyprstream-inference-replica-placement-v1",
1693                            )
1694                            .verifying_key()
1695                            .to_bytes(),
1696                        ),
1697                        transport: transport.clone(),
1698                        gpu_memory_free: 0,
1699                        active_sessions: 0,
1700                        last_heartbeat: Instant::now(),
1701                    }],
1702                    loaded_at: Instant::now(),
1703                    last_used: Instant::now(),
1704                    ttt_config,
1705                    generation_defaults,
1706                },
1707            );
1708        }
1709
1710        if let Some(ref artifact) = pinned_artifact {
1711            info!(model = %model_ref_str, commit = %artifact.commit(), files = artifact.file_count(),
1712                "loaded model from private disk exact-tree artifact");
1713        }
1714
1715        Ok(endpoint)
1716    }
1717
1718    async fn ensure_not_unloading(&self, instance: &InferenceInstanceId) -> Result<()> {
1719        anyhow::ensure!(
1720            !self.unloading_models.lock().await.contains(instance),
1721            "Model {} is unloading — please retry after teardown completes",
1722            instance.model_ref(),
1723        );
1724        Ok(())
1725    }
1726
1727    /// Unload a model.
1728    ///
1729    /// The teardown is owned by an internal task before this method awaits its
1730    /// result. Dropping an RPC continuation therefore cannot leave an absent
1731    /// cache entry while the old worker's thread still drains in the
1732    /// background: the instance remains in `unloading_models` until a
1733    /// successful shutdown/join. An ambiguous stop error retains that
1734    /// reservation fail-closed.
1735    async fn unload_model(&self, instance: &InferenceInstanceId) -> Result<()> {
1736        let (completion_tx, completion_rx) = tokio::sync::oneshot::channel();
1737        let inner = Arc::clone(&self.inner);
1738        let instance = instance.clone();
1739
1740        tokio::spawn(async move {
1741            let result = Self::unload_model_owned(Arc::clone(&inner), &instance).await;
1742            if let Err(Err(error)) = completion_tx.send(result) {
1743                warn!(
1744                    model = %instance.model_ref(),
1745                    error = %error,
1746                    "model unload caller cancelled; owned teardown completed with an error"
1747                );
1748            }
1749        });
1750
1751        completion_rx
1752            .await
1753            .map_err(|_| anyhow!("model unload task exited before reporting completion"))?
1754    }
1755
1756    async fn unload_model_owned(
1757        inner: Arc<ModelServiceInner>,
1758        instance: &InferenceInstanceId,
1759    ) -> Result<()> {
1760        let model_ref_str = instance.model_ref().to_owned();
1761        let (model, pending_load) = {
1762            let load_unload_gate = inner.load_unload_gate.lock().await;
1763            let mut unloading = inner.unloading_models.lock().await;
1764            let inserted = unloading.insert(instance.clone());
1765            drop(unloading);
1766            anyhow::ensure!(inserted, "Model {} is already unloading", model_ref_str,);
1767            let model = {
1768                let mut cache = inner.loaded_models.write().await;
1769                cache.pop_entry(instance).map(|(_, model)| model)
1770            };
1771            let pending_load = if model.is_none() {
1772                inner.pending_loads.lock().contains_key(instance)
1773            } else {
1774                false
1775            };
1776            drop(load_unload_gate);
1777            (model, pending_load)
1778        };
1779
1780        let Some(mut model) = model else {
1781            if pending_load {
1782                inner.unloading_models.lock().await.remove(instance);
1783                anyhow::bail!("Model {} is loading", model_ref_str);
1784            }
1785            inner.unloading_models.lock().await.remove(instance);
1786            anyhow::bail!("Model {} is not loaded", model_ref_str);
1787        };
1788
1789        info!("Unloading model {}", model_ref_str);
1790        if let Err(error) = model.service_handle.stop().await {
1791            // `SpawnedService::stop` can report a join-task failure after the
1792            // handle has moved to `spawn_blocking`; that does not establish
1793            // that the worker thread ended. Keep the reservation fail-closed
1794            // rather than admitting a replacement alongside an ambiguous old
1795            // worker. A process-level recovery can clear this poisoned state.
1796            warn!(
1797                model = %model_ref_str,
1798                error = %error,
1799                "model unload stop failed; retaining unload reservation"
1800            );
1801            return Err(anyhow!(
1802                "failed to stop model {} before unload: {error}",
1803                model_ref_str
1804            ));
1805        }
1806        // Acquire delivery order before releasing admission. A successful join
1807        // proves the worker is gone, so a listener may reload immediately; its
1808        // later loaded event must wait behind this unloaded completion, but no
1809        // lifecycle admission lock is held across the publish await.
1810        let _event_publish_gate = inner.lifecycle_event_publish_gate.lock().await;
1811        inner.unloading_models.lock().await.remove(instance);
1812        #[cfg(test)]
1813        if let Some(before_publish) = inner.before_unloaded_event.lock().await.take() {
1814            let _ = before_publish.send(());
1815        }
1816        let model_name = model_ref_str.split(':').next().unwrap_or(&model_ref_str);
1817        let scope = format!("serve:model:{}", model_name);
1818        let event = crate::events::EventEnvelope::new(
1819            crate::events::EventSource::Model,
1820            scope.clone(),
1821            crate::events::EventPayload::ModelUnloaded {
1822                model_ref: model_ref_str.clone(),
1823            },
1824        );
1825        if let Ok(payload) = serde_json::to_vec(&event) {
1826            let _ = inner
1827                .event_publisher
1828                .publish("lifecycle", "unloaded", &payload)
1829                .await;
1830        }
1831        Ok(())
1832    }
1833
1834    /// Convert a TTTConfig to a generated OnlineTrainingConfig wire type.
1835    fn ttt_config_to_wire(cfg: &crate::training::ttt::TTTConfig) -> GenOnlineTrainingConfig {
1836        GenOnlineTrainingConfig {
1837            enabled: cfg.enabled,
1838            learning_rate: cfg.learning_rate,
1839            gradient_steps: cfg.gradient_steps,
1840            max_grad_norm: cfg.max_grad_norm,
1841            min_input_length: cfg.min_input_length,
1842            max_ttt_context: cfg.max_ttt_context,
1843        }
1844    }
1845
1846    /// Convert SamplingParams to a generated GenerationDefaults wire type.
1847    fn sampling_params_to_wire(params: &crate::config::SamplingParams) -> GenGenerationDefaults {
1848        GenGenerationDefaults {
1849            temperature: params.temperature,
1850            top_p: params.top_p,
1851            top_k: params.top_k.map(|v| v as u32),
1852            max_tokens: params.max_tokens.map(|v| v as u32),
1853            repeat_penalty: params.repeat_penalty,
1854            stop_tokens: params.stop_tokens.clone().unwrap_or_default(),
1855            do_sample: params.do_sample,
1856        }
1857    }
1858
1859    /// Return status entries for all known models (loaded, loading, and unloading).
1860    /// Absence from this list means unloaded.
1861    async fn model_status_all(&self, verified_tenant: &str) -> Vec<GenModelStatusEntry> {
1862        // Take the same gate as load admission and unload reservation, then
1863        // snapshot every lifecycle store. A reservation is authoritative even
1864        // during the small inner-load interval where the cache and pending set
1865        // are both populated.
1866        let _load_unload_gate = self.load_unload_gate.lock().await;
1867        let unloading: HashSet<_> = self
1868            .unloading_models
1869            .lock()
1870            .await
1871            .iter()
1872            .filter(|instance| instance.tenant() == verified_tenant)
1873            .cloned()
1874            .collect();
1875        let (mut entries, loaded): (Vec<_>, HashSet<_>) = {
1876            let cache = self.loaded_models.read().await;
1877            (
1878                cache
1879                    .iter()
1880                    .filter(|(instance, _)| {
1881                        instance.tenant() == verified_tenant && !unloading.contains(*instance)
1882                    })
1883                    .map(|(_, model)| GenModelStatusEntry {
1884                        model_ref: model.model_ref.clone(),
1885                        status: "loaded".to_owned(),
1886                        reach: Self::model_reach(&model.network_transport),
1887                        loaded_at: model.loaded_at.elapsed().as_millis() as i64,
1888                        last_used: model.last_used.elapsed().as_millis() as i64,
1889                        online_training_config: model
1890                            .ttt_config
1891                            .as_ref()
1892                            .map(Self::ttt_config_to_wire)
1893                            .unwrap_or_default(),
1894                        generation_defaults: Self::sampling_params_to_wire(
1895                            &model.generation_defaults,
1896                        ),
1897                    })
1898                    .collect(),
1899                cache
1900                    .iter()
1901                    .filter(|(instance, _)| {
1902                        instance.tenant() == verified_tenant && !unloading.contains(*instance)
1903                    })
1904                    .map(|(instance, _)| instance.clone())
1905                    .collect(),
1906            )
1907        };
1908        for instance in &unloading {
1909            entries.push(GenModelStatusEntry {
1910                model_ref: instance.model_ref().to_owned(),
1911                status: "unloading".to_owned(),
1912                reach: Vec::new(),
1913                loaded_at: 0,
1914                last_used: 0,
1915                online_training_config: GenOnlineTrainingConfig::default(),
1916                generation_defaults: GenGenerationDefaults::default(),
1917            });
1918        }
1919        let pending: HashSet<_> = {
1920            self.pending_loads
1921                .lock()
1922                .keys()
1923                .filter(|instance| instance.tenant() == verified_tenant)
1924                .cloned()
1925                .collect()
1926        };
1927        for instance in &pending {
1928            if !loaded.contains(instance) && !unloading.contains(instance) {
1929                entries.push(GenModelStatusEntry {
1930                    model_ref: instance.model_ref().to_owned(),
1931                    status: "loading".to_owned(),
1932                    reach: Vec::new(),
1933                    loaded_at: 0,
1934                    last_used: 0,
1935                    online_training_config: GenOnlineTrainingConfig::default(),
1936                    generation_defaults: GenGenerationDefaults::default(),
1937                });
1938            }
1939        }
1940        entries
1941    }
1942
1943    /// Return status entry for a specific model ref (0 or 1 element).
1944    async fn model_status_single(
1945        &self,
1946        instance: &InferenceInstanceId,
1947    ) -> Vec<GenModelStatusEntry> {
1948        let _load_unload_gate = self.load_unload_gate.lock().await;
1949        if self.unloading_models.lock().await.contains(instance) {
1950            return vec![GenModelStatusEntry {
1951                model_ref: instance.model_ref().to_owned(),
1952                status: "unloading".to_owned(),
1953                reach: Vec::new(),
1954                loaded_at: 0,
1955                last_used: 0,
1956                online_training_config: GenOnlineTrainingConfig::default(),
1957                generation_defaults: GenGenerationDefaults::default(),
1958            }];
1959        }
1960        {
1961            let cache = self.loaded_models.read().await;
1962            if let Some(model) = cache.peek(instance) {
1963                return vec![GenModelStatusEntry {
1964                    model_ref: instance.model_ref().to_owned(),
1965                    status: "loaded".to_owned(),
1966                    reach: Self::model_reach(&model.network_transport),
1967                    loaded_at: model.loaded_at.elapsed().as_millis() as i64,
1968                    last_used: model.last_used.elapsed().as_millis() as i64,
1969                    online_training_config: model
1970                        .ttt_config
1971                        .as_ref()
1972                        .map(Self::ttt_config_to_wire)
1973                        .unwrap_or_default(),
1974                    generation_defaults: Self::sampling_params_to_wire(&model.generation_defaults),
1975                }];
1976            }
1977        }
1978        let pending = self.pending_loads.lock();
1979        if pending.contains_key(instance) {
1980            vec![GenModelStatusEntry {
1981                model_ref: instance.model_ref().to_owned(),
1982                status: "loading".to_owned(),
1983                reach: Vec::new(),
1984                loaded_at: 0,
1985                last_used: 0,
1986                online_training_config: GenOnlineTrainingConfig::default(),
1987                generation_defaults: GenGenerationDefaults::default(),
1988            }]
1989        } else {
1990            vec![]
1991        }
1992    }
1993
1994    /// Get model status
1995    async fn model_status(&self, instance: &InferenceInstanceId) -> ModelStatusResponse {
1996        let cache = self.loaded_models.read().await;
1997        if let Some(model) = cache.peek(instance) {
1998            ModelStatusResponse {
1999                loaded: true,
2000                reach: Self::model_reach(&model.network_transport),
2001                online_training_config: model
2002                    .ttt_config
2003                    .as_ref()
2004                    .map(Self::ttt_config_to_wire)
2005                    .unwrap_or_default(),
2006                ..Default::default()
2007            }
2008        } else {
2009            ModelStatusResponse {
2010                loaded: false,
2011                ..Default::default()
2012            }
2013        }
2014    }
2015    /// Mint the per-request internal execution work order for one
2016    /// already-authorized Model→Inference operation (user-approved D5
2017    /// single-service boundary).
2018    ///
2019    /// The caller credential does NOT cross this hop. The order is signed by
2020    /// Model, audience-pinned to the one allocated instance, bound to the
2021    /// verified tenant/model and the exact downstream dispatch coordinate, and
2022    /// carries the caller's verified claims snapshot so TTT state, stream
2023    /// ownership, and accounting keep attributing to the ORIGINAL caller.
2024    fn mint_internal_work_token(
2025        &self,
2026        instance: &InferenceInstanceId,
2027        ctx: &EnvelopeContext,
2028        scope: &hyprstream_rpc::auth::internal_work::InternalWorkScope,
2029        live_work_audience: &str,
2030    ) -> Result<String> {
2031        use hyprstream_rpc::auth::internal_work::{
2032            encode_internal_work, InternalWorkClaims, INTERNAL_WORK_ISSUER, MAX_LIFETIME_SECS,
2033        };
2034        let caller = ctx.claims().ok_or_else(|| {
2035            anyhow!("internal work requires a verified caller identity at Model ingress")
2036        })?;
2037        let subject = ctx.subject();
2038        anyhow::ensure!(
2039            !subject.is_anonymous(),
2040            "internal work requires an authenticated original subject"
2041        );
2042        let now = chrono::Utc::now().timestamp();
2043        let exp = caller.exp.min(now + MAX_LIFETIME_SECS);
2044        anyhow::ensure!(
2045            exp > now,
2046            "caller credential is expired; refusing to mint internal work"
2047        );
2048        // The caller snapshot's subject is canonicalized to the RESOLVED
2049        // subject string so the snapshot and the work-order subject are
2050        // provably identical (including federated callers, whose raw token
2051        // `sub` differs from the resolved Casbin label) — K3 finding m1.
2052        let subject_string = subject.to_string();
2053        let mut caller_snapshot = caller.clone();
2054        caller_snapshot.sub = subject_string.clone();
2055        let claims = InternalWorkClaims {
2056            iss: INTERNAL_WORK_ISSUER.to_owned(),
2057            sub: subject_string,
2058            // The EXACT audience of the currently live worker incarnation
2059            // (Sol bounded incarnation plan) — never a recomputed
2060            // deterministic name, which would survive a worker restart.
2061            aud: live_work_audience.to_owned(),
2062            tenant: instance.tenant().to_owned(),
2063            model: instance.model_ref().to_owned(),
2064            resource: scope.resource.clone(),
2065            operation: scope.operation.clone(),
2066            iat: now,
2067            exp,
2068            jti: hex::encode(hyprstream_rpc::envelope::generate_nonce()),
2069            cnf: InternalWorkClaims::controller_cnf(&self.signing_key.verifying_key()),
2070            owner_did: ctx
2071                .authenticated_pairwise_did()
2072                .map(|d| d.as_str().to_owned()),
2073            caller: caller_snapshot,
2074        };
2075        Ok(encode_internal_work(&claims, &self.signing_key))
2076    }
2077
2078    /// The authorize coordinate the generated Inference dispatch enforces.
2079    ///
2080    /// Generated dispatch builds the per-method authorize resource as
2081    /// `format!("{}:{}", service, variant_pascal)` where `variant_pascal` is
2082    /// the schema variant name through `to_pascal_case` (codegen/handler.rs).
2083    /// Text-payload variants instead embed the PAYLOAD value
2084    /// (`{service}:{payload}`) — see the `loadLora` call site.
2085    ///
2086    /// SCOPE NOTE (K3 follow-up): this helper uppercases the first character
2087    /// ONLY, which matches `to_pascal_case` exactly for the CURRENT camelCase
2088    /// schema leaves (no `_`/`-` separators — `hasLora` → `HasLora`). It is
2089    /// NOT a general `to_pascal_case`: the generator also removes `_`/`-`
2090    /// separators and capitalizes the following character
2091    /// (hyprstream-rpc-build/src/util.rs). If a future leaf carries
2092    /// separators, use the generator's own `to_pascal_case` spelling here —
2093    /// the worker's exact-match PEP denies any other spelling.
2094    pub(crate) fn inference_dispatch_resource(variant: &str) -> String {
2095        let mut chars = variant.chars();
2096        match chars.next() {
2097            Some(first) => {
2098                format!("inference:{}{}", first.to_ascii_uppercase(), chars.as_str())
2099            }
2100            None => "inference:".to_owned(),
2101        }
2102    }
2103
2104    /// Mint-side scope for one forwarded struct/Void Inference operation:
2105    /// the exact dispatch coordinate for `variant` plus its scope operation.
2106    fn inference_scope(
2107        variant: &str,
2108        operation: &str,
2109    ) -> hyprstream_rpc::auth::internal_work::InternalWorkScope {
2110        hyprstream_rpc::auth::internal_work::InternalWorkScope::new(
2111            Self::inference_dispatch_resource(variant),
2112            operation,
2113        )
2114    }
2115
2116    async fn get_inference_client(
2117        &self,
2118        model_ref_str: &str,
2119        ctx: &EnvelopeContext,
2120        work: hyprstream_rpc::auth::internal_work::InternalWorkScope,
2121    ) -> Result<InferenceClient> {
2122        let instance = Self::inference_instance(ctx, model_ref_str)?;
2123        let _endpoint = self.load_model(&instance, None, None).await?;
2124        let mut cache = self.loaded_models.write().await;
2125        let model = cache
2126            .get_mut(&instance)
2127            .ok_or_else(|| anyhow!("Model {} not found after loading", model_ref_str))?;
2128        // Bounded worker incarnation binding (Sol plan): a cached entry whose
2129        // worker DIED is fail-closed — evict and deny until a fresh run
2130        // reports ready with a fresh incarnation. Never mint against a dead
2131        // worker, even if its (old) receipt would still be time-valid.
2132        if !model.service_handle.is_running() {
2133            cache.pop(&instance);
2134            anyhow::bail!(
2135                "pinned worker for model {model_ref_str} is not running; \
2136                 the model must be loaded again before internal work"
2137            );
2138        }
2139        model.last_used = Instant::now();
2140        // The live binding for THIS running incarnation. Minting uses exactly
2141        // this audience; the worker denies anything else.
2142        let live_work_audience = model.work_audience.clone();
2143        let live_generation = model.load_generation;
2144        // #322 placement key. The envelope does not yet carry an explicit
2145        // session_id; use the authenticated subject as a stable per-caller key
2146        // (keeps HRW affinity effective for repeat requests from the same
2147        // caller). When a real session_id is plumbed through `EnvelopeContext`,
2148        // swap it in here — the router body is session_id-keyed, not
2149        // subject-keyed, by design.
2150        let placement_key = ctx.subject().to_string();
2151        // The co-located pinned instance executes Model-origin internal work —
2152        // the caller credential is never relayed. A remote replica is a
2153        // different controller's service: it may receive an end-user bearer as
2154        // delegated authority, but never a holder-bound service credential.
2155        // For service callers the remote arm therefore fails its no-bearer
2156        // gate and the router retries a healthy co-located candidate; when no
2157        // such candidate exists, the request denies rather than transferring a
2158        // credential whose cnf belongs to the original service signer.
2159        let relay_bearer = Self::remote_relay_bearer(ctx).map(ToOwned::to_owned);
2160        let routed = self
2161            .select_inference_routed(model, &placement_key, relay_bearer.as_deref())
2162            .await?;
2163        let work_token =
2164            self.mint_internal_work_token(&instance, ctx, &work, &live_work_audience)?;
2165        #[cfg(test)]
2166        self.capture_test_work_order(&work_token).await;
2167        tracing::debug!(
2168            generation = live_generation,
2169            "minted internal work order against live worker incarnation"
2170        );
2171        Ok(Self::attach_work_order(routed, work_token))
2172    }
2173
2174    /// Attach the internal work order to the routed client BY LOCALITY
2175    /// (K3 finding M1): the co-located subprocessor gets the work order as
2176    /// its DIRECT bearer; a remote-selected replica keeps the
2177    /// delegated-relay client the remote arm already built. Overlaying both
2178    /// on one client is a hard wire error (an envelope cannot carry a direct
2179    /// JWT and a delegated bearer), and a remote instance would deny a
2180    /// foreign-controller work order anyway — fail-closed on both sides.
2181    fn attach_work_order(
2182        routed: RoutedReplica<InferenceClient>,
2183        work_token: String,
2184    ) -> InferenceClient {
2185        match routed.locality {
2186            ReplicaLocality::CoLocated => routed.value.with_bearer(work_token),
2187            ReplicaLocality::Remote => routed.value,
2188        }
2189    }
2190
2191    fn remote_relay_bearer(ctx: &EnvelopeContext) -> Option<&str> {
2192        // The reserved Federate credential is holder-bound at this ingress.
2193        // Never send it to a remote replica as a transferable bearer; only a
2194        // co-located worker receives Model's audience-pinned internal order.
2195        if ctx.is_federate_admitted() {
2196            return None;
2197        }
2198        Self::remote_relay_bearer_for_claims(ctx.claims(), ctx.jwt_token())
2199    }
2200
2201    fn remote_relay_bearer_for_claims<'a>(
2202        claims: Option<&hyprstream_rpc::auth::Claims>,
2203        bearer: Option<&'a str>,
2204    ) -> Option<&'a str> {
2205        if claims.is_some_and(|claims| claims.sub.starts_with("service:")) {
2206            None
2207        } else {
2208            bearer
2209        }
2210    }
2211
2212    /// Load a LoRA adapter from a file
2213    async fn load_lora(
2214        &self,
2215        model_ref_str: &str,
2216        ctx: &EnvelopeContext,
2217        path: &str,
2218    ) -> Result<()> {
2219        // Text-payload variant (K3 finding B2): the generated dispatch
2220        // resource embeds the PAYLOAD (`inference:{path}`), so the work order
2221        // binds the exact adapter path rather than the method name.
2222        let client = self
2223            .get_inference_client(
2224                model_ref_str,
2225                ctx,
2226                hyprstream_rpc::auth::internal_work::InternalWorkScope::new(
2227                    format!("inference:{path}"),
2228                    "write",
2229                ),
2230            )
2231            .await?;
2232        client.load_lora(path).await
2233    }
2234
2235    /// Unload the current LoRA adapter
2236    async fn unload_lora(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2237        let client = self
2238            .get_inference_client(
2239                model_ref_str,
2240                ctx,
2241                Self::inference_scope("unloadLora", "write"),
2242            )
2243            .await?;
2244        client.unload_lora().await
2245    }
2246
2247    /// Check if a LoRA adapter is loaded
2248    async fn has_lora(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<bool> {
2249        let client = self
2250            .get_inference_client(
2251                model_ref_str,
2252                ctx,
2253                Self::inference_scope("hasLora", "query"),
2254            )
2255            .await?;
2256        client.has_lora().await
2257    }
2258
2259    // Training loop control - forward to InferenceService via ZMQ
2260    async fn writeback_adaptation(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2261        let client = self
2262            .get_inference_client(
2263                model_ref_str,
2264                ctx,
2265                Self::inference_scope("tttWriteback", "train"),
2266            )
2267            .await?;
2268        client.ttt_writeback().await
2269    }
2270
2271    async fn evict_adaptation(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2272        let client = self
2273            .get_inference_client(
2274                model_ref_str,
2275                ctx,
2276                Self::inference_scope("tttEvict", "train"),
2277            )
2278            .await?;
2279        client.ttt_evict().await
2280    }
2281
2282    async fn zero_delta(&self, model_ref_str: &str, ctx: &EnvelopeContext) -> Result<()> {
2283        let client = self
2284            .get_inference_client(
2285                model_ref_str,
2286                ctx,
2287                Self::inference_scope("tttZero", "manage"),
2288            )
2289            .await?;
2290        client.ttt_zero().await
2291    }
2292
2293    async fn get_delta_status_forward(
2294        &self,
2295        model_ref_str: &str,
2296        ctx: &EnvelopeContext,
2297    ) -> Result<hyprstream_rpc_std::inference_client::DeltaStatusResult> {
2298        let client = self
2299            .get_inference_client(
2300                model_ref_str,
2301                ctx,
2302                Self::inference_scope("getDeltaStatus", "query"),
2303            )
2304            .await?;
2305        client.get_delta_status().await
2306    }
2307
2308    async fn snapshot_delta_forward(
2309        &self,
2310        model_ref_str: &str,
2311        ctx: &EnvelopeContext,
2312    ) -> Result<hyprstream_rpc_std::inference_client::SnapshotDeltaResult> {
2313        let client = self
2314            .get_inference_client(
2315                model_ref_str,
2316                ctx,
2317                Self::inference_scope("snapshotDelta", "write"),
2318            )
2319            .await?;
2320        client.snapshot_delta().await
2321    }
2322
2323    async fn export_peft_adapter_forward(
2324        &self,
2325        model_ref_str: &str,
2326        ctx: &EnvelopeContext,
2327        data: &ExportPeftRequest,
2328    ) -> Result<ExportPeftResult> {
2329        let client = self
2330            .get_inference_client(
2331                model_ref_str,
2332                ctx,
2333                Self::inference_scope("exportPeftAdapter", "write"),
2334            )
2335            .await?;
2336        client.export_peft_adapter(data).await
2337    }
2338}
2339
2340// ═══════════════════════════════════════════════════════════════════════════════
2341// ModelHandler Implementation — generated dispatch for top-level + typed scope traits
2342// ═══════════════════════════════════════════════════════════════════════════════
2343
2344use crate::services::generated::model_client::{
2345    dispatch_model, serialize_response, AdapterHandler, InferHandler, ModelHandler, TttHandler,
2346};
2347use hyprstream_rpc_std::model_client::{
2348    // Adapter types
2349    AdapterInfo,
2350    ChatTemplateRequest,
2351    DeltaStatusResult,
2352    EmbedImagesRequest,
2353    EmbedImagesResponse,
2354    ErrorInfo,
2355    ExportPeftRequest,
2356    ExportPeftResult,
2357    GenerationDefaults as GenGenerationDefaults,
2358    // Infer types (GenerationRequest follows inference.capnp name)
2359    GenerationRequest,
2360    // Top-level request types
2361    LoadModelRequest,
2362    LoadedModelResponse,
2363    // TTT types (names follow inference.capnp via using-import)
2364    LoraConfig,
2365    MergeLoraRequest,
2366    ModelHealthStatus,
2367    ModelResponseVariant,
2368    ModelStatusEntry as GenModelStatusEntry,
2369    ModelStatusResponse,
2370    OnlineTrainingConfig as GenOnlineTrainingConfig,
2371    SaveAdaptationRequest,
2372    SaveAdaptationResult,
2373    SnapshotDeltaResult,
2374    StatusRequest,
2375    TrainStepRequest,
2376    TrainStepResult,
2377    UnloadModelRequest,
2378    WriteTttConfigRequest,
2379};
2380// AdaptationStrategy is now from inference_client (canonical source via using-import).
2381// model_client types reference it directly — no conversion needed.
2382
2383#[async_trait::async_trait(?Send)]
2384impl TttHandler for ModelService {
2385    async fn handle_init(
2386        &self,
2387        ctx: &EnvelopeContext,
2388        _request_id: u64,
2389        model_ref: &str,
2390        data: &LoraConfig,
2391    ) -> Result<()> {
2392        let client = self
2393            .get_inference_client(model_ref, ctx, Self::inference_scope("createLora", "write"))
2394            .await?;
2395        client.create_lora(data).await
2396    }
2397
2398    async fn handle_train(
2399        &self,
2400        ctx: &EnvelopeContext,
2401        _request_id: u64,
2402        model_ref: &str,
2403        data: &TrainStepRequest,
2404    ) -> Result<TrainStepResult> {
2405        let client = self
2406            .get_inference_client(model_ref, ctx, Self::inference_scope("trainStep", "train"))
2407            .await?;
2408        client.train_step(data).await
2409    }
2410
2411    async fn handle_train_stream(
2412        &self,
2413        ctx: &EnvelopeContext,
2414        _request_id: u64,
2415        model_ref: &str,
2416        data: &TrainStepRequest,
2417    ) -> Result<(
2418        hyprstream_rpc_std::model_client::StreamInfo,
2419        hyprstream_rpc::service::Continuation,
2420    )> {
2421        let client = self
2422            .get_inference_client(
2423                model_ref,
2424                ctx,
2425                Self::inference_scope("trainStepStream", "train"),
2426            )
2427            .await?;
2428        let ephemeral_pubkey = ctx.ephemeral_pubkey().ok_or_else(|| {
2429            anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
2430        })?;
2431        let stream_info = client.train_step_stream(data, ephemeral_pubkey).await?;
2432        Ok((stream_info, Box::pin(async {})))
2433    }
2434
2435    async fn handle_writeback(
2436        &self,
2437        ctx: &EnvelopeContext,
2438        _request_id: u64,
2439        model_ref: &str,
2440    ) -> Result<()> {
2441        self.writeback_adaptation(model_ref, ctx).await
2442    }
2443
2444    async fn handle_evict(
2445        &self,
2446        ctx: &EnvelopeContext,
2447        _request_id: u64,
2448        model_ref: &str,
2449    ) -> Result<()> {
2450        self.evict_adaptation(model_ref, ctx).await
2451    }
2452
2453    async fn handle_zero(
2454        &self,
2455        ctx: &EnvelopeContext,
2456        _request_id: u64,
2457        model_ref: &str,
2458    ) -> Result<()> {
2459        self.zero_delta(model_ref, ctx).await
2460    }
2461
2462    async fn handle_status(
2463        &self,
2464        ctx: &EnvelopeContext,
2465        _request_id: u64,
2466        model_ref: &str,
2467    ) -> Result<DeltaStatusResult> {
2468        self.get_delta_status_forward(model_ref, ctx).await
2469    }
2470
2471    async fn handle_save(
2472        &self,
2473        ctx: &EnvelopeContext,
2474        _request_id: u64,
2475        model_ref: &str,
2476        data: &SaveAdaptationRequest,
2477    ) -> Result<SaveAdaptationResult> {
2478        let client = self
2479            .get_inference_client(
2480                model_ref,
2481                ctx,
2482                Self::inference_scope("saveAdaptation", "write"),
2483            )
2484            .await?;
2485        client.save_adaptation(data).await
2486    }
2487
2488    async fn handle_snapshot(
2489        &self,
2490        ctx: &EnvelopeContext,
2491        _request_id: u64,
2492        model_ref: &str,
2493    ) -> Result<SnapshotDeltaResult> {
2494        self.snapshot_delta_forward(model_ref, ctx).await
2495    }
2496
2497    async fn handle_export(
2498        &self,
2499        ctx: &EnvelopeContext,
2500        _request_id: u64,
2501        model_ref: &str,
2502        data: &ExportPeftRequest,
2503    ) -> Result<ExportPeftResult> {
2504        self.export_peft_adapter_forward(model_ref, ctx, data).await
2505    }
2506
2507    async fn handle_write_ttt_config(
2508        &self,
2509        ctx: &EnvelopeContext,
2510        _request_id: u64,
2511        model_ref: &str,
2512        data: &WriteTttConfigRequest,
2513    ) -> Result<()> {
2514        // 1. Parse/resolve model ref and resolve worktree path
2515        let parsed = self.resolve_model_ref(model_ref).await?;
2516        let tracked = self
2517            .registry
2518            .get_by_name(parsed.name())
2519            .await
2520            .map_err(|e| anyhow!("Model '{}' not found in registry: {}", parsed.name(), e))?;
2521        let repo_client = self.registry.repo(&tracked.id);
2522
2523        let branch_name = match &parsed.git_ref {
2524            crate::storage::GitRef::Branch(name) => name.clone(),
2525            _ => repo_client
2526                .get_head()
2527                .await
2528                .unwrap_or_else(|_| "main".to_owned()),
2529        };
2530
2531        let storage_paths = crate::storage::StoragePaths::new()?;
2532        let model_path = storage_paths.worktree_path(parsed.name(), &branch_name)?;
2533
2534        if !model_path.exists() {
2535            return Err(anyhow!("Model worktree not found for {}", model_ref));
2536        }
2537
2538        // 2. Build HyprstreamTrainingConfig from request
2539        let ttt_config = crate::config::TTTTrainingConfig {
2540            learning_rate: if data.learning_rate > 0.0 {
2541                data.learning_rate
2542            } else {
2543                3e-4
2544            },
2545            gradient_steps: if data.gradient_steps > 0 {
2546                data.gradient_steps
2547            } else {
2548                3
2549            },
2550            max_grad_norm: if data.max_grad_norm > 0.0 {
2551                data.max_grad_norm
2552            } else {
2553                1.0
2554            },
2555            min_input_length: if data.min_input_length > 0 {
2556                data.min_input_length
2557            } else {
2558                32
2559            },
2560            max_ttt_context: if data.max_ttt_context > 0 {
2561                data.max_ttt_context
2562            } else {
2563                512
2564            },
2565            rank_oracle: None,
2566            gradient_gating: None,
2567        };
2568
2569        let training_config = crate::config::HyprstreamTrainingConfig {
2570            mode: crate::config::TrainingMode::TestTimeTraining,
2571            ttt: ttt_config,
2572            lora_rank: if data.lora_rank > 0 {
2573                data.lora_rank as usize
2574            } else {
2575                crate::config::default_lora_rank()
2576            },
2577            lora_alpha: if data.lora_alpha > 0.0 {
2578                Some(data.lora_alpha)
2579            } else {
2580                None
2581            },
2582            target_modules: if data.target_modules.is_empty() {
2583                crate::config::default_target_modules()
2584            } else {
2585                data.target_modules.clone()
2586            },
2587            ..Default::default()
2588        };
2589
2590        // 3. Write config.json
2591        crate::runtime::model_config::ModelConfig::save_training_config(
2592            &model_path,
2593            &training_config,
2594        )?;
2595
2596        // 4. Stage and commit via worktree-scoped API
2597        let wt = repo_client.worktree(&branch_name);
2598        wt.stage_files(&StageFilesRequest {
2599            files: vec!["config.json".to_owned()],
2600        })
2601        .await?;
2602        wt.commit_with_author(&CommitWithAuthorRequest {
2603            message: "Update hyprstream_training config via RPC".to_owned(),
2604            author_name: "hyprstream".to_owned(),
2605            author_email: "noreply@hyprstream.dev".to_owned(),
2606        })
2607        .await?;
2608
2609        info!("TTT config written for {}", model_ref);
2610
2611        // 5. Auto-reload if requested and model is loaded
2612        if data.auto_reload {
2613            let instance = Self::inference_instance(ctx, model_ref)?;
2614            let is_loaded = {
2615                let cache = self.loaded_models.read().await;
2616                cache.contains(&instance)
2617            };
2618            if is_loaded {
2619                info!("Auto-reloading {} after TTT config change", model_ref);
2620                self.unload_model(&instance).await?;
2621                self.load_model(&instance, None, None).await?;
2622            }
2623        }
2624
2625        Ok(())
2626    }
2627}
2628
2629#[async_trait::async_trait(?Send)]
2630impl AdapterHandler for ModelService {
2631    async fn handle_load(
2632        &self,
2633        ctx: &EnvelopeContext,
2634        _request_id: u64,
2635        model_ref: &str,
2636        value: &str,
2637    ) -> Result<()> {
2638        self.load_lora(model_ref, ctx, value).await
2639    }
2640
2641    async fn handle_unload(
2642        &self,
2643        ctx: &EnvelopeContext,
2644        _request_id: u64,
2645        model_ref: &str,
2646    ) -> Result<()> {
2647        self.unload_lora(model_ref, ctx).await
2648    }
2649
2650    async fn handle_status(
2651        &self,
2652        ctx: &EnvelopeContext,
2653        _request_id: u64,
2654        model_ref: &str,
2655    ) -> Result<bool> {
2656        self.has_lora(model_ref, ctx).await
2657    }
2658
2659    async fn handle_inspect(
2660        &self,
2661        _ctx: &EnvelopeContext,
2662        _request_id: u64,
2663        model_ref: &str,
2664        value: &str,
2665    ) -> Result<AdapterInfo> {
2666        // Resolve model_ref to a worktree client (does NOT require model loaded in memory)
2667        let parsed = self.resolve_model_ref(model_ref).await?;
2668        let tracked = self
2669            .registry
2670            .get_by_name(parsed.name())
2671            .await
2672            .map_err(|e| anyhow!("Model '{}' not found in registry: {}", parsed.name(), e))?;
2673        let repo_client = self.registry.repo(&tracked.id);
2674        let branch_name = match &parsed.git_ref {
2675            crate::storage::GitRef::Branch(name) => name.clone(),
2676            _ => repo_client
2677                .get_head()
2678                .await
2679                .unwrap_or_else(|_| "main".to_owned()),
2680        };
2681        let fs = repo_client.worktree(&branch_name);
2682
2683        // Read adapter_config.json from the adapter directory
2684        let config_path = format!("{}/adapter_config.json", value);
2685        let config_bytes = fs
2686            .read_file_chunked(&config_path)
2687            .await
2688            .map_err(|e| anyhow!("Failed to read {}: {}", config_path, e))?;
2689        let config_json: serde_json::Value = serde_json::from_slice(&config_bytes)
2690            .map_err(|e| anyhow!("Failed to parse adapter_config.json: {}", e))?;
2691
2692        // Verify adapter_model.safetensors exists
2693        let model_path = format!("{}/adapter_model.safetensors", value);
2694        let stat = fs
2695            .stat_path(&model_path)
2696            .await
2697            .map_err(|e| anyhow!("Failed to stat {}: {}", model_path, e))?;
2698        if !stat.exists {
2699            anyhow::bail!("adapter_model.safetensors not found in {}", value);
2700        }
2701
2702        // Extract PEFT fields from config
2703        let rank = config_json
2704            .get("r")
2705            .and_then(serde_json::Value::as_u64)
2706            .unwrap_or(0) as u32;
2707        let lora_alpha = config_json
2708            .get("lora_alpha")
2709            .and_then(serde_json::Value::as_f64)
2710            .unwrap_or(0.0) as f32;
2711        let target_modules = config_json
2712            .get("target_modules")
2713            .and_then(|v| v.as_array())
2714            .map(|arr| {
2715                arr.iter()
2716                    .filter_map(|v| v.as_str().map(String::from))
2717                    .collect()
2718            })
2719            .unwrap_or_default();
2720        let base_model = config_json
2721            .get("base_model_name_or_path")
2722            .and_then(|v| v.as_str())
2723            .unwrap_or("")
2724            .to_owned();
2725
2726        // Extract directory name from path
2727        let name = value.rsplit('/').next().unwrap_or(value).to_owned();
2728
2729        Ok(AdapterInfo {
2730            name,
2731            path: value.to_owned(),
2732            rank,
2733            lora_alpha,
2734            target_modules,
2735            base_model,
2736        })
2737    }
2738
2739    async fn handle_merge(
2740        &self,
2741        ctx: &EnvelopeContext,
2742        _request_id: u64,
2743        model_ref: &str,
2744        data: &MergeLoraRequest,
2745    ) -> Result<()> {
2746        let client = self
2747            .get_inference_client(model_ref, ctx, Self::inference_scope("mergeLora", "write"))
2748            .await?;
2749        client.merge_lora(data).await
2750    }
2751}
2752
2753#[async_trait::async_trait(?Send)]
2754impl InferHandler for ModelService {
2755    async fn handle_generate_stream(
2756        &self,
2757        ctx: &EnvelopeContext,
2758        _request_id: u64,
2759        model_ref: &str,
2760        data: &GenerationRequest,
2761    ) -> Result<(
2762        hyprstream_rpc_std::model_client::StreamInfo,
2763        hyprstream_rpc::service::Continuation,
2764    )> {
2765        let client = self
2766            .get_inference_client(
2767                model_ref,
2768                ctx,
2769                Self::inference_scope("generateStream", "infer"),
2770            )
2771            .await?;
2772        let ephemeral_pubkey = ctx.ephemeral_pubkey().ok_or_else(|| {
2773            anyhow!("Streaming requires client ephemeral pubkey for E2E authentication")
2774        })?;
2775        let stream_info = client.generate_stream(data, ephemeral_pubkey).await?;
2776        Ok((stream_info, Box::pin(async {})))
2777    }
2778
2779    async fn handle_apply_chat_template(
2780        &self,
2781        ctx: &EnvelopeContext,
2782        _request_id: u64,
2783        model_ref: &str,
2784        data: &ChatTemplateRequest,
2785    ) -> Result<String> {
2786        let client = self
2787            .get_inference_client(
2788                model_ref,
2789                ctx,
2790                Self::inference_scope("applyChatTemplate", "query"),
2791            )
2792            .await?;
2793        client.apply_chat_template(data).await
2794    }
2795
2796    async fn handle_embed(
2797        &self,
2798        ctx: &EnvelopeContext,
2799        _request_id: u64,
2800        model_ref: &str,
2801        data: &EmbedImagesRequest,
2802    ) -> Result<EmbedImagesResponse> {
2803        let client = self
2804            .get_inference_client(model_ref, ctx, Self::inference_scope("embed", "infer"))
2805            .await?;
2806        client.embed(data).await
2807    }
2808
2809    async fn handle_status(
2810        &self,
2811        ctx: &EnvelopeContext,
2812        _request_id: u64,
2813        model_ref: &str,
2814    ) -> Result<ModelStatusResponse> {
2815        let instance = Self::inference_instance(ctx, model_ref)?;
2816        Ok(self.model_status(&instance).await)
2817    }
2818}
2819
2820#[async_trait::async_trait(?Send)]
2821impl ModelHandler for ModelService {
2822    async fn authorize(
2823        &self,
2824        ctx: &EnvelopeContext,
2825        resource: &str,
2826        operation: &str,
2827    ) -> Result<()> {
2828        let subject = ctx.subject();
2829        let domain = ctx.domain()?;
2830        let object_label = crate::services::inference::inference_object_label();
2831        crate::services::inference::enforce_inference_mac(ctx, &object_label)?;
2832        let request = PolicyCheck {
2833            subject: subject.to_string(),
2834            domain,
2835            resource: resource.to_owned(),
2836            operation: operation.to_owned(),
2837        };
2838        if ctx.is_federate_admitted() {
2839            anyhow::ensure!(
2840                ctx.admitted_federate_operation(resource, operation),
2841                "Federate request-local operation mismatch"
2842            );
2843            // The H2 request-use RPC has already performed the fresh Policy
2844            // check for this exact original holder, resource and operation,
2845            // and consumed the shared replay key. Generic Policy.check would
2846            // authorize this service's identity, not the original caller.
2847            return Ok(());
2848        }
2849        let allowed = crate::services::policy::check_with_holder_evidence(
2850            &self.policy_client,
2851            &request,
2852            ctx.jwt_token(),
2853            &ctx.subject(),
2854            ctx.original_holder_evidence(),
2855        )
2856        .await
2857        .unwrap_or_else(|e| {
2858            warn!(
2859                "Policy check failed for {} on {}: {} - denying access",
2860                subject, resource, e
2861            );
2862            false
2863        });
2864        if allowed {
2865            Ok(())
2866        } else {
2867            anyhow::bail!(
2868                "Unauthorized: {} cannot {} on {}",
2869                subject,
2870                operation,
2871                resource
2872            )
2873        }
2874    }
2875
2876    async fn handle_load(
2877        &self,
2878        ctx: &EnvelopeContext,
2879        _request_id: u64,
2880        data: &LoadModelRequest,
2881    ) -> Result<ModelResponseVariant> {
2882        let max_ctx = data.max_context.filter(|&n| n != 0);
2883        let kv_q = data.kv_quant.filter(|q| *q != KVQuantType::None);
2884        let model_ref = &data.model_ref;
2885        let instance = Self::inference_instance(ctx, model_ref)?;
2886        match self.load_model(&instance, max_ctx, kv_q).await {
2887            Ok(_endpoint) => {
2888                let cache = self.loaded_models.read().await;
2889                let model = cache
2890                    .peek(&instance)
2891                    .ok_or_else(|| anyhow!("model missing after successful load"))?;
2892                Ok(ModelResponseVariant::LoadResult(LoadedModelResponse {
2893                    model_ref: model_ref.to_owned(),
2894                    reach: Self::model_reach(&model.network_transport),
2895                }))
2896            }
2897            Err(e) => Ok(ModelResponseVariant::Error(ErrorInfo {
2898                message: format!("Failed to load model: {e}"),
2899                code: "LOAD_FAILED".into(),
2900                details: String::new(),
2901            })),
2902        }
2903    }
2904
2905    async fn handle_unload(
2906        &self,
2907        ctx: &EnvelopeContext,
2908        _request_id: u64,
2909        data: &UnloadModelRequest,
2910    ) -> Result<ModelResponseVariant> {
2911        let model_ref = &data.model_ref;
2912        let instance = Self::inference_instance(ctx, model_ref)?;
2913        match self.unload_model(&instance).await {
2914            Ok(()) => Ok(ModelResponseVariant::UnloadResult),
2915            Err(e) => Ok(ModelResponseVariant::Error(ErrorInfo {
2916                message: format!("Failed to unload model: {e}"),
2917                code: "UNLOAD_FAILED".into(),
2918                details: String::new(),
2919            })),
2920        }
2921    }
2922
2923    async fn handle_status(
2924        &self,
2925        ctx: &EnvelopeContext,
2926        _request_id: u64,
2927        data: &StatusRequest,
2928    ) -> Result<ModelResponseVariant> {
2929        let tenant = ctx.domain()?;
2930        let entries = if data.model_ref.is_empty() {
2931            self.model_status_all(&tenant).await
2932        } else {
2933            let instance = InferenceInstanceId::new(&tenant, &data.model_ref, 0)?;
2934            self.model_status_single(&instance).await
2935        };
2936        Ok(ModelResponseVariant::StatusResult(entries))
2937    }
2938
2939    async fn handle_health_check(
2940        &self,
2941        ctx: &EnvelopeContext,
2942        _request_id: u64,
2943    ) -> Result<ModelResponseVariant> {
2944        let tenant = ctx.domain()?;
2945        let cache = self.loaded_models.read().await;
2946        let loaded_count = cache
2947            .iter()
2948            .filter(|(instance, _)| instance.tenant() == tenant.as_str())
2949            .count() as u32;
2950        let max_models = self.config.max_models as u32;
2951        drop(cache);
2952        Ok(ModelResponseVariant::HealthCheckResult(ModelHealthStatus {
2953            status: "healthy".into(),
2954            loaded_model_count: loaded_count,
2955            max_models,
2956            total_memory_bytes: 0,
2957        }))
2958    }
2959}
2960
2961// ═══════════════════════════════════════════════════════════════════════════════
2962// 9P Filesystem Handler (FsHandler trait)
2963// ═══════════════════════════════════════════════════════════════════════════════
2964
2965use crate::services::fs::{SyntheticNode, SyntheticQid, SyntheticTree};
2966use crate::services::generated::model_client::FsHandler;
2967use hyprstream_rpc_std::model_client::{
2968    NpClunk, NpCreate, NpOpen, NpRead, NpRemove, NpStat as GenNpStat, NpStatReq, NpWalk, NpWrite,
2969    Qid as GenQid, ROpen, RRead, RStat, RWalk, RWrite,
2970};
2971use hyprstream_vfs::DirEntry;
2972
2973impl ModelService {
2974    /// Get the persistent tenant-partitioned synthetic 9P tree.
2975    fn fs_tree(&self, verified_tenant: &str) -> Arc<SyntheticTree> {
2976        if let Some(tree) = self.inner.fs_trees.get(verified_tenant) {
2977            return Arc::clone(tree.value());
2978        }
2979        let tree = Arc::new(self.build_fs_tree(verified_tenant.to_owned()));
2980        let entry = self
2981            .inner
2982            .fs_trees
2983            .entry(verified_tenant.to_owned())
2984            .or_insert(tree);
2985        Arc::clone(entry.value())
2986    }
2987
2988    /// Build a synthetic 9P tree from current model service state.
2989    fn build_fs_tree(&self, verified_tenant: String) -> SyntheticTree {
2990        let inner_list = Arc::clone(&self.inner);
2991        let inner_resolve = Arc::clone(&self.inner);
2992        let list_tenant = verified_tenant.clone();
2993        let resolve_tenant = verified_tenant;
2994
2995        SyntheticTree::new(SyntheticNode::DynamicDir {
2996            list: Box::new(move || {
2997                let cache = inner_list.loaded_models.blocking_read();
2998                let pending = inner_list.pending_loads.lock();
2999                let mut entries: Vec<DirEntry> = cache
3000                    .iter()
3001                    .filter(|(instance, _)| instance.tenant() == list_tenant)
3002                    .map(|(instance, _)| DirEntry {
3003                        name: instance.service_name(),
3004                        is_dir: true,
3005                        size: 0,
3006                        stat: None,
3007                    })
3008                    .collect();
3009                for instance in pending
3010                    .keys()
3011                    .filter(|instance| instance.tenant() == list_tenant)
3012                {
3013                    if !cache.contains(instance) {
3014                        entries.push(DirEntry {
3015                            name: instance.service_name(),
3016                            is_dir: true,
3017                            size: 0,
3018                            stat: None,
3019                        });
3020                    }
3021                }
3022                entries
3023            }),
3024            resolve: Box::new(move |ref_name| {
3025                let cache = inner_resolve.loaded_models.blocking_read();
3026                let model = cache
3027                    .iter()
3028                    .find(|(instance, _)| {
3029                        instance.tenant() == resolve_tenant && instance.service_name() == ref_name
3030                    })
3031                    .map(|(_, model)| model);
3032                let is_loaded = model.is_some();
3033                let defaults_json = if let Some(model) = model {
3034                    serde_json::to_vec_pretty(&model.generation_defaults).unwrap_or_default()
3035                } else {
3036                    b"{}".to_vec()
3037                };
3038                let status_str = if is_loaded { "loaded" } else { "unloaded" };
3039
3040                let mut children = std::collections::HashMap::new();
3041                let status_owned = status_str.to_owned();
3042                children.insert(
3043                    "status".to_owned(),
3044                    SyntheticNode::ReadFile(Box::new(move || {
3045                        format!("{status_owned}\n").into_bytes()
3046                    })),
3047                );
3048                children.insert(
3049                    "defaults".to_owned(),
3050                    SyntheticNode::ReadFile(Box::new(move || defaults_json.clone())),
3051                );
3052                children.insert(
3053                    "ctl".to_owned(),
3054                    SyntheticNode::CtlFile {
3055                        handler: Box::new(|data, _subject| {
3056                            let cmd = String::from_utf8_lossy(data).trim().to_owned();
3057                            Ok(format!("ctl: {cmd}\n").into_bytes())
3058                        }),
3059                    },
3060                );
3061                Some(SyntheticNode::Dir { children })
3062            }),
3063        })
3064    }
3065
3066    fn qid_to_gen(qid: &SyntheticQid) -> GenQid {
3067        GenQid {
3068            qtype: qid.qtype,
3069            version: qid.version,
3070            path: qid.path,
3071        }
3072    }
3073}
3074
3075#[async_trait::async_trait(?Send)]
3076impl FsHandler for ModelService {
3077    async fn handle_walk(
3078        &self,
3079        ctx: &EnvelopeContext,
3080        _request_id: u64,
3081        _model_ref: &str,
3082        data: &NpWalk,
3083    ) -> Result<RWalk> {
3084        let tree = self.fs_tree(&ctx.domain()?);
3085        let owner = ctx.subject().to_string();
3086        let (fid, qid) = tree
3087            .walk(&data.wnames, &owner, Some(data.newfid))
3088            .map_err(|e| anyhow::anyhow!(e))?;
3089        Ok(RWalk {
3090            qid: Self::qid_to_gen(&qid),
3091            fid,
3092        })
3093    }
3094
3095    async fn handle_open(
3096        &self,
3097        ctx: &EnvelopeContext,
3098        _request_id: u64,
3099        _model_ref: &str,
3100        data: &NpOpen,
3101    ) -> Result<ROpen> {
3102        let tree = self.fs_tree(&ctx.domain()?);
3103        let owner = ctx.subject().to_string();
3104        // Re-walk to get the fid, then open.
3105        let (qid, iounit) = tree
3106            .open(data.fid, data.mode, &owner)
3107            .map_err(|e| anyhow::anyhow!(e))?;
3108        Ok(ROpen {
3109            qid: Self::qid_to_gen(&qid),
3110            iounit,
3111        })
3112    }
3113
3114    async fn handle_read(
3115        &self,
3116        ctx: &EnvelopeContext,
3117        _request_id: u64,
3118        _model_ref: &str,
3119        data: &NpRead,
3120    ) -> Result<RRead> {
3121        let tree = self.fs_tree(&ctx.domain()?);
3122        let owner = ctx.subject().to_string();
3123        let bytes = tree
3124            .read(data.fid, data.offset, data.count, &owner)
3125            .map_err(|e| anyhow::anyhow!(e))?;
3126        Ok(RRead { data: bytes })
3127    }
3128
3129    async fn handle_write(
3130        &self,
3131        ctx: &EnvelopeContext,
3132        _request_id: u64,
3133        _model_ref: &str,
3134        data: &NpWrite,
3135    ) -> Result<RWrite> {
3136        let tree = self.fs_tree(&ctx.domain()?);
3137        let subject = ctx.subject();
3138        let owner = subject.to_string();
3139        let count = tree
3140            .write(data.fid, data.offset, &data.data, &owner, &subject)
3141            .map_err(|e| anyhow::anyhow!(e))?;
3142        Ok(RWrite { count })
3143    }
3144
3145    async fn handle_clunk(
3146        &self,
3147        ctx: &EnvelopeContext,
3148        _request_id: u64,
3149        _model_ref: &str,
3150        data: &NpClunk,
3151    ) -> Result<()> {
3152        let tree = self.fs_tree(&ctx.domain()?);
3153        let owner = ctx.subject().to_string();
3154        tree.clunk(data.fid, &owner);
3155        Ok(())
3156    }
3157
3158    async fn handle_stat(
3159        &self,
3160        ctx: &EnvelopeContext,
3161        _request_id: u64,
3162        _model_ref: &str,
3163        data: &NpStatReq,
3164    ) -> Result<RStat> {
3165        let tree = self.fs_tree(&ctx.domain()?);
3166        let owner = ctx.subject().to_string();
3167        let (qid, name) = tree
3168            .stat(data.fid, &owner)
3169            .map_err(|e| anyhow::anyhow!(e))?;
3170        Ok(RStat {
3171            stat: GenNpStat {
3172                qid: Self::qid_to_gen(&qid),
3173                mode: if qid.qtype & 0x80 != 0 {
3174                    0o040755
3175                } else {
3176                    0o100644
3177                },
3178                atime: 0,
3179                mtime: 0,
3180                length: 0,
3181                name,
3182                uid: String::new(),
3183                gid: String::new(),
3184                muid: String::new(),
3185            },
3186        })
3187    }
3188
3189    async fn handle_create(
3190        &self,
3191        _ctx: &EnvelopeContext,
3192        _request_id: u64,
3193        _model_ref: &str,
3194        _data: &NpCreate,
3195    ) -> Result<ROpen> {
3196        anyhow::bail!("create not supported on model fs")
3197    }
3198
3199    async fn handle_remove(
3200        &self,
3201        _ctx: &EnvelopeContext,
3202        _request_id: u64,
3203        _model_ref: &str,
3204        _data: &NpRemove,
3205    ) -> Result<()> {
3206        anyhow::bail!("remove not supported on model fs")
3207    }
3208}
3209
3210// ═══════════════════════════════════════════════════════════════════════════════
3211// Load request interception — parse capnp to detect load before dispatch
3212// ═══════════════════════════════════════════════════════════════════════════════
3213
3214/// Parsed load request data extracted from Cap'n Proto payload.
3215struct ParsedLoadRequest {
3216    model_ref: String,
3217    max_context: u32,
3218    kv_quant: hyprstream_rpc_std::model_capnp::KVQuantType,
3219}
3220
3221impl ParsedLoadRequest {
3222    fn to_load_params(&self) -> (Option<u32>, Option<KVQuantType>) {
3223        use hyprstream_rpc_std::model_capnp::KVQuantType as CKV;
3224        let max_ctx = match self.max_context {
3225            0 => None,
3226            n => Some(n),
3227        };
3228        let kv_q = match self.kv_quant {
3229            CKV::Int8 => Some(KVQuantType::Int8),
3230            CKV::Nf4 => Some(KVQuantType::Nf4),
3231            CKV::Fp4 => Some(KVQuantType::Fp4),
3232            CKV::None => None,
3233        };
3234        (max_ctx, kv_q)
3235    }
3236}
3237
3238impl ModelService {
3239    /// Try to parse a load request from the ONE decoded request body.
3240    /// Returns `None` for all other request variants (list, unload, health, scoped, etc.).
3241    ///
3242    /// Reads from the already-decoded message (v16 §5.2) — pointer traversal,
3243    /// not a second decode of the signed bytes.
3244    fn try_parse_load_request(
3245        body: &hyprstream_rpc::service::DecodedRequestBody,
3246    ) -> Option<(u64, ParsedLoadRequest)> {
3247        use hyprstream_rpc::optional_capnp::option_uint32;
3248        use hyprstream_rpc_std::model_capnp::model_request;
3249        use hyprstream_rpc_std::model_capnp::option_k_v_quant_type;
3250        use hyprstream_rpc_std::model_capnp::KVQuantType as CKV;
3251        let req = body.root::<model_request::Reader>().ok()?;
3252        let request_id = req.get_id();
3253        match req.which().ok()? {
3254            model_request::Which::Load(data) => {
3255                let data = data.ok()?;
3256                let model_ref = data.get_model_ref().ok()?.to_str().ok()?.to_owned();
3257                let max_context = match data.get_max_context().ok()?.which().ok()? {
3258                    option_uint32::Which::None(()) => 0u32,
3259                    option_uint32::Which::Some(v) => v,
3260                };
3261                let kv_quant = match data.get_kv_quant().ok()?.which().ok()? {
3262                    option_k_v_quant_type::Which::None(()) => CKV::None,
3263                    option_k_v_quant_type::Which::Some(v) => v.unwrap_or(CKV::None),
3264                };
3265                Some((
3266                    request_id,
3267                    ParsedLoadRequest {
3268                        model_ref,
3269                        max_context,
3270                        kv_quant,
3271                    },
3272                ))
3273            }
3274            _ => None,
3275        }
3276    }
3277}
3278
3279// ═══════════════════════════════════════════════════════════════════════════════
3280// RequestService Implementation — delegates to generated dispatch_model
3281// ═══════════════════════════════════════════════════════════════════════════════
3282
3283/// Only the three staging acceptance methods can use request-local Federate
3284/// admission. Top-level resources and the scoped infer resource must exactly
3285/// match the generated handler's authorize coordinates. Every request names
3286/// the immutable staging model; an empty Status query cannot list other models.
3287#[cfg(feature = "postgres")]
3288pub(crate) const FEDERATE_MODEL_REQUEST_ROOT_TYPE_ID: u64 =
3289    <hyprstream_rpc_std::model_capnp::model_request::Reader<'static> as capnp::traits::HasTypeId>::TYPE_ID;
3290
3291#[cfg(feature = "postgres")]
3292fn federate_model_operation(
3293    body: &hyprstream_rpc::service::DecodedRequestBody,
3294    pin: &StagingModelPin,
3295) -> Result<(String, String)> {
3296    use hyprstream_rpc_std::model_capnp::{infer_request, model_request};
3297    let request = body.root::<model_request::Reader>()?;
3298    let (model_ref, resource, operation) = match request.which()? {
3299        model_request::Which::Load(inner) => (
3300            inner?.get_model_ref()?.to_str()?.to_owned(),
3301            "model:Load".to_owned(),
3302            "write",
3303        ),
3304        model_request::Which::Status(inner) => (
3305            inner?.get_model_ref()?.to_str()?.to_owned(),
3306            "model:Status".to_owned(),
3307            "query",
3308        ),
3309        model_request::Which::Infer(inner) => {
3310            let infer = inner?;
3311            anyhow::ensure!(
3312                matches!(infer.which()?, infer_request::Which::GenerateStream(_)),
3313                "unsupported Federate inference method"
3314            );
3315            let model_ref = infer.get_model_ref()?.to_str()?.to_owned();
3316            (model_ref.clone(), format!("model:{model_ref}"), "infer")
3317        }
3318        _ => anyhow::bail!("unsupported Federate Model method"),
3319    };
3320    anyhow::ensure!(
3321        pin == &StagingModelPin::reviewed()? && pin.admits(&model_ref),
3322        "Federate Model request is not the reviewed pinned staging model"
3323    );
3324    Ok((resource, operation.to_owned()))
3325}
3326
3327#[async_trait(?Send)]
3328impl crate::services::RequestService for ModelService {
3329    fn accept_deferred_federate_credential(&self) -> bool {
3330        #[cfg(feature = "postgres")]
3331        {
3332            self.federate_dispatch.is_some()
3333        }
3334        #[cfg(not(feature = "postgres"))]
3335        {
3336            false
3337        }
3338    }
3339
3340    async fn admit_deferred_federate_request(
3341        &self,
3342        ctx: &EnvelopeContext,
3343        body: &hyprstream_rpc::service::DecodedRequestBody,
3344        proof: &hyprstream_rpc::proof::parser::ParsedProof,
3345    ) -> anyhow::Result<(
3346        hyprstream_rpc::proof::verify::VerifiedProof,
3347        [u8; 32],
3348        String,
3349        String,
3350        hyprstream_rpc::proof::admission::ProofReplayKey,
3351        u64,
3352    )> {
3353        #[cfg(feature = "postgres")]
3354        {
3355            let adapter = self
3356                .federate_dispatch
3357                .as_ref()
3358                .ok_or_else(|| anyhow!("Federate dispatch is disabled"))?;
3359            let pin =
3360                self.config.staging_model_pin.as_ref().ok_or_else(|| {
3361                    anyhow!("Federate Model requires an immutable staging model pin")
3362                })?;
3363            let (resource, operation) = federate_model_operation(body, pin)?;
3364            let (verified, holder, replay_key, replay_expiry) = adapter
3365                .admit(
3366                    ctx,
3367                    proof,
3368                    "model",
3369                    FEDERATE_MODEL_REQUEST_ROOT_TYPE_ID,
3370                    body.bytes(),
3371                )
3372                .await?;
3373            Ok((
3374                verified,
3375                holder,
3376                resource,
3377                operation,
3378                replay_key,
3379                replay_expiry,
3380            ))
3381        }
3382        #[cfg(not(feature = "postgres"))]
3383        {
3384            let _ = (ctx, body, proof);
3385            anyhow::bail!("Federate dispatch is disabled")
3386        }
3387    }
3388
3389    async fn commit_deferred_federate_request(
3390        &self,
3391        ctx: &EnvelopeContext,
3392        resource: &str,
3393        operation: &str,
3394        replay_key: &hyprstream_rpc::proof::admission::ProofReplayKey,
3395        replay_expiry: u64,
3396    ) -> anyhow::Result<()> {
3397        #[cfg(feature = "postgres")]
3398        {
3399            let adapter = self
3400                .federate_dispatch
3401                .as_ref()
3402                .ok_or_else(|| anyhow!("Federate dispatch is disabled"))?;
3403            adapter
3404                .commit_request_use(ctx, resource, operation, replay_key, replay_expiry)
3405                .await
3406                .map_err(anyhow::Error::new)
3407        }
3408        #[cfg(not(feature = "postgres"))]
3409        {
3410            let _ = (ctx, resource, operation, replay_key, replay_expiry);
3411            anyhow::bail!("Federate dispatch is disabled")
3412        }
3413    }
3414
3415    fn decode_request_body(
3416        &self,
3417        signed_body: &[u8],
3418    ) -> anyhow::Result<hyprstream_rpc::service::DecodedRequestBody> {
3419        // The ONE bounded decode (v16 §5.2): the generated decoder derives
3420        // the full method leaf and returns the decoded message that policy,
3421        // MAC, the load fast-path, and dispatch below all consume.
3422        crate::services::generated::model_client::decode_model_request_body(signed_body)
3423    }
3424
3425    async fn handle_request(
3426        &self,
3427        ctx: &EnvelopeContext,
3428        body: &hyprstream_rpc::service::DecodedRequestBody,
3429    ) -> Result<(Vec<u8>, Option<crate::services::Continuation>)> {
3430        debug!(
3431            "Model request from {} (id={})",
3432            ctx.subject(),
3433            ctx.request_id
3434        );
3435
3436        // Intercept load requests to avoid blocking the request loop.
3437        // Model loading can take 60s+ (weight transfer to GPU), which would
3438        // block all other model service requests (list, health, info, etc.).
3439        // Instead, return an immediate "accepted" response and do the actual
3440        // load in a Continuation (spawned via spawn_local after the REP is sent).
3441        if let Some((request_id, load_data)) = Self::try_parse_load_request(body) {
3442            // The intercepted fast path bypasses generated dispatch, so it
3443            // must perform the same operation authorization and audit before
3444            // parsing pin/ref details or consulting Registry/Git state.
3445            let audit_resource = "model:Load";
3446            let authorization =
3447                <Self as ModelHandler>::authorize(self, ctx, audit_resource, "write").await;
3448            ctx.audit_authz(audit_resource, "write", authorization.is_ok());
3449            authorization?;
3450
3451            self.validate_model_load_ref(&load_data.model_ref)?;
3452            let instance = Self::inference_instance(ctx, &load_data.model_ref)?;
3453            self.validate_staging_model_checkout(&instance).await?;
3454
3455            // This interception runs before generated dispatch, so derive the
3456            // instance only from the already-verified tenant in the envelope.
3457            let model_ref = load_data.model_ref.clone();
3458            let reservation = match self.reserve_intercepted_load(&instance).await? {
3459                InterceptedLoadAdmission::Loaded { reach } => {
3460                    let response = serialize_response(
3461                        request_id,
3462                        &ModelResponseVariant::LoadResult(LoadedModelResponse { model_ref, reach }),
3463                    )?;
3464                    return Ok((response, None));
3465                }
3466                InterceptedLoadAdmission::Pending => {
3467                    debug!("Model {} already being loaded, deduplicating", model_ref);
3468                    let response = serialize_response(
3469                        request_id,
3470                        &ModelResponseVariant::LoadResult(LoadedModelResponse {
3471                            model_ref,
3472                            reach: Vec::new(),
3473                        }),
3474                    )?;
3475                    return Ok((response, None));
3476                }
3477                InterceptedLoadAdmission::Accepted(reservation) => reservation,
3478            };
3479
3480            // The accepted response is sent only after the shared gate has
3481            // reserved `pending_loads`. Transfer its attempt-scoped RAII guard
3482            // into the continuation *before* response signing/serialization:
3483            // if that fallible response path returns early, dropping this
3484            // continuation synchronously releases only this attempt's marker.
3485            info!("Load request accepted for {} (async)", model_ref);
3486            let response_model_ref = model_ref.clone();
3487            let service = self.clone(); // Arc clone — cheap, 'static
3488            let (load_max_context, load_kv_quant) = load_data.to_load_params();
3489            let continuation: crate::services::Continuation = Box::pin(async move {
3490                let model_name = model_ref.split(':').next().unwrap_or(&model_ref);
3491                let scope = format!("serve:model:{}", model_name);
3492                match service
3493                    .run_intercepted_load(&instance, load_max_context, load_kv_quant, reservation)
3494                    .await
3495                {
3496                    Ok(endpoint) => {
3497                        info!("Model {} loaded successfully at {}", model_ref, endpoint);
3498                        let event = crate::events::EventEnvelope::new(
3499                            crate::events::EventSource::Model,
3500                            scope.clone(),
3501                            crate::events::EventPayload::ModelLoaded {
3502                                model_ref: model_ref.clone(),
3503                                endpoint,
3504                            },
3505                        );
3506                        if let Ok(payload) = serde_json::to_vec(&event) {
3507                            let _event_publish_gate =
3508                                service.lifecycle_event_publish_gate.lock().await;
3509                            let _ = service
3510                                .event_publisher
3511                                .publish("lifecycle", "loaded", &payload)
3512                                .await;
3513                            debug!("Published model.loaded event");
3514                        }
3515                    }
3516                    Err(e) => {
3517                        warn!("Model {} failed to load: {}", model_ref, e);
3518                        let event = crate::events::EventEnvelope::new(
3519                            crate::events::EventSource::Model,
3520                            scope.clone(),
3521                            crate::events::EventPayload::ModelFailed {
3522                                model_ref: model_ref.clone(),
3523                                error: e.to_string(),
3524                            },
3525                        );
3526                        if let Ok(payload) = serde_json::to_vec(&event) {
3527                            let _event_publish_gate =
3528                                service.lifecycle_event_publish_gate.lock().await;
3529                            let _ = service
3530                                .event_publisher
3531                                .publish("lifecycle", "failed", &payload)
3532                                .await;
3533                        }
3534                    }
3535                }
3536            });
3537
3538            let response = serialize_response(
3539                request_id,
3540                &ModelResponseVariant::LoadResult(LoadedModelResponse {
3541                    model_ref: response_model_ref,
3542                    reach: Vec::new(),
3543                }),
3544            )?;
3545
3546            return Ok((response, Some(continuation)));
3547        }
3548
3549        dispatch_model(self, ctx, body).await
3550    }
3551
3552    fn name(&self) -> &str {
3553        "model"
3554    }
3555
3556    fn transport(&self) -> &TransportConfig {
3557        &self.transport
3558    }
3559
3560    fn signing_key(&self) -> SigningKey {
3561        self.signing_key.clone()
3562    }
3563
3564    fn expected_audience(&self) -> Option<&str> {
3565        self.expected_audience.as_deref()
3566    }
3567
3568    fn accept_delegated_bearer(&self, signer: &[u8; 32]) -> bool {
3569        let Ok(key) = hyprstream_rpc::prelude::VerifyingKey::from_bytes(signer) else {
3570            return false;
3571        };
3572        let trust = hyprstream_service::global_trust_store();
3573        (self.delegated_oai_audience.is_some() && trust.is_authorized(&key, "oai"))
3574            || (self.delegated_mcp_audience.is_some() && trust.is_authorized(&key, "mcp"))
3575    }
3576
3577    fn delegated_user_audience(
3578        &self,
3579        signer: &[u8; 32],
3580        untrusted_aud: Option<&str>,
3581    ) -> Option<&str> {
3582        let key = hyprstream_rpc::prelude::VerifyingKey::from_bytes(signer).ok()?;
3583        let trust = hyprstream_service::global_trust_store();
3584        if let Some(aud) = self.delegated_oai_audience.as_deref() {
3585            if untrusted_aud == Some(aud) && trust.is_authorized(&key, "oai") {
3586                return Some(aud);
3587            }
3588        }
3589        if let Some(aud) = self.delegated_mcp_audience.as_deref() {
3590            if untrusted_aud == Some(aud) && trust.is_authorized(&key, "mcp") {
3591                return Some(aud);
3592            }
3593        }
3594        self.expected_audience()
3595    }
3596
3597    fn accept_delegated_resource_leaf(
3598        &self,
3599        ctx: &EnvelopeContext,
3600        body: &hyprstream_rpc::service::DecodedRequestBody,
3601    ) -> bool {
3602        use hyprstream_rpc_std::model_capnp::{infer_request, model_request};
3603
3604        let Some(claims) = ctx.claims() else {
3605            return false;
3606        };
3607        let audience = claims.aud.as_deref();
3608        let mcp = audience.is_some() && self.delegated_mcp_audience.as_deref() == audience;
3609        let oai = audience.is_some() && self.delegated_oai_audience.as_deref() == audience;
3610        if !mcp && !oai {
3611            return false;
3612        }
3613        let Ok(request) = body.root::<model_request::Reader>() else {
3614            return false;
3615        };
3616        match (body.leaf_path(), request.which()) {
3617            (Some([2]), Ok(model_request::Which::Status(Ok(_)))) if mcp || oai => {
3618                claims.has_scope("read:model:Status") || claims.has_scope("read:model:*")
3619            }
3620            (
3621                Some([6, 0] | [6, 1] | [6, 2] | [6, 3]),
3622                Ok(model_request::Which::Infer(Ok(infer))),
3623            ) if oai => {
3624                let Ok(model_ref) = infer.get_model_ref() else {
3625                    return false;
3626                };
3627                let Ok(model_ref) = model_ref.to_str() else {
3628                    return false;
3629                };
3630                if model_ref.is_empty() {
3631                    return false;
3632                }
3633                let has_scope = |operation| {
3634                    claims.has_scope(&format!("{operation}:model:{model_ref}"))
3635                        || claims.has_scope(&format!("{operation}:model:*"))
3636                };
3637                match (body.leaf_path(), infer.which()) {
3638                    (Some([6, 0]), Ok(infer_request::Which::GenerateStream(_)))
3639                    | (Some([6, 3]), Ok(infer_request::Which::Embed(_))) => has_scope("infer"),
3640                    (Some([6, 1]), Ok(infer_request::Which::ApplyChatTemplate(_))) => {
3641                        // The OAI generation route calls the template step before infer.
3642                        // A query grant may use that step alone, but cannot infer.
3643                        has_scope("read") || has_scope("infer")
3644                    }
3645                    (Some([6, 2]), Ok(infer_request::Which::Status(_))) => has_scope("read"),
3646                    _ => false,
3647                }
3648            }
3649            _ => false,
3650        }
3651    }
3652
3653    fn jwt_key_source(&self) -> Option<std::sync::Arc<dyn hyprstream_rpc::auth::JwtKeySource>> {
3654        self.inner.jwt_key_source.clone()
3655    }
3656
3657    fn producer_reach_config_handle(
3658        &self,
3659    ) -> Option<hyprstream_rpc::moq_stream::ProducerReachConfigHandle> {
3660        Some(Arc::clone(&self.producer_reach_config))
3661    }
3662
3663    fn moq_origin_handle(&self) -> Option<hyprstream_rpc::moq_stream::MoqStreamOriginHandle> {
3664        Some(Arc::clone(&self.moq_origin))
3665    }
3666
3667    fn build_error_payload(&self, request_id: u64, error: &str) -> Vec<u8> {
3668        let variant = ModelResponseVariant::Error(ErrorInfo {
3669            message: error.to_owned(),
3670            code: "INTERNAL".to_owned(),
3671            details: String::new(),
3672        });
3673        serialize_response(request_id, &variant).unwrap_or_default()
3674    }
3675}
3676
3677// ============================================================================
3678// Helper types
3679// ============================================================================
3680
3681// ModelStatusEntry, OnlineTrainingConfigInfo, ModelStatusInfo deleted — use generated types directly:
3682// - GenModelStatusEntry = generated::model_client::ModelStatusEntry
3683// - GenOnlineTrainingConfig = generated::model_client::OnlineTrainingConfig
3684// - ModelStatusResponse = generated::model_client::ModelStatusResponse (infer-scoped)
3685
3686// ModelZmqClient removed — use generated ModelClient directly.
3687// Convenience methods (load/unload/status/infer_stream) are now inlined at call sites.
3688
3689#[cfg(test)]
3690mod tests {
3691    #[test]
3692    fn clef_projection_contract_keeps_a0_default_and_rejects_rebinding() -> anyhow::Result<()> {
3693        let a0 = super::StagingModelPin::reviewed()?;
3694        assert_eq!(
3695            a0.capture_policy()?,
3696            git2db::pinned_disk::ProjectionPolicy::default()
3697        );
3698        let mut clef = super::StagingModelPin::reviewed_clef_flash(Default::default())?;
3699        assert_ne!(
3700            clef.capture_policy()?,
3701            git2db::pinned_disk::ProjectionPolicy::default()
3702        );
3703        clef.commit = a0.commit;
3704        assert!(clef.capture_policy().is_err());
3705        let mut clef = super::StagingModelPin::reviewed_clef_flash(Default::default())?;
3706        clef.model_ref = a0.model_ref;
3707        assert!(clef.capture_policy().is_err());
3708        Ok(())
3709    }
3710
3711    use super::*;
3712    use hyprstream_rpc::registry::{global as registry, SocketKind};
3713
3714    #[cfg(feature = "postgres")]
3715    #[test]
3716    fn federate_model_route_is_exact_load_status_stream_and_pinned_ref() -> Result<()> {
3717        let pin = StagingModelPin::from_settings("staging", None, None)?
3718            .ok_or_else(|| anyhow!("staging pin missing"))?;
3719        let model_ref = pin.model_ref.as_str();
3720        assert_eq!(
3721            model_ref.split_once(':').map(|(name, _)| name),
3722            Some(StagingModelPin::reviewed_repo_name())
3723        );
3724        let bytes = hyprstream_rpc::serialize_message(|message| {
3725            let mut request =
3726                message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3727            request.set_id(1);
3728            let mut infer = request.init_infer();
3729            infer.set_model_ref(model_ref);
3730            infer.init_generate_stream();
3731        })?;
3732        let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3733        assert_eq!(
3734            federate_model_operation(&body, &pin)?,
3735            (format!("model:{model_ref}"), "infer".into())
3736        );
3737        let wrong_oid = StagingModelPin::new(
3738            model_ref,
3739            git2::Oid::from_str("0000000000000000000000000000000000000001")?,
3740        )?;
3741        assert!(federate_model_operation(&body, &wrong_oid).is_err());
3742
3743        let bytes = hyprstream_rpc::serialize_message(|message| {
3744            let mut request =
3745                message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3746            request.set_id(2);
3747            let mut infer = request.init_infer();
3748            infer.set_model_ref(model_ref);
3749            infer.set_status(());
3750        })?;
3751        let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3752        assert!(federate_model_operation(&body, &pin).is_err());
3753
3754        let bytes = hyprstream_rpc::serialize_message(|message| {
3755            let mut request =
3756                message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3757            request.set_id(3);
3758            request.init_load().set_model_ref(model_ref);
3759        })?;
3760        let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3761        assert_eq!(
3762            federate_model_operation(&body, &pin)?,
3763            ("model:Load".into(), "write".into())
3764        );
3765
3766        let bytes = hyprstream_rpc::serialize_message(|message| {
3767            let mut request =
3768                message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3769            request.set_id(4);
3770            request.init_status().set_model_ref(model_ref);
3771        })?;
3772        let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3773        assert_eq!(
3774            federate_model_operation(&body, &pin)?,
3775            ("model:Status".into(), "query".into())
3776        );
3777
3778        for rejected in ["", "qwen2.5-0.5b-instruct:other", "attacker:main"] {
3779            let bytes = hyprstream_rpc::serialize_message(|message| {
3780                let mut request =
3781                    message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3782                request.set_id(5);
3783                request.init_status().set_model_ref(rejected);
3784            })?;
3785            let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3786            assert!(
3787                federate_model_operation(&body, &pin).is_err(),
3788                "status accepted {rejected:?}"
3789            );
3790        }
3791        let bytes = hyprstream_rpc::serialize_message(|message| {
3792            let mut request =
3793                message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3794            request.set_id(6);
3795            request.init_load().set_model_ref("attacker:main");
3796        })?;
3797        let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3798        assert!(federate_model_operation(&body, &pin).is_err());
3799
3800        let bytes = hyprstream_rpc::serialize_message(|message| {
3801            let mut request =
3802                message.init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>();
3803            request.set_id(7);
3804            let mut infer = request.init_infer();
3805            infer.set_model_ref("attacker:main");
3806            infer.init_generate_stream();
3807        })?;
3808        let body = crate::services::generated::model_client::decode_model_request_body(&bytes)?;
3809        assert!(federate_model_operation(&body, &pin).is_err());
3810        Ok(())
3811    }
3812
3813    #[tokio::test]
3814    async fn delegated_resource_model_leaves_intersect_signed_oauth_scope() -> Result<()> {
3815        use hyprstream_rpc::auth::{jwt, Claims};
3816        use hyprstream_rpc::service::RequestService;
3817        use hyprstream_rpc_std::model_capnp::model_request;
3818
3819        let oai = "https://oai.example";
3820        let mcp = "https://mcp.example";
3821        let service = selector_model_service()
3822            .await
3823            .with_delegated_resource_audiences(oai.to_owned(), mcp.to_owned());
3824        let relay = SigningKey::from_bytes(&[0x7f; 32]);
3825        hyprstream_service::global_trust_store().insert(
3826            relay.verifying_key(),
3827            hyprstream_service::Attestation {
3828                scopes: ["oai".to_owned(), "mcp".to_owned()].into_iter().collect(),
3829                subject: Some("service:oai".to_owned()),
3830                jwt: None,
3831                expires_at: 0,
3832                attested_by: None,
3833            },
3834        );
3835        assert!(service.accept_delegated_bearer(relay.verifying_key().as_bytes()));
3836        let signer = SigningKey::from_bytes(&[0x7e; 32]);
3837        let model_ref = "fixture-model:main";
3838        let body_for = |method: &str| -> Result<hyprstream_rpc::service::DecodedRequestBody> {
3839            let bytes = hyprstream_rpc::serialize_message(|message| {
3840                let mut request = message.init_root::<model_request::Builder>();
3841                request.set_id(1);
3842                match method {
3843                    "status" => request.init_status().set_model_ref(model_ref),
3844                    "generate" => {
3845                        let mut infer = request.init_infer();
3846                        infer.set_model_ref(model_ref);
3847                        infer.init_generate_stream();
3848                    }
3849                    "template" => {
3850                        let mut infer = request.init_infer();
3851                        infer.set_model_ref(model_ref);
3852                        infer.init_apply_chat_template();
3853                    }
3854                    _ => unreachable!(),
3855                }
3856            })?;
3857            crate::services::generated::model_client::decode_model_request_body(&bytes)
3858        };
3859        let status = body_for("status")?;
3860        let generate = body_for("generate")?;
3861        let template = body_for("template")?;
3862        assert_eq!(status.leaf_path(), Some([2].as_slice()));
3863        assert_eq!(generate.leaf_path(), Some([6, 0].as_slice()));
3864        assert_eq!(template.leaf_path(), Some([6, 1].as_slice()));
3865        let now = chrono::Utc::now().timestamp();
3866        for (audience, scope, body, allowed) in [
3867            (oai, "infer:model:fixture-model:main", &generate, true),
3868            (oai, "infer:model:*", &generate, true),
3869            (oai, "openid", &generate, false),
3870            (oai, "read:model:fixture-model:main", &generate, false),
3871            (oai, "infer:model:other", &generate, false),
3872            (oai, "infer:model:fixture-model:main", &template, true),
3873            (oai, "read:model:fixture-model:main", &template, true),
3874            (oai, "infer:model:fixture-model:main", &status, false),
3875            (oai, "read:model:Status", &status, true),
3876            (oai, "read:model:*", &status, true),
3877            (mcp, "read:model:Status", &status, true),
3878            (mcp, "read:model:*", &status, true),
3879            (mcp, "query:model:*", &status, false),
3880            (mcp, "openid", &status, false),
3881            (mcp, "infer:model:fixture-model:main", &status, false),
3882            (mcp, "read:model:other", &status, false),
3883            (mcp, "read:model:Status", &generate, false),
3884        ] {
3885            let claims = Claims::new("alice".to_owned(), now, now + 3600)
3886                .with_audience(Some(audience.to_owned()))
3887                .with_scope(Some(scope.to_owned()))
3888                .with_tenant("fixture-tenant".to_owned())
3889                .with_clearance(crate::services::inference::inference_object_label());
3890            let token = jwt::encode(&claims, &signer);
3891            let signed = hyprstream_rpc::auth::decode_with_key(
3892                &token,
3893                &signer.verifying_key(),
3894                Some(audience),
3895            )?;
3896            let ctx = EnvelopeContext::for_test_authenticated_subject_with_claims(
3897                hyprstream_rpc::envelope::Subject::new("alice"),
3898                "fixture-tenant",
3899                relay.verifying_key(),
3900                signed,
3901            );
3902            assert_eq!(ctx.domain()?, "fixture-tenant");
3903            assert!(crate::services::inference::enforce_inference_mac(
3904                &ctx,
3905                &crate::services::inference::inference_object_label(),
3906            )
3907            .is_ok());
3908            assert_eq!(
3909                service.accept_delegated_resource_leaf(&ctx, body),
3910                allowed,
3911                "audience={audience} scope={scope} leaf={:?}",
3912                body.leaf_path()
3913            );
3914        }
3915        for audience in [oai, mcp] {
3916            let claims = Claims::new("alice".to_owned(), now, now + 3600)
3917                .with_audience(Some(audience.to_owned()))
3918                .with_tenant("fixture-tenant".to_owned())
3919                .with_clearance(crate::services::inference::inference_object_label());
3920            let token = jwt::encode(&claims, &signer);
3921            let signed = hyprstream_rpc::auth::decode_with_key(
3922                &token,
3923                &signer.verifying_key(),
3924                Some(audience),
3925            )?;
3926            let ctx = EnvelopeContext::for_test_authenticated_subject_with_claims(
3927                hyprstream_rpc::envelope::Subject::new("alice"),
3928                "fixture-tenant",
3929                relay.verifying_key(),
3930                signed,
3931            );
3932            assert!(!service.accept_delegated_resource_leaf(&ctx, &status));
3933            assert!(!service.accept_delegated_resource_leaf(&ctx, &generate));
3934        }
3935        Ok(())
3936    }
3937
3938    #[test]
3939    fn test_config_defaults() {
3940        let config = ModelServiceConfig::default();
3941        assert_eq!(config.max_models, 5);
3942        assert_eq!(config.max_context, None);
3943        assert_eq!(config.kv_quant, KVQuantType::None);
3944        assert_eq!(
3945            config.inference_deployment.isolation,
3946            InferenceIsolationProfile::InProcess
3947        );
3948    }
3949
3950    #[test]
3951    fn staging_pin_rejects_wrong_name_and_ref() -> Result<()> {
3952        let oid = git2::Oid::from_str("18c562db6830c2ef6636b8eaf42fb479272052f7")?;
3953        let pin = StagingModelPin::new("qwen2.5-0.5b-instruct:main", oid)?;
3954        assert!(pin.admits("qwen2.5-0.5b-instruct:main"));
3955        for wrong in [
3956            "other-model:main",
3957            "qwen2.5-0.5b-instruct:other",
3958            "qwen2.5-0.5b-instruct",
3959        ] {
3960            assert!(!pin.admits(wrong));
3961        }
3962        assert!(StagingModelPin::new("qwen2.5-0.5b-instruct:refs/heads/main", oid).is_err());
3963        Ok(())
3964    }
3965
3966    #[tokio::test]
3967    async fn denied_model_load_is_audited_before_pin_or_registry_io() -> Result<()> {
3968        use tracing::field::{Field, Visit};
3969        use tracing_subscriber::layer::{Context, Layer, SubscriberExt};
3970
3971        #[derive(Clone, Default, Debug)]
3972        struct AuditObservation {
3973            target: String,
3974            resource: String,
3975            action: String,
3976            decision: String,
3977        }
3978
3979        #[derive(Default)]
3980        struct AuditVisitor(AuditObservation);
3981
3982        impl Visit for AuditVisitor {
3983            fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
3984                self.record_str(field, format!("{value:?}").trim_matches('"'));
3985            }
3986
3987            fn record_str(&mut self, field: &Field, value: &str) {
3988                match field.name() {
3989                    "resource" => self.0.resource = value.to_owned(),
3990                    "action" => self.0.action = value.to_owned(),
3991                    "decision" => self.0.decision = value.to_owned(),
3992                    _ => {}
3993                }
3994            }
3995        }
3996
3997        #[derive(Clone)]
3998        struct AuditLayer(Arc<parking_lot::Mutex<Vec<AuditObservation>>>);
3999
4000        impl<S: tracing::Subscriber> Layer<S> for AuditLayer {
4001            fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
4002                if event.metadata().target() != "audit" {
4003                    return;
4004                }
4005                let mut visitor = AuditVisitor::default();
4006                event.record(&mut visitor);
4007                visitor.0.target = event.metadata().target().to_owned();
4008                self.0.lock().push(visitor.0);
4009            }
4010        }
4011
4012        let registry_calls = Arc::new(BoundaryDialState::default());
4013        let policy_calls = Arc::new(BoundaryDialState::default());
4014        let registry_rpc = Arc::new(BoundaryRpcClient::new(
4015            TransportConfig::inproc("pin-admission-registry"),
4016            Arc::clone(&registry_calls),
4017        ));
4018        let policy_rpc = Arc::new(BoundaryRpcClient::new(
4019            TransportConfig::inproc("pin-admission-policy"),
4020            Arc::clone(&policy_calls),
4021        ));
4022        let mut config = ModelServiceConfig::default();
4023        config.staging_model_pin = Some(StagingModelPin::new(
4024            "qwen2.5-0.5b-instruct:main",
4025            git2::Oid::from_str("18c562db6830c2ef6636b8eaf42fb479272052f7")?,
4026        )?);
4027        let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
4028            hyprstream_rpc::moq_event::MoqEventOrigin::new(),
4029        );
4030        let service = ModelService::new(
4031            config,
4032            SigningKey::from_bytes(&[0x45; 32]),
4033            PolicyClient::new(policy_rpc),
4034            RegistryClient::new(registry_rpc),
4035            TransportConfig::inproc("pin-admission-model"),
4036            TransportConfig::inproc("pin-admission-policy-transport"),
4037        )
4038        .await?;
4039
4040        let context = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4041            hyprstream_rpc::envelope::Subject::new("unauthorized-caller"),
4042            "pin-admission-tenant",
4043            SigningKey::from_bytes(&[0x46; 32]).verifying_key(),
4044        );
4045        let records = Arc::new(parking_lot::Mutex::new(Vec::new()));
4046        let subscriber =
4047            tracing_subscriber::Registry::default().with(AuditLayer(Arc::clone(&records)));
4048        tracing::subscriber::with_default(subscriber, || -> Result<()> {
4049            for model_ref in ["other-model:main", "qwen2.5-0.5b-instruct:main"] {
4050                let bytes = hyprstream_rpc::serialize_message(|message| {
4051                    let mut request = message
4052                        .init_root::<hyprstream_rpc_std::model_capnp::model_request::Builder>(
4053                    );
4054                    request.set_id(91);
4055                    request.init_load().set_model_ref(model_ref);
4056                })?;
4057                let body =
4058                    crate::services::generated::model_client::decode_model_request_body(&bytes)?;
4059                let error =
4060                    match futures::executor::block_on(service.handle_request(&context, &body)) {
4061                        Err(error) => error,
4062                        Ok(_) => anyhow::bail!(
4063                        "missing MAC clearance must deny the load before pin/ref status handling"
4064                    ),
4065                    };
4066                anyhow::ensure!(
4067                    error.to_string().contains(
4068                        "authorization denied: inference caller has no verified MAC clearance"
4069                    ),
4070                    "denial must be authorization-specific, not a pin or registry error: {error}"
4071                );
4072                anyhow::ensure!(
4073                    !error.to_string().contains("configured staging modelRef"),
4074                    "unauthorized callers must not receive pin-specific diagnostics"
4075                );
4076            }
4077            Ok(())
4078        })?;
4079
4080        let observations = records.lock().clone();
4081        anyhow::ensure!(
4082            observations.len() == 2,
4083            "each denied load must emit one authz audit event: {observations:?}"
4084        );
4085        for observation in observations {
4086            anyhow::ensure!(
4087                observation.target == "audit",
4088                "authorization denial must use the audit target"
4089            );
4090            anyhow::ensure!(
4091                observation.resource == "model:Load",
4092                "unexpected audited resource: {observation:?}"
4093            );
4094            anyhow::ensure!(
4095                observation.action == "write",
4096                "unexpected audited action: {observation:?}"
4097            );
4098            anyhow::ensure!(
4099                observation.decision == "deny",
4100                "denied caller must be audited as denied: {observation:?}"
4101            );
4102        }
4103        Ok(())
4104    }
4105
4106    #[test]
4107    fn staging_pin_settings_fail_closed_on_missing_or_partial_override() -> Result<()> {
4108        let pin = StagingModelPin::from_settings("staging", None, None)?
4109            .ok_or_else(|| anyhow!("staging profile must have reviewed pin"))?;
4110        assert!(pin.admits("qwen2.5-0.5b-instruct:main"));
4111        assert!(StagingModelPin::from_settings(
4112            "staging",
4113            Some("qwen2.5-0.5b-instruct:main"),
4114            None
4115        )
4116        .is_err());
4117        assert!(StagingModelPin::from_settings(
4118            "staging",
4119            None,
4120            Some("18c562db6830c2ef6636b8eaf42fb479272052f7")
4121        )
4122        .is_err());
4123        assert!(StagingModelPin::from_settings(
4124            "staging",
4125            Some("other:main"),
4126            Some("18c562db6830c2ef6636b8eaf42fb479272052f7")
4127        )
4128        .is_err());
4129        assert!(StagingModelPin::from_settings("default", None, None)?.is_none());
4130        assert!(StagingModelPin::from_settings("default", Some("x:main"), None).is_err());
4131        assert!(
4132            StagingModelPin::from_settings("default", Some("x:main"), Some("deadbeef")).is_err()
4133        );
4134        Ok(())
4135    }
4136
4137    #[tokio::test]
4138    async fn unpublished_spawn_guard_stops_worker_after_forced_failure() -> Result<()> {
4139        let temp = tempfile::tempdir()?;
4140        let stopped = temp.path().join("worker-stopped");
4141        let witness = stopped.clone();
4142        let shutdown = Arc::new(tokio::sync::Notify::new());
4143        let worker_shutdown = Arc::clone(&shutdown);
4144        let worker = std::thread::spawn(move || {
4145            let Ok(runtime) = tokio::runtime::Builder::new_current_thread()
4146                .enable_all()
4147                .build()
4148            else {
4149                return;
4150            };
4151            runtime.block_on(worker_shutdown.notified());
4152            let _ = std::fs::write(witness, b"stopped");
4153        });
4154        let handle = hyprstream_service::SpawnedService::thread(
4155            "model-pin-guard-test".to_owned(),
4156            Some(worker),
4157            shutdown,
4158            None,
4159        );
4160        let result: Result<()> = async {
4161            let _guard = SpawnPublicationGuard::new(handle);
4162            anyhow::bail!("forced post-spawn pre-publication failure")
4163        }
4164        .await;
4165        assert!(result.is_err());
4166        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4167            while !stopped.exists() {
4168                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4169            }
4170        })
4171        .await?;
4172        assert_eq!(std::fs::read(stopped)?, b"stopped");
4173        Ok(())
4174    }
4175
4176    #[test]
4177    fn inference_instance_denies_unlabeled_tenant_before_load() {
4178        let signer = SigningKey::from_bytes(&[44u8; 32]).verifying_key();
4179        let ctx = EnvelopeContext::for_test_authenticated_subject_in_tenant(
4180            hyprstream_rpc::envelope::Subject::new("alice"),
4181            "did-hosted-account",
4182            signer,
4183        );
4184        assert!(
4185            ModelService::inference_instance(&ctx, "tiny-llama:main").is_err(),
4186            "an authority-bound tenant without MAC clearance must not reach engine loading"
4187        );
4188    }
4189
4190    // ========================================================================
4191    // #649 — latched load terminal (EV7 host-side retain)
4192    //
4193    // `ModelLoadTerminal` + `ModelLoadKey` latch directly into a
4194    // `TerminalStore`; these tests prove the per-load-attempt / monotonic-
4195    // write-once contract without constructing a full `ModelService` (which
4196    // needs a registry + transport). The wiring (`allocate_load_epoch` +
4197    // `latch_load_terminal` called from `load_model`) is thin glue over these
4198    // primitives.
4199    // ========================================================================
4200
4201    fn loaded_term(endpoint: &str) -> Terminal<ModelLoadTerminal> {
4202        Terminal {
4203            value: ModelLoadTerminal::Loaded {
4204                endpoint: endpoint.to_owned(),
4205            },
4206            latched_by: "model-service".to_owned(),
4207        }
4208    }
4209
4210    fn failed_term(err: &str) -> Terminal<ModelLoadTerminal> {
4211        Terminal {
4212            value: ModelLoadTerminal::LoadFailed {
4213                error: err.to_owned(),
4214            },
4215            latched_by: "model-service".to_owned(),
4216        }
4217    }
4218
4219    // Read back a latched terminal without `.unwrap()` (clippy denies it here).
4220    fn latched_value(
4221        store: &TerminalStore<ModelLoadKey, ModelLoadTerminal>,
4222        key: &ModelLoadKey,
4223    ) -> ModelLoadTerminal {
4224        match store.get(key) {
4225            Some(t) => t.value,
4226            None => panic!("expected a terminal latched for {key:?}"),
4227        }
4228    }
4229
4230    /// Monotonic write-once within a single load attempt (single epoch): a
4231    /// second latch on the same key is rejected and the first terminal wins.
4232    #[test]
4233    fn load_terminal_is_monotonic_write_once_per_epoch() {
4234        let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4235        let key = ModelLoadKey {
4236            tenant: "tenant-a".to_owned(),
4237            model_ref: "qwen3:main".to_owned(),
4238            epoch: 1,
4239        };
4240        assert!(store.latch(key.clone(), loaded_term("ep1")));
4241        // A late failure for the SAME attempt cannot overwrite the real Loaded.
4242        assert!(!store.latch(key.clone(), failed_term("race")));
4243        assert_eq!(
4244            latched_value(&store, &key),
4245            ModelLoadTerminal::Loaded {
4246                endpoint: "ep1".to_owned()
4247            }
4248        );
4249    }
4250
4251    /// Per-load-attempt keying: a reload allocates a fresh epoch and latches a
4252    /// NEW terminal — the EV7 "reload = new latch" contract. Without per-epoch
4253    /// keying the first Loaded would shadow every subsequent reload.
4254    #[test]
4255    fn reload_latches_new_terminal_under_new_epoch() {
4256        let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4257        // First load attempt (epoch 1): Loaded.
4258        let k1 = ModelLoadKey {
4259            tenant: "tenant-a".to_owned(),
4260            model_ref: "qwen3:main".to_owned(),
4261            epoch: 1,
4262        };
4263        assert!(store.latch(k1.clone(), loaded_term("ep1")));
4264        // Reload (epoch 2): a distinct key, so it latches a fresh terminal.
4265        let k2 = ModelLoadKey {
4266            tenant: "tenant-a".to_owned(),
4267            model_ref: "qwen3:main".to_owned(),
4268            epoch: 2,
4269        };
4270        assert!(store.latch(k2.clone(), loaded_term("ep2")));
4271        // Both attempts' terminals are retained (monotonic per-epoch).
4272        assert_eq!(
4273            latched_value(&store, &k1),
4274            ModelLoadTerminal::Loaded {
4275                endpoint: "ep1".to_owned()
4276            }
4277        );
4278        assert_eq!(
4279            latched_value(&store, &k2),
4280            ModelLoadTerminal::Loaded {
4281                endpoint: "ep2".to_owned()
4282            }
4283        );
4284    }
4285
4286    /// A failed attempt latches `LoadFailed` under its epoch; a later
4287    /// successful reload (new epoch) latches `Loaded`. The failure is retained,
4288    /// not overwritten — each attempt has its own authoritative terminal.
4289    #[test]
4290    fn failed_then_reload_latches_distinct_terminals() {
4291        let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4292        let kf = ModelLoadKey {
4293            tenant: "tenant-a".to_owned(),
4294            model_ref: "qwen3:main".to_owned(),
4295            epoch: 1,
4296        };
4297        assert!(store.latch(kf.clone(), failed_term("oom")));
4298        let ks = ModelLoadKey {
4299            tenant: "tenant-a".to_owned(),
4300            model_ref: "qwen3:main".to_owned(),
4301            epoch: 2,
4302        };
4303        assert!(store.latch(ks.clone(), loaded_term("ep2")));
4304        assert!(matches!(
4305            latched_value(&store, &kf),
4306            ModelLoadTerminal::LoadFailed { .. }
4307        ));
4308        assert!(matches!(
4309            latched_value(&store, &ks),
4310            ModelLoadTerminal::Loaded { .. }
4311        ));
4312    }
4313
4314    #[test]
4315    fn load_terminals_are_tenant_scoped() {
4316        let store: TerminalStore<ModelLoadKey, ModelLoadTerminal> = TerminalStore::new();
4317        let tenant_a = ModelLoadKey {
4318            tenant: "tenant-a".to_owned(),
4319            model_ref: "qwen3:main".to_owned(),
4320            epoch: 1,
4321        };
4322        let tenant_b = ModelLoadKey {
4323            tenant: "tenant-b".to_owned(),
4324            model_ref: "qwen3:main".to_owned(),
4325            epoch: 1,
4326        };
4327        assert!(store.latch(tenant_a.clone(), loaded_term("a")));
4328        assert!(store.latch(tenant_b.clone(), loaded_term("b")));
4329        assert_ne!(
4330            latched_value(&store, &tenant_a),
4331            latched_value(&store, &tenant_b)
4332        );
4333    }
4334
4335    #[test]
4336    fn in_process_profile_never_admits_two_tenants() {
4337        let admitted = std::sync::OnceLock::new();
4338        assert!(admit_in_process_tenant(&admitted, "tenant-a").is_ok());
4339        assert!(admit_in_process_tenant(&admitted, "tenant-a").is_ok());
4340        assert!(admit_in_process_tenant(&admitted, "tenant-b").is_err());
4341    }
4342
4343    /// A threaded worker stays Inproc even when standalone Model uses global IPC.
4344    /// Isolate the first-write registry so other tests cannot mask the profile.
4345    #[test]
4346    fn inference_transport_is_inproc_co_located() -> Result<()> {
4347        const CHILD: &str = "HYPRSTREAM_TEST_INFERENCE_IPC_PROFILE_CHILD";
4348        if std::env::var_os(CHILD).is_none() {
4349            let output = std::process::Command::new(std::env::current_exe()?)
4350                .args([
4351                    "--exact",
4352                    "services::model::tests::inference_transport_is_inproc_co_located",
4353                    "--nocapture",
4354                ])
4355                .env(CHILD, "1")
4356                .output()?;
4357            assert!(
4358                output.status.success(),
4359                "{}{}",
4360                String::from_utf8_lossy(&output.stdout),
4361                String::from_utf8_lossy(&output.stderr)
4362            );
4363            return Ok(());
4364        }
4365        hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Ipc, None);
4366        let instance = InferenceInstanceId::new("tenant-a", "qwen3-small:main", 0)
4367            .unwrap_or_else(|e| panic!("valid instance: {e}"));
4368        assert!(matches!(
4369            registry()
4370                .endpoint(&instance.service_name(), SocketKind::Rep)
4371                .endpoint,
4372            hyprstream_rpc::transport::EndpointType::Ipc { .. }
4373        ));
4374        let t = ModelService::inference_transport(&instance);
4375        match &t.endpoint {
4376            hyprstream_rpc::transport::EndpointType::Inproc { endpoint } => {
4377                assert!(
4378                    endpoint.contains("inference-"),
4379                    "opaque deterministic per-tenant/model inproc name, got {endpoint}"
4380                );
4381                assert_eq!(endpoint, &format!("hyprstream/{}", instance.service_name()));
4382            }
4383            other => panic!("expected Inproc co-located transport, got {other:?}"),
4384        }
4385        let other = InferenceInstanceId::new("tenant-b", "qwen3-small:main", 0)?;
4386        assert_ne!(
4387            t.endpoint_string(),
4388            ModelService::inference_endpoint(&other)
4389        );
4390        assert_eq!(
4391            t.endpoint_string(),
4392            ModelService::inference_endpoint(&instance)
4393        );
4394        Ok(())
4395    }
4396
4397    /// #320: a co-located (`Inproc`) service publishes an EMPTY wire reach list —
4398    /// same-host endpoints are never advertised; the co-located caller uses the
4399    /// in-process fast path. (This is what the router's single-select consumes:
4400    /// empty list ⇒ co-located fast path.)
4401    #[test]
4402    fn model_reach_co_located_is_empty() {
4403        let inproc = TransportConfig::inproc("hyprstream/inference-x");
4404        assert!(
4405            ModelService::model_reach(&inproc).is_empty(),
4406            "co-located Inproc reach must not be wire-advertised"
4407        );
4408    }
4409
4410    /// #320: a networked (Iroh) inference reach maps to exactly ONE wire arm —
4411    /// the seam the router single-selects an Iroh reach from when no co-located
4412    /// fast path is present. The carrier target (`nodeId`) is preserved only for
4413    /// routing; it does not grant an application role.
4414    #[test]
4415    fn model_reach_networked_iroh_single_select() {
4416        let node_id = [0xEEu8; 32];
4417        let iroh = TransportConfig::iroh(
4418            node_id,
4419            Vec::new(),
4420            Some("https://relay.example".to_owned()),
4421        );
4422        let reach = ModelService::model_reach(&iroh);
4423        assert_eq!(
4424            reach.len(),
4425            1,
4426            "single-select: exactly one reach per service"
4427        );
4428        match &reach[0] {
4429            WireTransportConfig::Iroh(i) => assert_eq!(i.node_id, node_id),
4430            other => panic!("expected wire Iroh reach, got {other:?}"),
4431        }
4432    }
4433
4434    // ---- #395 ModelRef prefix-dispatch grammar ----
4435
4436    #[test]
4437    fn model_ref_dispatch_at_uri_is_federated() {
4438        // at:// → federated arm, scheme captured, prefix stripped from `rest`.
4439        assert_eq!(
4440            model_ref_dispatch("at://did:plc:x123/hyprstream.models/qwen3/v1"),
4441            ModelRefDispatch::Federated {
4442                scheme: "at://",
4443                rest: "did:plc:x123/hyprstream.models/qwen3/v1",
4444            }
4445        );
4446    }
4447
4448    #[test]
4449    fn model_ref_dispatch_bare_did_is_federated() {
4450        // did: → federated arm (bare-DID form). Note `did:` (no `//`) is matched
4451        // literally, and `at://` is NOT — `did:plc:...` must not be confused with
4452        // an at-uri.
4453        assert_eq!(
4454            model_ref_dispatch("did:plc:abcdef"),
4455            ModelRefDispatch::Federated {
4456                scheme: "did:",
4457                rest: "plc:abcdef",
4458            }
4459        );
4460        // did:web variant too.
4461        assert_eq!(
4462            model_ref_dispatch("did:web:hyprstream.example.com"),
4463            ModelRefDispatch::Federated {
4464                scheme: "did:",
4465                rest: "web:hyprstream.example.com",
4466            }
4467        );
4468    }
4469
4470    #[test]
4471    fn model_ref_dispatch_local_name_is_local() {
4472        // Bare name, no federated prefix → local ModelRef arm.
4473        assert_eq!(model_ref_dispatch("qwen3"), ModelRefDispatch::Local);
4474        // name:ref — still local (the colon does not make it federated).
4475        assert_eq!(model_ref_dispatch("qwen3:main"), ModelRefDispatch::Local);
4476        assert_eq!(
4477            model_ref_dispatch("Qwen3-0.6B:tags/v1.0"),
4478            ModelRefDispatch::Local
4479        );
4480        // HuggingFace-style repo id (slash, no colon).
4481        assert_eq!(
4482            model_ref_dispatch("org/model-name"),
4483            ModelRefDispatch::Local
4484        );
4485    }
4486
4487    #[test]
4488    fn model_ref_dispatch_uuid_is_local() {
4489        // UUID (backwards-compat) has no federated prefix → local arm.
4490        assert_eq!(
4491            model_ref_dispatch("550e8400-e29b-41d4-a716-446655440000"),
4492            ModelRefDispatch::Local
4493        );
4494    }
4495
4496    #[test]
4497    fn model_ref_dispatch_local_arms_parse_unchanged() {
4498        // Every `Local` dispatch must parse via ModelRef::parse exactly as it did
4499        // pre-#395 — backward compatibility contract.
4500        for s in &["qwen3", "qwen3:main", "Qwen3-0.6B:tags/v1.0"] {
4501            assert_eq!(model_ref_dispatch(s), ModelRefDispatch::Local);
4502            assert!(
4503                ModelRef::parse(s).is_ok(),
4504                "local ModelRef '{s}' must still parse after #395"
4505            );
4506        }
4507    }
4508
4509    // ========================================================================
4510    // #523 P3 — route_decision (HRW/least-loaded + session affinity ranking)
4511    //
4512    // `route_decision` is pure (no `InferenceClient`/`SpawnedService`), so
4513    // these tests build a `CellRouter` + synthetic `load_state` directly,
4514    // proving the ranking substrate without needing a live `LoadedModel`.
4515    // The `CellRouter`/`InferenceServerInfo` ranking machinery itself is
4516    // already exhaustively covered in `services::router::tests`; these tests
4517    // cover `route_decision`'s CoLocated/Remote/NoHealthyCandidate
4518    // classification specifically, which is new in this PR.
4519    // ========================================================================
4520
4521    use crate::services::router::{CellRouter, InferenceServerInfo};
4522
4523    fn server(id: u8, mem_free: u64, active: u64) -> InferenceServerInfo {
4524        InferenceServerInfo {
4525            replica_id: crate::services::router::ReplicaId::from_bytes([id; 32]),
4526            transport: TransportConfig::inproc("test"),
4527            gpu_memory_free: mem_free,
4528            active_sessions: active,
4529            last_heartbeat: Instant::now(),
4530        }
4531    }
4532
4533    const SELECTOR_BEARER: &str = "fixture-delegated-bearer";
4534    static SELECTOR_BOUNDARY_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
4535    static SELECTOR_DIAL_STATE: std::sync::OnceLock<Arc<BoundaryDialState>> =
4536        std::sync::OnceLock::new();
4537    static SELECTOR_FIXTURE: std::sync::OnceLock<hyprstream_discovery::ProductionInferenceFixture> =
4538        std::sync::OnceLock::new();
4539
4540    #[test]
4541    fn holder_bound_service_credentials_are_not_relayed_to_remote_replicas() {
4542        let service =
4543            hyprstream_rpc::auth::Claims::new("service:registry".to_owned(), 0, 9_999_999_999);
4544        let user = hyprstream_rpc::auth::Claims::new("alice".to_owned(), 0, 9_999_999_999);
4545
4546        assert_eq!(
4547            ModelService::remote_relay_bearer_for_claims(Some(&service), Some("service-jwt")),
4548            None,
4549            "a service credential is holder-bound and must not be delegated to another controller"
4550        );
4551        assert_eq!(
4552            ModelService::remote_relay_bearer_for_claims(Some(&user), Some("user-jwt")),
4553            Some("user-jwt"),
4554            "an authenticated end-user bearer remains relayable"
4555        );
4556        assert_eq!(
4557            ModelService::remote_relay_bearer_for_claims(None, None),
4558            None,
4559            "an unauthenticated caller cannot select a remote relay path"
4560        );
4561    }
4562
4563    #[derive(Clone, Debug)]
4564    struct ReadinessObservation {
4565        transport: TransportConfig,
4566        service: String,
4567        method_discriminator: u16,
4568        delegated_bearer: Option<String>,
4569        jwt: Option<String>,
4570    }
4571
4572    #[derive(Default)]
4573    struct BoundaryDialBehavior {
4574        failures_remaining: usize,
4575        fail_all: bool,
4576    }
4577
4578    #[derive(Default)]
4579    struct BoundaryDialState {
4580        dials: parking_lot::Mutex<Vec<TransportConfig>>,
4581        readiness: parking_lot::Mutex<Vec<ReadinessObservation>>,
4582        expected_bearer: parking_lot::Mutex<Option<String>>,
4583        expected_jwt: parking_lot::Mutex<Option<String>>,
4584        behavior: parking_lot::Mutex<BoundaryDialBehavior>,
4585    }
4586
4587    impl BoundaryDialState {
4588        fn reset(&self, expected_bearer: Option<&str>) {
4589            self.dials.lock().clear();
4590            self.readiness.lock().clear();
4591            *self.expected_bearer.lock() = expected_bearer.map(ToOwned::to_owned);
4592            *self.behavior.lock() = BoundaryDialBehavior::default();
4593        }
4594
4595        fn fail_next(&self, count: usize) {
4596            self.behavior.lock().failures_remaining = count;
4597        }
4598
4599        fn fail_all(&self) {
4600            self.behavior.lock().fail_all = true;
4601        }
4602    }
4603
4604    struct BoundaryRpcClient {
4605        transport: TransportConfig,
4606        state: Arc<BoundaryDialState>,
4607        request_id: std::sync::atomic::AtomicU64,
4608    }
4609
4610    impl BoundaryRpcClient {
4611        fn new(transport: TransportConfig, state: Arc<BoundaryDialState>) -> Self {
4612            Self {
4613                transport,
4614                state,
4615                request_id: std::sync::atomic::AtomicU64::new(1),
4616            }
4617        }
4618
4619        fn unsupported() -> Result<Vec<u8>> {
4620            anyhow::bail!("selector fixture received an unexpected RPC method")
4621        }
4622    }
4623
4624    #[async_trait]
4625    impl hyprstream_rpc::RpcClient for BoundaryRpcClient {
4626        async fn call(&self, _payload: Vec<u8>) -> Result<Vec<u8>> {
4627            Self::unsupported()
4628        }
4629
4630        async fn call_for_service(
4631            &self,
4632            _service_domain: &str,
4633            _payload: Vec<u8>,
4634        ) -> Result<Vec<u8>> {
4635            Self::unsupported()
4636        }
4637
4638        async fn call_for_service_with_method(
4639            &self,
4640            _service_domain: &str,
4641            _method_discriminator: u16,
4642            _payload: Vec<u8>,
4643        ) -> Result<Vec<u8>> {
4644            Self::unsupported()
4645        }
4646
4647        async fn call_with_options(
4648            &self,
4649            _payload: Vec<u8>,
4650            _options: hyprstream_rpc::CallOptions,
4651        ) -> Result<Vec<u8>> {
4652            Self::unsupported()
4653        }
4654
4655        async fn call_with_options_for_service(
4656            &self,
4657            _service_domain: &str,
4658            _payload: Vec<u8>,
4659            _options: hyprstream_rpc::CallOptions,
4660        ) -> Result<Vec<u8>> {
4661            Self::unsupported()
4662        }
4663
4664        async fn call_with_options_for_service_with_method(
4665            &self,
4666            service_domain: &str,
4667            method_discriminator: u16,
4668            payload: Vec<u8>,
4669            options: hyprstream_rpc::CallOptions,
4670        ) -> Result<Vec<u8>> {
4671            let reader = capnp::serialize::read_message(
4672                &mut std::io::Cursor::new(&payload),
4673                capnp::message::ReaderOptions::new(),
4674            )?;
4675            let request = reader
4676                .get_root::<hyprstream_rpc_std::inference_capnp::inference_request::Reader>()?;
4677            anyhow::ensure!(
4678                matches!(
4679                    request.which()?,
4680                    hyprstream_rpc_std::inference_capnp::inference_request::Which::IsReady(())
4681                ),
4682                "selector fixture received a non-readiness request"
4683            );
4684            let expected_bearer = self.state.expected_bearer.lock().clone();
4685            let expected_jwt = self.state.expected_jwt.lock().clone();
4686            self.state.readiness.lock().push(ReadinessObservation {
4687                transport: self.transport.clone(),
4688                service: service_domain.to_owned(),
4689                method_discriminator,
4690                delegated_bearer: options.delegated_bearer.clone(),
4691                jwt: options.jwt.clone(),
4692            });
4693            anyhow::ensure!(
4694                service_domain == InferenceClient::SERVICE_NAME,
4695                "readiness used the wrong generated service authority"
4696            );
4697            anyhow::ensure!(
4698                method_discriminator == 2,
4699                "readiness used the wrong schema method discriminator"
4700            );
4701            anyhow::ensure!(
4702                options.jwt == expected_jwt && options.delegated_bearer == expected_bearer,
4703                "readiness did not carry the expected bearer shape"
4704            );
4705            let should_fail = {
4706                let mut behavior = self.state.behavior.lock();
4707                if behavior.fail_all {
4708                    true
4709                } else if behavior.failures_remaining > 0 {
4710                    behavior.failures_remaining -= 1;
4711                    true
4712                } else {
4713                    false
4714                }
4715            };
4716            if should_fail {
4717                anyhow::bail!("fixture readiness failure");
4718            }
4719            crate::services::generated::inference_client::serialize_response(
4720                request.get_id(),
4721                &hyprstream_rpc_std::inference_client::InferenceResponseVariant::IsReadyResult(
4722                    true,
4723                ),
4724            )
4725        }
4726
4727        async fn call_streaming(
4728            &self,
4729            _payload: Vec<u8>,
4730            _ephemeral_pubkey: [u8; 32],
4731        ) -> Result<Vec<u8>> {
4732            Self::unsupported()
4733        }
4734
4735        async fn call_streaming_for_service(
4736            &self,
4737            _service_domain: &str,
4738            _payload: Vec<u8>,
4739            _ephemeral_pubkey: [u8; 32],
4740        ) -> Result<Vec<u8>> {
4741            Self::unsupported()
4742        }
4743
4744        async fn call_streaming_for_service_with_method(
4745            &self,
4746            _service_domain: &str,
4747            _method_discriminator: u16,
4748            _payload: Vec<u8>,
4749            _ephemeral_pubkey: [u8; 32],
4750        ) -> Result<Vec<u8>> {
4751            Self::unsupported()
4752        }
4753
4754        async fn open_stream(
4755            &self,
4756            _payload: Vec<u8>,
4757        ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
4758            anyhow::bail!("selector fixture received an unexpected streaming RPC")
4759        }
4760
4761        async fn open_stream_from_info(
4762            &self,
4763            _stream_info: hyprstream_rpc::stream_info::StreamInfo,
4764            _client_secret: [u8; 32],
4765            _client_pubkey: [u8; 32],
4766        ) -> Result<Box<dyn hyprstream_rpc::stream_consumer::StreamHandle>> {
4767            anyhow::bail!("selector fixture received an unexpected stream-open RPC")
4768        }
4769
4770        fn next_id(&self) -> u64 {
4771            self.request_id.fetch_add(1, Ordering::Relaxed)
4772        }
4773    }
4774
4775    fn selector_instance() -> InferenceInstanceId {
4776        InferenceInstanceId::new("fixture-tenant", "fixture-model:main", 0)
4777            .unwrap_or_else(|error| panic!("fixture inference identity failed: {error}"))
4778    }
4779
4780    fn remote_transport(id: u8) -> TransportConfig {
4781        TransportConfig::iroh([id; 32], Vec::new(), None)
4782    }
4783
4784    fn selector_fixture() -> (
4785        &'static hyprstream_discovery::ProductionInferenceFixture,
4786        &'static Arc<BoundaryDialState>,
4787    ) {
4788        let state = SELECTOR_DIAL_STATE.get_or_init(|| Arc::new(BoundaryDialState::default()));
4789        let fixture = SELECTOR_FIXTURE.get_or_init(|| {
4790            let dial_state = Arc::clone(state);
4791            let dial = Arc::new(move |transport: &TransportConfig| {
4792                dial_state.dials.lock().push(transport.clone());
4793                Ok(Arc::new(BoundaryRpcClient::new(
4794                    transport.clone(),
4795                    Arc::clone(&dial_state),
4796                )) as Arc<dyn hyprstream_rpc::RpcClient>)
4797            });
4798            hyprstream_discovery::install_production_inference_fixture(
4799                &selector_instance().service_name(),
4800                &[remote_transport(0x01)],
4801                dial,
4802            )
4803            .unwrap_or_else(|error| panic!("install selector fixture failed: {error}"))
4804        });
4805        (fixture, state)
4806    }
4807
4808    async fn selector_model_service() -> ModelService {
4809        let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
4810            hyprstream_rpc::moq_event::MoqEventOrigin::new(),
4811        );
4812        let infrastructure_state = Arc::new(BoundaryDialState::default());
4813        let infrastructure_rpc = Arc::new(BoundaryRpcClient::new(
4814            TransportConfig::inproc("selector-infrastructure"),
4815            infrastructure_state,
4816        )) as Arc<dyn hyprstream_rpc::RpcClient>;
4817        ModelService::new(
4818            ModelServiceConfig::default(),
4819            SigningKey::from_bytes(&[0x41; 32]),
4820            PolicyClient::new(Arc::clone(&infrastructure_rpc)),
4821            RegistryClient::new(infrastructure_rpc),
4822            TransportConfig::inproc("selector-model-service"),
4823            TransportConfig::inproc("policy"),
4824        )
4825        .await
4826        .unwrap_or_else(|error| panic!("construct model-free selector service failed: {error}"))
4827    }
4828
4829    fn selector_loaded_model(
4830        load_state: Vec<InferenceServerInfo>,
4831        local_state: Arc<BoundaryDialState>,
4832    ) -> LoadedModel {
4833        let local_transport = TransportConfig::inproc("selector-local-inference");
4834        let network_transport = load_state
4835            .iter()
4836            .find(|candidate| {
4837                matches!(
4838                    &candidate.transport.endpoint,
4839                    EndpointType::Quic { .. } | EndpointType::Iroh { .. }
4840                )
4841            })
4842            .map_or_else(
4843                || local_transport.clone(),
4844                |candidate| candidate.transport.clone(),
4845            );
4846        LoadedModel {
4847            instance: selector_instance(),
4848            model_ref: "fixture-model:main".to_owned(),
4849            pinned_commit: None,
4850            incarnation: "fixture-incarnation-0000".to_owned(),
4851            work_audience: crate::services::inference::internal_work_audience(
4852                &selector_instance().service_name(),
4853                "fixture-incarnation-0000",
4854            ),
4855            load_generation: 1,
4856            transport: local_transport.clone(),
4857            network_transport,
4858            service_handle: hyprstream_service::SpawnedService::dummy(),
4859            client: InferenceClient::new(Arc::new(BoundaryRpcClient::new(
4860                local_transport,
4861                local_state,
4862            ))),
4863            router: CellRouter::default(),
4864            load_state,
4865            loaded_at: Instant::now(),
4866            last_used: Instant::now(),
4867            ttt_config: None,
4868            generation_defaults: crate::config::SamplingParams::default(),
4869        }
4870    }
4871
4872    fn server_at(id: u8, transport: TransportConfig) -> InferenceServerInfo {
4873        InferenceServerInfo {
4874            replica_id: crate::services::router::ReplicaId::from_bytes([id; 32]),
4875            transport,
4876            gpu_memory_free: 1024,
4877            active_sessions: 0,
4878            last_heartbeat: Instant::now(),
4879        }
4880    }
4881
4882    fn exclude_local(model: &mut LoadedModel) -> crate::services::router::ReplicaId {
4883        let local = ModelService::co_located_replica_id(model);
4884        model.router.report_dial_fail(local, Instant::now());
4885        local
4886    }
4887
4888    #[tokio::test]
4889    async fn selector_boundary_fixed_authority_exact_reach_and_delegated_readiness() {
4890        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4891        let (fixture, dial_state) = selector_fixture();
4892        let advertised = remote_transport(0x11);
4893        fixture
4894            .reset(std::slice::from_ref(&advertised))
4895            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4896        dial_state.reset(Some(SELECTOR_BEARER));
4897        let local_state = Arc::new(BoundaryDialState::default());
4898        let local = server_at(0x10, TransportConfig::inproc("selector-local"));
4899        let remote = server_at(0x20, advertised.clone());
4900        let mut model = selector_loaded_model(vec![local, remote], local_state);
4901        exclude_local(&mut model);
4902        let service = selector_model_service().await;
4903
4904        service
4905            .select_inference_routed(&mut model, "selector-positive", Some(SELECTOR_BEARER))
4906            .await
4907            .unwrap_or_else(|error| panic!("production selector rejected exact reach: {error}"));
4908
4909        let dials = dial_state.dials.lock().clone();
4910        assert_eq!(dials, vec![advertised.clone()]);
4911        let readiness = dial_state.readiness.lock().clone();
4912        assert_eq!(readiness.len(), 1);
4913        assert_eq!(readiness[0].transport, advertised);
4914        assert_eq!(readiness[0].service, "inference");
4915        assert_eq!(readiness[0].method_discriminator, 2);
4916        assert_eq!(
4917            readiness[0].delegated_bearer.as_deref(),
4918            Some(SELECTOR_BEARER)
4919        );
4920        assert!(readiness[0].jwt.is_none());
4921        assert_ne!(selector_instance().service_name(), "inference");
4922    }
4923
4924    #[tokio::test]
4925    async fn selector_boundary_rejects_unadvertised_reach_before_bearer_disclosure() {
4926        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4927        let (fixture, dial_state) = selector_fixture();
4928        fixture
4929            .reset(&[remote_transport(0x21)])
4930            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4931        dial_state.reset(Some(SELECTOR_BEARER));
4932        let local_state = Arc::new(BoundaryDialState::default());
4933        let selected_id = crate::services::router::ReplicaId::from_bytes([0x22; 32]);
4934        let mut model = selector_loaded_model(
4935            vec![
4936                server_at(0x10, TransportConfig::inproc("selector-local")),
4937                server_at(0x22, remote_transport(0x22)),
4938            ],
4939            local_state,
4940        );
4941        exclude_local(&mut model);
4942        let error = selector_model_service()
4943            .await
4944            .select_inference_routed(&mut model, "selector-unadvertised", Some(SELECTOR_BEARER))
4945            .await
4946            .err()
4947            .unwrap_or_else(|| panic!("unadvertised reach was accepted"));
4948        assert!(error
4949            .to_string()
4950            .contains("not a current authorized candidate"));
4951        assert!(dial_state.dials.lock().is_empty());
4952        assert!(dial_state.readiness.lock().is_empty());
4953        assert!(model
4954            .router
4955            .is_excluded(selected_id, Instant::now())
4956            .is_some());
4957    }
4958
4959    #[tokio::test]
4960    async fn selector_boundary_rejects_stale_reach_before_bearer_disclosure() {
4961        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4962        let (fixture, dial_state) = selector_fixture();
4963        let advertised = remote_transport(0x31);
4964        fixture
4965            .reset(std::slice::from_ref(&advertised))
4966            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4967        fixture.mark_stale();
4968        dial_state.reset(Some(SELECTOR_BEARER));
4969        let selected_id = crate::services::router::ReplicaId::from_bytes([0x23; 32]);
4970        let mut model = selector_loaded_model(
4971            vec![
4972                server_at(0x10, TransportConfig::inproc("selector-local")),
4973                server_at(0x23, advertised),
4974            ],
4975            Arc::new(BoundaryDialState::default()),
4976        );
4977        exclude_local(&mut model);
4978        assert!(selector_model_service()
4979            .await
4980            .select_inference_routed(&mut model, "selector-stale", Some(SELECTOR_BEARER))
4981            .await
4982            .is_err());
4983        assert!(dial_state.dials.lock().is_empty());
4984        assert!(dial_state.readiness.lock().is_empty());
4985        assert!(model
4986            .router
4987            .is_excluded(selected_id, Instant::now())
4988            .is_some());
4989    }
4990
4991    #[tokio::test]
4992    async fn selector_boundary_rejects_cross_authority_retry_set_before_bearer_disclosure() {
4993        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
4994        let (fixture, dial_state) = selector_fixture();
4995        let selected = remote_transport(0x41);
4996        fixture
4997            .reset(std::slice::from_ref(&selected))
4998            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
4999        fixture
5000            .add_foreign_authority(&remote_transport(0x42))
5001            .unwrap_or_else(|error| panic!("add foreign fixture authority failed: {error}"));
5002        dial_state.reset(Some(SELECTOR_BEARER));
5003        let selected_id = crate::services::router::ReplicaId::from_bytes([0x24; 32]);
5004        let mut model = selector_loaded_model(
5005            vec![
5006                server_at(0x10, TransportConfig::inproc("selector-local")),
5007                server_at(0x24, selected),
5008            ],
5009            Arc::new(BoundaryDialState::default()),
5010        );
5011        exclude_local(&mut model);
5012        let error = selector_model_service()
5013            .await
5014            .select_inference_routed(&mut model, "selector-authority", Some(SELECTOR_BEARER))
5015            .await
5016            .err()
5017            .unwrap_or_else(|| panic!("cross-authority retry set was accepted"));
5018        assert!(error
5019            .to_string()
5020            .contains("ambiguous validated service authority"));
5021        assert!(dial_state.dials.lock().is_empty());
5022        assert!(dial_state.readiness.lock().is_empty());
5023        assert!(model
5024            .router
5025            .is_excluded(selected_id, Instant::now())
5026            .is_some());
5027    }
5028
5029    #[tokio::test]
5030    async fn selector_boundary_rejects_non_network_transport_before_bearer_disclosure() {
5031        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5032        let (fixture, dial_state) = selector_fixture();
5033        fixture
5034            .reset(&[remote_transport(0x51)])
5035            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5036        dial_state.reset(Some(SELECTOR_BEARER));
5037        let selected_id = crate::services::router::ReplicaId::from_bytes([0x25; 32]);
5038        let mut model = selector_loaded_model(
5039            vec![
5040                server_at(0x10, TransportConfig::inproc("selector-local")),
5041                server_at(0x25, TransportConfig::ipc("/run/hyprstream/attacker.sock")),
5042            ],
5043            Arc::new(BoundaryDialState::default()),
5044        );
5045        exclude_local(&mut model);
5046        let error = selector_model_service()
5047            .await
5048            .select_inference_routed(&mut model, "selector-transport", Some(SELECTOR_BEARER))
5049            .await
5050            .err()
5051            .unwrap_or_else(|| panic!("non-network remote transport was accepted"));
5052        assert!(error.to_string().contains("non-network transport"));
5053        assert!(dial_state.dials.lock().is_empty());
5054        assert!(dial_state.readiness.lock().is_empty());
5055        assert!(model
5056            .router
5057            .is_excluded(selected_id, Instant::now())
5058            .is_some());
5059    }
5060
5061    #[tokio::test]
5062    async fn selector_boundary_failure_marks_exact_selected_replica_and_reselects_once() {
5063        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5064        let (fixture, dial_state) = selector_fixture();
5065        let first = remote_transport(0x61);
5066        let second = remote_transport(0x62);
5067        fixture
5068            .reset(&[first.clone(), second.clone()])
5069            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5070        dial_state.reset(Some(SELECTOR_BEARER));
5071        dial_state.fail_next(1);
5072        let first_id = crate::services::router::ReplicaId::from_bytes([0x26; 32]);
5073        let second_id = crate::services::router::ReplicaId::from_bytes([0x27; 32]);
5074        let mut model = selector_loaded_model(
5075            vec![
5076                server_at(0x10, TransportConfig::inproc("selector-local")),
5077                server_at(0x26, first.clone()),
5078                server_at(0x27, second.clone()),
5079            ],
5080            Arc::new(BoundaryDialState::default()),
5081        );
5082        exclude_local(&mut model);
5083        selector_model_service()
5084            .await
5085            .select_inference_routed(&mut model, "selector-reselect", Some(SELECTOR_BEARER))
5086            .await
5087            .unwrap_or_else(|error| panic!("bounded reselection did not recover: {error}"));
5088        let dials = dial_state.dials.lock().clone();
5089        assert_eq!(dials.len(), 2);
5090        assert_ne!(dials[0], dials[1]);
5091        let (failed_id, surviving_id) = if dials[0] == first {
5092            (first_id, second_id)
5093        } else {
5094            assert_eq!(dials[0], second);
5095            (second_id, first_id)
5096        };
5097        assert!(model
5098            .router
5099            .is_excluded(failed_id, Instant::now())
5100            .is_some());
5101        assert!(model
5102            .router
5103            .is_excluded(surviving_id, Instant::now())
5104            .is_none());
5105        assert_eq!(dial_state.readiness.lock().len(), 2);
5106    }
5107
5108    #[tokio::test]
5109    async fn selector_boundary_explicit_colocation_returns_local_client_without_remote_dial() {
5110        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5111        let (fixture, dial_state) = selector_fixture();
5112        fixture
5113            .reset(&[remote_transport(0x71)])
5114            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5115        dial_state.reset(Some(SELECTOR_BEARER));
5116        let local_state = Arc::new(BoundaryDialState::default());
5117        local_state.reset(None);
5118        let mut model = selector_loaded_model(
5119            vec![server_at(0x10, TransportConfig::inproc("selector-local"))],
5120            Arc::clone(&local_state),
5121        );
5122        let client = selector_model_service()
5123            .await
5124            .select_inference_routed(&mut model, "selector-local", Some(SELECTOR_BEARER))
5125            .await
5126            .unwrap_or_else(|error| panic!("explicit co-location failed: {error}"))
5127            .value;
5128        assert!(dial_state.dials.lock().is_empty());
5129        assert!(dial_state.readiness.lock().is_empty());
5130        assert!(client
5131            .is_ready()
5132            .await
5133            .unwrap_or_else(|error| panic!("returned local client failed: {error}")));
5134        assert_eq!(local_state.readiness.lock().len(), 1);
5135    }
5136
5137    #[tokio::test]
5138    async fn selector_boundary_exhaustion_fails_without_local_fallback() {
5139        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5140        let (fixture, dial_state) = selector_fixture();
5141        let first = remote_transport(0x81);
5142        let second = remote_transport(0x82);
5143        fixture
5144            .reset(&[first.clone(), second.clone()])
5145            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5146        dial_state.reset(Some(SELECTOR_BEARER));
5147        dial_state.fail_all();
5148        let first_id = crate::services::router::ReplicaId::from_bytes([0x28; 32]);
5149        let second_id = crate::services::router::ReplicaId::from_bytes([0x29; 32]);
5150        let local_state = Arc::new(BoundaryDialState::default());
5151        local_state.reset(None);
5152        let mut model = selector_loaded_model(
5153            vec![
5154                server_at(0x10, TransportConfig::inproc("selector-local")),
5155                server_at(0x28, first),
5156                server_at(0x29, second),
5157            ],
5158            Arc::clone(&local_state),
5159        );
5160        exclude_local(&mut model);
5161        let error = selector_model_service()
5162            .await
5163            .select_inference_routed(&mut model, "selector-exhausted", Some(SELECTOR_BEARER))
5164            .await
5165            .err()
5166            .unwrap_or_else(|| panic!("exhausted remote candidates fell back locally"));
5167        assert!(error.to_string().contains("candidates exhausted"));
5168        assert_eq!(dial_state.dials.lock().len(), 2);
5169        assert_eq!(dial_state.readiness.lock().len(), 2);
5170        assert!(local_state.readiness.lock().is_empty());
5171        assert!(model.router.is_excluded(first_id, Instant::now()).is_some());
5172        assert!(model
5173            .router
5174            .is_excluded(second_id, Instant::now())
5175            .is_some());
5176    }
5177
5178    /// Degenerate/solo case (acceptance criterion): a single-entry replica set
5179    /// (today's actual `load_state` shape at load time) always classifies as
5180    /// `CoLocated` — no regression from pre-P3 behavior.
5181    #[test]
5182    fn route_decision_single_replica_is_co_located() {
5183        let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5184        let load_state = vec![server(0x10, 1024, 0)];
5185        let mut router = CellRouter::default();
5186        let decision = ModelService::route_decision(
5187            &mut router,
5188            &load_state,
5189            co_located,
5190            "sess-A",
5191            Instant::now(),
5192        );
5193        assert_eq!(decision, RouteDecision::CoLocated);
5194    }
5195
5196    /// Multi-replica (synthetic `load_state`, per the PR notes on the P1
5197    /// candidate-population gap): HRW must be able to select a NON-co-located
5198    /// entry, classified as `Remote`, distributing distinct sessions across
5199    /// more than one node — the acceptance criterion's "requests distribute
5200    /// by HRW/least-loaded over live candidates" proven at this layer.
5201    #[test]
5202    fn route_decision_multi_replica_distributes_and_detects_remote() {
5203        let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5204        let load_state = vec![
5205            server(0x10, 16 * 1024 * 1024 * 1024, 0), // co-located
5206            server(0x20, 16 * 1024 * 1024 * 1024, 0), // remote
5207            server(0x30, 16 * 1024 * 1024 * 1024, 0), // remote
5208        ];
5209        let mut saw_co_located = false;
5210        let mut saw_remote = false;
5211        for i in 0..150u32 {
5212            // Fresh router per session avoids affinity stickiness masking the
5213            // underlying HRW spread this test wants to observe.
5214            let mut router = CellRouter::default();
5215            match ModelService::route_decision(
5216                &mut router,
5217                &load_state,
5218                co_located,
5219                &format!("sess-{i}"),
5220                Instant::now(),
5221            ) {
5222                RouteDecision::CoLocated => saw_co_located = true,
5223                RouteDecision::Remote(_) => saw_remote = true,
5224                RouteDecision::NoHealthyCandidate => panic!("healthy replica set must place"),
5225            }
5226        }
5227        assert!(
5228            saw_co_located,
5229            "some sessions must land on the co-located replica"
5230        );
5231        assert!(
5232            saw_remote,
5233            "some sessions must land on a remote replica (not always co-located)"
5234        );
5235    }
5236
5237    /// Session affinity holds: the same session_id must classify identically
5238    /// across repeated calls on the SAME router instance (KV-cache stickiness),
5239    /// whether it lands co-located or remote.
5240    #[test]
5241    fn route_decision_session_affinity_holds() {
5242        let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5243        let load_state = vec![
5244            server(0x10, 16 * 1024 * 1024 * 1024, 0),
5245            server(0x20, 16 * 1024 * 1024 * 1024, 0),
5246            server(0x30, 16 * 1024 * 1024 * 1024, 0),
5247        ];
5248        let mut router = CellRouter::default();
5249        let now = Instant::now();
5250        let first =
5251            ModelService::route_decision(&mut router, &load_state, co_located, "sess-sticky", now);
5252        for _ in 0..5 {
5253            let again = ModelService::route_decision(
5254                &mut router,
5255                &load_state,
5256                co_located,
5257                "sess-sticky",
5258                now,
5259            );
5260            assert_eq!(
5261                first, again,
5262                "same session must stick to the same replica within its lease"
5263            );
5264        }
5265    }
5266
5267    /// Fail-over on replica loss: once the affinity-bound node is marked down
5268    /// (dial-fail), the NEXT `route_decision` for the same session must
5269    /// reassign to a different node rather than returning the excluded one.
5270    #[test]
5271    fn route_decision_fails_over_on_replica_loss() {
5272        let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5273        let load_state = vec![
5274            server(0x10, 16 * 1024 * 1024 * 1024, 0),
5275            server(0x20, 16 * 1024 * 1024 * 1024, 0),
5276            server(0x30, 16 * 1024 * 1024 * 1024, 0),
5277        ];
5278        let mut router = CellRouter::default();
5279        let now = Instant::now();
5280        let first =
5281            ModelService::route_decision(&mut router, &load_state, co_located, "sess-B", now);
5282        let first_node = match first {
5283            RouteDecision::CoLocated => co_located,
5284            RouteDecision::Remote(n) => n,
5285            RouteDecision::NoHealthyCandidate => panic!("healthy replica set must place"),
5286        };
5287        router.report_dial_fail(first_node, now);
5288        let second =
5289            ModelService::route_decision(&mut router, &load_state, co_located, "sess-B", now);
5290        let second_node = match second {
5291            RouteDecision::CoLocated => co_located,
5292            RouteDecision::Remote(n) => n,
5293            RouteDecision::NoHealthyCandidate => {
5294                panic!("two healthy replicas remain after one failure")
5295            }
5296        };
5297        assert_ne!(
5298            second_node, first_node,
5299            "must fail over off the down replica"
5300        );
5301    }
5302
5303    /// Empty/all-excluded replica set classifies as `NoHealthyCandidate` (the
5304    /// caller's fallback-to-co-located-client path), never a panic.
5305    #[test]
5306    fn route_decision_empty_replica_set_is_no_healthy_candidate() {
5307        let co_located = crate::services::router::ReplicaId::from_bytes([0x10; 32]);
5308        let load_state: Vec<InferenceServerInfo> = vec![];
5309        let mut router = CellRouter::default();
5310        let decision = ModelService::route_decision(
5311            &mut router,
5312            &load_state,
5313            co_located,
5314            "sess-A",
5315            Instant::now(),
5316        );
5317        assert_eq!(decision, RouteDecision::NoHealthyCandidate);
5318    }
5319
5320    #[tokio::test]
5321    async fn selector_boundary_colocated_client_carries_internal_work_bearer_not_relay() {
5322        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5323        let local_state = Arc::new(BoundaryDialState::default());
5324        // The co-located instance expects the minted work order as the DIRECT
5325        // jwt — never as a delegated relay of a caller credential.
5326        let work_token = "iw-minted-work-order-fixture".to_owned();
5327        *local_state.expected_jwt.lock() = Some(work_token.clone());
5328        local_state.reset(None);
5329        let mut model = selector_loaded_model(
5330            vec![server_at(0x10, TransportConfig::inproc("selector-local"))],
5331            Arc::clone(&local_state),
5332        );
5333        let service = selector_model_service().await;
5334        let routed = service
5335            .select_inference_routed(&mut model, "selector-internal", None)
5336            .await
5337            .unwrap_or_else(|error| panic!("co-location without a relay bearer failed: {error}"));
5338        // The co-located arm carries the work order as its DIRECT bearer via
5339        // the same locality gate get_inference_client uses (K3 finding M1).
5340        let bearer_client = ModelService::attach_work_order(routed, work_token.clone());
5341        assert!(
5342            bearer_client
5343                .is_ready()
5344                .await
5345                .unwrap_or_else(|error| panic!("returned local client failed: {error}")),
5346            "readiness with the internal work bearer must succeed"
5347        );
5348        let readiness = local_state.readiness.lock().clone();
5349        assert_eq!(readiness.len(), 1);
5350        assert_eq!(readiness[0].jwt.as_deref(), Some(work_token.as_str()));
5351        assert!(
5352            readiness[0].delegated_bearer.is_none(),
5353            "the internal leg must never relay a caller credential as delegated bearer"
5354        );
5355    }
5356
5357    /// K3 finding M1 regression: the REMOTE arm must never carry the internal
5358    /// work order. The routed client keeps its delegated-relay shape; the
5359    /// locality gate must not overlay a direct JWT (an envelope carrying both
5360    /// is a hard wire error, and a remote instance would deny a
5361    /// foreign-controller work order anyway).
5362    #[tokio::test]
5363    async fn selector_boundary_remote_arm_never_overlays_internal_work_order() {
5364        let _guard = SELECTOR_BOUNDARY_LOCK.lock().await;
5365        let (fixture, dial_state) = selector_fixture();
5366        let advertised = remote_transport(0x91);
5367        fixture
5368            .reset(std::slice::from_ref(&advertised))
5369            .unwrap_or_else(|error| panic!("reset selector fixture failed: {error}"));
5370        dial_state.reset(Some(SELECTOR_BEARER));
5371        // The remote relay must present the delegated bearer and NO direct
5372        // jwt on its first forwarded call.
5373        *dial_state.expected_jwt.lock() = None;
5374        let local_state = Arc::new(BoundaryDialState::default());
5375        local_state.reset(None);
5376        let mut model = selector_loaded_model(
5377            vec![
5378                server_at(0x10, TransportConfig::inproc("selector-local")),
5379                server_at(0x91, advertised.clone()),
5380            ],
5381            Arc::clone(&local_state),
5382        );
5383        exclude_local(&mut model);
5384        let service = selector_model_service().await;
5385        let routed = service
5386            .select_inference_routed(
5387                &mut model,
5388                "selector-remote-internal",
5389                Some(SELECTOR_BEARER),
5390            )
5391            .await
5392            .unwrap_or_else(|error| panic!("remote selection failed: {error}"));
5393        // The same locality gate get_inference_client applies: Remote ⇒ no
5394        // work-order overlay, the delegated relay shape is untouched.
5395        let client =
5396            ModelService::attach_work_order(routed, "iw-foreign-controller-order".to_owned());
5397        assert!(
5398            client
5399                .is_ready()
5400                .await
5401                .unwrap_or_else(|error| panic!("remote client failed: {error}")),
5402            "remote relay readiness must succeed with the delegated bearer only"
5403        );
5404        // Two observations: the remote arm's selection readiness probe AND the
5405        // post-attach forwarded call. BOTH must carry the delegated relay and
5406        // never the work order.
5407        let readiness = dial_state.readiness.lock().clone();
5408        assert_eq!(readiness.len(), 2);
5409        for observation in &readiness {
5410            assert_eq!(
5411                observation.delegated_bearer.as_deref(),
5412                Some(SELECTOR_BEARER),
5413                "remote relay must carry the delegated bearer"
5414            );
5415            assert!(
5416                observation.jwt.is_none(),
5417                "the remote arm must not carry the internal work order as a direct jwt"
5418            );
5419        }
5420    }
5421
5422    // ========================================================================
5423    // User-approved single-service Model→Inference boundary (D5): mint side.
5424    //
5425    // Proves the work order Model mints per forwarded operation is bound to
5426    // the pinned instance, the verified tenant/model, the exact downstream
5427    // dispatch coordinate, the caller's own expiry, and Model's key — and
5428    // that it verifies against the PINNED controller key the spawned
5429    // instance enforces.
5430    // ========================================================================
5431
5432    use base64::Engine as _;
5433    use hyprstream_rpc::auth::internal_work::{
5434        verify_internal_work, InternalWorkScope, MAX_LIFETIME_SECS,
5435    };
5436
5437    const BOUNDARY_CALLER_KEY: [u8; 32] = [0xCB; 32];
5438
5439    /// The exact live worker audience a mint must bind (Sol incarnation plan):
5440    /// the versioned audience of the fixture instance's test incarnation.
5441    fn audience_fixture() -> String {
5442        crate::services::inference::internal_work_audience(
5443            &selector_instance().service_name(),
5444            "test-incarnation-0000",
5445        )
5446    }
5447
5448    // ── Sol bounded incarnation plan: live binding + dead-worker fail-close ─
5449
5450    fn running_worker() -> hyprstream_service::SpawnedService {
5451        // A live parked thread keeps the handle "running" for the guard; the
5452        // parked thread exits when the test process ends (no join needed).
5453        let handle = std::thread::spawn(|| loop {
5454            std::thread::park_timeout(std::time::Duration::from_secs(3600));
5455        });
5456        hyprstream_service::SpawnedService::thread(
5457            "s1-alive-worker".to_owned(),
5458            Some(handle),
5459            Arc::new(tokio::sync::Notify::new()),
5460            None,
5461        )
5462    }
5463
5464    fn dead_worker() -> hyprstream_service::SpawnedService {
5465        hyprstream_service::SpawnedService::thread(
5466            "s1-dead-worker".to_owned(),
5467            None,
5468            Arc::new(tokio::sync::Notify::new()),
5469            None,
5470        )
5471    }
5472
5473    fn blocking_unload_worker(
5474        armed: std::sync::mpsc::Sender<()>,
5475        shutdown_observed: tokio::sync::oneshot::Sender<()>,
5476        release: Arc<tokio::sync::Notify>,
5477    ) -> hyprstream_service::SpawnedService {
5478        let shutdown = Arc::new(tokio::sync::Notify::new());
5479        let worker_shutdown = Arc::clone(&shutdown);
5480        let handle = std::thread::spawn(move || {
5481            let runtime = tokio::runtime::Builder::new_current_thread()
5482                .enable_all()
5483                .build()
5484                .unwrap_or_else(|error| panic!("build blocking-unload runtime: {error}"));
5485            runtime.block_on(async move {
5486                let notified = worker_shutdown.notified();
5487                tokio::pin!(notified);
5488                notified.as_mut().enable();
5489                armed
5490                    .send(())
5491                    .unwrap_or_else(|error| panic!("signal blocking worker readiness: {error}"));
5492                notified.await;
5493                let _ = shutdown_observed.send(());
5494                release.notified().await;
5495            });
5496        });
5497        hyprstream_service::SpawnedService::thread(
5498            "s1-blocking-unload-worker".to_owned(),
5499            Some(handle),
5500            shutdown,
5501            None,
5502        )
5503    }
5504
5505    fn panicking_unload_worker() -> hyprstream_service::SpawnedService {
5506        let handle = std::thread::spawn(|| panic!("test worker panics before join"));
5507        hyprstream_service::SpawnedService::thread(
5508            "s1-panicking-unload-worker".to_owned(),
5509            Some(handle),
5510            Arc::new(tokio::sync::Notify::new()),
5511            None,
5512        )
5513    }
5514
5515    #[tokio::test]
5516    async fn cancelled_unload_keeps_instance_reserved_until_thread_join() {
5517        let service = selector_model_service().await;
5518        let instance = selector_instance();
5519        let local_state = Arc::new(BoundaryDialState::default());
5520        let (armed_tx, armed_rx) = std::sync::mpsc::channel();
5521        let (shutdown_observed_tx, shutdown_observed_rx) = tokio::sync::oneshot::channel();
5522        let release = Arc::new(tokio::sync::Notify::new());
5523        let mut model = selector_loaded_model(Vec::new(), local_state);
5524        model.service_handle =
5525            blocking_unload_worker(armed_tx, shutdown_observed_tx, Arc::clone(&release));
5526        service
5527            .loaded_models
5528            .write()
5529            .await
5530            .put(instance.clone(), model);
5531        // This is the real inner-load/outer-load handoff overlap: a worker is
5532        // visible in the cache while the load reservation still exists. The
5533        // unloading worker below blocks its join so both status APIs must
5534        // prefer the reservation for the whole drain interval.
5535        service.pending_loads.lock().insert(instance.clone(), 1);
5536
5537        tokio::task::spawn_blocking(move || {
5538            armed_rx
5539                .recv_timeout(std::time::Duration::from_secs(5))
5540                .map_err(|error| anyhow!("blocking worker did not arm: {error}"))
5541        })
5542        .await
5543        .unwrap_or_else(|error| panic!("wait for blocking worker readiness task: {error}"))
5544        .unwrap_or_else(|error| panic!("wait for blocking worker readiness: {error}"));
5545
5546        let unload_service = service.clone();
5547        let unload_instance = instance.clone();
5548        let caller =
5549            tokio::spawn(async move { unload_service.unload_model(&unload_instance).await });
5550
5551        tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_observed_rx)
5552            .await
5553            .unwrap_or_else(|_| panic!("unload did not notify the blocking worker"))
5554            .unwrap_or_else(|error| {
5555                panic!("blocking worker dropped shutdown observation: {error}")
5556            });
5557
5558        caller.abort();
5559        assert!(
5560            matches!(caller.await, Err(error) if error.is_cancelled()),
5561            "the caller task must be cancelled while owned teardown continues"
5562        );
5563        assert!(
5564            service.unloading_models.lock().await.contains(&instance),
5565            "cancellation must retain the unloading reservation until join completes"
5566        );
5567        assert!(
5568            !service.loaded_models.read().await.contains(&instance),
5569            "the cache must not expose a worker after its shutdown begins"
5570        );
5571        let (all_status, single_status) = tokio::join!(
5572            service.model_status_all(instance.tenant()),
5573            service.model_status_single(&instance),
5574        );
5575        assert_eq!(
5576            all_status
5577                .iter()
5578                .filter(|entry| entry.model_ref == instance.model_ref())
5579                .map(|entry| entry.status.as_str())
5580                .collect::<Vec<_>>(),
5581            vec!["unloading"],
5582            "all-status must retain an unloading worker after its caller is cancelled"
5583        );
5584        assert_eq!(
5585            single_status
5586                .iter()
5587                .map(|entry| entry.status.as_str())
5588                .collect::<Vec<_>>(),
5589            vec!["unloading"],
5590            "single-status must not report the draining worker as absent"
5591        );
5592        let error = match service.load_model(&instance, None, None).await {
5593            Err(error) => error,
5594            Ok(_) => panic!("a replacement load must not race the draining worker"),
5595        };
5596        assert!(
5597            error.to_string().contains("is unloading"),
5598            "replacement denial must disclose only lifecycle state: {error:#}"
5599        );
5600
5601        // In production the outer load completion clears this marker. Keep the
5602        // worker drain blocked but model that completion before asserting final
5603        // status absence below.
5604        service.pending_loads.lock().remove(&instance);
5605        release.notify_one();
5606        tokio::time::timeout(std::time::Duration::from_secs(5), async {
5607            loop {
5608                if !service.unloading_models.lock().await.contains(&instance) {
5609                    break;
5610                }
5611                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5612            }
5613        })
5614        .await
5615        .unwrap_or_else(|_| panic!("owned teardown did not clear its reservation after join"));
5616        let (all_status, single_status) = tokio::join!(
5617            service.model_status_all(instance.tenant()),
5618            service.model_status_single(&instance),
5619        );
5620        assert!(
5621            all_status.is_empty() && single_status.is_empty(),
5622            "status absence is permitted only after teardown releases its reservation"
5623        );
5624    }
5625
5626    #[tokio::test]
5627    async fn unloading_reservation_precedes_loaded_and_pending_status_snapshot() {
5628        let service = selector_model_service().await;
5629        let instance = selector_instance();
5630
5631        // `load_model_inner` publishes the cache entry before outer
5632        // `load_model` clears this reservation. If unload reserves and pops in
5633        // that interval, both sets are deliberately present: teardown is the
5634        // authoritative lifecycle state, never a second loading attempt.
5635        service.loaded_models.write().await.put(
5636            instance.clone(),
5637            selector_loaded_model(Vec::new(), Arc::new(BoundaryDialState::default())),
5638        );
5639        service.pending_loads.lock().insert(instance.clone(), 1);
5640        service
5641            .unloading_models
5642            .lock()
5643            .await
5644            .insert(instance.clone());
5645
5646        let (all_status, single_status) = tokio::join!(
5647            service.model_status_all(instance.tenant()),
5648            service.model_status_single(&instance),
5649        );
5650        assert_eq!(
5651            all_status
5652                .iter()
5653                .filter(|entry| entry.model_ref == instance.model_ref())
5654                .map(|entry| entry.status.as_str())
5655                .collect::<Vec<_>>(),
5656            vec!["unloading"],
5657            "list status must not let pending_loads conceal an active teardown"
5658        );
5659        assert_eq!(
5660            single_status
5661                .iter()
5662                .map(|entry| entry.status.as_str())
5663                .collect::<Vec<_>>(),
5664            vec!["unloading"],
5665            "single status must not let pending_loads conceal an active teardown"
5666        );
5667    }
5668
5669    #[tokio::test]
5670    async fn intercepted_load_reservation_rejects_unload_before_continuation_runs() {
5671        let service = selector_model_service().await;
5672        let instance = selector_instance();
5673
5674        // Force the former check-to-response gap: handler admission reserves a
5675        // pending load, then unload attempts to win before the continuation is
5676        // polled. Both transitions use `load_unload_gate`, so the unload must
5677        // fail closed and leave the accepted continuation as sole owner.
5678        let reservation = match service.reserve_intercepted_load(&instance).await {
5679            Ok(InterceptedLoadAdmission::Accepted(reservation)) => reservation,
5680            Ok(_) => panic!("fixture load must receive a new accepted reservation"),
5681            Err(error) => panic!("fixture load admission failed: {error:#}"),
5682        };
5683        let unload_error = match service.unload_model(&instance).await {
5684            Err(error) => error,
5685            Ok(()) => panic!("unload must not cancel an accepted pending load"),
5686        };
5687        assert!(
5688            unload_error.to_string().contains("is loading"),
5689            "unload must reject against the accepted pending reservation: {unload_error:#}"
5690        );
5691        assert!(
5692            service.pending_loads.lock().contains_key(&instance),
5693            "the continuation must retain ownership of the accepted reservation"
5694        );
5695        assert!(
5696            !service.unloading_models.lock().await.contains(&instance),
5697            "a rejected unload must not leave a false teardown reservation"
5698        );
5699
5700        // The selector fixture fails normal model resolution immediately; this
5701        // drives the continuation's ordinary failure completion without
5702        // starting a worker and proves it releases its own reservation.
5703        assert!(
5704            service
5705                .run_intercepted_load(&instance, None, None, reservation)
5706                .await
5707                .is_err(),
5708            "fixture resolution must reach the continuation failure path"
5709        );
5710        assert!(
5711            !service.pending_loads.lock().contains_key(&instance),
5712            "continuation completion must release its accepted reservation"
5713        );
5714    }
5715
5716    #[tokio::test]
5717    async fn dropped_intercepted_load_reservation_rolls_back_pending_marker() {
5718        let service = selector_model_service().await;
5719        let instance = selector_instance();
5720
5721        // This is the response-serialization / unpolled-continuation failure
5722        // path: admission succeeded, the continuation owns its guard, but the
5723        // response path drops that continuation before its first poll. Drop
5724        // must synchronously remove the marker so neither status nor a later
5725        // admission sees a permanent false load.
5726        let reservation = match service.reserve_intercepted_load(&instance).await {
5727            Ok(InterceptedLoadAdmission::Accepted(reservation)) => reservation,
5728            Ok(_) => panic!("fixture load must receive a new accepted reservation"),
5729            Err(error) => panic!("fixture load admission failed: {error:#}"),
5730        };
5731        assert!(service.pending_loads.lock().contains_key(&instance));
5732        let unpolled: crate::services::Continuation = Box::pin(async move {
5733            let _reservation = reservation;
5734            std::future::pending::<()>().await;
5735        });
5736        drop(unpolled);
5737        assert!(
5738            !service.pending_loads.lock().contains_key(&instance),
5739            "dropping an unpolled accepted continuation must roll back pending"
5740        );
5741
5742        // A late Drop from an earlier handoff must not erase a newer attempt
5743        // for the same instance. This is the attempt identity contract, not a
5744        // best-effort instance-wide cleanup.
5745        let stale =
5746            InterceptedLoadReservation::new(Arc::clone(&service.inner), instance.clone(), 7);
5747        service.pending_loads.lock().insert(instance.clone(), 8);
5748        drop(stale);
5749        assert_eq!(
5750            service.pending_loads.lock().get(&instance),
5751            Some(&8),
5752            "a stale guard must release only its own attempt marker"
5753        );
5754        service.pending_loads.lock().remove(&instance);
5755        let retry = service.reserve_intercepted_load(&instance).await;
5756        assert!(
5757            matches!(retry, Ok(InterceptedLoadAdmission::Accepted(_))),
5758            "a later load must not deduplicate against a dropped reservation"
5759        );
5760    }
5761
5762    #[tokio::test]
5763    async fn successful_unload_releases_admission_before_unloaded_event() {
5764        let service = selector_model_service().await;
5765        let instance = selector_instance();
5766        service.loaded_models.write().await.put(
5767            instance.clone(),
5768            selector_loaded_model(Vec::new(), Arc::new(BoundaryDialState::default())),
5769        );
5770        let (before_publish_tx, before_publish_rx) = tokio::sync::oneshot::channel();
5771        *service.before_unloaded_event.lock().await = Some(before_publish_tx);
5772
5773        let unload_service = service.clone();
5774        let unload_instance = instance.clone();
5775        let unload =
5776            tokio::spawn(async move { unload_service.unload_model(&unload_instance).await });
5777        before_publish_rx
5778            .await
5779            .unwrap_or_else(|error| panic!("unload did not reach event boundary: {error}"));
5780
5781        assert!(
5782            !service.unloading_models.lock().await.contains(&instance),
5783            "successful join must release admission before model.unloaded is published"
5784        );
5785        let reload = service.reserve_intercepted_load(&instance).await;
5786        assert!(
5787            matches!(reload, Ok(InterceptedLoadAdmission::Accepted(_))),
5788            "an event-driven reload must be admitted after a successful join"
5789        );
5790        unload
5791            .await
5792            .unwrap_or_else(|error| panic!("unload task panicked: {error}"))
5793            .unwrap_or_else(|error| panic!("dummy worker unload failed: {error:#}"));
5794    }
5795
5796    #[tokio::test]
5797    async fn unload_stop_error_retains_instance_reservation() {
5798        let service = selector_model_service().await;
5799        let instance = selector_instance();
5800        let local_state = Arc::new(BoundaryDialState::default());
5801        let mut model = selector_loaded_model(Vec::new(), local_state);
5802        model.service_handle = panicking_unload_worker();
5803        service
5804            .loaded_models
5805            .write()
5806            .await
5807            .put(instance.clone(), model);
5808
5809        let error = match service.unload_model(&instance).await {
5810            Err(error) => error,
5811            Ok(()) => panic!("a thread join panic must make unload fail"),
5812        };
5813        assert!(
5814            error.to_string().contains("failed to stop model"),
5815            "unload must return the stop failure: {error:#}"
5816        );
5817        assert!(
5818            service.unloading_models.lock().await.contains(&instance),
5819            "an ambiguous stop failure must retain the replacement-load reservation"
5820        );
5821        assert!(
5822            !service.loaded_models.read().await.contains(&instance),
5823            "a failed stop must not expose the popped worker as live"
5824        );
5825        let error = match service.load_model(&instance, None, None).await {
5826            Err(error) => error,
5827            Ok(_) => panic!("a failed stop must not admit a replacement worker"),
5828        };
5829        assert!(
5830            error.to_string().contains("is unloading"),
5831            "replacement denial must remain a lifecycle-only message: {error:#}"
5832        );
5833    }
5834
5835    #[tokio::test]
5836    async fn dead_worker_cache_entry_cannot_mint_and_is_evicted() {
5837        let service = selector_model_service().await;
5838        let instance = selector_instance();
5839        let mut cache = service.loaded_models.write().await;
5840        cache.put(
5841            instance.clone(),
5842            LoadedModel {
5843                instance: instance.clone(),
5844                model_ref: instance.model_ref().to_owned(),
5845                pinned_commit: None,
5846                transport: TransportConfig::inproc("dead-worker-test"),
5847                network_transport: TransportConfig::inproc("dead-worker-test"),
5848                service_handle: dead_worker(),
5849                client: InferenceClient::new(Arc::new(BoundaryRpcClient::new(
5850                    TransportConfig::inproc("dead-worker-test"),
5851                    Arc::new(BoundaryDialState::default()),
5852                ))),
5853                incarnation: "dead-incarnation".to_owned(),
5854                work_audience: "iw1/dead/dead".to_owned(),
5855                load_generation: 1,
5856                router: CellRouter::default(),
5857                load_state: Vec::new(),
5858                loaded_at: Instant::now(),
5859                last_used: Instant::now(),
5860                ttt_config: None,
5861                generation_defaults: crate::config::SamplingParams::default(),
5862            },
5863        );
5864        drop(cache);
5865
5866        let ctx = boundary_caller_ctx(chrono::Utc::now().timestamp() + 3600);
5867        let scope = InternalWorkScope::new("inference:GenerateStream", "infer");
5868        let error = match service
5869            .get_inference_client(instance.model_ref(), &ctx, scope)
5870            .await
5871        {
5872            Ok(_) => panic!("a dead worker cache entry must not mint work orders"),
5873            Err(error) => error,
5874        };
5875        assert!(
5876            error.to_string().contains("not running"),
5877            "dead-worker denial must name the liveness guard: {error:#}"
5878        );
5879        // The dead entry was evicted: the cache no longer holds it.
5880        assert!(!service.loaded_models.read().await.contains(&instance));
5881    }
5882
5883    #[test]
5884    fn running_worker_liveness_guard_passes() {
5885        // Positive control for the liveness guard itself: an entry whose
5886        // service handle reports running passes the guard.
5887        let worker = running_worker();
5888        assert!(
5889            worker.is_running(),
5890            "live thread fixture must report running"
5891        );
5892        let dead = dead_worker();
5893        assert!(!dead.is_running(), "dead fixture must report not-running");
5894        drop(worker);
5895    }
5896
5897    #[test]
5898    fn incarnation_handoff_validation_rejects_stale_or_mismatched() {
5899        // The model-side handoff validation ensures the reported handoff
5900        // matches THIS spawn attempt exactly. Exercised through the same
5901        // ensure-conditions the load path applies (generation/instance/
5902        // controller), via direct value checks mirroring model.rs load logic.
5903        let instance = selector_instance();
5904        let expected_generation: u64 = 3;
5905        let expected_controller = SigningKey::from_bytes(&[0x41; 32])
5906            .verifying_key()
5907            .to_bytes();
5908
5909        let handoff = crate::services::inference::IncarnationHandoff {
5910            instance_service_name: instance.service_name(),
5911            controller_pubkey: expected_controller,
5912            generation: expected_generation,
5913            incarnation: "aa11".to_owned(),
5914            audience: "iw1/x/aa11".to_owned(),
5915        };
5916        assert_eq!(handoff.generation, expected_generation);
5917        assert_eq!(handoff.instance_service_name, instance.service_name());
5918        assert_eq!(handoff.controller_pubkey, expected_controller);
5919
5920        let stale = crate::services::inference::IncarnationHandoff {
5921            generation: expected_generation - 1,
5922            ..handoff.clone()
5923        };
5924        assert_ne!(stale.generation, expected_generation, "stale handoff");
5925    }
5926
5927    fn boundary_caller_ctx(caller_exp: i64) -> EnvelopeContext {
5928        let caller_key = SigningKey::from_bytes(&BOUNDARY_CALLER_KEY);
5929        let now = chrono::Utc::now().timestamp();
5930        let claims = hyprstream_rpc::auth::Claims::new("alice".to_owned(), now, caller_exp)
5931            .with_tenant("fixture-tenant".to_owned())
5932            .with_cnf_jwk(caller_key.verifying_key().as_bytes())
5933            // Model's ingress MAC gate (inference_instance →
5934            // enforce_inference_mac) runs inside get_inference_client before
5935            // the worker binding is consulted; the caller must carry the same
5936            // clearance the real ingress path requires.
5937            .with_clearance(crate::services::inference::inference_object_label());
5938        EnvelopeContext::for_test_authenticated_subject_with_claims(
5939            hyprstream_rpc::envelope::Subject::new("alice"),
5940            "fixture-tenant",
5941            caller_key.verifying_key(),
5942            claims,
5943        )
5944    }
5945
5946    fn restart_acceptance_caller_ctx(
5947        subject: &str,
5948        caller_key: &SigningKey,
5949        tenant: &str,
5950    ) -> EnvelopeContext {
5951        let now = chrono::Utc::now().timestamp();
5952        let claims = hyprstream_rpc::auth::Claims::new(subject.to_owned(), now, now + 300)
5953            .with_tenant(tenant.to_owned())
5954            .with_cnf_jwk(caller_key.verifying_key().as_bytes())
5955            .with_clearance(crate::services::inference::inference_object_label());
5956        EnvelopeContext::for_test_authenticated_subject_with_claims(
5957            hyprstream_rpc::envelope::Subject::new(subject),
5958            tenant,
5959            caller_key.verifying_key(),
5960            claims,
5961        )
5962    }
5963
5964    fn resolve_gitdir_reference(
5965        reference_file: &std::path::Path,
5966        gitfile_format: bool,
5967    ) -> Result<std::path::PathBuf> {
5968        let contents = std::fs::read_to_string(reference_file)?;
5969        let reference = if gitfile_format {
5970            contents
5971                .strip_prefix("gitdir: ")
5972                .ok_or_else(|| anyhow!("fixture Git file is malformed"))?
5973        } else {
5974            contents.as_str()
5975        }
5976        .trim();
5977        anyhow::ensure!(
5978            !reference.is_empty() && !reference.contains(['\n', '\r']),
5979            "fixture Git link is malformed"
5980        );
5981        let path = std::path::PathBuf::from(reference);
5982        let resolved = if path.is_absolute() {
5983            path
5984        } else {
5985            reference_file
5986                .parent()
5987                .ok_or_else(|| anyhow!("fixture Git link has no parent"))?
5988                .join(path)
5989        };
5990        Ok(resolved.canonicalize()?)
5991    }
5992
5993    fn validate_restart_fixture_worktree_link(
5994        repo: &std::path::Path,
5995        worktree: &std::path::Path,
5996    ) -> Result<()> {
5997        let admin_worktree = repo.join("worktrees").join("main");
5998        let expected_gitfile = worktree.join(".git");
5999        anyhow::ensure!(
6000            expected_gitfile.is_file(),
6001            "fixture main worktree has no Git file"
6002        );
6003        anyhow::ensure!(
6004            resolve_gitdir_reference(&admin_worktree.join("gitdir"), false)?
6005                == expected_gitfile.canonicalize()?,
6006            "fixture repository main worktree does not name the isolated StoragePaths worktree"
6007        );
6008        anyhow::ensure!(
6009            resolve_gitdir_reference(&expected_gitfile, true)? == admin_worktree.canonicalize()?,
6010            "isolated StoragePaths worktree does not link back to fixture repository main"
6011        );
6012        Ok(())
6013    }
6014
6015    /// A guest-provisioned, local-only fixture for the real restart acceptance
6016    /// test.  The repository must be bare and its linked `main` worktree must
6017    /// already be the XDG-isolated path that `StoragePaths` resolves.  The test
6018    /// never clones, downloads, or copies model weights.
6019    struct RestartAcceptanceFixture {
6020        model: ModelService,
6021        model_key: SigningKey,
6022        model_ref: String,
6023        instance: InferenceInstanceId,
6024        _auth: RestartAuthServices,
6025    }
6026
6027    const RESTART_FIXTURE_ISSUER: &str = "http://127.0.0.1:6791";
6028
6029    /// Shared by the guest acceptance and the focused dispatch regression.
6030    /// Own the spawned services for as long as either test uses their clients.
6031    struct RestartAuthServices {
6032        model_key: SigningKey,
6033        policy_key: SigningKey,
6034        registry_key: SigningKey,
6035        model_token: String,
6036        policy_endpoint: String,
6037        registry_endpoint: String,
6038        policy_transport: TransportConfig,
6039        jwt_source: Arc<hyprstream_rpc::auth::ClusterKeySource>,
6040        _registry_handle: hyprstream_service::SpawnedService,
6041        _policy_handle: hyprstream_service::SpawnedService,
6042        _registry_base: tempfile::TempDir,
6043        _policy_base: tempfile::TempDir,
6044        _credentials: tempfile::TempDir,
6045    }
6046
6047    fn restart_acceptance_inproc_endpoint(kind: &str, tag: &str) -> (TransportConfig, String) {
6048        let transport = TransportConfig::inproc(format!("restart-acceptance-{kind}-{tag}"));
6049        let uri = transport.endpoint_string();
6050        (transport, uri)
6051    }
6052
6053    fn install_restart_fixture_auth(keys: &[SigningKey]) -> Result<()> {
6054        use hyprstream_rpc::crypto::CryptoPolicy;
6055        use hyprstream_rpc::envelope::{
6056            EnvelopeVerifyConfig, KeyedPqTrustStore, ResponseVerifyConfig,
6057        };
6058
6059        let _ = hyprstream_rpc::proof::admission::set_global_proof_replay_store(Box::new(
6060            hyprstream_rpc::proof::admission::InMemoryProofReplayStore::single_verifier_instance(
6061                10_000,
6062            ),
6063        ));
6064        if hyprstream_rpc::auth::global_credential_revocation_store().is_none() {
6065            let _ = hyprstream_rpc::auth::set_global_credential_revocation_store(Arc::new(
6066                hyprstream_rpc::auth::InMemoryCredentialRevocationStore::new(),
6067            ));
6068        }
6069        let mut pq_store = KeyedPqTrustStore::new();
6070        for key in keys {
6071            let pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(key);
6072            let pq_vk = hyprstream_rpc::crypto::pq::ml_dsa_vk_from_bytes(
6073                &hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk_bytes(&pq),
6074            )?;
6075            pq_store.bind(key.verifying_key().to_bytes(), &pq_vk);
6076        }
6077        let pq_store = Arc::new(pq_store);
6078        hyprstream_rpc::envelope::install_verify_config(EnvelopeVerifyConfig {
6079            policy: CryptoPolicy::Hybrid,
6080            pq_store: Some(pq_store.clone()),
6081        })?;
6082        hyprstream_rpc::envelope::install_response_verify_config(ResponseVerifyConfig {
6083            policy: CryptoPolicy::Hybrid,
6084            pq_store: Some(pq_store),
6085        })?;
6086        crate::mac::install_production_rpc_dispatch_pep()?;
6087        anyhow::ensure!(
6088            hyprstream_rpc::auth::mac::global_mac_dispatch_pep().is_some(),
6089            "restart fixture has no production dispatch PEP"
6090        );
6091        Ok(())
6092    }
6093
6094    fn restart_fixture_service_jwt(
6095        credentials: &tempfile::TempDir,
6096        name: &str,
6097        ca: &SigningKey,
6098        key: &SigningKey,
6099    ) -> Result<String> {
6100        let bootstrap = crate::auth::identity_store::BootstrapPubkey::for_service_key(key)?;
6101        crate::auth::service_jwt::issue_or_load_service_jwt(
6102            credentials.path(),
6103            name,
6104            ca,
6105            &bootstrap,
6106            RESTART_FIXTURE_ISSUER,
6107            chrono::Utc::now().timestamp(),
6108            Some(&crate::mac::dispatch_labels::BOOTSTRAP_SERVICE_CLEARANCE),
6109        )
6110    }
6111
6112    impl RestartAuthServices {
6113        async fn new(repo: &std::path::Path, model_name: &str) -> Result<Self> {
6114            use hyprstream_service::ServiceManager as _;
6115
6116            let tag = hex::encode(hyprstream_rpc::envelope::generate_nonce());
6117            let policy_key = SigningKey::from_bytes(&[0x91; 32]);
6118            let registry_key = SigningKey::from_bytes(&[0x92; 32]);
6119            let model_key = SigningKey::from_bytes(&[0x93; 32]);
6120            // The fourth key is anchored only for the wrong-holder regression:
6121            // its denial must be a credential binding failure, not a missing PQ anchor.
6122            let wrong_holder = SigningKey::from_bytes(&[0x94; 32]);
6123            install_restart_fixture_auth(&[
6124                policy_key.clone(),
6125                registry_key.clone(),
6126                model_key.clone(),
6127                wrong_holder,
6128            ])?;
6129            let ca =
6130                hyprstream_rpc::node_identity::derive_purpose_key(&policy_key, "hyprstream-jwt-v1");
6131            let ca_pq = hyprstream_rpc::node_identity::derive_mesh_mldsa_key(&ca);
6132            let jwt_source = Arc::new(
6133                hyprstream_rpc::auth::ClusterKeySource::new(
6134                    ca.verifying_key(),
6135                    RESTART_FIXTURE_ISSUER.to_owned(),
6136                )
6137                .with_ca_composite_key(hyprstream_rpc::crypto::pq::ml_dsa_sk_to_vk(&ca_pq)),
6138            );
6139            let credentials = tempfile::TempDir::new()?;
6140            let model_token = restart_fixture_service_jwt(&credentials, "model", &ca, &model_key)?;
6141            let registry_token =
6142                restart_fixture_service_jwt(&credentials, "registry", &ca, &registry_key)?;
6143            for (key, name) in [(&policy_key, "policy"), (&registry_key, "registry")] {
6144                hyprstream_service::global_trust_store().insert(
6145                    key.verifying_key(),
6146                    hyprstream_service::Attestation {
6147                        scopes: std::iter::once(name.to_owned()).collect(),
6148                        subject: Some(format!("service:{name}")),
6149                        jwt: None,
6150                        expires_at: 0,
6151                        attested_by: None,
6152                    },
6153                );
6154            }
6155
6156            let policy_base = tempfile::TempDir::new()?;
6157            let policy_manager = Arc::new(crate::auth::PolicyManager::permissive().await?);
6158            let (policy_transport, policy_endpoint) =
6159                restart_acceptance_inproc_endpoint("policy", &tag);
6160            let policy_service = crate::services::PolicyService::new(
6161                policy_manager,
6162                Arc::new(policy_key.clone()),
6163                crate::config::TokenConfig::default(),
6164                Arc::new(RwLock::new(git2db::Git2DB::open(policy_base.path()).await?)),
6165                policy_transport.clone(),
6166            )
6167            .with_jwt_key_source(jwt_source.clone());
6168            crate::services::restart_diag::phase("policy.spawn.enter");
6169            let manager = hyprstream_service::InprocManager::new();
6170            let policy_handle = manager.spawn(Box::new(policy_service)).await?;
6171            crate::services::restart_diag::phase("policy.spawn.done");
6172
6173            let registry_base = tempfile::TempDir::new()?;
6174            {
6175                let mut registry_store = git2db::Git2DB::open(registry_base.path()).await?;
6176                registry_store
6177                    .register(git2db::RepoId::new())
6178                    .name(model_name)
6179                    .worktree_path(repo)
6180                    .url(String::new())
6181                    .exec()
6182                    .await?;
6183            }
6184            let (registry_transport, registry_endpoint) =
6185                restart_acceptance_inproc_endpoint("registry", &tag);
6186            let registry_policy = PolicyClient::for_local_endpoint_bootstrap(
6187                &policy_endpoint,
6188                registry_key.clone(),
6189                policy_key.verifying_key(),
6190                Some(registry_token),
6191            )?;
6192            let registry_service = crate::services::RegistryService::new(
6193                registry_base.path(),
6194                registry_policy,
6195                registry_transport,
6196                registry_key.clone(),
6197            )
6198            .await?
6199            .with_expected_audience(RESTART_FIXTURE_ISSUER.to_owned())
6200            .with_jwt_key_source(jwt_source.clone());
6201            crate::services::restart_diag::phase("registry.spawn.enter");
6202            let registry_handle = manager.spawn(Box::new(registry_service)).await?;
6203            crate::services::restart_diag::phase("registry.spawn.done");
6204            Ok(Self {
6205                model_key,
6206                policy_key,
6207                registry_key,
6208                model_token,
6209                policy_endpoint,
6210                registry_endpoint,
6211                policy_transport,
6212                jwt_source,
6213                _registry_handle: registry_handle,
6214                _policy_handle: policy_handle,
6215                _registry_base: registry_base,
6216                _policy_base: policy_base,
6217                _credentials: credentials,
6218            })
6219        }
6220
6221        fn model_registry_client(&self) -> Result<RegistryClient> {
6222            RegistryClient::for_local_endpoint_bootstrap(
6223                &self.registry_endpoint,
6224                self.model_key.clone(),
6225                self.registry_key.verifying_key(),
6226                Some(self.model_token.clone()),
6227            )
6228        }
6229
6230        fn model_policy_client(&self) -> Result<PolicyClient> {
6231            PolicyClient::for_local_endpoint_bootstrap(
6232                &self.policy_endpoint,
6233                self.model_key.clone(),
6234                self.policy_key.verifying_key(),
6235                Some(self.model_token.clone()),
6236            )
6237        }
6238    }
6239
6240    #[test]
6241    fn restart_acceptance_inproc_endpoints_match_client_dial_names() {
6242        for kind in ["policy", "registry"] {
6243            let (server, uri) = restart_acceptance_inproc_endpoint(kind, "fixture");
6244            assert_eq!(
6245                server.endpoint,
6246                TransportConfig::from_endpoint(&uri).endpoint,
6247                "{kind} server registration and client dial must use the same name"
6248            );
6249            assert_eq!(uri, format!("inproc://restart-acceptance-{kind}-fixture"));
6250        }
6251    }
6252
6253    struct ClassicalRestartSigner(SigningKey);
6254
6255    #[async_trait::async_trait]
6256    impl hyprstream_rpc::transport_traits::Signer for ClassicalRestartSigner {
6257        fn pubkey(&self) -> [u8; 32] {
6258            self.0.verifying_key().to_bytes()
6259        }
6260
6261        async fn sign(&self, bytes: &[u8]) -> Result<[u8; 64]> {
6262            use ed25519_dalek::Signer as _;
6263            Ok(self.0.sign(bytes).to_bytes())
6264        }
6265    }
6266
6267    fn assert_restart_dispatch_denied<T>(result: Result<T>, case: &str) -> Result<()> {
6268        let error = result
6269            .err()
6270            .ok_or_else(|| anyhow!("{case} reached the Registry handler"))?;
6271        anyhow::ensure!(
6272            error
6273                .to_string()
6274                .contains(hyprstream_rpc::service::dispatch::DISPATCH_DENIED),
6275            "{case} failed outside the uniform dispatch boundary: {error}"
6276        );
6277        Ok(())
6278    }
6279
6280    /// Run by exact filter in its own test process: request/response verify
6281    /// configs are first-write globals, while the dispatch PEP is swappable.
6282    #[tokio::test]
6283    #[ignore = "requires first-write verify-config globals; run with --exact --test-threads=1"]
6284    async fn restart_fixture_authenticated_hybrid_get_by_name_requires_model_holder() -> Result<()>
6285    {
6286        use hyprstream_rpc::rpc_client::RpcClientImpl;
6287        use hyprstream_rpc::transport::in_memory::InMemoryTransport;
6288
6289        let repo = tempfile::TempDir::new()?;
6290        let model_name = "restart-auth-regression";
6291        let auth = RestartAuthServices::new(repo.path(), model_name).await?;
6292        let model = auth.model_registry_client()?;
6293        let policy_check = hyprstream_rpc_std::policy_client::PolicyCheck {
6294            subject: "service:model".to_owned(),
6295            domain: "*".to_owned(),
6296            resource: "registry:*".to_owned(),
6297            operation: "query".to_owned(),
6298        };
6299        anyhow::ensure!(
6300            auth.model_policy_client()?.check(&policy_check).await?,
6301            "model holder must reach real Policy dispatch"
6302        );
6303        let found = model.get_by_name(model_name).await?;
6304        anyhow::ensure!(
6305            found.name == model_name,
6306            "the authenticated model lookup returned the wrong repository"
6307        );
6308        anyhow::ensure!(
6309            model
6310                .list()
6311                .await?
6312                .iter()
6313                .any(|entry| entry.name == model_name),
6314            "Registry-to-Policy mediation lost the authenticated model caller"
6315        );
6316
6317        let client = |key: SigningKey, token: Option<String>| {
6318            RegistryClient::for_local_endpoint_bootstrap(
6319                &auth.registry_endpoint,
6320                key,
6321                auth.registry_key.verifying_key(),
6322                token,
6323            )
6324        };
6325        let missing = client(auth.model_key.clone(), None)?
6326            .get_by_name(model_name)
6327            .await;
6328        assert_restart_dispatch_denied(missing, "tokenless model lookup")?;
6329        let wrong_holder = client(
6330            SigningKey::from_bytes(&[0x94; 32]),
6331            Some(auth.model_token.clone()),
6332        )?
6333        .get_by_name(model_name)
6334        .await;
6335        assert_restart_dispatch_denied(wrong_holder, "wrong holder with model token")?;
6336
6337        let processor = hyprstream_rpc::dial::lookup_inproc(
6338            auth.registry_endpoint
6339                .strip_prefix("inproc://")
6340                .ok_or_else(|| anyhow!("fixture Registry endpoint is not in-process"))?,
6341        )
6342        .ok_or_else(|| anyhow!("fixture Registry handle was not retained"))?;
6343        let classical = RpcClientImpl::new(
6344            ClassicalRestartSigner(auth.model_key.clone()),
6345            InMemoryTransport::new(processor),
6346            Some(auth.registry_key.verifying_key()),
6347        )
6348        .with_default_jwt(auth.model_token.clone());
6349        let classical = RegistryClient::new(Arc::new(classical));
6350        let classical_error = classical
6351            .get_by_name(model_name)
6352            .await
6353            .err()
6354            .ok_or_else(|| anyhow!("classical signer reached Registry"))?;
6355        anyhow::ensure!(
6356            classical_error
6357                .to_string()
6358                .contains("mandatory Hybrid suite requires an ML-DSA-65 signer key"),
6359            "classical signer failed for an unrelated reason: {classical_error}"
6360        );
6361
6362        // This signer has its own valid model certificate, but no PQ anchor in
6363        // the process verifier. Credential authority cannot fill that gap.
6364        let unanchored_key = SigningKey::from_bytes(&[0x95; 32]);
6365        let ca = hyprstream_rpc::node_identity::derive_purpose_key(
6366            &auth.policy_key,
6367            "hyprstream-jwt-v1",
6368        );
6369        let unanchored_credentials = tempfile::TempDir::new()?;
6370        let unanchored_token =
6371            restart_fixture_service_jwt(&unanchored_credentials, "model", &ca, &unanchored_key)?;
6372        let unanchored = client(unanchored_key, Some(unanchored_token))?
6373            .get_by_name(model_name)
6374            .await;
6375        let unanchored_error = unanchored
6376            .err()
6377            .ok_or_else(|| anyhow!("unanchored PQ signer reached Registry"))?;
6378        anyhow::ensure!(
6379            unanchored_error.to_string() == "registry envelope admission failed",
6380            "unanchored PQ signer failed outside envelope admission: {unanchored_error:#}"
6381        );
6382        anyhow::ensure!(
6383            unanchored_error.chain().any(|cause| matches!(
6384                cause.downcast_ref::<hyprstream_rpc::EnvelopeError>(),
6385                Some(hyprstream_rpc::EnvelopeError::PqSignatureInvalid(reason))
6386                    if reason == "mandatory Hybrid suite requires an anchored ML-DSA-65 signer key"
6387            )),
6388            "unanchored PQ signer failed for an unrelated admission reason: {unanchored_error:#}"
6389        );
6390        Ok(())
6391    }
6392
6393    impl RestartAcceptanceFixture {
6394        async fn from_guest_env() -> Result<Self> {
6395            let repo = std::path::PathBuf::from(std::env::var(
6396                "HYPRSTREAM_RESTART_ACCEPTANCE_MODEL_REPO",
6397            ).map_err(|_| anyhow!(
6398                "HYPRSTREAM_RESTART_ACCEPTANCE_MODEL_REPO must name the sealed local bare model repository"
6399            ))?);
6400            anyhow::ensure!(
6401                repo.is_dir(),
6402                "restart acceptance model repository is not a directory"
6403            );
6404            let model_name = std::env::var("HYPRSTREAM_RESTART_ACCEPTANCE_MODEL_NAME")
6405                .unwrap_or_else(|_| "restart-acceptance-fixture".to_owned());
6406            let instance_name = std::env::var("HYPRSTREAM_INSTANCE").map_err(|_| {
6407                anyhow!("HYPRSTREAM_INSTANCE must be set to an isolated synthetic-guest namespace")
6408            })?;
6409            anyhow::ensure!(
6410                instance_name.starts_with("restart-acceptance-"),
6411                "HYPRSTREAM_INSTANCE must use the restart-acceptance-* isolated namespace"
6412            );
6413
6414            let storage = crate::storage::StoragePaths::new()?;
6415            let worktree = storage.worktree_path(&model_name, "main")?;
6416            anyhow::ensure!(worktree.is_dir(), "fixture main worktree is absent");
6417            anyhow::ensure!(
6418                worktree.join("config.json").is_file()
6419                    && worktree.join("tokenizer.json").is_file()
6420                    && std::fs::read_dir(&worktree)?.flatten().any(|entry| {
6421                        entry
6422                            .file_name()
6423                            .to_string_lossy()
6424                            .ends_with(".safetensors")
6425                    }),
6426                "fixture must provide config.json, tokenizer.json, and safetensors weights"
6427            );
6428            validate_restart_fixture_worktree_link(&repo, &worktree)?;
6429
6430            let _ = hyprstream_rpc::moq_event::init_global_moq_event_origin(
6431                hyprstream_rpc::moq_event::MoqEventOrigin::new(),
6432            );
6433            hyprstream_rpc::registry::init(hyprstream_rpc::registry::EndpointMode::Inproc, None);
6434            anyhow::ensure!(
6435                hyprstream_rpc::registry::global().mode()
6436                    == hyprstream_rpc::registry::EndpointMode::Inproc,
6437                "restart fixture endpoint registry must use Inproc mode"
6438            );
6439            let auth = RestartAuthServices::new(&repo, &model_name).await?;
6440            let mut config = ModelServiceConfig::default();
6441            config.inference_deployment.compute = InferenceCompute::Cpu;
6442            let model = ModelService::new(
6443                config,
6444                auth.model_key.clone(),
6445                auth.model_policy_client()?,
6446                auth.model_registry_client()?,
6447                TransportConfig::inproc(format!(
6448                    "restart-acceptance-model-{}",
6449                    hex::encode(hyprstream_rpc::envelope::generate_nonce())
6450                )),
6451                auth.policy_transport.clone(),
6452            )
6453            .await?
6454            .with_expected_audience(RESTART_FIXTURE_ISSUER.to_owned())
6455            .with_jwt_key_source(auth.jwt_source.clone());
6456            let model_ref = format!("{model_name}:main");
6457            let instance = InferenceInstanceId::new("restart-acceptance-tenant", &model_ref, 0)?;
6458            Ok(Self {
6459                model,
6460                model_key: auth.model_key.clone(),
6461                model_ref,
6462                instance,
6463                _auth: auth,
6464            })
6465        }
6466    }
6467
6468    #[test]
6469    fn restart_fixture_gitdir_backlink_requires_expected_checkout() -> Result<()> {
6470        fn git(args: &[&str]) -> Result<()> {
6471            let status = std::process::Command::new("git")
6472                .args(args)
6473                .status()
6474                .map_err(|error| anyhow!("temporary Git fixture command did not start: {error}"))?;
6475            anyhow::ensure!(
6476                status.success(),
6477                "temporary Git fixture command failed: git {args:?}"
6478            );
6479            Ok(())
6480        }
6481
6482        let temp = tempfile::tempdir()?;
6483        let source = temp.path().join("source");
6484        let repo = temp.path().join("fixture.git");
6485        let expected_checkout = temp.path().join("isolated/models/worktrees/main");
6486        let other_checkout = temp.path().join("other-worktree");
6487        let source_arg = source
6488            .to_str()
6489            .ok_or_else(|| anyhow!("temporary source path is not UTF-8"))?;
6490        let repo_arg = repo
6491            .to_str()
6492            .ok_or_else(|| anyhow!("temporary bare repository path is not UTF-8"))?;
6493        let expected_arg = expected_checkout
6494            .to_str()
6495            .ok_or_else(|| anyhow!("temporary expected checkout path is not UTF-8"))?;
6496        let other_arg = other_checkout
6497            .to_str()
6498            .ok_or_else(|| anyhow!("temporary mismatched checkout path is not UTF-8"))?;
6499
6500        git(&["init", "-b", "main", source_arg])?;
6501        git(&[
6502            "-C",
6503            source_arg,
6504            "config",
6505            "user.name",
6506            "restart-fixture-test",
6507        ])?;
6508        git(&[
6509            "-C",
6510            source_arg,
6511            "config",
6512            "user.email",
6513            "restart-fixture-test@example.invalid",
6514        ])?;
6515        std::fs::write(source.join("fixture.txt"), "fixture\n")?;
6516        git(&["-C", source_arg, "add", "fixture.txt"])?;
6517        git(&["-C", source_arg, "commit", "-m", "temporary fixture"])?;
6518        git(&["clone", "--bare", source_arg, repo_arg])?;
6519        std::fs::create_dir_all(
6520            expected_checkout
6521                .parent()
6522                .ok_or_else(|| anyhow!("temporary expected checkout has no parent"))?,
6523        )?;
6524        git(&[
6525            "--git-dir",
6526            repo_arg,
6527            "worktree",
6528            "add",
6529            expected_arg,
6530            "main",
6531        ])?;
6532        git(&[
6533            "--git-dir",
6534            repo_arg,
6535            "worktree",
6536            "add",
6537            "-b",
6538            "other",
6539            other_arg,
6540            "main",
6541        ])?;
6542
6543        validate_restart_fixture_worktree_link(&repo, &expected_checkout)?;
6544        let mismatch = match validate_restart_fixture_worktree_link(&repo, &other_checkout) {
6545            Ok(()) => anyhow::bail!("a different linked checkout must be rejected"),
6546            Err(error) => error,
6547        };
6548        anyhow::ensure!(
6549            mismatch
6550                .to_string()
6551                .contains("does not name the isolated StoragePaths worktree"),
6552            "mismatched checkout error must identify the administrative backlink"
6553        );
6554        Ok(())
6555    }
6556
6557    fn assert_restart_order_holder(
6558        order: &str,
6559        controller: &SigningKey,
6560        audience: &str,
6561        subject: &str,
6562        caller_key: &SigningKey,
6563    ) -> Result<()> {
6564        let claims = verify_internal_work(
6565            order,
6566            &controller.verifying_key(),
6567            audience,
6568            chrono::Utc::now().timestamp(),
6569        )?;
6570        anyhow::ensure!(claims.sub == subject && claims.caller.sub == subject);
6571        anyhow::ensure!(claims.tenant == "restart-acceptance-tenant");
6572        anyhow::ensure!(claims.resource == "inference:HasLora" && claims.operation == "query");
6573        anyhow::ensure!(
6574            claims.owner_did.as_deref()
6575                == Some(
6576                    hyprstream_rpc::identity::Did::from_ed25519(
6577                        &caller_key.verifying_key().to_bytes(),
6578                    )
6579                    .as_str()
6580                ),
6581            "work order owner DID must remain bound to the verified caller"
6582        );
6583        Ok(())
6584    }
6585
6586    /// Test-only observer for the one server-side denial class that identifies
6587    /// an old worker order at its replacement.  It retains only a boolean: no
6588    /// bearer, subject, audience, request ID, or formatted event is kept.
6589    #[derive(Clone)]
6590    struct RestartAudienceDenialProbe {
6591        seen: Arc<std::sync::atomic::AtomicBool>,
6592    }
6593
6594    impl RestartAudienceDenialProbe {
6595        fn install() -> Result<Arc<std::sync::atomic::AtomicBool>> {
6596            use tracing_subscriber::prelude::*;
6597
6598            let seen = Arc::new(std::sync::atomic::AtomicBool::new(false));
6599            let subscriber = tracing_subscriber::registry().with(Self {
6600                seen: Arc::clone(&seen),
6601            });
6602            // The prescribed invocation is one ignored test in its own
6603            // process (`--exact --test-threads=1`). A global dispatcher is
6604            // necessary because the real worker dispatches on its own service
6605            // task; a thread-local subscriber would not observe that task.
6606            tracing::subscriber::set_global_default(subscriber).map_err(|_| {
6607                anyhow!(
6608                    "restart acceptance requires an isolated test process without a preinstalled tracing subscriber"
6609                )
6610            })?;
6611            Ok(seen)
6612        }
6613    }
6614
6615    struct RestartAudienceDenialVisitor<'a> {
6616        seen: &'a std::sync::atomic::AtomicBool,
6617    }
6618
6619    impl RestartAudienceDenialVisitor<'_> {
6620        fn record_message(&self, field: &tracing::field::Field, message: &str) {
6621            if field.name() == "message"
6622                && message.contains("claims verification failed")
6623                && message.contains("audience")
6624            {
6625                self.seen.store(true, std::sync::atomic::Ordering::SeqCst);
6626            }
6627        }
6628    }
6629
6630    impl tracing::field::Visit for RestartAudienceDenialVisitor<'_> {
6631        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
6632            self.record_message(field, &format!("{value:?}"));
6633        }
6634
6635        fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
6636            self.record_message(field, value);
6637        }
6638    }
6639
6640    impl<S> tracing_subscriber::Layer<S> for RestartAudienceDenialProbe
6641    where
6642        S: tracing::Subscriber + for<'lookup> tracing_subscriber::registry::LookupSpan<'lookup>,
6643    {
6644        fn on_event(
6645            &self,
6646            event: &tracing::Event<'_>,
6647            _context: tracing_subscriber::layer::Context<'_, S>,
6648        ) {
6649            if event.metadata().target() == "hyprstream_rpc::service::dispatch" {
6650                event.record(&mut RestartAudienceDenialVisitor { seen: &self.seen });
6651            }
6652        }
6653    }
6654
6655    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6656    #[ignore = "requires the sealed local CPU-model repository and isolated XDG guest fixture"]
6657    async fn real_cpu_worker_restart_rejects_old_order_and_preserves_holders() -> Result<()> {
6658        crate::services::restart_diag::enable();
6659        crate::services::restart_diag::phase("test.enter");
6660        let audience_denial = RestartAudienceDenialProbe::install()?;
6661        crate::services::restart_diag::phase("fixture.enter");
6662        let fixture = tokio::time::timeout(
6663            std::time::Duration::from_secs(180),
6664            RestartAcceptanceFixture::from_guest_env(),
6665        )
6666        .await
6667        .map_err(|_| anyhow!("RESTART_DIAG_PHASE_TIMEOUT=fixture"))??;
6668        crate::services::restart_diag::phase("fixture.done");
6669        let caller_a_key = SigningKey::from_bytes(&[0xA1; 32]);
6670        let caller_b_key = SigningKey::from_bytes(&[0xB1; 32]);
6671        let caller_a = restart_acceptance_caller_ctx(
6672            "restart-caller-a",
6673            &caller_a_key,
6674            "restart-acceptance-tenant",
6675        );
6676        let caller_b = restart_acceptance_caller_ctx(
6677            "restart-caller-b",
6678            &caller_b_key,
6679            "restart-acceptance-tenant",
6680        );
6681        let scope = ModelService::inference_scope("hasLora", "query");
6682
6683        // A is a real CPU Model load, then the normal Model forwarding path.
6684        crate::services::restart_diag::phase("worker_a.load.enter");
6685        tokio::time::timeout(
6686            std::time::Duration::from_secs(600),
6687            fixture.model.load_model(&fixture.instance, None, None),
6688        )
6689        .await
6690        .map_err(|_| anyhow!("RESTART_DIAG_PHASE_TIMEOUT=worker_a.load"))??;
6691        crate::services::restart_diag::phase("worker_a.load.done");
6692        let (audience_a, incarnation_a) = {
6693            let cache = fixture.model.loaded_models.read().await;
6694            let loaded = cache
6695                .peek(&fixture.instance)
6696                .ok_or_else(|| anyhow!("worker A missing after successful load"))?;
6697            (loaded.work_audience.clone(), loaded.incarnation.clone())
6698        };
6699        let a_client = fixture
6700            .model
6701            .get_inference_client(&fixture.model_ref, &caller_a, scope.clone())
6702            .await?;
6703        let mut old_order = fixture
6704            .model
6705            .take_test_work_order()
6706            .await
6707            .ok_or_else(|| anyhow!("test-only capture missed worker A work order"))?;
6708        assert_restart_order_holder(
6709            &old_order,
6710            &fixture.model_key,
6711            &audience_a,
6712            "restart-caller-a",
6713            &caller_a_key,
6714        )?;
6715        crate::services::restart_diag::phase("worker_a.call.enter");
6716        let _ = a_client.has_lora().await?;
6717        crate::services::restart_diag::phase("worker_a.call.done");
6718
6719        // Model's actual unload owns the cached SpawnedService stop/join. A
6720        // subsequent load creates B through the same production init closure.
6721        crate::services::restart_diag::phase("worker_a.unload.enter");
6722        fixture.model.unload_model(&fixture.instance).await?;
6723        crate::services::restart_diag::phase("worker_a.unload.done");
6724        anyhow::ensure!(
6725            !fixture
6726                .model
6727                .loaded_models
6728                .read()
6729                .await
6730                .contains(&fixture.instance),
6731            "unload must remove worker A before B is created"
6732        );
6733        crate::services::restart_diag::phase("worker_b.load.enter");
6734        tokio::time::timeout(
6735            std::time::Duration::from_secs(600),
6736            fixture.model.load_model(&fixture.instance, None, None),
6737        )
6738        .await
6739        .map_err(|_| anyhow!("RESTART_DIAG_PHASE_TIMEOUT=worker_b.load"))??;
6740        crate::services::restart_diag::phase("worker_b.load.done");
6741        let (audience_b, incarnation_b, worker_b) = {
6742            let cache = fixture.model.loaded_models.read().await;
6743            let loaded = cache
6744                .peek(&fixture.instance)
6745                .ok_or_else(|| anyhow!("worker B missing after successful reload"))?;
6746            (
6747                loaded.work_audience.clone(),
6748                loaded.incarnation.clone(),
6749                loaded.client.clone(),
6750            )
6751        };
6752        anyhow::ensure!(
6753            audience_a != audience_b && incarnation_a != incarnation_b,
6754            "reload must install a fresh worker-generated incarnation"
6755        );
6756
6757        // This is the causal assertion: B receives A's real captured bearer
6758        // through generated dispatch. Its client response stays uniformly
6759        // opaque, while the in-process test probe records only the server-side
6760        // audience-denial class before the `hasLora` handler can succeed.
6761        crate::services::restart_diag::phase("old_order.denial.enter");
6762        let replay_error = match worker_b
6763            .with_bearer(std::mem::take(&mut *old_order))
6764            .has_lora()
6765            .await
6766        {
6767            Ok(_) => anyhow::bail!("worker B accepted worker A's order"),
6768            Err(error) => error,
6769        };
6770        anyhow::ensure!(
6771            replay_error.to_string() == hyprstream_rpc::service::dispatch::DISPATCH_DENIED,
6772            "old order must receive the uniform dispatch denial"
6773        );
6774        anyhow::ensure!(
6775            audience_denial.load(std::sync::atomic::Ordering::SeqCst),
6776            "replacement worker must classify the old order as an audience mismatch"
6777        );
6778        crate::services::restart_diag::phase("old_order.denial.done");
6779
6780        // Fresh A work succeeds at B and carries the same original holder.
6781        let b_client_for_a = fixture
6782            .model
6783            .get_inference_client(&fixture.model_ref, &caller_a, scope.clone())
6784            .await?;
6785        let fresh_a_order = fixture
6786            .model
6787            .take_test_work_order()
6788            .await
6789            .ok_or_else(|| anyhow!("test-only capture missed worker B fresh order"))?;
6790        assert_restart_order_holder(
6791            &fresh_a_order,
6792            &fixture.model_key,
6793            &audience_b,
6794            "restart-caller-a",
6795            &caller_a_key,
6796        )?;
6797        crate::services::restart_diag::phase("fresh_a.call.enter");
6798        let _ = b_client_for_a.has_lora().await?;
6799        crate::services::restart_diag::phase("fresh_a.call.done");
6800
6801        // Caller B gets a different holder-bound order; it cannot inherit A's
6802        // subject, caller snapshot, or pairwise owner DID.
6803        let b_client_for_b = fixture
6804            .model
6805            .get_inference_client(&fixture.model_ref, &caller_b, scope)
6806            .await?;
6807        let fresh_b_order = fixture
6808            .model
6809            .take_test_work_order()
6810            .await
6811            .ok_or_else(|| anyhow!("test-only capture missed caller B order"))?;
6812        assert_restart_order_holder(
6813            &fresh_b_order,
6814            &fixture.model_key,
6815            &audience_b,
6816            "restart-caller-b",
6817            &caller_b_key,
6818        )?;
6819        anyhow::ensure!(
6820            fresh_a_order != fresh_b_order,
6821            "distinct caller requests require distinct work orders"
6822        );
6823        crate::services::restart_diag::phase("fresh_b.call.enter");
6824        let _ = b_client_for_b.has_lora().await?;
6825        crate::services::restart_diag::phase("fresh_b.call.done");
6826        crate::services::restart_diag::phase("holders.done");
6827
6828        crate::services::restart_diag::phase("worker_b.unload.enter");
6829        fixture.model.unload_model(&fixture.instance).await?;
6830        crate::services::restart_diag::phase("worker_b.unload.done");
6831        Ok(())
6832    }
6833
6834    #[tokio::test]
6835    async fn internal_work_mint_binds_instance_tenant_model_operation_and_caller() {
6836        let service = selector_model_service().await;
6837        let instance = selector_instance();
6838        let scope = ModelService::inference_scope("generateStream", "infer");
6839        let caller_exp = chrono::Utc::now().timestamp() + 3600;
6840        let ctx = boundary_caller_ctx(caller_exp);
6841        let token = service
6842            .mint_internal_work_token(&instance, &ctx, &scope, &audience_fixture())
6843            .unwrap_or_else(|e| panic!("mint failed: {e}"));
6844
6845        let model_key = SigningKey::from_bytes(&[0x41; 32]);
6846        let now = chrono::Utc::now().timestamp();
6847        let claims =
6848            verify_internal_work(&token, &model_key.verifying_key(), &audience_fixture(), now)
6849                .unwrap_or_else(|e| {
6850                    panic!("minted order must verify against the pinned controller: {e}")
6851                });
6852        assert_eq!(claims.sub, "alice");
6853        assert_eq!(claims.tenant, "fixture-tenant");
6854        assert_eq!(claims.model, "fixture-model:main");
6855        assert_eq!(claims.resource, "inference:GenerateStream");
6856        assert_eq!(claims.operation, "infer");
6857        assert!(
6858            claims.exp <= caller_exp && claims.exp <= now + MAX_LIFETIME_SECS,
6859            "exp must be capped by the caller credential and the hard bound"
6860        );
6861        assert!(
6862            claims.exp > now,
6863            "a live caller credential must mint a live work order"
6864        );
6865        assert!(!claims.jti.is_empty());
6866        let expected_x = base64::engine::general_purpose::URL_SAFE_NO_PAD
6867            .encode(model_key.verifying_key().as_bytes());
6868        assert_eq!(
6869            claims.cnf.jwk.as_ref().map(|j| j.x.as_str()),
6870            Some(expected_x.as_str()),
6871            "cnf must bind the pinned controller key"
6872        );
6873        // The original caller's pairwise DID (from the verified caller envelope
6874        // key) rides for ledger attribution.
6875        let caller_key = SigningKey::from_bytes(&BOUNDARY_CALLER_KEY);
6876        assert_eq!(
6877            claims.owner_did.as_deref(),
6878            Some(
6879                hyprstream_rpc::identity::Did::from_ed25519(&caller_key.verifying_key().to_bytes())
6880                    .as_str()
6881            )
6882        );
6883        // The caller snapshot carries the verified claims WITHOUT the raw
6884        // bearer string (serde skip).
6885        assert_eq!(claims.caller.sub, "alice");
6886        assert!(claims.caller.token.is_none());
6887    }
6888
6889    #[tokio::test]
6890    async fn internal_work_mint_requires_verified_caller_identity_and_live_expiry() {
6891        let service = selector_model_service().await;
6892        let instance = selector_instance();
6893        // Text-payload variant grammar: the payload IS the coordinate.
6894        let scope = InternalWorkScope::new("inference:probe/adapters/k3", "write");
6895
6896        // No verified caller claims (e.g. a bearer-less service caller): never
6897        // minted — the deny outcome matches the pre-boundary behavior.
6898        let bare = EnvelopeContext::for_test_authenticated_subject(
6899            hyprstream_rpc::envelope::Subject::anonymous(),
6900            SigningKey::from_bytes(&BOUNDARY_CALLER_KEY).verifying_key(),
6901        );
6902        assert!(
6903            service
6904                .mint_internal_work_token(&instance, &bare, &scope, &audience_fixture())
6905                .is_err(),
6906            "internal work without a verified caller identity must not mint"
6907        );
6908
6909        // Expired caller credential: never minted.
6910        let expired = boundary_caller_ctx(chrono::Utc::now().timestamp() - 10);
6911        assert!(
6912            service
6913                .mint_internal_work_token(&instance, &expired, &scope, &audience_fixture())
6914                .is_err(),
6915            "internal work must not outlive an expired caller credential"
6916        );
6917    }
6918}